From ffee068fd029afe0d4ceaecb582f537b8cf7e865 Mon Sep 17 00:00:00 2001
From: Matthieu Gautier <mgautier@kymeria.fr>
Date: Wed, 22 Feb 2017 10:44:01 +0100
Subject: [PATCH 01/25] Split the too long kiwix-build.py file into several
 smaller ones.

---
 dependencies.py     | 229 +++++++++++++
 dependency_utils.py | 292 +++++++++++++++++
 kiwix-build.py      | 767 ++++++--------------------------------------
 utils.py            |  90 ++++++
 4 files changed, 708 insertions(+), 670 deletions(-)
 create mode 100644 dependencies.py
 create mode 100644 dependency_utils.py
 create mode 100644 utils.py

diff --git a/dependencies.py b/dependencies.py
new file mode 100644
index 0000000..f630e71
--- /dev/null
+++ b/dependencies.py
@@ -0,0 +1,229 @@
+import shutil
+
+from dependency_utils import (
+    Dependency,
+    ReleaseDownload,
+    GitClone,
+    MakeBuilder,
+    CMakeBuilder,
+    MesonBuilder)
+
+from utils import Remotefile, pj, SkipCommand
+
+# *************************************
+# Missing dependencies
+# Is this ok to assume that those libs
+# exist in your "distri" (linux/mac) ?
+# If not, we need to compile them here
+# *************************************
+# Zlib
+# LZMA
+# aria2
+# Argtable
+# MSVirtual
+# Android
+# libiconv
+# gettext
+# *************************************
+
+
+class zlib(Dependency):
+    name = 'zlib'
+    version = '1.2.8'
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('zlib-1.2.8.tar.gz',
+                             '36658cb768a54c1d4dec43c3116c27ed893e88b02ecfcb44f2166f9c0b7f2a0d')
+        patches = ['zlib_std_libname.patch']
+
+    class Builder(CMakeBuilder):
+        @property
+        def configure_option(self):
+            return "-DINSTALL_PKGCONFIG_DIR={}".format(pj(self.buildEnv.install_dir, self.buildEnv.libprefix, 'pkgconfig'))
+
+
+class UUID(Dependency):
+    name = 'uuid'
+    version = "1.43.4"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('e2fsprogs-libs-1.43.4.tar.gz',
+                             'eed4516325768255c9745e7b82c9d7d0393abce302520a5b2cde693204b0e419',
+                             'https://www.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v1.43.4/e2fsprogs-libs-1.43.4.tar.gz')
+        extract_dir = 'e2fsprogs-libs-1.43.4'
+
+    class Builder(MakeBuilder):
+        configure_option = "--enable-libuuid"
+        configure_env = {'_format_CFLAGS': "{env.CFLAGS} -fPIC"}
+        make_target = 'libs'
+        make_install_target = 'install-libs'
+
+
+class Xapian(Dependency):
+    name = "xapian-core"
+    version = "1.4.2"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('xapian-core-1.4.2.tar.xz',
+                             'aec2c4352998127a2f2316218bf70f48cef0a466a87af3939f5f547c5246e1ce')
+        patches = ["xapian_pkgconfig.patch"]
+
+    class Builder(MakeBuilder):
+        configure_option = "--disable-sse --disable-backend-inmemory --disable-documentation"
+        dynamic_configure_option = "--enable-shared --disable-static"
+        static_configure_option = "--enable-static --disable-shared"
+        configure_env = {'_format_LDFLAGS': "-L{buildEnv.install_dir}/{buildEnv.libprefix}",
+                         '_format_CXXFLAGS': "-I{buildEnv.install_dir}/include"}
+
+    @property
+    def dependencies(self):
+        deps = ['zlib']
+        if self.buildEnv.build_target == 'win32':
+            return deps
+        return deps + ['UUID']
+
+
+class CTPP2(Dependency):
+    name = "ctpp2"
+    version = "2.8.3"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('ctpp2-2.8.3.tar.gz',
+                             'a83ffd07817adb575295ef40fbf759892512e5a63059c520f9062d9ab8fb42fc')
+        patches = ["ctpp2_include.patch",
+                   "ctpp2_no_src_modification.patch",
+                   "ctpp2_fix-static-libname.patch",
+                   "ctpp2_mingw32.patch",
+                   "ctpp2_dll_export_VMExecutable.patch",
+                   "ctpp2_win_install_lib_in_lib_dir.patch",
+                   "ctpp2_iconv_support.patch"]
+
+    class Builder(CMakeBuilder):
+        configure_option = "-DMD5_SUPPORT=OFF"
+
+
+class Pugixml(Dependency):
+    name = "pugixml"
+    version = "1.2"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('pugixml-1.2.tar.gz',
+                             '0f422dad86da0a2e56a37fb2a88376aae6e931f22cc8b956978460c9db06136b')
+        patches = ["pugixml_meson.patch"]
+
+    Builder = MesonBuilder
+
+
+class MicroHttpd(Dependency):
+    name = "libmicrohttpd"
+    version = "0.9.46"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('libmicrohttpd-0.9.46.tar.gz',
+                             '06dbd2654f390fa1e8196fe063fc1449a6c2ed65a38199a49bf29ad8a93b8979',
+                             'http://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.46.tar.gz')
+
+    class Builder(MakeBuilder):
+        configure_option = "--disable-https --without-libgcrypt --without-libcurl"
+        dynamic_configure_option = "--enable-shared --disable-static"
+        static_configure_option = "--enable-static --disable-shared"
+
+
+class Icu(Dependency):
+    name = "icu4c"
+    version = "56_1"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('icu4c-56_1-src.tgz',
+                             '3a64e9105c734dcf631c0b3ed60404531bce6c0f5a64bfe1a6402a4cc2314816')
+        patches = ["icu4c_fix_static_lib_name_mingw.patch"]
+        data = Remotefile('icudt56l.dat',
+                          'e23d85eee008f335fc49e8ef37b1bc2b222db105476111e3d16f0007d371cbca')
+
+        def _download_data(self, context):
+            self.buildEnv.download(self.data)
+
+        def _copy_data(self, context):
+            context.try_skip(self.extract_path)
+            shutil.copyfile(pj(self.buildEnv.archive_dir, self.data.name), pj(self.extract_path, 'source', 'data', 'in', self.data.name))
+
+        def prepare(self):
+            super().prepare()
+            self.command("download_data", self._download_data)
+            self.command("copy_data", self._copy_data)
+
+    class Builder(MakeBuilder):
+        subsource_dir = "source"
+        configure_option = "--disable-samples --disable-tests --disable-extras --disable-dyload"
+        dynamic_configure_option = "--enable-shared --disable-static"
+        static_configure_option = "--enable-static --disable-shared"
+
+
+class Icu_native(Icu):
+    force_native_build = True
+
+    class Builder(Icu.Builder):
+        @property
+        def build_path(self):
+            return super().build_path+"_native"
+
+        def _install(self, context):
+            raise SkipCommand()
+
+
+class Icu_cross_compile(Icu):
+    dependencies = ['Icu_native']
+
+    class Builder(Icu.Builder):
+        @property
+        def configure_option(self):
+            Icu_native = self.buildEnv.targetsDict['Icu_native']
+            return super().configure_option + " --with-cross-build=" + Icu_native.builder.build_path
+
+
+class Zimlib(Dependency):
+    name = "zimlib"
+
+    class Source(GitClone):
+        #git_remote = "https://gerrit.wikimedia.org/r/p/openzim.git"
+        git_remote = "https://github.com/mgautierfr/openzim"
+        git_dir = "openzim"
+        git_ref = "meson"
+
+    class Builder(MesonBuilder):
+        subsource_dir = "zimlib"
+
+
+class Kiwixlib(Dependency):
+    name = "kiwix-lib"
+    dependencies = ['zlib']
+
+    @property
+    def dependencies(self):
+        if self.buildEnv.build_target == 'win32':
+            return ["Xapian", "CTPP2", "Pugixml", "Icu_cross_compile", "Zimlib"]
+        return ["Xapian", "CTPP2", "Pugixml", "Icu", "Zimlib"]
+
+    class Source(GitClone):
+        git_remote = "https://github.com/kiwix/kiwix-lib.git"
+        git_dir = "kiwix-lib"
+
+    class Builder(MesonBuilder):
+        configure_option = "-Dctpp2-install-prefix={buildEnv.install_dir}"
+
+
+class KiwixTools(Dependency):
+    name = "kiwix-tools"
+    dependencies = ["Kiwixlib", "MicroHttpd", "zlib"]
+
+    class Source(GitClone):
+        git_remote = "https://github.com/kiwix/kiwix-tools.git"
+        git_dir = "kiwix-tools"
+
+    class Builder(MesonBuilder):
+        @property
+        def configure_option(self):
+            base_options = "-Dctpp2-install-prefix={buildEnv.install_dir}"
+            if self.buildEnv.build_static:
+                base_options += " -Dstatic-linkage=true"
+            return base_options
diff --git a/dependency_utils.py b/dependency_utils.py
new file mode 100644
index 0000000..20bfb89
--- /dev/null
+++ b/dependency_utils.py
@@ -0,0 +1,292 @@
+import subprocess
+import os
+import shutil
+
+from utils import pj, Context, SkipCommand, extract_archive, Defaultdict, StopBuild
+
+SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
+
+
+class _MetaDependency(type):
+    def __new__(cls, name, bases, dct):
+        _class = type.__new__(cls, name, bases, dct)
+        if name != 'Dependency':
+            Dependency.all_deps[name] = _class
+        return _class
+
+
+class Dependency(metaclass=_MetaDependency):
+    all_deps = {}
+    dependencies = []
+    force_native_build = False
+    version = None
+
+    def __init__(self, buildEnv):
+        self.buildEnv = buildEnv
+        self.source = self.Source(self)
+        self.builder = self.Builder(self)
+        self.skip = False
+
+    @property
+    def full_name(self):
+        if self.version:
+            return "{}-{}".format(self.name, self.version)
+        return self.name
+
+    @property
+    def source_path(self):
+        return pj(self.buildEnv.source_dir, self.source.source_dir)
+
+    @property
+    def _log_dir(self):
+        return self.buildEnv.log_dir
+
+    def command(self, name, function, *args):
+        print("  {} {} : ".format(name, self.name), end="", flush=True)
+        log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
+        context = Context(name, log)
+        try:
+            ret = function(*args, context=context)
+            context._finalise()
+            print("OK")
+            return ret
+        except SkipCommand:
+            print("SKIP")
+        except subprocess.CalledProcessError:
+            print("ERROR")
+            try:
+                with open(log, 'r') as f:
+                    print(f.read())
+            except:
+                pass
+            raise StopBuild()
+        except:
+            print("ERROR")
+            raise
+
+
+class Source:
+    """Base Class to the real preparator
+       A source preparator must install source in the self.source_dir attribute
+       inside the buildEnv.source_dir."""
+    def __init__(self, target):
+        self.target = target
+        self.buildEnv = target.buildEnv
+
+    @property
+    def name(self):
+        return self.target.name
+
+    @property
+    def source_dir(self):
+        return self.target.full_name
+
+    def command(self, *args, **kwargs):
+        return self.target.command(*args, **kwargs)
+
+
+class ReleaseDownload(Source):
+    archive_top_dir = None
+
+    @property
+    def extract_path(self):
+        return pj(self.buildEnv.source_dir, self.source_dir)
+
+    def _download(self, context):
+        self.buildEnv.download(self.archive)
+
+    def _extract(self, context):
+        context.try_skip(self.extract_path)
+        if os.path.exists(self.extract_path):
+            shutil.rmtree(self.extract_path)
+        extract_archive(pj(self.buildEnv.archive_dir, self.archive.name),
+                        self.buildEnv.source_dir,
+                        topdir=self.archive_top_dir,
+                        name=self.source_dir)
+
+    def _patch(self, context):
+        context.try_skip(self.extract_path)
+        for p in self.patches:
+            with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input:
+                self.buildEnv.run_command("patch -p1", self.extract_path, context, input=patch_input, allow_wrapper=False)
+
+    def prepare(self):
+        self.command('download', self._download)
+        self.command('extract', self._extract)
+        if hasattr(self, 'patches'):
+            self.command('patch', self._patch)
+
+
+class GitClone(Source):
+    git_ref = "master"
+
+    @property
+    def source_dir(self):
+        return self.git_dir
+
+    @property
+    def git_path(self):
+        return pj(self.buildEnv.source_dir, self.git_dir)
+
+    def _git_clone(self, context):
+        if os.path.exists(self.git_path):
+            raise SkipCommand()
+        command = "git clone " + self.git_remote
+        self.buildEnv.run_command(command, self.buildEnv.source_dir, context)
+
+    def _git_update(self, context):
+        self.buildEnv.run_command("git fetch", self.git_path, context)
+        self.buildEnv.run_command("git checkout "+self.git_ref, self.git_path, context)
+
+    def prepare(self):
+        self.command('gitclone', self._git_clone)
+        self.command('gitupdate', self._git_update)
+
+
+class Builder:
+    subsource_dir = None
+
+    def __init__(self, target):
+        self.target = target
+        self.buildEnv = target.buildEnv
+
+    @property
+    def name(self):
+        return self.target.name
+
+    @property
+    def source_path(self):
+        base_source_path = self.target.source_path
+        if self.subsource_dir:
+            return pj(base_source_path, self.subsource_dir)
+        return base_source_path
+
+    @property
+    def build_path(self):
+        return pj(self.buildEnv.build_dir, self.target.full_name)
+
+    def command(self, *args, **kwargs):
+        return self.target.command(*args, **kwargs)
+
+    def build(self):
+        self.command('configure', self._configure)
+        self.command('compile', self._compile)
+        self.command('install', self._install)
+
+
+class MakeBuilder(Builder):
+    configure_option = static_configure_option = dynamic_configure_option = ""
+    make_option = ""
+    install_option = ""
+    configure_script = "configure"
+    configure_env = None
+    make_target = ""
+    make_install_target = "install"
+
+    def _configure(self, context):
+        context.try_skip(self.build_path)
+        configure_option = "{} {} {}".format(
+            self.configure_option,
+            self.static_configure_option if self.buildEnv.build_static else self.dynamic_configure_option,
+            self.buildEnv.configure_option)
+        command = "{configure_script} {configure_option} --prefix {install_dir} --libdir {libdir}"
+        command = command.format(
+            configure_script=pj(self.source_path, self.configure_script),
+            configure_option=configure_option,
+            install_dir=self.buildEnv.install_dir,
+            libdir=pj(self.buildEnv.install_dir, self.buildEnv.libprefix)
+        )
+        env = Defaultdict(str, os.environ)
+        if self.buildEnv.build_static:
+            env['CFLAGS'] = env['CFLAGS'] + ' -fPIC'
+        if self.configure_env:
+            for k in self.configure_env:
+                if k.startswith('_format_'):
+                    v = self.configure_env.pop(k)
+                    v = v.format(buildEnv=self.buildEnv, env=env)
+                    self.configure_env[k[8:]] = v
+            env.update(self.configure_env)
+        self.buildEnv.run_command(command, self.build_path, context, env=env)
+
+    def _compile(self, context):
+        context.try_skip(self.build_path)
+        command = "make -j4 {make_target} {make_option}".format(
+            make_target=self.make_target,
+            make_option=self.make_option
+        )
+        self.buildEnv.run_command(command, self.build_path, context)
+
+    def _install(self, context):
+        context.try_skip(self.build_path)
+        command = "make {make_install_target} {make_option}".format(
+            make_install_target=self.make_install_target,
+            make_option=self.make_option
+        )
+        self.buildEnv.run_command(command, self.build_path, context)
+
+
+class CMakeBuilder(MakeBuilder):
+    def _configure(self, context):
+        context.try_skip(self.build_path)
+        command = ("cmake {configure_option}"
+                   " -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON"
+                   " -DCMAKE_INSTALL_PREFIX={install_dir}"
+                   " -DCMAKE_INSTALL_LIBDIR={libdir}"
+                   " {source_path}"
+                   " {cross_option}")
+        command = command.format(
+            configure_option="{} {}".format(self.buildEnv.cmake_option, self.configure_option),
+            install_dir=self.buildEnv.install_dir,
+            libdir=self.buildEnv.libprefix,
+            source_path=self.source_path,
+            cross_option="-DCMAKE_TOOLCHAIN_FILE={}".format(self.buildEnv.cmake_crossfile) if self.buildEnv.cmake_crossfile else ""
+        )
+        env = Defaultdict(str, os.environ)
+        if self.buildEnv.build_static:
+            env['CFLAGS'] = env['CFLAGS'] + ' -fPIC'
+        if self.configure_env:
+            for k in self.configure_env:
+                if k.startswith('_format_'):
+                    v = self.configure_env.pop(k)
+                    v = v.format(buildEnv=self.buildEnv, env=env)
+                    self.configure_env[k[8:]] = v
+            env.update(self.configure_env)
+        self.buildEnv.run_command(command, self.build_path, context, env=env, allow_wrapper=False)
+
+
+class MesonBuilder(Builder):
+    configure_option = ""
+
+    def _configure(self, context):
+        context.try_skip(self.build_path)
+        if os.path.exists(self.build_path):
+            shutil.rmtree(self.build_path)
+        os.makedirs(self.build_path)
+        if self.buildEnv.build_static:
+            library_type = 'static'
+        else:
+            library_type = 'shared'
+        configure_option = self.configure_option.format(buildEnv=self.buildEnv)
+        command = ("{command} . {build_path}"
+                   " --default-library={library_type}"
+                   " {configure_option}"
+                   " --prefix={buildEnv.install_dir}"
+                   " --libdir={buildEnv.libprefix}"
+                   " {cross_option}")
+        command = command.format(
+            command=self.buildEnv.meson_command,
+            library_type=library_type,
+            configure_option=configure_option,
+            build_path=self.build_path,
+            buildEnv=self.buildEnv,
+            cross_option="--cross-file {}".format(self.buildEnv.meson_crossfile) if self.buildEnv.meson_crossfile else ""
+        )
+        self.buildEnv.run_command(command, self.source_path, context, allow_wrapper=False)
+
+    def _compile(self, context):
+        command = "{} -v".format(self.buildEnv.ninja_command)
+        self.buildEnv.run_command(command, self.build_path, context, allow_wrapper=False)
+
+    def _install(self, context):
+        command = "{} -v install".format(self.buildEnv.ninja_command)
+        self.buildEnv.run_command(command, self.build_path, context, allow_wrapper=False)
diff --git a/kiwix-build.py b/kiwix-build.py
index 9fbab10..6279401 100755
--- a/kiwix-build.py
+++ b/kiwix-build.py
@@ -4,71 +4,72 @@ import os, sys, stat
 import argparse
 import ssl
 import urllib.request
-import tarfile
 import subprocess
-import hashlib
-import shutil
-import tempfile
-import configparser
 import platform
-from collections import defaultdict, namedtuple, OrderedDict
+from collections import OrderedDict
 
-
-pj = os.path.join
+from dependencies import Dependency
+from utils import (
+    pj,
+    remove_duplicates,
+    get_sha256,
+    StopBuild,
+    SkipCommand,
+    Defaultdict)
 
 REMOTE_PREFIX = 'http://download.kiwix.org/dev/'
 
 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
 
 CROSS_ENV = {
-    'fedora_win32' : {
-        'root_path' : '/usr/i686-w64-mingw32/sys-root/mingw',
-        'binaries' : {
-            'c' : 'i686-w64-mingw32-gcc',
-            'cpp' : 'i686-w64-mingw32-g++',
-            'ar' : 'i686-w64-mingw32-ar',
-            'strip' : 'i686-w64-mingw32-strip',
-            'windres' : 'i686-w64-mingw32-windres',
-            'ranlib'  : 'i686-w64-mingw32-ranlib',
-            'exe_wrapper' : 'wine'
+    'fedora_win32': {
+        'root_path': '/usr/i686-w64-mingw32/sys-root/mingw',
+        'binaries': {
+            'c': 'i686-w64-mingw32-gcc',
+            'cpp': 'i686-w64-mingw32-g++',
+            'ar': 'i686-w64-mingw32-ar',
+            'strip': 'i686-w64-mingw32-strip',
+            'windres': 'i686-w64-mingw32-windres',
+            'ranlib': 'i686-w64-mingw32-ranlib',
+            'exe_wrapper': 'wine'
         },
-        'properties' : {
+        'properties': {
             'c_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4'],
             'cpp_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4']
         },
-        'host_machine' : {
-            'system' : 'windows',
-            'cpu_family' : 'x86',
-            'cpu' : 'i686',
-            'endian' : 'little'
+        'host_machine': {
+            'system': 'windows',
+            'cpu_family': 'x86',
+            'cpu': 'i686',
+            'endian': 'little'
         },
         'env': {
-            '_format_PKG_CONFIG_LIBDIR' : '{root_path}/lib/pkgconfig'
+            '_format_PKG_CONFIG_LIBDIR': '{root_path}/lib/pkgconfig'
         }
     },
-    'debian_win32' : {
-        'root_path' : '/usr/i686-w64-mingw32/',
-        'binaries' : {
-            'c' : 'i686-w64-mingw32-gcc',
-            'cpp' : 'i686-w64-mingw32-g++',
-            'ar' : 'i686-w64-mingw32-ar',
-            'strip' : 'i686-w64-mingw32-strip',
-            'windres' : 'i686-w64-mingw32-windres',
-            'ranlib'  : 'i686-w64-mingw32-ranlib',
-            'exe_wrapper' : 'wine'
+    'debian_win32': {
+        'root_path': '/usr/i686-w64-mingw32/',
+        'binaries': {
+            'c': 'i686-w64-mingw32-gcc',
+            'cpp': 'i686-w64-mingw32-g++',
+            'ar': 'i686-w64-mingw32-ar',
+            'strip': 'i686-w64-mingw32-strip',
+            'windres': 'i686-w64-mingw32-windres',
+            'ranlib': 'i686-w64-mingw32-ranlib',
+            'exe_wrapper': 'wine'
         },
-        'properties' : {
+        'properties': {
             'c_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4'],
             'cpp_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4']
         },
-        'host_machine' : {
-            'system' : 'windows',
-            'cpu_family' : 'x86',
-            'cpu' : 'i686',
-            'endian' : 'little'
+        'host_machine': {
+            'system': 'windows',
+            'cpu_family': 'x86',
+            'cpu': 'i686',
+            'endian': 'little'
         },
         'env': {
-            '_format_PKG_CONFIG_LIBDIR' : '{root_path}/lib/pkgconfig'
+            '_format_PKG_CONFIG_LIBDIR': '{root_path}/lib/pkgconfig'
         }
     }
 }
@@ -76,57 +77,53 @@ CROSS_ENV = {
 
 PACKAGE_NAME_MAPPERS = {
     'fedora_native_dyn': {
-        'COMMON' : ['gcc-c++', 'cmake', 'automake', 'ccache'],
+        'COMMON': ['gcc-c++', 'cmake', 'automake', 'ccache'],
         'uuid': ['libuuid-devel'],
-        'xapian-core' : None, # Not the right version on fedora 25
-        'ctpp2' : None,
-        'pugixml' : None, # ['pugixml-devel'] but package doesn't provide pkg-config file
-        'libmicrohttpd' : ['libmicrohttpd-devel'],
-        'zlib' : ['zlib-devel'],
+        'xapian-core': None, # Not the right version on fedora 25
+        'ctpp2': None,
+        'pugixml': None, # ['pugixml-devel'] but package doesn't provide pkg-config file
+        'libmicrohttpd': ['libmicrohttpd-devel'],
+        'zlib': ['zlib-devel'],
         'icu4c': None,
         'zimlib': None,
     },
-    'fedora_native_static' : {
-        'COMMON' : ['gcc-c++', 'cmake', 'automake', 'glibc-static', 'libstdc++-static', 'ccache'],
-        'zlib' : ['zlib-devel', 'zlib-static']
+    'fedora_native_static': {
+        'COMMON': ['gcc-c++', 'cmake', 'automake', 'glibc-static', 'libstdc++-static', 'ccache'],
+        'zlib': ['zlib-devel', 'zlib-static']
         # Either there is no packages, or no static or too old
     },
-    'fedora_win32_dyn' : {
-        'COMMON' : ['mingw32-gcc-c++', 'mingw32-bzip2', 'mingw32-win-iconv', 'mingw32-winpthreads', 'wine', 'ccache'],
-        'zlib' : ['mingw32-zlib'],
-        'libmicrohttpd' : ['mingw32-libmicrohttpd'],
+    'fedora_win32_dyn': {
+        'COMMON': ['mingw32-gcc-c++', 'mingw32-bzip2', 'mingw32-win-iconv', 'mingw32-winpthreads', 'wine', 'ccache'],
+        'zlib': ['mingw32-zlib'],
+        'libmicrohttpd': ['mingw32-libmicrohttpd'],
     },
-    'fedora_win32_static' : {
-        'COMMON' : ['mingw32-gcc-c++', 'mingw32-bzip2-static', 'mingw32-win-iconv-static', 'mingw32-winpthreads-static', 'wine', 'ccache'],
-        'zlib' : ['mingw32-zlib-static'],
-        'libmicrohttpd' : None, # ['mingw32-libmicrohttpd-static'] packaging dependecy seems buggy, and some static lib are name libfoo.dll.a and
-                                # gcc cannot found them.
+    'fedora_win32_static': {
+        'COMMON': ['mingw32-gcc-c++', 'mingw32-bzip2-static', 'mingw32-win-iconv-static', 'mingw32-winpthreads-static', 'wine', 'ccache'],
+        'zlib': ['mingw32-zlib-static'],
+        'libmicrohttpd': None, # ['mingw32-libmicrohttpd-static'] packaging dependecy seems buggy, and some static lib are name libfoo.dll.a and
+                               # gcc cannot found them.
     },
-    'debian_native_dyn' : {
-        'COMMON' : ['gcc', 'cmake', 'libbz2-dev', 'ccache'],
-        'zlib' : ['zlib1g-dev'],
-        'uuid' : ['uuid-dev'],
+    'debian_native_dyn': {
+        'COMMON': ['gcc', 'cmake', 'libbz2-dev', 'ccache'],
+        'zlib': ['zlib1g-dev'],
+        'uuid': ['uuid-dev'],
         'ctpp2': ['libctpp2-dev'],
-        'libmicrohttpd' : ['libmicrohttpd-dev', 'ccache']
+        'libmicrohttpd': ['libmicrohttpd-dev', 'ccache']
     },
-    'debian_native_static' : {
-        'COMMON' : ['gcc', 'cmake', 'libbz2-dev', 'ccache'],
-        'zlib' : ['zlib1g-dev'],
-        'uuid' : ['uuid-dev'],
+    'debian_native_static': {
+        'COMMON': ['gcc', 'cmake', 'libbz2-dev', 'ccache'],
+        'zlib': ['zlib1g-dev'],
+        'uuid': ['uuid-dev'],
         'ctpp2': ['libctpp2-dev'],
     },
-    'debian_win32_dyn' : {
-        'COMMON' : ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools', 'ccache']
+    'debian_win32_dyn': {
+        'COMMON': ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools', 'ccache']
     },
-    'debian_win32_static' : {
-        'COMMON' : ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools', 'ccache']
+    'debian_win32_static': {
+        'COMMON': ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools', 'ccache']
     },
 }
 
-class Defaultdict(defaultdict):
-    def __getattr__(self, name):
-        return self[name]
-
 
 class Which():
     def __getattr__(self, name):
@@ -137,77 +134,6 @@ class Which():
     def __format__(self, format_spec):
         return getattr(self, format_spec)
 
-def remove_duplicates(iterable, key_function=None):
-    seen = set()
-    if key_function is None:
-        key_function = lambda e:e
-    for elem in iterable:
-        key = key_function(elem)
-        if key in seen:
-            continue
-        seen.add(key)
-        yield elem
-
-def get_sha256(path):
-    sha256 = hashlib.sha256()
-    with open(path, 'br') as f:
-        sha256.update(f.read())
-    return sha256.hexdigest()
-
-class SkipCommand(Exception):
-    pass
-
-class StopBuild(Exception):
-    pass
-
-class Remotefile(namedtuple('Remotefile', ('name', 'sha256', 'url'))):
-    def __new__(cls, name, sha256, url=None):
-        return super().__new__(cls, name, sha256, url)
-
-class Context:
-    def __init__(self, command_name, log_file):
-        self.command_name = command_name
-        self.log_file = log_file
-        self.autoskip_file = None
-
-    def try_skip(self, path):
-        self.autoskip_file = pj(path, ".{}_ok".format(self.command_name))
-        if os.path.exists(self.autoskip_file):
-            raise SkipCommand()
-
-    def _finalise(self):
-        if self.autoskip_file is not None:
-            with open(self.autoskip_file, 'w') as f: pass
-
-
-def extract_archive(archive_path, dest_dir, topdir=None, name=None):
-    with tarfile.open(archive_path) as archive:
-        members = archive.getmembers()
-        if not topdir:
-            for d in (m for m in members if m.isdir()):
-                if not os.path.dirname(d.name):
-                    if topdir:
-                        # There is already a top dir.
-                        # Two topdirs in the same archive.
-                        # Extract all
-                        topdir = None
-                        break
-                    topdir = d
-        else:
-            topdir = archive.getmember(topdir)
-        if topdir:
-            members_to_extract = [m for m in members if m.name.startswith(topdir.name+'/')]
-            os.makedirs(dest_dir, exist_ok=True)
-            with tempfile.TemporaryDirectory(prefix=os.path.basename(archive_path), dir=dest_dir) as tmpdir:
-                archive.extractall(path=tmpdir, members=members_to_extract)
-                name = name or topdir.name
-                os.rename(pj(tmpdir, topdir.name), pj(dest_dir, name))
-        else:
-            if name:
-                dest_dir = pj(dest_dir, name)
-                os.makedirs(dest_dir)
-            archive.extractall(path=dest_dir)
-
 
 class BuildEnv:
     build_targets = ['native', 'win32']
@@ -282,16 +208,16 @@ class BuildEnv:
 
     def setup_win32(self):
         cross_name = "{host}_{target}".format(
-            host = self.distname,
-            target = self.build_target)
+            host=self.distname,
+            target=self.build_target)
         try:
             self.cross_env = CROSS_ENV[cross_name]
         except KeyError:
             sys.exit("ERROR : We don't know how to set env to compile"
                      " a {target} version on a {host} host.".format(
-                        target = self.build_target,
-                        host = self.distname
-                    ))
+                        target=self.build_target,
+                        host=self.distname
+                     ))
 
         self.wrapper = self._gen_crossfile('bash_wrapper.sh')
         current_permissions = stat.S_IMODE(os.lstat(self.wrapper).st_mode)
@@ -357,11 +283,11 @@ class BuildEnv:
         env['PKG_CONFIG_PATH'] = (env['PKG_CONFIG_PATH'] + ':' + pkgconfig_path
                                   if env['PKG_CONFIG_PATH']
                                   else pkgconfig_path
-                                 )
+                                  )
         # Add ccache path
         for p in ('/usr/lib/ccache', '/usr/lib64/ccache'):
             if os.path.isdir(p):
-                ccache_path=[p]
+                ccache_path = [p]
                 break
         else:
             ccache_path = []
@@ -373,7 +299,7 @@ class BuildEnv:
         env['LD_LIBRARY_PATH'] = (env['LD_LIBRARY_PATH'] + ':' + ld_library_path
                                   if env['LD_LIBRARY_PATH']
                                   else ld_library_path
-                                 )
+                                  )
         env['CPPFLAGS'] = '-I'+pj(self.install_dir, 'include')
         env['LDFLAGS'] = '-L'+pj(self.install_dir, 'lib')
         return env
@@ -388,7 +314,7 @@ class BuildEnv:
             if not self.options.verbose:
                 log = open(context.log_file, 'w')
             print("run command '{}'".format(command), file=log)
-            print("env is :",file=log)
+            print("env is :", file=log)
             for k, v in env.items():
                 print("  {} : {!r}".format(k, v), file=log)
 
@@ -436,18 +362,17 @@ class BuildEnv:
         elif self.distname in ('debian', 'Ubuntu'):
             package_installer = 'apt-get'
         mapper_name = "{host}_{target}_{build_type}".format(
-            host = self.distname,
-            target = self.build_target,
-            build_type = 'static' if self.options.build_static else 'dyn')
+            host=self.distname,
+            target=self.build_target,
+            build_type='static' if self.options.build_static else 'dyn')
         try:
             package_name_mapper = PACKAGE_NAME_MAPPERS[mapper_name]
         except KeyError:
             print("SKIP : We don't know which packages we must install to compile"
                   " a {target} {build_type} version on a {host} host.".format(
-                      target = self.build_target,
-                      build_type = 'static' if self.options.build_static else 'dyn',
-                      host = self.distname
-                 ))
+                      target=self.build_target,
+                      build_type='static' if self.options.build_static else 'dyn',
+                      host=self.distname))
             return
 
         packages_list = package_name_mapper.get('COMMON', [])
@@ -461,516 +386,16 @@ class BuildEnv:
             return
         if packages_list:
             command = "sudo {package_installer} install {packages_list}".format(
-                package_installer = package_installer,
-                packages_list = " ".join(packages_list)
+                package_installer=package_installer,
+                packages_list=" ".join(packages_list)
             )
             print(command)
             subprocess.check_call(command, shell=True)
         else:
             print("SKIP, No package to install.")
 
-        with open(autoskip_file, 'w') as f: pass
-
-################################################################################
-##### PROJECT
-################################################################################
-class _MetaDependency(type):
-    def __new__(cls, name, bases, dct):
-        _class = type.__new__(cls, name, bases, dct)
-        if name != 'Dependency':
-            Dependency.all_deps[name] = _class
-        return _class
-
-
-class Dependency(metaclass=_MetaDependency):
-    all_deps = {}
-    dependencies = []
-    force_native_build = False
-    version = None
-    def __init__(self, buildEnv):
-        self.buildEnv = buildEnv
-        self.source = self.Source(self)
-        self.builder = self.Builder(self)
-        self.skip = False
-
-    @property
-    def full_name(self):
-        if self.version:
-            return "{}-{}".format(self.name, self.version)
-        return self.name
-
-    @property
-    def source_path(self):
-        return pj(self.buildEnv.source_dir, self.source.source_dir)
-
-    @property
-    def _log_dir(self):
-        return self.buildEnv.log_dir
-
-    def command(self, name, function, *args):
-        print("  {} {} : ".format(name, self.name), end="", flush=True)
-        log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
-        context = Context(name, log)
-        try:
-            ret = function(*args, context=context)
-            context._finalise()
-            print("OK")
-            return ret
-        except SkipCommand:
-            print("SKIP")
-        except subprocess.CalledProcessError:
-            print("ERROR")
-            try:
-                with open(log, 'r') as f:
-                    print(f.read())
-            except:
-                pass
-            raise StopBuild()
-        except:
-            print("ERROR")
-            raise
-
-
-class Source:
-    """Base Class to the real preparator
-       A source preparator must install source in the self.source_dir attribute
-       inside the buildEnv.source_dir."""
-    def __init__(self, target):
-        self.target = target
-        self.buildEnv = target.buildEnv
-
-    @property
-    def name(self):
-        return self.target.name
-
-    @property
-    def source_dir(self):
-        return self.target.full_name
-
-    def command(self, *args, **kwargs):
-        return self.target.command(*args, **kwargs)
-
-
-class ReleaseDownload(Source):
-    archive_top_dir = None
-    @property
-    def extract_path(self):
-        return pj(self.buildEnv.source_dir, self.source_dir)
-
-    def _download(self, context):
-        self.buildEnv.download(self.archive)
-
-    def _extract(self, context):
-        context.try_skip(self.extract_path)
-        if os.path.exists(self.extract_path):
-            shutil.rmtree(self.extract_path)
-        extract_archive(pj(self.buildEnv.archive_dir, self.archive.name),
-                           self.buildEnv.source_dir,
-                           topdir=self.archive_top_dir,
-                           name=self.source_dir)
-
-    def _patch(self, context):
-        context.try_skip(self.extract_path)
-        for p in self.patches:
-            with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input:
-                self.buildEnv.run_command("patch -p1", self.extract_path, context, input=patch_input, allow_wrapper=False)
-
-    def prepare(self):
-        self.command('download', self._download)
-        self.command('extract', self._extract)
-        if hasattr(self, 'patches'):
-            self.command('patch', self._patch)
-
-
-class GitClone(Source):
-    git_ref = "master"
-    @property
-    def source_dir(self):
-        return self.git_dir
-
-    @property
-    def git_path(self):
-        return pj(self.buildEnv.source_dir, self.git_dir)
-
-    def _git_clone(self, context):
-        if os.path.exists(self.git_path):
-            raise SkipCommand()
-        command = "git clone " + self.git_remote
-        self.buildEnv.run_command(command, self.buildEnv.source_dir, context)
-
-    def _git_update(self, context):
-        self.buildEnv.run_command("git fetch", self.git_path, context)
-        self.buildEnv.run_command("git checkout "+self.git_ref, self.git_path, context)
-
-    def prepare(self):
-        self.command('gitclone', self._git_clone)
-        self.command('gitupdate', self._git_update)
-
-
-class Builder:
-    subsource_dir = None
-    def __init__(self, target):
-        self.target = target
-        self.buildEnv = target.buildEnv
-
-    @property
-    def name(self):
-        return self.target.name
-
-    @property
-    def source_path(self):
-        base_source_path = self.target.source_path
-        if self.subsource_dir:
-            return pj(base_source_path, self.subsource_dir)
-        return base_source_path
-
-    @property
-    def build_path(self):
-        return pj(self.buildEnv.build_dir, self.target.full_name)
-
-    def command(self, *args, **kwargs):
-        return self.target.command(*args, **kwargs)
-
-    def build(self):
-        self.command('configure', self._configure)
-        self.command('compile', self._compile)
-        self.command('install', self._install)
-
-
-class MakeBuilder(Builder):
-    configure_option = static_configure_option = dynamic_configure_option = ""
-    make_option = ""
-    install_option = ""
-    configure_script = "configure"
-    configure_env = None
-    make_target = ""
-    make_install_target = "install"
-
-    def _configure(self, context):
-        context.try_skip(self.build_path)
-        configure_option = "{} {} {}".format(
-            self.configure_option,
-            self.static_configure_option if self.buildEnv.build_static else self.dynamic_configure_option,
-            self.buildEnv.configure_option)
-        command = "{configure_script} {configure_option} --prefix {install_dir} --libdir {libdir}"
-        command = command.format(
-            configure_script = pj(self.source_path, self.configure_script),
-            configure_option = configure_option,
-            install_dir = self.buildEnv.install_dir,
-            libdir = pj(self.buildEnv.install_dir, self.buildEnv.libprefix)
-        )
-        env = Defaultdict(str, os.environ)
-        if self.buildEnv.build_static:
-            env['CFLAGS'] = env['CFLAGS'] + ' -fPIC'
-        if self.configure_env:
-           for k in self.configure_env:
-               if k.startswith('_format_'):
-                   v = self.configure_env.pop(k)
-                   v = v.format(buildEnv=self.buildEnv, env=env)
-                   self.configure_env[k[8:]] = v
-           env.update(self.configure_env)
-        self.buildEnv.run_command(command, self.build_path, context, env=env)
-
-    def _compile(self, context):
-        context.try_skip(self.build_path)
-        command = "make -j4 {make_target} {make_option}".format(
-            make_target = self.make_target,
-            make_option = self.make_option
-        )
-        self.buildEnv.run_command(command, self.build_path, context)
-
-    def  _install(self, context):
-        context.try_skip(self.build_path)
-        command = "make {make_install_target} {make_option}".format(
-            make_install_target = self.make_install_target,
-            make_option = self.make_option
-        )
-        self.buildEnv.run_command(command, self.build_path, context)
-
-
-class CMakeBuilder(MakeBuilder):
-    def _configure(self, context):
-        context.try_skip(self.build_path)
-        command = ("cmake {configure_option}"
-                   " -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON"
-                   " -DCMAKE_INSTALL_PREFIX={install_dir}"
-                   " -DCMAKE_INSTALL_LIBDIR={libdir}"
-                   " {source_path}"
-                   " {cross_option}")
-        command = command.format(
-            configure_option = "{} {}".format(self.buildEnv.cmake_option, self.configure_option),
-            install_dir = self.buildEnv.install_dir,
-            libdir = self.buildEnv.libprefix,
-            source_path = self.source_path,
-            cross_option = "-DCMAKE_TOOLCHAIN_FILE={}".format(self.buildEnv.cmake_crossfile) if self.buildEnv.cmake_crossfile else ""
-        )
-        env = Defaultdict(str, os.environ)
-        if self.buildEnv.build_static:
-            env['CFLAGS'] = env['CFLAGS'] + ' -fPIC'
-        if self.configure_env:
-           for k in self.configure_env:
-               if k.startswith('_format_'):
-                   v = self.configure_env.pop(k)
-                   v = v.format(buildEnv=self.buildEnv, env=env)
-                   self.configure_env[k[8:]] = v
-           env.update(self.configure_env)
-        self.buildEnv.run_command(command, self.build_path, context, env=env, allow_wrapper=False)
-
-
-class MesonBuilder(Builder):
-    configure_option = ""
-
-    def _configure(self, context):
-        context.try_skip(self.build_path)
-        if os.path.exists(self.build_path):
-            shutil.rmtree(self.build_path)
-        os.makedirs(self.build_path)
-        if self.buildEnv.build_static:
-            library_type = 'static'
-        else:
-            library_type = 'shared'
-        configure_option = self.configure_option.format(buildEnv=self.buildEnv)
-        command = ("{command} . {build_path}"
-                   " --default-library={library_type}"
-                   " {configure_option}"
-                   " --prefix={buildEnv.install_dir}"
-                   " --libdir={buildEnv.libprefix}"
-                   " {cross_option}")
-        command = command.format(
-            command = self.buildEnv.meson_command,
-            library_type=library_type,
-            configure_option=configure_option,
-            build_path = self.build_path,
-            buildEnv=self.buildEnv,
-            cross_option = "--cross-file {}".format(self.buildEnv.meson_crossfile) if self.buildEnv.meson_crossfile else ""
-        )
-        self.buildEnv.run_command(command, self.source_path, context, allow_wrapper=False)
-
-    def _compile(self, context):
-        command = "{} -v".format(self.buildEnv.ninja_command)
-        self.buildEnv.run_command(command, self.build_path, context, allow_wrapper=False)
-
-    def _install(self, context):
-        command = "{} -v install".format(self.buildEnv.ninja_command)
-        self.buildEnv.run_command(command, self.build_path, context, allow_wrapper=False)
-
-
-# *************************************
-# Missing dependencies
-# Is this ok to assume that those libs
-# exist in your "distri" (linux/mac) ?
-# If not, we need to compile them here
-# *************************************
-# Zlib
-# LZMA
-# aria2
-# Argtable
-# MSVirtual
-# Android
-# libiconv
-# gettext
-# *************************************
-
-class zlib(Dependency):
-    name = 'zlib'
-    version = '1.2.8'
-
-    class Source(ReleaseDownload):
-        archive = Remotefile('zlib-1.2.8.tar.gz',
-                             '36658cb768a54c1d4dec43c3116c27ed893e88b02ecfcb44f2166f9c0b7f2a0d')
-        patches = ['zlib_std_libname.patch']
-
-    class Builder(CMakeBuilder):
-        @property
-        def configure_option(self):
-            return "-DINSTALL_PKGCONFIG_DIR={}".format(pj(self.buildEnv.install_dir, self.buildEnv.libprefix, 'pkgconfig'))
-
-class UUID(Dependency):
-    name = 'uuid'
-    version = "1.43.4"
-
-    class Source(ReleaseDownload):
-        archive = Remotefile('e2fsprogs-1.43.4.tar.gz',
-                             '1644db4fc58300c363ba1ab688cf9ca1e46157323aee1029f8255889be4bc856',
-                             'https://www.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v1.43.4/e2fsprogs-1.43.4.tar.gz')
-        extract_dir = 'e2fsprogs-1.43.4'
-
-    class Builder(MakeBuilder):
-        configure_option = "--enable-libuuid"
-        configure_env = {'_format_CFLAGS' : "{env.CFLAGS} -fPIC"}
-        make_target = 'libs'
-        make_install_target = 'install-libs'
-
-
-class Xapian(Dependency):
-    name = "xapian-core"
-    version = "1.4.2"
-
-    class Source(ReleaseDownload):
-        archive = Remotefile('xapian-core-1.4.2.tar.xz',
-                             'aec2c4352998127a2f2316218bf70f48cef0a466a87af3939f5f547c5246e1ce')
-        patches = ["xapian_pkgconfig.patch"]
-
-    class Builder(MakeBuilder):
-        configure_option = "--disable-sse --disable-backend-inmemory --disable-documentation"
-        dynamic_configure_option = "--enable-shared --disable-static"
-        static_configure_option = "--enable-static --disable-shared"
-        configure_env = {'_format_LDFLAGS' : "-L{buildEnv.install_dir}/{buildEnv.libprefix}",
-                         '_format_CXXFLAGS' : "-I{buildEnv.install_dir}/include"}
-
-    @property
-    def dependencies(self):
-        deps = ['zlib']
-        if self.buildEnv.build_target == 'win32':
-            return deps
-        return deps + ['UUID']
-
-
-class CTPP2(Dependency):
-    name = "ctpp2"
-    version = "2.8.3"
-
-    class Source(ReleaseDownload):
-         archive = Remotefile('ctpp2-2.8.3.tar.gz',
-                              'a83ffd07817adb575295ef40fbf759892512e5a63059c520f9062d9ab8fb42fc')
-         patches = ["ctpp2_include.patch",
-               "ctpp2_no_src_modification.patch",
-               "ctpp2_fix-static-libname.patch",
-               "ctpp2_mingw32.patch",
-               "ctpp2_dll_export_VMExecutable.patch",
-               "ctpp2_win_install_lib_in_lib_dir.patch",
-               "ctpp2_iconv_support.patch"]
-
-    class Builder(CMakeBuilder):
-        configure_option = "-DMD5_SUPPORT=OFF"
-
-
-class Pugixml(Dependency):
-    name = "pugixml"
-    version = "1.2"
-
-    class Source(ReleaseDownload):
-        archive = Remotefile('pugixml-1.2.tar.gz',
-                             '0f422dad86da0a2e56a37fb2a88376aae6e931f22cc8b956978460c9db06136b')
-        patches = ["pugixml_meson.patch"]
-
-    Builder = MesonBuilder
-
-
-class MicroHttpd(Dependency):
-    name = "libmicrohttpd"
-    version = "0.9.46"
-
-    class Source(ReleaseDownload):
-        archive = Remotefile('libmicrohttpd-0.9.46.tar.gz',
-                             '06dbd2654f390fa1e8196fe063fc1449a6c2ed65a38199a49bf29ad8a93b8979',
-                             'http://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.46.tar.gz')
-
-    class Builder(MakeBuilder):
-        configure_option = "--disable-https --without-libgcrypt --without-libcurl"
-        dynamic_configure_option = "--enable-shared --disable-static"
-        static_configure_option = "--enable-static --disable-shared"
-
-
-class Icu(Dependency):
-    name = "icu4c"
-    version = "56_1"
-
-    class Source(ReleaseDownload):
-        archive = Remotefile('icu4c-56_1-src.tgz',
-                             '3a64e9105c734dcf631c0b3ed60404531bce6c0f5a64bfe1a6402a4cc2314816'
-                            )
-        patches = ["icu4c_fix_static_lib_name_mingw.patch"]
-        data = Remotefile('icudt56l.dat',
-                          'e23d85eee008f335fc49e8ef37b1bc2b222db105476111e3d16f0007d371cbca')
-
-        def _download_data(self, context):
-            self.buildEnv.download(self.data)
-
-        def _copy_data(self, context):
-            context.try_skip(self.extract_path)
-            shutil.copyfile(pj(self.buildEnv.archive_dir, self.data.name), pj(self.extract_path, 'source', 'data', 'in', self.data.name))
-
-        def prepare(self):
-            super().prepare()
-            self.command("download_data", self._download_data)
-            self.command("copy_data", self._copy_data)
-
-    class Builder(MakeBuilder):
-        subsource_dir = "source"
-        configure_option = "--disable-samples --disable-tests --disable-extras --disable-dyload"
-        dynamic_configure_option = "--enable-shared --disable-static"
-        static_configure_option = "--enable-static --disable-shared"
-
-
-class Icu_native(Icu):
-    force_native_build = True
-
-    class Builder(Icu.Builder):
-        @property
-        def build_path(self):
-            return super().build_path+"_native"
-
-        def _install(self, context):
-            raise SkipCommand()
-
-
-class Icu_cross_compile(Icu):
-    dependencies = ['Icu_native']
-
-    class Builder(Icu.Builder):
-        @property
-        def configure_option(self):
-            Icu_native = self.buildEnv.targetsDict['Icu_native']
-            return super().configure_option + " --with-cross-build=" + Icu_native.builder.build_path
-
-
-class Zimlib(Dependency):
-    name = "zimlib"
-
-    class Source(GitClone):
-        #git_remote = "https://gerrit.wikimedia.org/r/p/openzim.git"
-        git_remote = "https://github.com/mgautierfr/openzim"
-        git_dir = "openzim"
-        git_ref = "meson"
-
-    class Builder(MesonBuilder):
-        subsource_dir = "zimlib"
-
-
-class Kiwixlib(Dependency):
-    name = "kiwix-lib"
-    dependencies = ['zlib']
-
-    @property
-    def dependencies(self):
-        if self.buildEnv.build_target == 'win32':
-            return ["Xapian", "CTPP2", "Pugixml", "Icu_cross_compile", "Zimlib"]
-        return ["Xapian", "CTPP2", "Pugixml", "Icu", "Zimlib"]
-
-    class Source(GitClone):
-        git_remote = "https://github.com/kiwix/kiwix-lib.git"
-        git_dir = "kiwix-lib"
-
-    class Builder(MesonBuilder):
-        configure_option = "-Dctpp2-install-prefix={buildEnv.install_dir}"
-
-
-class KiwixTools(Dependency):
-    name = "kiwix-tools"
-    dependencies = ["Kiwixlib", "MicroHttpd", "zlib"]
-
-    class Source(GitClone):
-        git_remote = "https://github.com/kiwix/kiwix-tools.git"
-        git_dir = "kiwix-tools"
-
-    class Builder(MesonBuilder):
-        @property
-        def configure_option(self):
-           base_options = "-Dctpp2-install-prefix={buildEnv.install_dir}"
-           if self.buildEnv.build_static:
-               base_options += " -Dstatic-linkage=true"
-           return base_options
+        with open(autoskip_file, 'w'):
+            pass
 
 
 class Builder:
@@ -989,7 +414,7 @@ class Builder:
         dependencies = list(remove_duplicates(dependencies))
 
         for dep in dependencies:
-             self.targets[dep] = _targets[dep]
+            self.targets[dep] = _targets[dep]
 
     def add_targets(self, targetName, targets):
         if targetName in targets:
@@ -1011,7 +436,7 @@ class Builder:
 
     def prepare_sources(self):
         sources = (dep.source for dep in self.targets.values() if not dep.skip)
-        sources = remove_duplicates(sources, lambda s:s.__class__)
+        sources = remove_duplicates(sources, lambda s: s.__class__)
         for source in sources:
             print("prepare sources {} :".format(source.name))
             source.prepare()
@@ -1036,6 +461,7 @@ class Builder:
         except StopBuild:
             sys.exit("Stopping build due to errors")
 
+
 def parse_args():
     parser = argparse.ArgumentParser()
     parser.add_argument('working_dir', default=".", nargs='?')
@@ -1049,6 +475,7 @@ def parse_args():
                         help="Skip SSL certificate verification during download")
     return parser.parse_args()
 
+
 if __name__ == "__main__":
     options = parse_args()
     options.working_dir = os.path.abspath(options.working_dir)
diff --git a/utils.py b/utils.py
new file mode 100644
index 0000000..9c1464b
--- /dev/null
+++ b/utils.py
@@ -0,0 +1,90 @@
+import os.path
+import hashlib
+import tarfile
+import tempfile
+from collections import namedtuple, defaultdict
+
+pj = os.path.join
+
+
+class Defaultdict(defaultdict):
+    def __getattr__(self, name):
+        return self[name]
+
+
+def remove_duplicates(iterable, key_function=None):
+    seen = set()
+    if key_function is None:
+        key_function = lambda e: e
+    for elem in iterable:
+        key = key_function(elem)
+        if key in seen:
+            continue
+        seen.add(key)
+        yield elem
+
+
+def get_sha256(path):
+    sha256 = hashlib.sha256()
+    with open(path, 'br') as f:
+        sha256.update(f.read())
+    return sha256.hexdigest()
+
+
+class SkipCommand(Exception):
+    pass
+
+
+class StopBuild(Exception):
+    pass
+
+
+class Remotefile(namedtuple('Remotefile', ('name', 'sha256', 'url'))):
+    def __new__(cls, name, sha256, url=None):
+        return super().__new__(cls, name, sha256, url)
+
+
+class Context:
+    def __init__(self, command_name, log_file):
+        self.command_name = command_name
+        self.log_file = log_file
+        self.autoskip_file = None
+
+    def try_skip(self, path):
+        self.autoskip_file = pj(path, ".{}_ok".format(self.command_name))
+        if os.path.exists(self.autoskip_file):
+            raise SkipCommand()
+
+    def _finalise(self):
+        if self.autoskip_file is not None:
+            with open(self.autoskip_file, 'w'):
+                pass
+
+
+def extract_archive(archive_path, dest_dir, topdir=None, name=None):
+    with tarfile.open(archive_path) as archive:
+        members = archive.getmembers()
+        if not topdir:
+            for d in (m for m in members if m.isdir()):
+                if not os.path.dirname(d.name):
+                    if topdir:
+                        # There is already a top dir.
+                        # Two topdirs in the same archive.
+                        # Extract all
+                        topdir = None
+                        break
+                    topdir = d
+        else:
+            topdir = archive.getmember(topdir)
+        if topdir:
+            members_to_extract = [m for m in members if m.name.startswith(topdir.name+'/')]
+            os.makedirs(dest_dir, exist_ok=True)
+            with tempfile.TemporaryDirectory(prefix=os.path.basename(archive_path), dir=dest_dir) as tmpdir:
+                archive.extractall(path=tmpdir, members=members_to_extract)
+                name = name or topdir.name
+                os.rename(pj(tmpdir, topdir.name), pj(dest_dir, name))
+        else:
+            if name:
+                dest_dir = pj(dest_dir, name)
+                os.makedirs(dest_dir)
+            archive.extractall(path=dest_dir)

From 759812c41d05f1f313e7dc129c23965d772e3000 Mon Sep 17 00:00:00 2001
From: Matthieu Gautier <mgautier@kymeria.fr>
Date: Mon, 27 Feb 2017 14:21:22 +0100
Subject: [PATCH 02/25] Factorize how we create working directories.

---
 kiwix-build.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/kiwix-build.py b/kiwix-build.py
index 6279401..bae05b5 100755
--- a/kiwix-build.py
+++ b/kiwix-build.py
@@ -148,11 +148,12 @@ class BuildEnv:
         self.archive_dir = pj(options.working_dir, "ARCHIVE")
         self.log_dir = pj(options.working_dir, 'LOGS')
         self.install_dir = pj(self.build_dir, "INSTALL")
-        os.makedirs(self.source_dir, exist_ok=True)
-        os.makedirs(self.archive_dir, exist_ok=True)
-        os.makedirs(self.build_dir, exist_ok=True)
-        os.makedirs(self.log_dir, exist_ok=True)
-        os.makedirs(self.install_dir, exist_ok=True)
+        for d in (self.source_dir,
+                  self.build_dir,
+                  self.archive_dir,
+                  self.log_dir,
+                  self.install_dir):
+            os.makedirs(d, exist_ok=True)
         self.detect_platform()
         self.ninja_command = self._detect_ninja()
         if not self.ninja_command:

From 843964e55e3d14cb04ead4414a6255ab439233b5 Mon Sep 17 00:00:00 2001
From: Matthieu Gautier <mgautier@kymeria.fr>
Date: Mon, 27 Feb 2017 14:17:36 +0100
Subject: [PATCH 03/25] Check if packages are installed before trying to
 install them.

This avoid to ask to user its (sudo) password if all packages are already
installed.
---
 kiwix-build.py | 26 +++++++++++++++++++-------
 1 file changed, 19 insertions(+), 7 deletions(-)

diff --git a/kiwix-build.py b/kiwix-build.py
index bae05b5..8a0e985 100755
--- a/kiwix-build.py
+++ b/kiwix-build.py
@@ -359,9 +359,11 @@ class BuildEnv:
     def install_packages(self):
         autoskip_file = pj(self.build_dir, ".install_packages_ok")
         if self.distname in ('fedora', 'redhat', 'centos'):
-            package_installer = 'dnf'
+            package_installer = 'sudo dnf install {}'
+            package_checker = 'rpm -q --quiet {}'
         elif self.distname in ('debian', 'Ubuntu'):
-            package_installer = 'apt-get'
+            package_installer = 'sudo apt-get install {}'
+            package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1'
         mapper_name = "{host}_{target}_{build_type}".format(
             host=self.distname,
             target=self.build_target,
@@ -385,11 +387,21 @@ class BuildEnv:
         if os.path.exists(autoskip_file):
             print("SKIP")
             return
-        if packages_list:
-            command = "sudo {package_installer} install {packages_list}".format(
-                package_installer=package_installer,
-                packages_list=" ".join(packages_list)
-            )
+
+        packages_to_install = []
+        for package in packages_list:
+            print(" - {} : ".format(package), end="")
+            command = package_checker.format(package)
+            try:
+                subprocess.check_call(command, shell=True)
+            except subprocess.CalledProcessError:
+                print("NEEDED")
+                packages_to_install.append(package)
+            else:
+                print("SKIP")
+
+        if packages_to_install:
+            command = package_installer.format(" ".join(packages_to_install))
             print(command)
             subprocess.check_call(command, shell=True)
         else:

From 77137e09087129c04965fca3e91bfa4332d8d1ab Mon Sep 17 00:00:00 2001
From: Matthieu Gautier <mgautier@kymeria.fr>
Date: Wed, 22 Feb 2017 12:30:25 +0100
Subject: [PATCH 04/25] Add lzma dependency.

---
 dependencies.py | 19 +++++++++++++++----
 kiwix-build.py  |  6 +++++-
 2 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/dependencies.py b/dependencies.py
index f630e71..6248be9 100644
--- a/dependencies.py
+++ b/dependencies.py
@@ -16,8 +16,6 @@ from utils import Remotefile, pj, SkipCommand
 # exist in your "distri" (linux/mac) ?
 # If not, we need to compile them here
 # *************************************
-# Zlib
-# LZMA
 # aria2
 # Argtable
 # MSVirtual
@@ -42,6 +40,19 @@ class zlib(Dependency):
             return "-DINSTALL_PKGCONFIG_DIR={}".format(pj(self.buildEnv.install_dir, self.buildEnv.libprefix, 'pkgconfig'))
 
 
+class lzma(Dependency):
+    name = 'lzma'
+    version = '5.0.4'
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('xz-5.0.4.tar.bz2',
+                             '5cd9b060d3a1ad396b3be52c9b9311046a1c369e6062aea752658c435629ce92')
+
+    class Builder(MakeBuilder):
+        @property
+        def configure_option(self):
+            return "--disable-assembler"
+
 class UUID(Dependency):
     name = 'uuid'
     version = "1.43.4"
@@ -77,7 +88,7 @@ class Xapian(Dependency):
 
     @property
     def dependencies(self):
-        deps = ['zlib']
+        deps = ['zlib', 'lzma']
         if self.buildEnv.build_target == 'win32':
             return deps
         return deps + ['UUID']
@@ -196,7 +207,7 @@ class Zimlib(Dependency):
 
 class Kiwixlib(Dependency):
     name = "kiwix-lib"
-    dependencies = ['zlib']
+    dependencies = ['zlib', 'lzma']
 
     @property
     def dependencies(self):
diff --git a/kiwix-build.py b/kiwix-build.py
index 8a0e985..371fdba 100755
--- a/kiwix-build.py
+++ b/kiwix-build.py
@@ -84,22 +84,26 @@ PACKAGE_NAME_MAPPERS = {
         'pugixml': None, # ['pugixml-devel'] but package doesn't provide pkg-config file
         'libmicrohttpd': ['libmicrohttpd-devel'],
         'zlib': ['zlib-devel'],
+        'lzma': ['xz-devel'],
         'icu4c': None,
         'zimlib': None,
     },
     'fedora_native_static': {
         'COMMON': ['gcc-c++', 'cmake', 'automake', 'glibc-static', 'libstdc++-static', 'ccache'],
-        'zlib': ['zlib-devel', 'zlib-static']
+        'zlib': ['zlib-devel', 'zlib-static'],
+        'lzma': ['xz-devel', 'xz-static']
         # Either there is no packages, or no static or too old
     },
     'fedora_win32_dyn': {
         'COMMON': ['mingw32-gcc-c++', 'mingw32-bzip2', 'mingw32-win-iconv', 'mingw32-winpthreads', 'wine', 'ccache'],
         'zlib': ['mingw32-zlib'],
+        'lzma': ['mingw32-xz-libs'],
         'libmicrohttpd': ['mingw32-libmicrohttpd'],
     },
     'fedora_win32_static': {
         'COMMON': ['mingw32-gcc-c++', 'mingw32-bzip2-static', 'mingw32-win-iconv-static', 'mingw32-winpthreads-static', 'wine', 'ccache'],
         'zlib': ['mingw32-zlib-static'],
+        'lzma': ['mingw32-xz-libs-static'],
         'libmicrohttpd': None, # ['mingw32-libmicrohttpd-static'] packaging dependecy seems buggy, and some static lib are name libfoo.dll.a and
                                # gcc cannot found them.
     },

From 00acba9ee933a3cc87c01b840d16883615475681 Mon Sep 17 00:00:00 2001
From: Matthieu Gautier <mgautier@kymeria.fr>
Date: Wed, 22 Feb 2017 12:42:53 +0100
Subject: [PATCH 05/25] Add the notion of toolschain.

- For now, only the win32 is here.
- There no more nativeBuilEnv as the buildEnv can launch command without
  using the toolschain.
---
 dependency_utils.py            |  17 +--
 kiwix-build.py                 | 209 +++++++++++++++++++++------------
 templates/cmake_cross_file.txt |  12 +-
 templates/meson_cross_file.txt |  20 ++--
 utils.py                       |   3 +-
 5 files changed, 165 insertions(+), 96 deletions(-)

diff --git a/dependency_utils.py b/dependency_utils.py
index 20bfb89..291f118 100644
--- a/dependency_utils.py
+++ b/dependency_utils.py
@@ -44,7 +44,7 @@ class Dependency(metaclass=_MetaDependency):
     def command(self, name, function, *args):
         print("  {} {} : ".format(name, self.name), end="", flush=True)
         log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
-        context = Context(name, log)
+        context = Context(name, log, self.force_native_build)
         try:
             ret = function(*args, context=context)
             context._finalise()
@@ -106,9 +106,10 @@ class ReleaseDownload(Source):
 
     def _patch(self, context):
         context.try_skip(self.extract_path)
+        context.force_native_build = True
         for p in self.patches:
             with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input:
-                self.buildEnv.run_command("patch -p1", self.extract_path, context, input=patch_input, allow_wrapper=False)
+                self.buildEnv.run_command("patch -p1", self.extract_path, context, input=patch_input)
 
     def prepare(self):
         self.command('download', self._download)
@@ -129,12 +130,14 @@ class GitClone(Source):
         return pj(self.buildEnv.source_dir, self.git_dir)
 
     def _git_clone(self, context):
+        context.force_native_build = True
         if os.path.exists(self.git_path):
             raise SkipCommand()
         command = "git clone " + self.git_remote
         self.buildEnv.run_command(command, self.buildEnv.source_dir, context)
 
     def _git_update(self, context):
+        context.force_native_build = True
         self.buildEnv.run_command("git fetch", self.git_path, context)
         self.buildEnv.run_command("git checkout "+self.git_ref, self.git_path, context)
 
@@ -188,7 +191,7 @@ class MakeBuilder(Builder):
         configure_option = "{} {} {}".format(
             self.configure_option,
             self.static_configure_option if self.buildEnv.build_static else self.dynamic_configure_option,
-            self.buildEnv.configure_option)
+            self.buildEnv.configure_option if not self.target.force_native_build else "")
         command = "{configure_script} {configure_option} --prefix {install_dir} --libdir {libdir}"
         command = command.format(
             configure_script=pj(self.source_path, self.configure_script),
@@ -251,7 +254,7 @@ class CMakeBuilder(MakeBuilder):
                     v = v.format(buildEnv=self.buildEnv, env=env)
                     self.configure_env[k[8:]] = v
             env.update(self.configure_env)
-        self.buildEnv.run_command(command, self.build_path, context, env=env, allow_wrapper=False)
+        self.buildEnv.run_command(command, self.build_path, context, env=env, cross_path_only=True)
 
 
 class MesonBuilder(Builder):
@@ -281,12 +284,12 @@ class MesonBuilder(Builder):
             buildEnv=self.buildEnv,
             cross_option="--cross-file {}".format(self.buildEnv.meson_crossfile) if self.buildEnv.meson_crossfile else ""
         )
-        self.buildEnv.run_command(command, self.source_path, context, allow_wrapper=False)
+        self.buildEnv.run_command(command, self.source_path, context, cross_path_only=True)
 
     def _compile(self, context):
         command = "{} -v".format(self.buildEnv.ninja_command)
-        self.buildEnv.run_command(command, self.build_path, context, allow_wrapper=False)
+        self.buildEnv.run_command(command, self.build_path, context)
 
     def _install(self, context):
         command = "{} -v install".format(self.buildEnv.ninja_command)
-        self.buildEnv.run_command(command, self.build_path, context, allow_wrapper=False)
+        self.buildEnv.run_command(command, self.build_path, context)
diff --git a/kiwix-build.py b/kiwix-build.py
index 371fdba..6d48cce 100755
--- a/kiwix-build.py
+++ b/kiwix-build.py
@@ -23,16 +23,8 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
 
 CROSS_ENV = {
     'fedora_win32': {
+        'toolchain_names': ['mingw32_toolchain'],
         'root_path': '/usr/i686-w64-mingw32/sys-root/mingw',
-        'binaries': {
-            'c': 'i686-w64-mingw32-gcc',
-            'cpp': 'i686-w64-mingw32-g++',
-            'ar': 'i686-w64-mingw32-ar',
-            'strip': 'i686-w64-mingw32-strip',
-            'windres': 'i686-w64-mingw32-windres',
-            'ranlib': 'i686-w64-mingw32-ranlib',
-            'exe_wrapper': 'wine'
-        },
         'properties': {
             'c_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4'],
             'cpp_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4']
@@ -48,16 +40,8 @@ CROSS_ENV = {
         }
     },
     'debian_win32': {
+        'toolchain_names': ['mingw32_toolchain'],
         'root_path': '/usr/i686-w64-mingw32/',
-        'binaries': {
-            'c': 'i686-w64-mingw32-gcc',
-            'cpp': 'i686-w64-mingw32-g++',
-            'ar': 'i686-w64-mingw32-ar',
-            'strip': 'i686-w64-mingw32-strip',
-            'windres': 'i686-w64-mingw32-windres',
-            'ranlib': 'i686-w64-mingw32-ranlib',
-            'exe_wrapper': 'wine'
-        },
         'properties': {
             'c_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4'],
             'cpp_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4']
@@ -129,14 +113,10 @@ PACKAGE_NAME_MAPPERS = {
 }
 
 
-class Which():
-    def __getattr__(self, name):
-        command = "which {}".format(name)
-        output = subprocess.check_output(command, shell=True)
-        return output[:-1].decode()
-
-    def __format__(self, format_spec):
-        return getattr(self, format_spec)
+def which(name):
+    command = "which {}".format(name)
+    output = subprocess.check_output(command, shell=True)
+    return output[:-1].decode()
 
 
 class BuildEnv:
@@ -165,6 +145,8 @@ class BuildEnv:
         self.meson_command = self._detect_meson()
         if not self.meson_command:
             sys.exit("ERROR: meson command not fount")
+        self.setup_build(options.build_target)
+        self.setup_toolchains()
         self.options = options
         self.libprefix = options.libprefix or self._detect_libdir()
         self.targetsDict = targetsDict
@@ -187,14 +169,32 @@ class BuildEnv:
             if self.distname == 'ubuntu':
                 self.distname = 'debian'
 
+    def setup_build(self, target):
+        self.build_target = target
+        if target == 'native':
+            self.cross_env = {}
+        else:
+            cross_name = "{host}_{target}".format(
+                host = self.distname,
+                target = self.build_target)
+            try:
+                self.cross_env = CROSS_ENV[cross_name]
+            except KeyError:
+                sys.exit("ERROR : We don't know how to set env to compile"
+                         " a {target} version on a {host} host.".format(
+                            target = self.build_target,
+                            host = self.distname
+                        ))
+
+    def setup_toolchains(self):
+        toolchain_names = self.cross_env.get('toolchain_names', [])
+        self.toolchains =[Toolchain.all_toolchains[toolchain_name](self)
+                              for toolchain_name in toolchain_names]
+
     def finalize_setup(self):
         getattr(self, 'setup_{}'.format(self.build_target))()
 
     def setup_native(self):
-        self.cross_env = {}
-        self.wrapper = None
-        self.configure_option = ""
-        self.cmake_option = ""
         self.cmake_crossfile = None
         self.meson_crossfile = None
 
@@ -204,7 +204,7 @@ class BuildEnv:
         with open(template_file, 'r') as f:
             template = f.read()
         content = template.format(
-            which=Which(),
+            toolchain=self.toolchains[0],
             **self.cross_env
         )
         with open(crossfile, 'w') as outfile:
@@ -212,23 +212,6 @@ class BuildEnv:
         return crossfile
 
     def setup_win32(self):
-        cross_name = "{host}_{target}".format(
-            host=self.distname,
-            target=self.build_target)
-        try:
-            self.cross_env = CROSS_ENV[cross_name]
-        except KeyError:
-            sys.exit("ERROR : We don't know how to set env to compile"
-                     " a {target} version on a {host} host.".format(
-                        target=self.build_target,
-                        host=self.distname
-                     ))
-
-        self.wrapper = self._gen_crossfile('bash_wrapper.sh')
-        current_permissions = stat.S_IMODE(os.lstat(self.wrapper).st_mode)
-        os.chmod(self.wrapper, current_permissions | stat.S_IXUSR)
-        self.configure_option = "--host=i686-w64-mingw32"
-        self.cmake_option = ""
         self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
         self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
 
@@ -276,14 +259,33 @@ class BuildEnv:
             if retcode == 0:
                 return n
 
-    def _set_env(self, env):
+    @property
+    def configure_option(self):
+        configure_options = [tlc.configure_option for tlc in self.toolchains]
+        return " ".join(configure_options)
+
+    @property
+    def cmake_option(self):
+        cmake_options = [tlc.cmake_option for tlc in self.toolchains]
+        return " ".join(cmake_options)
+
+    def _set_env(self, env, cross_compile_env, cross_compile_path):
         if env is None:
             env = Defaultdict(str, os.environ)
-        for k, v in self.cross_env.get('env', {}).items():
-            if k.startswith('_format_'):
-                v = v.format(**self.cross_env)
-                k = k[8:]
-            env[k] = v
+
+        bin_dirs = []
+        if cross_compile_env:
+            for k, v in self.cross_env.get('env', {}).items():
+                if k.startswith('_format_'):
+                    v = v.format(**self.cross_env)
+                    k = k[8:]
+                env[k] = v
+            for toolchain in self.toolchains:
+                toolchain.set_env(env)
+        if cross_compile_path:
+            for tlc in self.toolchains:
+                bin_dirs += tlc.get_bin_dir()
+
         pkgconfig_path = pj(self.install_dir, self.libprefix, 'pkgconfig')
         env['PKG_CONFIG_PATH'] = (env['PKG_CONFIG_PATH'] + ':' + pkgconfig_path
                                   if env['PKG_CONFIG_PATH']
@@ -296,7 +298,10 @@ class BuildEnv:
                 break
         else:
             ccache_path = []
-        env['PATH'] = ':'.join([pj(self.install_dir, 'bin')] + ccache_path + [env['PATH']])
+        env['PATH'] = ':'.join(bin_dirs +
+                               [pj(self.install_dir, 'bin')] +
+                               ccache_path +
+                               [env['PATH']])
         ld_library_path = ':'.join([
             pj(self.install_dir, 'lib'),
             pj(self.install_dir, 'lib64')
@@ -305,15 +310,20 @@ class BuildEnv:
                                   if env['LD_LIBRARY_PATH']
                                   else ld_library_path
                                   )
-        env['CPPFLAGS'] = '-I'+pj(self.install_dir, 'include')
-        env['LDFLAGS'] = '-L'+pj(self.install_dir, 'lib')
+        env['CPPFLAGS'] = " ".join(['-I'+pj(self.install_dir, 'include'), env['CPPFLAGS']])
+        env['LDFLAGS'] = " ".join(['-L'+pj(self.install_dir, 'lib'), env['LDFLAGS']])
         return env
 
-    def run_command(self, command, cwd, context, env=None, input=None, allow_wrapper=True):
+    def run_command(self, command, cwd, context, env=None, input=None, cross_path_only=False):
         os.makedirs(cwd, exist_ok=True)
-        if allow_wrapper and self.wrapper:
-            command = "{} {}".format(self.wrapper, command)
-        env = self._set_env(env)
+        cross_compile_env = True
+        cross_compile_path = True
+        if context.force_native_build:
+            cross_compile_env = False
+            cross_compile_path = False
+        if cross_path_only:
+            cross_compile_env = False
+        env = self._set_env(env, cross_compile_env, cross_compile_path)
         log = None
         try:
             if not self.options.verbose:
@@ -414,16 +424,71 @@ class BuildEnv:
         with open(autoskip_file, 'w'):
             pass
 
+class _MetaToolchain(type):
+    def __new__(cls, name, bases, dct):
+        _class = type.__new__(cls, name, bases, dct)
+        if name != 'Toolchain':
+            Toolchain.all_toolchains[name] = _class
+        return _class
+
+
+class Toolchain(metaclass=_MetaToolchain):
+    all_toolchains = {}
+    configure_option = ""
+    cmake_option = ""
+
+    def __init__(self, buildEnv):
+        self.buildEnv = buildEnv
+
+    @property
+    def full_name(self):
+        return "{name}-{version}".format(
+            name = self.name,
+            version = self.version)
+
+    def set_env(self, env):
+        pass
+
+
+class mingw32_toolchain(Toolchain):
+    name = 'mingw32'
+    arch_full = 'i686-w64-mingw32'
+
+    @property
+    def root_path(self):
+        return self.buildEnv.cross_env['root_path']
+
+    @property
+    def binaries(self):
+        return {k:which('{}-{}'.format(self.arch_full, v))
+                for k, v in (('CC', 'gcc'),
+                             ('CXX', 'g++'),
+                             ('AR', 'ar'),
+                             ('STRIP', 'strip'),
+                             ('WINDRES', 'windres'),
+                             ('RANLIB', 'ranlib'))
+               }
+
+    @property
+    def configure_option(self):
+        return '--host={}'.format(self.arch_full)
+
+    def get_bin_dir(self):
+        return [pj(self.root_path, 'bin')]
+
+    def set_env(self, env):
+        for k, v in self.binaries.items():
+            env[k] = v
+
+        env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
+        env['CFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS']
+        env['CXXFLAGS'] =" -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
+
 
 class Builder:
     def __init__(self, options, targetDef='KiwixTools'):
         self.targets = OrderedDict()
         self.buildEnv = buildEnv = BuildEnv(options, self.targets)
-        if buildEnv.build_target != 'native':
-            self.nativeBuildEnv = BuildEnv(options, self.targets)
-            self.nativeBuildEnv.build_target = 'native'
-        else:
-            self.nativeBuildEnv = buildEnv
 
         _targets = {}
         self.add_targets(targetDef, _targets)
@@ -437,10 +502,7 @@ class Builder:
         if targetName in targets:
             return
         targetClass = Dependency.all_deps[targetName]
-        if targetClass.force_native_build:
-            target = targetClass(self.nativeBuildEnv)
-        else:
-            target = targetClass(self.buildEnv)
+        target = targetClass(self.buildEnv)
         targets[targetName] = target
         for dep in target.dependencies:
             self.add_targets(dep, targets)
@@ -452,6 +514,11 @@ class Builder:
         yield targetName
 
     def prepare_sources(self):
+        toolchain_sources = (tlc.source for tlc in self.buildEnv.toolchains if tlc.source)
+        for toolchain_source in toolchain_sources:
+            print("prepare sources for toolchain {} :".format(toolchain_source.name))
+            toolchain_source.prepare()
+
         sources = (dep.source for dep in self.targets.values() if not dep.skip)
         sources = remove_duplicates(sources, lambda s: s.__class__)
         for source in sources:
@@ -469,8 +536,6 @@ class Builder:
             print("[INSTALL PACKAGES]")
             self.buildEnv.install_packages()
             self.buildEnv.finalize_setup()
-            if self.nativeBuildEnv != self.buildEnv:
-                self.nativeBuildEnv.finalize_setup()
             print("[PREPARE]")
             self.prepare_sources()
             print("[BUILD]")
diff --git a/templates/cmake_cross_file.txt b/templates/cmake_cross_file.txt
index 8410a07..75a98a0 100644
--- a/templates/cmake_cross_file.txt
+++ b/templates/cmake_cross_file.txt
@@ -2,11 +2,11 @@ SET(CMAKE_SYSTEM_NAME Windows)
 SET(CMAKE_SYSTEM_PROCESSOR x86)
 
 # specify the cross compiler
-SET(CMAKE_C_COMPILER "{which:{binaries[c]}}")
-SET(CMAKE_CXX_COMPILER "{which:{binaries[cpp]}}")
-SET(CMAKE_RC_COMPILER {which:{binaries[windres]}})
-SET(CMAKE_AR:FILEPATH {which:{binaries[ar]}})
-SET(CMAKE_RANLIB:FILEPATH {which:{binaries[ranlib]}})
+SET(CMAKE_C_COMPILER "{toolchain.binaries[CC]}")
+SET(CMAKE_CXX_COMPILER "{toolchain.binaries[CXX]}")
+SET(CMAKE_RC_COMPILER {toolchain.binaries[WINDRES]})
+SET(CMAKE_AR:FILEPATH {toolchain.binaries[AR]})
+SET(CMAKE_RANLIB:FILEPATH {toolchain.binaries[RANLIB]})
 
 find_program(CCACHE_FOUND ccache)
 if(CCACHE_FOUND)
@@ -15,7 +15,7 @@ if(CCACHE_FOUND)
 endif(CCACHE_FOUND)
 
 # where is the target environment
-SET(CMAKE_FIND_ROOT_PATH {root_path})
+SET(CMAKE_FIND_ROOT_PATH {toolchain.root_path})
 
 # search for programs in the build host directories
 SET(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
diff --git a/templates/meson_cross_file.txt b/templates/meson_cross_file.txt
index edea054..464cbfe 100644
--- a/templates/meson_cross_file.txt
+++ b/templates/meson_cross_file.txt
@@ -1,16 +1,16 @@
 [binaries]
 pkgconfig = 'pkg-config'
-c = '{which:{binaries[c]}}'
-ar = '{which:{binaries[ar]}}'
-cpp = '{which:{binaries[cpp]}}'
-strip = '{which:{binaries[strip]}}'
+c = '{toolchain.binaries[CC]}'
+ar = '{toolchain.binaries[AR]}'
+cpp = '{toolchain.binaries[CXX]}'
+strip = '{toolchain.binaries[STRIP]}'
 
 [properties]
-c_link_args = ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4']
-cpp_link_args = ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4']
+c_link_args = {properties[c_link_args]!r}
+cpp_link_args = {properties[cpp_link_args]!r}
 
 [host_machine]
-cpu_family = 'x86'
-cpu = 'i586'
-system = 'windows'
-endian = 'little'
+cpu_family = '{host_machine[cpu_family]}'
+cpu = '{host_machine[cpu]}'
+system = '{host_machine[system]}'
+endian = '{host_machine[endian]}'
diff --git a/utils.py b/utils.py
index 9c1464b..af51cc8 100644
--- a/utils.py
+++ b/utils.py
@@ -45,9 +45,10 @@ class Remotefile(namedtuple('Remotefile', ('name', 'sha256', 'url'))):
 
 
 class Context:
-    def __init__(self, command_name, log_file):
+    def __init__(self, command_name, log_file, force_native_build):
         self.command_name = command_name
         self.log_file = log_file
+        self.force_native_build = force_native_build
         self.autoskip_file = None
 
     def try_skip(self, path):

From f2c841df6d544338f7b33ccc9d9da123cd269d7c Mon Sep 17 00:00:00 2001
From: Matthieu Gautier <mgautier@kymeria.fr>
Date: Wed, 22 Feb 2017 14:19:50 +0100
Subject: [PATCH 06/25] extract_archive can now extract zip format.

---
 utils.py | 50 ++++++++++++++++++++++++++++++++++++++------------
 1 file changed, 38 insertions(+), 12 deletions(-)

diff --git a/utils.py b/utils.py
index af51cc8..e3ef32c 100644
--- a/utils.py
+++ b/utils.py
@@ -1,7 +1,8 @@
 import os.path
 import hashlib
-import tarfile
+import tarfile, zipfile
 import tempfile
+import os
 from collections import namedtuple, defaultdict
 
 pj = os.path.join
@@ -63,29 +64,54 @@ class Context:
 
 
 def extract_archive(archive_path, dest_dir, topdir=None, name=None):
-    with tarfile.open(archive_path) as archive:
-        members = archive.getmembers()
+    is_zip_archive = archive_path.endswith('.zip')
+    try:
+        if is_zip_archive:
+            archive = zipfile.ZipFile(archive_path)
+            members = archive.infolist()
+            getname = lambda info : info.filename
+            isdir = lambda info: info.filename.endswith('/')
+        else:
+            archive = tarfile.open(archive_path)
+            members = archive.getmembers()
+            getname = lambda info : getattr(info, 'name')
+            isdir = lambda info: info.isdir()
         if not topdir:
-            for d in (m for m in members if m.isdir()):
-                if not os.path.dirname(d.name):
+            for d in (m for m in members if isdir(m)):
+                _name = getname(d)
+                if _name.endswith('/'):
+                    _name = _name[:-1]
+                if not os.path.dirname(_name):
                     if topdir:
                         # There is already a top dir.
                         # Two topdirs in the same archive.
                         # Extract all
                         topdir = None
                         break
-                    topdir = d
-        else:
-            topdir = archive.getmember(topdir)
+                    topdir = _name
         if topdir:
-            members_to_extract = [m for m in members if m.name.startswith(topdir.name+'/')]
+            members_to_extract = [m for m in members if getname(m).startswith(topdir+'/')]
             os.makedirs(dest_dir, exist_ok=True)
             with tempfile.TemporaryDirectory(prefix=os.path.basename(archive_path), dir=dest_dir) as tmpdir:
-                archive.extractall(path=tmpdir, members=members_to_extract)
-                name = name or topdir.name
-                os.rename(pj(tmpdir, topdir.name), pj(dest_dir, name))
+                if is_zip_archive:
+                    _members_to_extract = [getname(m) for m in members_to_extract]
+                else:
+                    _members_to_extract = members_to_extract
+                archive.extractall(path=tmpdir, members=_members_to_extract)
+                if is_zip_archive:
+                    for member in members_to_extract:
+                        if isdir(member):
+                            continue
+                        perm = (member.external_attr >> 16) & 0x1FF
+                        os.chmod(pj(tmpdir, getname(member)), perm)
+                name = name or topdir
+                os.rename(pj(tmpdir, topdir), pj(dest_dir, name))
         else:
             if name:
                 dest_dir = pj(dest_dir, name)
                 os.makedirs(dest_dir)
             archive.extractall(path=dest_dir)
+    finally:
+        if archive is not None:
+            archive.close()
+

From 6d69f056a43da812859123ca83bb05a824d88399 Mon Sep 17 00:00:00 2001
From: Matthieu Gautier <mgautier@kymeria.fr>
Date: Wed, 22 Feb 2017 14:20:48 +0100
Subject: [PATCH 07/25] Rewrite how we set environment. (Simpler way to write
 the same thing)

---
 kiwix-build.py | 20 ++++++++------------
 1 file changed, 8 insertions(+), 12 deletions(-)

diff --git a/kiwix-build.py b/kiwix-build.py
index 6d48cce..5e860a0 100755
--- a/kiwix-build.py
+++ b/kiwix-build.py
@@ -287,10 +287,8 @@ class BuildEnv:
                 bin_dirs += tlc.get_bin_dir()
 
         pkgconfig_path = pj(self.install_dir, self.libprefix, 'pkgconfig')
-        env['PKG_CONFIG_PATH'] = (env['PKG_CONFIG_PATH'] + ':' + pkgconfig_path
-                                  if env['PKG_CONFIG_PATH']
-                                  else pkgconfig_path
-                                  )
+        env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path])
+
         # Add ccache path
         for p in ('/usr/lib/ccache', '/usr/lib64/ccache'):
             if os.path.isdir(p):
@@ -302,14 +300,12 @@ class BuildEnv:
                                [pj(self.install_dir, 'bin')] +
                                ccache_path +
                                [env['PATH']])
-        ld_library_path = ':'.join([
-            pj(self.install_dir, 'lib'),
-            pj(self.install_dir, 'lib64')
-        ])
-        env['LD_LIBRARY_PATH'] = (env['LD_LIBRARY_PATH'] + ':' + ld_library_path
-                                  if env['LD_LIBRARY_PATH']
-                                  else ld_library_path
-                                  )
+
+        env['LD_LIBRARY_PATH'] = ':'.join([env['LD_LIBRARY_PATH'],
+                                          pj(self.install_dir, 'lib'),
+                                          pj(self.install_dir, 'lib64')
+                                          ])
+
         env['CPPFLAGS'] = " ".join(['-I'+pj(self.install_dir, 'include'), env['CPPFLAGS']])
         env['LDFLAGS'] = " ".join(['-L'+pj(self.install_dir, 'lib'), env['LDFLAGS']])
         return env

From bd4e7d3550c1a167bcdbaec6392c0bdb1be3ff7f Mon Sep 17 00:00:00 2001
From: Matthieu Gautier <mgautier@kymeria.fr>
Date: Wed, 22 Feb 2017 14:22:41 +0100
Subject: [PATCH 08/25] Move some common options for static/shared libs in the
 common class.

---
 dependencies.py     | 7 +------
 dependency_utils.py | 4 +++-
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/dependencies.py b/dependencies.py
index 6248be9..43c6b93 100644
--- a/dependencies.py
+++ b/dependencies.py
@@ -66,6 +66,7 @@ class UUID(Dependency):
     class Builder(MakeBuilder):
         configure_option = "--enable-libuuid"
         configure_env = {'_format_CFLAGS': "{env.CFLAGS} -fPIC"}
+        static_configure_option = dynamic_configure_option = ""
         make_target = 'libs'
         make_install_target = 'install-libs'
 
@@ -81,8 +82,6 @@ class Xapian(Dependency):
 
     class Builder(MakeBuilder):
         configure_option = "--disable-sse --disable-backend-inmemory --disable-documentation"
-        dynamic_configure_option = "--enable-shared --disable-static"
-        static_configure_option = "--enable-static --disable-shared"
         configure_env = {'_format_LDFLAGS': "-L{buildEnv.install_dir}/{buildEnv.libprefix}",
                          '_format_CXXFLAGS': "-I{buildEnv.install_dir}/include"}
 
@@ -136,8 +135,6 @@ class MicroHttpd(Dependency):
 
     class Builder(MakeBuilder):
         configure_option = "--disable-https --without-libgcrypt --without-libcurl"
-        dynamic_configure_option = "--enable-shared --disable-static"
-        static_configure_option = "--enable-static --disable-shared"
 
 
 class Icu(Dependency):
@@ -166,8 +163,6 @@ class Icu(Dependency):
     class Builder(MakeBuilder):
         subsource_dir = "source"
         configure_option = "--disable-samples --disable-tests --disable-extras --disable-dyload"
-        dynamic_configure_option = "--enable-shared --disable-static"
-        static_configure_option = "--enable-static --disable-shared"
 
 
 class Icu_native(Icu):
diff --git a/dependency_utils.py b/dependency_utils.py
index 291f118..721b4b5 100644
--- a/dependency_utils.py
+++ b/dependency_utils.py
@@ -178,7 +178,9 @@ class Builder:
 
 
 class MakeBuilder(Builder):
-    configure_option = static_configure_option = dynamic_configure_option = ""
+    configure_option = ""
+    dynamic_configure_option = "--enable-shared --disable-static"
+    static_configure_option = "--enable-static --disable-shared"
     make_option = ""
     install_option = ""
     configure_script = "configure"

From 92c1a5513ad150d599d8c2454ba8ec3985c68617 Mon Sep 17 00:00:00 2001
From: Matthieu Gautier <mgautier@kymeria.fr>
Date: Wed, 22 Feb 2017 14:24:47 +0100
Subject: [PATCH 09/25] Move some variables definition in a property instead of
 classic calculus.

---
 dependency_utils.py | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/dependency_utils.py b/dependency_utils.py
index 721b4b5..9819077 100644
--- a/dependency_utils.py
+++ b/dependency_utils.py
@@ -188,16 +188,19 @@ class MakeBuilder(Builder):
     make_target = ""
     make_install_target = "install"
 
-    def _configure(self, context):
-        context.try_skip(self.build_path)
-        configure_option = "{} {} {}".format(
+    @property
+    def all_configure_option(self):
+        return "{} {} {}".format(
             self.configure_option,
             self.static_configure_option if self.buildEnv.build_static else self.dynamic_configure_option,
             self.buildEnv.configure_option if not self.target.force_native_build else "")
+
+    def _configure(self, context):
+        context.try_skip(self.build_path)
         command = "{configure_script} {configure_option} --prefix {install_dir} --libdir {libdir}"
         command = command.format(
             configure_script=pj(self.source_path, self.configure_script),
-            configure_option=configure_option,
+            configure_option=self.all_configure_option,
             install_dir=self.buildEnv.install_dir,
             libdir=pj(self.buildEnv.install_dir, self.buildEnv.libprefix)
         )
@@ -262,15 +265,15 @@ class CMakeBuilder(MakeBuilder):
 class MesonBuilder(Builder):
     configure_option = ""
 
+    @property
+    def library_type(self):
+        return 'static' if self.buildEnv.build_static else 'shared'
+
     def _configure(self, context):
         context.try_skip(self.build_path)
         if os.path.exists(self.build_path):
             shutil.rmtree(self.build_path)
         os.makedirs(self.build_path)
-        if self.buildEnv.build_static:
-            library_type = 'static'
-        else:
-            library_type = 'shared'
         configure_option = self.configure_option.format(buildEnv=self.buildEnv)
         command = ("{command} . {build_path}"
                    " --default-library={library_type}"
@@ -280,7 +283,7 @@ class MesonBuilder(Builder):
                    " {cross_option}")
         command = command.format(
             command=self.buildEnv.meson_command,
-            library_type=library_type,
+            library_type=self.library_type,
             configure_option=configure_option,
             build_path=self.build_path,
             buildEnv=self.buildEnv,

From dd7a0660c0dbb711e503cb7b46f0a92ced5ec40e Mon Sep 17 00:00:00 2001
From: Matthieu Gautier <mgautier@kymeria.fr>
Date: Wed, 22 Feb 2017 14:25:23 +0100
Subject: [PATCH 10/25] Allow the use of pre/post build script in dependencies.

---
 dependency_utils.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/dependency_utils.py b/dependency_utils.py
index 9819077..559cc07 100644
--- a/dependency_utils.py
+++ b/dependency_utils.py
@@ -172,9 +172,13 @@ class Builder:
         return self.target.command(*args, **kwargs)
 
     def build(self):
+        if hasattr(self, '_pre_build_script'):
+            self.command('pre_build_script', self._pre_build_script)
         self.command('configure', self._configure)
         self.command('compile', self._compile)
         self.command('install', self._install)
+        if hasattr(self, '_post_build_script'):
+            self.command('post_build_script', self._post_build_script)
 
 
 class MakeBuilder(Builder):

From 960f6c9e821981e76b0e8fdc0bb80c18aea23757 Mon Sep 17 00:00:00 2001
From: Matthieu Gautier <mgautier@kymeria.fr>
Date: Wed, 22 Feb 2017 14:28:33 +0100
Subject: [PATCH 11/25] zlib now compile using configure/make instead of cmake.

With cmake it is not possible to install only one kind of library
(static or shared).
---
 dependencies.py | 36 ++++++++++++++++++++++++++++++++++--
 1 file changed, 34 insertions(+), 2 deletions(-)

diff --git a/dependencies.py b/dependencies.py
index 43c6b93..ee80e46 100644
--- a/dependencies.py
+++ b/dependencies.py
@@ -34,11 +34,43 @@ class zlib(Dependency):
                              '36658cb768a54c1d4dec43c3116c27ed893e88b02ecfcb44f2166f9c0b7f2a0d')
         patches = ['zlib_std_libname.patch']
 
-    class Builder(CMakeBuilder):
+    class Builder(MakeBuilder):
+        dynamic_configure_option = "--shared"
+        static_configure_option = "--static"
+
+        def _pre_build_script(self, context):
+            context.try_skip(self.build_path)
+            shutil.copytree(self.source_path, self.build_path)
+
+        @property
+        def all_configure_option(self):
+            return '--static' if self.buildEnv.build_static else '--shared'
+
         @property
         def configure_option(self):
-            return "-DINSTALL_PKGCONFIG_DIR={}".format(pj(self.buildEnv.install_dir, self.buildEnv.libprefix, 'pkgconfig'))
+            options = "-DINSTALL_PKGCONFIG_DIR={}".format(pj(self.buildEnv.install_dir, self.buildEnv.libprefix, 'pkgconfig'))
+            if self.buildEnv.build_static:
+                options += " -DBUILD_SHARED_LIBS=false"
+            else:
+                options += " -DBUILD_SHARED_LIBS=true"
+            return options
 
+        def _configure(self, context):
+            if self.buildEnv.target_info.build == 'win32':
+                raise SkipCommand()
+            return super()._configure(context)
+
+        @property
+        def make_option(self):
+            if self.buildEnv.target_info.build == 'win32':
+                return "--makefile win32/Makefile.gcc PREFIX={host}- SHARED_MODE={static} INCLUDE_PATH={include_path} LIBRARY_PATH={library_path} BINARY_PATH={binary_path}".format(
+                    host='i686-w64-mingw32',
+                    static="0" if self.buildEnv.target_info.static else "1",
+                    include_path=pj(self.buildEnv.install_dir, 'include'),
+                    library_path=pj(self.buildEnv.install_dir, self.buildEnv.libprefix),
+                    binary_path=pj(self.buildEnv.install_dir, 'bin'),
+                    )
+            return ""
 
 class lzma(Dependency):
     name = 'lzma'

From cbb0ce83b5e1b0564fd9a28cd70f1ae0a98e9de4 Mon Sep 17 00:00:00 2001
From: Matthieu Gautier <mgautier@kymeria.fr>
Date: Wed, 22 Feb 2017 14:40:18 +0100
Subject: [PATCH 12/25] Disable compilation of unused stuff in UUID.

---
 dependencies.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/dependencies.py b/dependencies.py
index ee80e46..38ca6f9 100644
--- a/dependencies.py
+++ b/dependencies.py
@@ -96,7 +96,8 @@ class UUID(Dependency):
         extract_dir = 'e2fsprogs-libs-1.43.4'
 
     class Builder(MakeBuilder):
-        configure_option = "--enable-libuuid"
+        configure_option = ("--enable-libuuid --disable-debugfs --disable-imager --disable-resizer --disable-defrag --enable-fsck"
+                            " --disable-uuidd")
         configure_env = {'_format_CFLAGS': "{env.CFLAGS} -fPIC"}
         static_configure_option = dynamic_configure_option = ""
         make_target = 'libs'

From 83d6d00acddb0b13468aaae9d5d7a6d2ea2514bb Mon Sep 17 00:00:00 2001
From: Matthieu Gautier <mgautier@kymeria.fr>
Date: Wed, 22 Feb 2017 14:40:53 +0100
Subject: [PATCH 13/25] Force name of ICU to differentiate icu_native from
 icu_cross-compile.

---
 dependencies.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/dependencies.py b/dependencies.py
index 38ca6f9..1a014f7 100644
--- a/dependencies.py
+++ b/dependencies.py
@@ -202,6 +202,7 @@ class Icu_native(Icu):
     force_native_build = True
 
     class Builder(Icu.Builder):
+        name = "icu_native"
         @property
         def build_path(self):
             return super().build_path+"_native"
@@ -214,6 +215,7 @@ class Icu_cross_compile(Icu):
     dependencies = ['Icu_native']
 
     class Builder(Icu.Builder):
+        name = "icu_cross-compile"
         @property
         def configure_option(self):
             Icu_native = self.buildEnv.targetsDict['Icu_native']

From c31a29488a2022af2686a758bbcd30f6f2242005 Mon Sep 17 00:00:00 2001
From: Matthieu Gautier <mgautier@kymeria.fr>
Date: Wed, 22 Feb 2017 14:43:21 +0100
Subject: [PATCH 14/25] [API Change] User can specify the target to build.

This is a API change, the working directory is set using the
--working-dir option, not as an argument.
---
 kiwix-build.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/kiwix-build.py b/kiwix-build.py
index 5e860a0..980eeb4 100755
--- a/kiwix-build.py
+++ b/kiwix-build.py
@@ -482,11 +482,12 @@ class mingw32_toolchain(Toolchain):
 
 
 class Builder:
-    def __init__(self, options, targetDef='KiwixTools'):
+    def __init__(self, options):
         self.targets = OrderedDict()
         self.buildEnv = buildEnv = BuildEnv(options, self.targets)
 
         _targets = {}
+        targetDef = options.targets
         self.add_targets(targetDef, _targets)
         dependencies = self.order_dependencies(_targets, targetDef)
         dependencies = list(remove_duplicates(dependencies))
@@ -542,7 +543,8 @@ class Builder:
 
 def parse_args():
     parser = argparse.ArgumentParser()
-    parser.add_argument('working_dir', default=".", nargs='?')
+    parser.add_argument('targets', default='KiwixTools', nargs='?')
+    parser.add_argument('--working-dir', default=".")
     parser.add_argument('--libprefix', default=None)
     parser.add_argument('--build-static', action="store_true")
     parser.add_argument('--build-target', default="native", choices=BuildEnv.build_targets)

From 29bba313bc932dc4d44e3cea3f0c5784b13803e4 Mon Sep 17 00:00:00 2001
From: Matthieu Gautier <mgautier@kymeria.fr>
Date: Wed, 22 Feb 2017 15:26:24 +0100
Subject: [PATCH 15/25] [API change] Use just one option to specify the build
 target.

There is no more option for static or share build. This value is
integrated in the build-target.
---
 .travis.yml           | 10 +++++-----
 dependencies.py       | 16 +++++++--------
 dependency_utils.py   |  8 ++++----
 kiwix-build.py        | 46 +++++++++++++++++++++++++------------------
 travis/compile_all.sh |  7 +------
 travis/deploy.sh      | 10 ++--------
 6 files changed, 47 insertions(+), 50 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 8f792ce..2056eca 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -12,12 +12,12 @@ deploy:
   skip_cleanup: true
   script: travis/deploy.sh
   on:
-    condition: ( "$STATIC_BUILD" = "true" ) && ( "$TRAVIS_EVENT_TYPE" = "cron" )
+    condition: ( "$DEPLOY" = "true" ) && ( "$TRAVIS_EVENT_TYPE" = "cron" )
 env:
- - STATIC_BUILD=true BUILD_TARGET=native
- - STATIC_BUILD=true BUILD_TARGET=win32
- - STATIC_BUILD=false BUILD_TARGET=native
- - STATIC_BUILD=false BUILD_TARGET=win32
+ - BUILD_OPTION="--target-platform=native_dyn"
+ - BUILD_OPTION="--target-platform=native_static" ARCHIVE_TYPE="--tar" DEPLOY=true
+ - BUILD_OPTION="--target-platform=win32_dyn"
+ - BUILD_OPTION="--target-platform=win32_static" ARCHIVE_TYPE="--zip" DEPLOY=true
 notifications:
   irc:
     channels:
diff --git a/dependencies.py b/dependencies.py
index 1a014f7..dd22697 100644
--- a/dependencies.py
+++ b/dependencies.py
@@ -44,28 +44,28 @@ class zlib(Dependency):
 
         @property
         def all_configure_option(self):
-            return '--static' if self.buildEnv.build_static else '--shared'
+            return '--static' if self.buildEnv.platform_info.static else '--shared'
 
         @property
         def configure_option(self):
             options = "-DINSTALL_PKGCONFIG_DIR={}".format(pj(self.buildEnv.install_dir, self.buildEnv.libprefix, 'pkgconfig'))
-            if self.buildEnv.build_static:
+            if self.buildEnv.platform_info.static:
                 options += " -DBUILD_SHARED_LIBS=false"
             else:
                 options += " -DBUILD_SHARED_LIBS=true"
             return options
 
         def _configure(self, context):
-            if self.buildEnv.target_info.build == 'win32':
+            if self.buildEnv.platform_info.build == 'win32':
                 raise SkipCommand()
             return super()._configure(context)
 
         @property
         def make_option(self):
-            if self.buildEnv.target_info.build == 'win32':
+            if self.buildEnv.platform_info.build == 'win32':
                 return "--makefile win32/Makefile.gcc PREFIX={host}- SHARED_MODE={static} INCLUDE_PATH={include_path} LIBRARY_PATH={library_path} BINARY_PATH={binary_path}".format(
                     host='i686-w64-mingw32',
-                    static="0" if self.buildEnv.target_info.static else "1",
+                    static="0" if self.buildEnv.platform_info.static else "1",
                     include_path=pj(self.buildEnv.install_dir, 'include'),
                     library_path=pj(self.buildEnv.install_dir, self.buildEnv.libprefix),
                     binary_path=pj(self.buildEnv.install_dir, 'bin'),
@@ -121,7 +121,7 @@ class Xapian(Dependency):
     @property
     def dependencies(self):
         deps = ['zlib', 'lzma']
-        if self.buildEnv.build_target == 'win32':
+        if self.buildEnv.platform_info.build == 'win32':
             return deps
         return deps + ['UUID']
 
@@ -241,7 +241,7 @@ class Kiwixlib(Dependency):
 
     @property
     def dependencies(self):
-        if self.buildEnv.build_target == 'win32':
+        if self.buildEnv.platform_info.build == 'win32':
             return ["Xapian", "CTPP2", "Pugixml", "Icu_cross_compile", "Zimlib"]
         return ["Xapian", "CTPP2", "Pugixml", "Icu", "Zimlib"]
 
@@ -265,6 +265,6 @@ class KiwixTools(Dependency):
         @property
         def configure_option(self):
             base_options = "-Dctpp2-install-prefix={buildEnv.install_dir}"
-            if self.buildEnv.build_static:
+            if self.buildEnv.platform_info.static:
                 base_options += " -Dstatic-linkage=true"
             return base_options
diff --git a/dependency_utils.py b/dependency_utils.py
index 559cc07..677d638 100644
--- a/dependency_utils.py
+++ b/dependency_utils.py
@@ -196,7 +196,7 @@ class MakeBuilder(Builder):
     def all_configure_option(self):
         return "{} {} {}".format(
             self.configure_option,
-            self.static_configure_option if self.buildEnv.build_static else self.dynamic_configure_option,
+            self.static_configure_option if self.buildEnv.platform_info.static else self.dynamic_configure_option,
             self.buildEnv.configure_option if not self.target.force_native_build else "")
 
     def _configure(self, context):
@@ -209,7 +209,7 @@ class MakeBuilder(Builder):
             libdir=pj(self.buildEnv.install_dir, self.buildEnv.libprefix)
         )
         env = Defaultdict(str, os.environ)
-        if self.buildEnv.build_static:
+        if self.buildEnv.platform_info.static:
             env['CFLAGS'] = env['CFLAGS'] + ' -fPIC'
         if self.configure_env:
             for k in self.configure_env:
@@ -254,7 +254,7 @@ class CMakeBuilder(MakeBuilder):
             cross_option="-DCMAKE_TOOLCHAIN_FILE={}".format(self.buildEnv.cmake_crossfile) if self.buildEnv.cmake_crossfile else ""
         )
         env = Defaultdict(str, os.environ)
-        if self.buildEnv.build_static:
+        if self.buildEnv.platform_info.static:
             env['CFLAGS'] = env['CFLAGS'] + ' -fPIC'
         if self.configure_env:
             for k in self.configure_env:
@@ -271,7 +271,7 @@ class MesonBuilder(Builder):
 
     @property
     def library_type(self):
-        return 'static' if self.buildEnv.build_static else 'shared'
+        return 'static' if self.buildEnv.platform_info.static else 'shared'
 
     def _configure(self, context):
         context.try_skip(self.build_path)
diff --git a/kiwix-build.py b/kiwix-build.py
index 980eeb4..7236d8a 100755
--- a/kiwix-build.py
+++ b/kiwix-build.py
@@ -119,15 +119,26 @@ def which(name):
     return output[:-1].decode()
 
 
+class TargetInfo:
+    def __init__(self, build, static):
+        self.build = build
+        self.static = static
+
+    def __str__(self):
+        return "{}_{}".format(self.build, 'static' if self.static else 'dyn')
+
+
 class BuildEnv:
-    build_targets = ['native', 'win32']
+    target_platforms = {
+        'native_dyn': TargetInfo('native', False),
+        'native_static': TargetInfo('native', True),
+        'win32_dyn': TargetInfo('win32', False),
+        'win32_static': TargetInfo('win32', True)
+    }
 
     def __init__(self, options, targetsDict):
         self.source_dir = pj(options.working_dir, "SOURCE")
-        build_dir = "BUILD_{target}_{libmod}".format(
-            target=options.build_target,
-            libmod='static' if options.build_static else 'dyn'
-        )
+        build_dir = "BUILD_{}".format(options.target_platform)
         self.build_dir = pj(options.working_dir, build_dir)
         self.archive_dir = pj(options.working_dir, "ARCHIVE")
         self.log_dir = pj(options.working_dir, 'LOGS')
@@ -145,7 +156,7 @@ class BuildEnv:
         self.meson_command = self._detect_meson()
         if not self.meson_command:
             sys.exit("ERROR: meson command not fount")
-        self.setup_build(options.build_target)
+        self.setup_build(options.target_platform)
         self.setup_toolchains()
         self.options = options
         self.libprefix = options.libprefix or self._detect_libdir()
@@ -169,20 +180,20 @@ class BuildEnv:
             if self.distname == 'ubuntu':
                 self.distname = 'debian'
 
-    def setup_build(self, target):
-        self.build_target = target
-        if target == 'native':
+    def setup_build(self, target_platform):
+        self.platform_info = platform_info = self.target_platforms[target_platform]
+        if platform_info.build == 'native':
             self.cross_env = {}
         else:
             cross_name = "{host}_{target}".format(
                 host = self.distname,
-                target = self.build_target)
+                target = platform_info.build)
             try:
                 self.cross_env = CROSS_ENV[cross_name]
             except KeyError:
                 sys.exit("ERROR : We don't know how to set env to compile"
                          " a {target} version on a {host} host.".format(
-                            target = self.build_target,
+                            target = platform_info.build,
                             host = self.distname
                         ))
 
@@ -192,7 +203,7 @@ class BuildEnv:
                               for toolchain_name in toolchain_names]
 
     def finalize_setup(self):
-        getattr(self, 'setup_{}'.format(self.build_target))()
+        getattr(self, 'setup_{}'.format(self.platform_info.build))()
 
     def setup_native(self):
         self.cmake_crossfile = None
@@ -374,17 +385,15 @@ class BuildEnv:
         elif self.distname in ('debian', 'Ubuntu'):
             package_installer = 'sudo apt-get install {}'
             package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1'
-        mapper_name = "{host}_{target}_{build_type}".format(
+        mapper_name = "{host}_{target}".format(
             host=self.distname,
-            target=self.build_target,
-            build_type='static' if self.options.build_static else 'dyn')
+            target=self.platform_info)
         try:
             package_name_mapper = PACKAGE_NAME_MAPPERS[mapper_name]
         except KeyError:
             print("SKIP : We don't know which packages we must install to compile"
                   " a {target} {build_type} version on a {host} host.".format(
-                      target=self.build_target,
-                      build_type='static' if self.options.build_static else 'dyn',
+                      target=self.platform_info,
                       host=self.distname))
             return
 
@@ -546,8 +555,7 @@ def parse_args():
     parser.add_argument('targets', default='KiwixTools', nargs='?')
     parser.add_argument('--working-dir', default=".")
     parser.add_argument('--libprefix', default=None)
-    parser.add_argument('--build-static', action="store_true")
-    parser.add_argument('--build-target', default="native", choices=BuildEnv.build_targets)
+    parser.add_argument('--target-platform', default="native_dyn", choices=BuildEnv.target_platforms)
     parser.add_argument('--verbose', '-v', action="store_true",
                         help=("Print all logs on stdout instead of in specific"
                               " log files per commands"))
diff --git a/travis/compile_all.sh b/travis/compile_all.sh
index 7358c69..7cc09b4 100755
--- a/travis/compile_all.sh
+++ b/travis/compile_all.sh
@@ -2,9 +2,4 @@
 
 set -e
 
-OPTION=""
-if [ "${STATIC_BUILD}" = "true" ]; then
-    OPTION="--build-static"
-fi
-
-./kiwix-build.py --build-target=${BUILD_TARGET} ${OPTION}
+./kiwix-build.py $BUILD_OPTION
diff --git a/travis/deploy.sh b/travis/deploy.sh
index 997637f..c3dfa61 100755
--- a/travis/deploy.sh
+++ b/travis/deploy.sh
@@ -6,14 +6,8 @@ SSH_KEY=travis/travisci_builder_id_key
 
 chmod 600 ${SSH_KEY}
 
-
-BASE_DIR="BUILD_${BUILD_TARGET}_static/INSTALL"
-if [ "${BUILD_TARGET}" = "win32" ]; then
-    ARCHIVE_OPTION="--zip"
-else
-    ARCHIVE_OPTION="--tar"
-fi
-./kiwix-deploy.py ${BASE_DIR} ${ARCHIVE_OPTION} \
+BASE_DIR="BUILD_*/INSTALL"
+./kiwix-deploy.py ${BASE_DIR} ${ARCHIVE_TYPE} \
     --deploy \
     --ssh_private_key=${SSH_KEY} \
     --server=nightlybot@download.kiwix.org \

From 940cea290d211d27b66d2208be58cb47bf6c8e6d Mon Sep 17 00:00:00 2001
From: Matthieu Gautier <mgautier@kymeria.fr>
Date: Tue, 7 Mar 2017 22:06:19 +0100
Subject: [PATCH 16/25] Use a 'extra_libs' key instead of 'properties'.

'properties' is too close of the meson definition and we need to duplicate
the entries for 'c_link_args' and 'cpp_link_args'.

By using 'extra_libs' we have a more functional definition of what we want.
---
 kiwix-build.py                 | 13 ++++---------
 templates/meson_cross_file.txt |  4 ++--
 2 files changed, 6 insertions(+), 11 deletions(-)

diff --git a/kiwix-build.py b/kiwix-build.py
index 7236d8a..d06647e 100755
--- a/kiwix-build.py
+++ b/kiwix-build.py
@@ -25,10 +25,7 @@ CROSS_ENV = {
     'fedora_win32': {
         'toolchain_names': ['mingw32_toolchain'],
         'root_path': '/usr/i686-w64-mingw32/sys-root/mingw',
-        'properties': {
-            'c_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4'],
-            'cpp_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4']
-        },
+        'extra_libs': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4', '-lmsvcr90'],
         'host_machine': {
             'system': 'windows',
             'cpu_family': 'x86',
@@ -42,10 +39,7 @@ CROSS_ENV = {
     'debian_win32': {
         'toolchain_names': ['mingw32_toolchain'],
         'root_path': '/usr/i686-w64-mingw32/',
-        'properties': {
-            'c_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4'],
-            'cpp_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4']
-        },
+        'extra_libs': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4', '-lmsvcr90'],
         'host_machine': {
             'system': 'windows',
             'cpu_family': 'x86',
@@ -487,7 +481,8 @@ class mingw32_toolchain(Toolchain):
 
         env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
         env['CFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS']
-        env['CXXFLAGS'] =" -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
+        env['CXXFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
+        env['LIBS'] = " ".join(self.buildEnv.cross_env['extra_libs']) + " " +env['LIBS']
 
 
 class Builder:
diff --git a/templates/meson_cross_file.txt b/templates/meson_cross_file.txt
index 464cbfe..6edddf4 100644
--- a/templates/meson_cross_file.txt
+++ b/templates/meson_cross_file.txt
@@ -6,8 +6,8 @@ cpp = '{toolchain.binaries[CXX]}'
 strip = '{toolchain.binaries[STRIP]}'
 
 [properties]
-c_link_args = {properties[c_link_args]!r}
-cpp_link_args = {properties[cpp_link_args]!r}
+c_link_args = {extra_libs!r}
+cpp_link_args = {extra_libs!r}
 
 [host_machine]
 cpu_family = '{host_machine[cpu_family]}'

From e3c0c5db82d9e30ffc9e6b10acf37a765a12c6dc Mon Sep 17 00:00:00 2001
From: Matthieu Gautier <mgautier@kymeria.fr>
Date: Tue, 7 Mar 2017 22:11:52 +0100
Subject: [PATCH 17/25] Toolchains now can have a builder and a source.

---
 kiwix-build.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/kiwix-build.py b/kiwix-build.py
index d06647e..53429b7 100755
--- a/kiwix-build.py
+++ b/kiwix-build.py
@@ -435,9 +435,13 @@ class Toolchain(metaclass=_MetaToolchain):
     all_toolchains = {}
     configure_option = ""
     cmake_option = ""
+    Builder = None
+    Source = None
 
     def __init__(self, buildEnv):
         self.buildEnv = buildEnv
+        self.source = self.Source(self) if self.Source else None
+        self.builder = self.Builder(self) if self.Builder else None
 
     @property
     def full_name(self):
@@ -527,7 +531,12 @@ class Builder:
             source.prepare()
 
     def build(self):
-        builders = (dep.builder for dep in self.targets.values() if not dep.skip)
+        toolchain_builders = (tlc.builder for tlc in self.buildEnv.toolchains if tlc.builder)
+        for toolchain_builder in toolchain_builders:
+            print("build toolchain {} :".format(toolchain_builder.name))
+            toolchain_builder.build()
+
+        builders = (dep.builder for dep in self.targets.values() if (dep.builder and not dep.skip))
         for builder in builders:
             print("build {} :".format(builder.name))
             builder.build()

From 600f1b55aa2a6f16cd6ad6e40d46413074919d8a Mon Sep 17 00:00:00 2001
From: Matthieu Gautier <mgautier@kymeria.fr>
Date: Tue, 7 Mar 2017 22:15:13 +0100
Subject: [PATCH 18/25] Add the android ndk toolchains and compile android on
 travis.

---
 .travis.yml                            |   6 +
 dependencies.py                        |  23 ++-
 kiwix-build.py                         | 226 ++++++++++++++++++++++++-
 templates/cmake_android_cross_file.txt |   7 +
 4 files changed, 250 insertions(+), 12 deletions(-)
 create mode 100644 templates/cmake_android_cross_file.txt

diff --git a/.travis.yml b/.travis.yml
index 2056eca..78778a4 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -18,6 +18,12 @@ env:
  - BUILD_OPTION="--target-platform=native_static" ARCHIVE_TYPE="--tar" DEPLOY=true
  - BUILD_OPTION="--target-platform=win32_dyn"
  - BUILD_OPTION="--target-platform=win32_static" ARCHIVE_TYPE="--zip" DEPLOY=true
+ - BUILD_OPTION="--target-platform=android_arm Kiwixlib"
+ - BUILD_OPTION="--target-platform=android_arm64 Kiwixlib"
+ - BUILD_OPTION="--target-platform=android_mips Kiwixlib"
+ - BUILD_OPTION="--target-platform=android_mips64 Kiwixlib"
+ - BUILD_OPTION="--target-platform=android_x86 Kiwixlib"
+ - BUILD_OPTION="--target-platform=android_x86_64 Kiwixlib"
 notifications:
   irc:
     channels:
diff --git a/dependencies.py b/dependencies.py
index dd22697..fcd87a7 100644
--- a/dependencies.py
+++ b/dependencies.py
@@ -241,16 +241,31 @@ class Kiwixlib(Dependency):
 
     @property
     def dependencies(self):
-        if self.buildEnv.platform_info.build == 'win32':
-            return ["Xapian", "CTPP2", "Pugixml", "Icu_cross_compile", "Zimlib"]
-        return ["Xapian", "CTPP2", "Pugixml", "Icu", "Zimlib"]
+        base_dependencies = ["Xapian", "Pugixml", "Zimlib"]
+        if self.buildEnv.platform_info.build != 'android':
+            base_dependencies += ['CTPP2']
+        if self.buildEnv.platform_info.build != 'native':
+            return base_dependencies + ["Icu_cross_compile"]
+        else:
+            return base_dependencies + ["Icu"]
 
     class Source(GitClone):
         git_remote = "https://github.com/kiwix/kiwix-lib.git"
         git_dir = "kiwix-lib"
 
     class Builder(MesonBuilder):
-        configure_option = "-Dctpp2-install-prefix={buildEnv.install_dir}"
+        @property
+        def configure_option(self):
+            base_option = "-Dctpp2-install-prefix={buildEnv.install_dir}"
+            if self.buildEnv.platform_info.build == 'android':
+                base_option += ' -Dandroid=true'
+            return base_option
+
+        @property
+        def library_type(self):
+            if self.buildEnv.platform_info.build == 'android':
+                return 'shared'
+            return super().library_type
 
 
 class KiwixTools(Dependency):
diff --git a/kiwix-build.py b/kiwix-build.py
index 53429b7..2fb1760 100755
--- a/kiwix-build.py
+++ b/kiwix-build.py
@@ -9,13 +9,16 @@ import platform
 from collections import OrderedDict
 
 from dependencies import Dependency
+from dependency_utils import ReleaseDownload, Builder
 from utils import (
     pj,
     remove_duplicates,
     get_sha256,
     StopBuild,
     SkipCommand,
-    Defaultdict)
+    Defaultdict,
+    Remotefile,
+    Context)
 
 REMOTE_PREFIX = 'http://download.kiwix.org/dev/'
 
@@ -31,9 +34,16 @@ CROSS_ENV = {
             'cpu_family': 'x86',
             'cpu': 'i686',
             'endian': 'little'
-        },
-        'env': {
-            '_format_PKG_CONFIG_LIBDIR': '{root_path}/lib/pkgconfig'
+        }
+    },
+    'fedora_android': {
+        'toolchain_names': ['android_ndk'],
+        'extra_libs': [],
+        'host_machine': {
+            'system': 'Android',
+            'cpu_family': 'x86',
+            'cpu': 'i686',
+            'endian': 'little'
         }
     },
     'debian_win32': {
@@ -45,9 +55,16 @@ CROSS_ENV = {
             'cpu_family': 'x86',
             'cpu': 'i686',
             'endian': 'little'
-        },
-        'env': {
-            '_format_PKG_CONFIG_LIBDIR': '{root_path}/lib/pkgconfig'
+        }
+    },
+    'debian_android': {
+        'toolchain_names': ['android_ndk'],
+        'extra_libs': [],
+        'host_machine': {
+            'system': 'Android',
+            'cpu_family': 'x86',
+            'cpu': 'i686',
+            'endian': 'little'
         }
     }
 }
@@ -85,6 +102,9 @@ PACKAGE_NAME_MAPPERS = {
         'libmicrohttpd': None, # ['mingw32-libmicrohttpd-static'] packaging dependecy seems buggy, and some static lib are name libfoo.dll.a and
                                # gcc cannot found them.
     },
+    'fedora_android': {
+        'COMMON': ['gcc-c++', 'cmake', 'automake', 'ccache', 'java-1.8.0-openjdk-devel']
+    },
     'debian_native_dyn': {
         'COMMON': ['gcc', 'cmake', 'libbz2-dev', 'ccache'],
         'zlib': ['zlib1g-dev'],
@@ -104,6 +124,9 @@ PACKAGE_NAME_MAPPERS = {
     'debian_win32_static': {
         'COMMON': ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools', 'ccache']
     },
+    'debian_android': {
+        'COMMON': ['gcc', 'cmake', 'ccache']
+    },
 }
 
 
@@ -122,12 +145,37 @@ class TargetInfo:
         return "{}_{}".format(self.build, 'static' if self.static else 'dyn')
 
 
+class AndroidTargetInfo(TargetInfo):
+    __arch_infos = {
+        'arm' : ('arm-linux-androideabi'),
+        'arm64': ('aarch64-linux-android'),
+        'mips': ('mipsel-linux-android'),
+        'mips64': ('mips64el-linux-android'),
+        'x86': ('i686-linux-android'),
+        'x86_64': ('x86_64-linux-android'),
+    }
+
+    def __init__(self, arch):
+        super().__init__('android', True)
+        self.arch = arch
+        self.arch_full = self.__arch_infos[arch]
+
+    def __str__(self):
+        return "android"
+
+
 class BuildEnv:
     target_platforms = {
         'native_dyn': TargetInfo('native', False),
         'native_static': TargetInfo('native', True),
         'win32_dyn': TargetInfo('win32', False),
-        'win32_static': TargetInfo('win32', True)
+        'win32_static': TargetInfo('win32', True),
+        'android_arm': AndroidTargetInfo('arm'),
+        'android_arm64': AndroidTargetInfo('arm64'),
+        'android_mips': AndroidTargetInfo('mips'),
+        'android_mips64': AndroidTargetInfo('mips64'),
+        'android_x86': AndroidTargetInfo('x86'),
+        'android_x86_64': AndroidTargetInfo('x86_64'),
     }
 
     def __init__(self, options, targetsDict):
@@ -220,6 +268,10 @@ class BuildEnv:
         self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
         self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
 
+    def setup_android(self):
+        self.cmake_crossfile = self._gen_crossfile('cmake_android_cross_file.txt')
+        self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
+
     def __getattr__(self, name):
         return getattr(self.options, name)
 
@@ -449,9 +501,40 @@ class Toolchain(metaclass=_MetaToolchain):
             name = self.name,
             version = self.version)
 
+    @property
+    def source_path(self):
+        return pj(self.buildEnv.source_dir, self.source.source_dir)
+
+    @property
+    def _log_dir(self):
+        return self.buildEnv.log_dir
+
     def set_env(self, env):
         pass
 
+    def command(self, name, function, *args):
+        print("  {} {} : ".format(name, self.name), end="", flush=True)
+        log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
+        context = Context(name, log, True)
+        try:
+            ret = function(*args, context=context)
+            context._finalise()
+            print("OK")
+            return ret
+        except SkipCommand:
+            print("SKIP")
+        except subprocess.CalledProcessError:
+            print("ERROR")
+            try:
+                with open(log, 'r') as f:
+                    print(f.read())
+            except:
+                pass
+            raise StopBuild()
+        except:
+            print("ERROR")
+            raise
+
 
 class mingw32_toolchain(Toolchain):
     name = 'mingw32'
@@ -489,6 +572,133 @@ class mingw32_toolchain(Toolchain):
         env['LIBS'] = " ".join(self.buildEnv.cross_env['extra_libs']) + " " +env['LIBS']
 
 
+class android_ndk(Toolchain):
+    name = 'android-ndk'
+    version = 'r13b'
+    gccver = '4.9.x'
+
+    @property
+    def api(self):
+        return '21' if self.arch in ('arm64', 'mips64', 'x86_64') else '14'
+
+    @property
+    def platform(self):
+        return 'android-'+self.api
+
+    @property
+    def arch(self):
+        return self.buildEnv.platform_info.arch
+
+    @property
+    def arch_full(self):
+        return self.buildEnv.platform_info.arch_full
+
+    @property
+    def toolchain(self):
+        return self.arch_full+"-4.9"
+
+    @property
+    def root_path(self):
+        return pj(self.builder.install_path, 'sysroot')
+
+    @property
+    def binaries(self):
+        binaries = ((k,'{}-{}'.format(self.arch_full, v))
+                for k, v in (('CC', 'gcc'),
+                             ('CXX', 'g++'),
+                             ('AR', 'ar'),
+                             ('STRIP', 'strip'),
+                             ('WINDRES', 'windres'),
+                             ('RANLIB', 'ranlib'),
+                             ('LD', 'ld'))
+               )
+        return {k:pj(self.builder.install_path, 'bin', v)
+                for k,v in binaries}
+
+    @property
+    def configure_option(self):
+        return '--host={}'.format(self.arch_full)
+
+    @property
+    def full_name(self):
+        return "{name}-{version}-{arch}-{api}".format(
+            name = self.name,
+            version = self.version,
+            arch = self.arch,
+            api = self.api)
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('android-ndk-r13b-linux-x86_64.zip',
+                             '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c',
+                             'https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip')
+
+        @property
+        def source_dir(self):
+            return "{}-{}".format(
+                self.target.name,
+                self.target.version)
+
+
+    class Builder(Builder):
+
+        @property
+        def install_path(self):
+            return self.build_path
+
+        def _build_platform(self, context):
+            context.try_skip(self.build_path)
+            script = pj(self.source_path, 'build/tools/make_standalone_toolchain.py')
+            current_permissions = stat.S_IMODE(os.lstat(script).st_mode)
+            os.chmod(script, current_permissions | stat.S_IXUSR)
+            command = '{script} --arch={arch} --api={api} --install-dir={install_dir} --force'
+            command = command.format(
+                script=script,
+                arch=self.target.arch,
+                api=self.target.api,
+                install_dir=self.install_path
+            )
+            self.buildEnv.run_command(command, self.build_path, context)
+
+        def _fix_permission_right(self, context):
+            context.try_skip(self.build_path)
+            bin_dirs = [pj(self.install_path, 'bin'),
+                        pj(self.install_path, self.target.arch_full, 'bin'),
+                        pj(self.install_path, 'libexec', 'gcc', self.target.arch_full, self.target.gccver)
+                       ]
+            for root, dirs, files in os.walk(self.install_path):
+                if not root in bin_dirs:
+                    continue
+
+                for file_ in files:
+                    file_path = pj(root, file_)
+                    if os.path.islink(file_path):
+                        continue
+                    current_permissions = stat.S_IMODE(os.lstat(file_path).st_mode)
+                    os.chmod(file_path, current_permissions | stat.S_IXUSR)
+
+        def build(self):
+            self.command('build_platform', self._build_platform)
+            self.command('fix_permission_right', self._fix_permission_right)
+
+    def get_bin_dir(self):
+        return [pj(self.builder.install_path, 'bin')]
+
+    def set_env(self, env):
+        env['CC'] = self.binaries['CC']
+        env['CXX'] = self.binaries['CXX']
+
+        env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
+        env['CFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(self.root_path) + env['CFLAGS']
+        env['CXXFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(self.root_path) + env['CXXFLAGS']
+        env['LDFLAGS'] = ' -fPIE -pie --sysroot={} '.format(self.root_path) + env['LDFLAGS']
+        #env['CFLAGS'] = ' -fPIC -D_FILE_OFFSET_BITS=64 -O3 '+env['CFLAGS']
+        #env['CXXFLAGS'] = (' -D__OPTIMIZE__ -fno-strict-aliasing '
+        #                   ' -DU_HAVE_NL_LANGINFO_CODESET=0 '
+        #                   '-DU_STATIC_IMPLEMENTATION -O3 '
+        #                   '-DU_HAVE_STD_STRING -DU_TIMEZONE=0 ')+env['CXXFLAGS']
+        env['NDK_DEBUG'] = '0'
+
+
 class Builder:
     def __init__(self, options):
         self.targets = OrderedDict()
diff --git a/templates/cmake_android_cross_file.txt b/templates/cmake_android_cross_file.txt
new file mode 100644
index 0000000..7f80a34
--- /dev/null
+++ b/templates/cmake_android_cross_file.txt
@@ -0,0 +1,7 @@
+SET(CMAKE_SYSTEM_NAME {host_machine[system]})
+SET(CMAKE_ANDROID_STANDALONE_TOOLCHAIN {toolchain.builder.install_path})
+
+SET(CMAKE_C_COMPILER "{toolchain.binaries[CC]}")
+SET(CMAKE_CXX_COMPILER "{toolchain.binaries[CXX]}")
+
+SET(CMAKE_FIND_ROOT_PATH {toolchain.root_path})

From 6d7adf0f043ff2591203f80da6bc4d4774be5b49 Mon Sep 17 00:00:00 2001
From: Matthieu Gautier <mgautier@kymeria.fr>
Date: Mon, 27 Feb 2017 14:03:54 +0100
Subject: [PATCH 19/25] Add a patch for icu4c on android 64 bits.

On recent version of the NDK (r13b), `exec_elf.h` has been removed for
API level >= 20. On 64 bits, we need the API level 21, and then,
a defined (ELF64_ST_INFO) is missing.

We readd it where (and if) we need it.
---
 dependencies.py                           |  3 ++-
 patches/icu4c_android_elf64_st_info.patch | 13 +++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)
 create mode 100644 patches/icu4c_android_elf64_st_info.patch

diff --git a/dependencies.py b/dependencies.py
index fcd87a7..d9140c7 100644
--- a/dependencies.py
+++ b/dependencies.py
@@ -177,7 +177,8 @@ class Icu(Dependency):
     class Source(ReleaseDownload):
         archive = Remotefile('icu4c-56_1-src.tgz',
                              '3a64e9105c734dcf631c0b3ed60404531bce6c0f5a64bfe1a6402a4cc2314816')
-        patches = ["icu4c_fix_static_lib_name_mingw.patch"]
+        patches = ["icu4c_fix_static_lib_name_mingw.patch",
+                   "icu4c_android_elf64_st_info.patch"]
         data = Remotefile('icudt56l.dat',
                           'e23d85eee008f335fc49e8ef37b1bc2b222db105476111e3d16f0007d371cbca')
 
diff --git a/patches/icu4c_android_elf64_st_info.patch b/patches/icu4c_android_elf64_st_info.patch
new file mode 100644
index 0000000..12e901f
--- /dev/null
+++ b/patches/icu4c_android_elf64_st_info.patch
@@ -0,0 +1,13 @@
+diff -ur icu4c-58_2/source/tools/toolutil/pkg_genc.c icu4c-58_2.patched/source/tools/toolutil/pkg_genc.c
+--- icu4c-58_2/source/tools/toolutil/pkg_genc.c	2016-06-15 20:58:17.000000000 +0200
++++ icu4c-58_2.patched/source/tools/toolutil/pkg_genc.c	2017-02-27 10:23:39.985471339 +0100
+@@ -35,6 +35,9 @@
+ #       define EM_X86_64 62
+ #   endif
+ #   define ICU_ENTRY_OFFSET 0
++#   ifndef ELF64_ST_INFO
++#       define ELF64_ST_INFO(b,t)      (((b) << 4) + ((t) & 0xf))
++#   endif
+ #endif
+ 
+ #include <stdio.h>

From 6b85f7d012606df4774cde268ceb77fa34ed0cfb Mon Sep 17 00:00:00 2001
From: Matthieu Gautier <mgautier@kymeria.fr>
Date: Wed, 22 Feb 2017 18:11:28 +0100
Subject: [PATCH 20/25] Update xapian patch for pkg_config.

AC_SEARCH_LIBS try to compile and use a symbol (timer_create) using
a lib (rt).

But, in fact, it also try to use the symbol but without using the lib.
If it's work, it consider that the libs is found and set the ac_res
variable to "none required".

So, we must check the value of ac_res to know if we must add '-lrt' to
the pkg-config file.
---
 patches/xapian_pkgconfig.patch | 65 +++++++++++++++++++++-------------
 1 file changed, 41 insertions(+), 24 deletions(-)

diff --git a/patches/xapian_pkgconfig.patch b/patches/xapian_pkgconfig.patch
index b2d4dc9..1e7c2dc 100644
--- a/patches/xapian_pkgconfig.patch
+++ b/patches/xapian_pkgconfig.patch
@@ -1,33 +1,44 @@
-diff -ur xapian-core-1.4.0/configure.ac xapian-core-1.4.0.patched/configure.ac
+diff -ur xapian-core-1.4.0/configure.ac xapian-core-1.4.0-patched/configure.ac
 --- xapian-core-1.4.0/configure.ac	2016-06-25 17:36:49.000000000 +0200
-+++ xapian-core-1.4.0.patched/configure.ac	2017-01-17 14:33:42.268536542 +0100
-@@ -393,6 +393,7 @@
++++ xapian-core-1.4.0-patched/configure.ac	2017-02-22 17:45:57.066365636 +0100
+@@ -393,22 +393,25 @@
  esac
  
  dnl We use timer_create() if available to implement a search time limit.
 +use_rt_lib=0
  SAVE_LIBS=$LIBS
- AC_SEARCH_LIBS([timer_create], [rt],
- 	       [
-@@ -403,12 +404,14 @@
+-AC_SEARCH_LIBS([timer_create], [rt],
+-	       [
++AC_SEARCH_LIBS([timer_create], [rt], [
+     AC_MSG_CHECKING([for timer_create() usability])
+-    AC_COMPILE_IFELSE([AC_LANG_PROGRAM(
++    AC_COMPILE_IFELSE(
++      [AC_LANG_PROGRAM(
+ [[#if defined _AIX || defined __GNU__
+ #error timer_create known not to work
  #endif]])],
- 	[AC_MSG_RESULT([yes])
- 	XAPIAN_LIBS="$LIBS $XAPIAN_LIBS"
+-	[AC_MSG_RESULT([yes])
+-	XAPIAN_LIBS="$LIBS $XAPIAN_LIBS"
 -	AC_DEFINE([HAVE_TIMER_CREATE], [1], [Define to 1 if you have the 'timer_create' function.])]
-+	AC_DEFINE([HAVE_TIMER_CREATE], [1], [Define to 1 if you have the 'timer_create' function.])
-+        use_rt_lib=1]
- 	,
- 	[AC_MSG_RESULT([no])
-     ])
+-	,
+-	[AC_MSG_RESULT([no])
+-    ])
++	  [AC_MSG_RESULT([yes])
++	  XAPIAN_LIBS="$LIBS $XAPIAN_LIBS"
++	  AC_DEFINE([HAVE_TIMER_CREATE], [1], [Define to 1 if you have the 'timer_create' function.])
++	  AS_IF([test "$ac_res" != "none required"], use_rt_lib=1)
++      ],
++	  [AC_MSG_RESULT([no])]
++	)
  ])
  LIBS=$SAVE_LIBS
-+AM_CONDITIONAL([USE_RT_LIB], [test "$use_win32_uuid_api" = 1])
++AM_CONDITIONAL([USE_RT_LIB], [test "$use_rt_lib" = 1])
  
  dnl Used by tests/soaktest/soaktest.cc
  AC_CHECK_FUNCS([srandom random])
-diff -ur xapian-core-1.4.0/configure xapian-core-1.4.0.patched/configure
+diff -ur xapian-core-1.4.0/configure xapian-core-1.4.0-patched/configure
 --- xapian-core-1.4.0/configure	2016-06-25 17:39:25.000000000 +0200
-+++ xapian-core-1.4.0.patched/configure	2017-01-17 15:43:07.929290801 +0100
++++ xapian-core-1.4.0-patched/configure	2017-02-22 17:45:44.472585524 +0100
 @@ -671,6 +671,8 @@
  DOCUMENTATION_RULES_FALSE
  DOCUMENTATION_RULES_TRUE
@@ -45,15 +56,22 @@ diff -ur xapian-core-1.4.0/configure xapian-core-1.4.0.patched/configure
  SAVE_LIBS=$LIBS
  { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing timer_create" >&5
  $as_echo_n "checking for library containing timer_create... " >&6; }
-@@ -18324,6 +18327,7 @@
+@@ -18320,10 +18323,13 @@
+ if ac_fn_cxx_try_compile "$LINENO"; then :
+   { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+ $as_echo "yes" >&6; }
+-	XAPIAN_LIBS="$LIBS $XAPIAN_LIBS"
++	  XAPIAN_LIBS="$LIBS $XAPIAN_LIBS"
  
  $as_echo "#define HAVE_TIMER_CREATE 1" >>confdefs.h
  
-+        use_rt_lib=1
++	  if test "$ac_res" != "none required"; then :
++  use_rt_lib=1
++fi
  
  else
    { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
-@@ -18335,6 +18339,14 @@
+@@ -18335,6 +18341,14 @@
  fi
  
  LIBS=$SAVE_LIBS
@@ -68,7 +86,7 @@ diff -ur xapian-core-1.4.0/configure xapian-core-1.4.0.patched/configure
  
  for ac_func in srandom random
  do :
-@@ -20854,6 +20866,10 @@
+@@ -20854,6 +20868,10 @@
  Usually this means the macro was only invoked conditionally." "$LINENO" 5
  fi
  
@@ -79,9 +97,9 @@ diff -ur xapian-core-1.4.0/configure xapian-core-1.4.0.patched/configure
  if test -z "${DOCUMENTATION_RULES_TRUE}" && test -z "${DOCUMENTATION_RULES_FALSE}"; then
    as_fn_error $? "conditional \"DOCUMENTATION_RULES\" was never defined.
  Usually this means the macro was only invoked conditionally." "$LINENO" 5
-diff -ur xapian-core-1.4.0/pkgconfig/xapian-core.pc.in xapian-core-1.4.0.patched/pkgconfig/xapian-core.pc.in
---- xapian-core-1.4.0/pkgconfig/xapian-core.pc.in	2017-01-17 15:22:40.184786108 +0100
-+++ xapian-core-1.4.0.patched/pkgconfig/xapian-core.pc.in	2017-01-17 14:34:17.692972977 +0100
+diff -ur xapian-core-1.4.0/pkgconfig/xapian-core.pc.in xapian-core-1.4.0-patched/pkgconfig/xapian-core.pc.in
+--- xapian-core-1.4.0/pkgconfig/xapian-core.pc.in	2016-06-25 17:36:49.000000000 +0200
++++ xapian-core-1.4.0-patched/pkgconfig/xapian-core.pc.in	2017-02-22 17:09:12.488793901 +0100
 @@ -11,4 +11,6 @@
  URL: https://xapian.org/
  Version: @VERSION@
@@ -90,4 +108,3 @@ diff -ur xapian-core-1.4.0/pkgconfig/xapian-core.pc.in xapian-core-1.4.0.patched
 +@USE_RT_LIB_TRUE@Libs: @ldflags@ -L${libdir} -lxapian@LIBRARY_VERSION_SUFFIX@ -lrt
 +@USE_RT_LIB_FALSE@Libs: @ldflags@ -L${libdir} -lxapian@LIBRARY_VERSION_SUFFIX@
 +@USE_WIN32_UUID_API_FALSE@Requires: uuid
-

From 0845a5a53b030f900266a2e65fc0aad3782df053 Mon Sep 17 00:00:00 2001
From: Matthieu Gautier <mgautier@kymeria.fr>
Date: Wed, 22 Feb 2017 18:57:03 +0100
Subject: [PATCH 21/25] Move to last version of ICU.

There is a bug in 56_1 version when compiling resource.
This is fixed in 58_2.

However the last version 58_2 use _create_local and _free_locale symbols on
Windows and we need to add the right libs to the compilation command.
---
 dependencies.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/dependencies.py b/dependencies.py
index d9140c7..ef70f60 100644
--- a/dependencies.py
+++ b/dependencies.py
@@ -172,11 +172,12 @@ class MicroHttpd(Dependency):
 
 class Icu(Dependency):
     name = "icu4c"
-    version = "56_1"
+    version = "58_2"
 
     class Source(ReleaseDownload):
-        archive = Remotefile('icu4c-56_1-src.tgz',
-                             '3a64e9105c734dcf631c0b3ed60404531bce6c0f5a64bfe1a6402a4cc2314816')
+        archive = Remotefile('icu4c-58_2-src.tgz',
+                             '2b0a4410153a9b20de0e20c7d8b66049a72aef244b53683d0d7521371683da0c',
+                             'https://freefr.dl.sourceforge.net/project/icu/ICU4C/58.2/icu4c-58_2-src.tgz')
         patches = ["icu4c_fix_static_lib_name_mingw.patch",
                    "icu4c_android_elf64_st_info.patch"]
         data = Remotefile('icudt56l.dat',

From 5ae802b7074af182437d4b4a8d4701d3f8bd4652 Mon Sep 17 00:00:00 2001
From: Matthieu Gautier <mgautier@kymeria.fr>
Date: Mon, 27 Feb 2017 15:30:09 +0100
Subject: [PATCH 22/25] Do not pass -Dctpp2-install-prefix to KiwixTools.

We don't need it anymore with last version of KiwixTools.
---
 dependencies.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/dependencies.py b/dependencies.py
index ef70f60..1120763 100644
--- a/dependencies.py
+++ b/dependencies.py
@@ -281,7 +281,6 @@ class KiwixTools(Dependency):
     class Builder(MesonBuilder):
         @property
         def configure_option(self):
-            base_options = "-Dctpp2-install-prefix={buildEnv.install_dir}"
             if self.buildEnv.platform_info.static:
-                base_options += " -Dstatic-linkage=true"
-            return base_options
+                return "-Dstatic-linkage=true"
+            return ""

From 411f861b133f6a7a319e35d1a5c81c240917223e Mon Sep 17 00:00:00 2001
From: Matthieu Gautier <mgautier@kymeria.fr>
Date: Tue, 28 Feb 2017 10:01:48 +0100
Subject: [PATCH 23/25] Move log dir in a subdir of BUILD_*

---
 kiwix-build.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kiwix-build.py b/kiwix-build.py
index 2fb1760..5494179 100755
--- a/kiwix-build.py
+++ b/kiwix-build.py
@@ -183,7 +183,7 @@ class BuildEnv:
         build_dir = "BUILD_{}".format(options.target_platform)
         self.build_dir = pj(options.working_dir, build_dir)
         self.archive_dir = pj(options.working_dir, "ARCHIVE")
-        self.log_dir = pj(options.working_dir, 'LOGS')
+        self.log_dir = pj(self.build_dir, 'LOGS')
         self.install_dir = pj(self.build_dir, "INSTALL")
         for d in (self.source_dir,
                   self.build_dir,

From c8e2e14b5cc564b8cfc7517d394f7472dffe2b1b Mon Sep 17 00:00:00 2001
From: Matthieu Gautier <mgautier@kymeria.fr>
Date: Tue, 28 Feb 2017 10:24:38 +0100
Subject: [PATCH 24/25] Update README.md

---
 README.md | 126 ++++++++++++++++++++++++++++++++++++++----------------
 1 file changed, 90 insertions(+), 36 deletions(-)

diff --git a/README.md b/README.md
index 50f01d9..e2f36e4 100644
--- a/README.md
+++ b/README.md
@@ -20,67 +20,121 @@ Take care, the paragraphs are about the *target platforms*. If you
 want to build Kiwix for Android on a GNU/Linux system, you should
 follow the instructions of the "Android" paragraph.
 
-## GNU/Linux
-Install pre-requisties in your distro, eg, in Debian based:
+## Preparing environment
 
-    sudo apt install git cmake meson python3-virtualenv virtualenvwrapper zlib1g-dev libicu-dev aria2 libtool
+You will need a recent version of meson (0.34) and ninja (1.6)
+If your distribution provide a recent enough versions for them, just install
+them with your package manager.
 
-### Dynamic
+Else you can install them manually
 
-Pretty simple once all dependencies are installed :
+### Meson
 
-The archives and sources will be copied in your current working dir so
-I recommend you to create a working directory:
+```
+pip install meson --user # Install Meson
+```
 
-    $ mkdir <MY_WORKING_DIR>
-    $ cd <MY_WOKRING_DIR>
+(You may want to install meson in a virtualenv if you prefer)
 
-Once ready, just run the script with the install dir as argument:
+### ninja
 
-    $ <right/path/to/>kiwix-build.py <INSTALL_DIR>
+```
+git clone git://github.com/ninja-build/ninja.git
+cd ninja
+git checkout release
+./configure.py --bootstrap
+cp ninja <a_directory_in_your_path>
+```
 
-The script will download and install all the needed dependencies and
-kiwix related repository.
+## Buildings
 
-At the end of the script you will found the binaries in <INSTALL_DIR>/bin.
+This is the simple one.
 
-As it is a dynamic linked binary, you will need to add the lib directory to
-LD_LIBRARY_PATH :
+```
+./kiwix-build.py
+```
 
-    $ LD_LIBRARY_PATH=<INSTALL_DIR>/lib <INSTALL_DIR>/bin/kiwix-serve
+It will compile everything.
+If you are using a supported platform (redhad or debian based) it will install
+missing packages using sudo.
+If you don't want to trust kiwix-build.py and give it the root right, just
+launch yourself the the printed command.
 
-### Static
+### Outputs
 
-To build a static binary simply add the --build_static option to the 
-kiwix-build.py script :
+Kiwix-build.py will create several directories:
 
-    $ kiwix-build.py --build_static <INSTALL_DIR>
+- ARCHIVES : All the downloaded archives go there.
+- SOURCES : All the sources (extracted from archives and patched) go there.
+- BUILD_native_dyn : All the build files go there.
+- BUILD_native_dyn/INSTALL : The installed files go there.
+- BUILD_native_dyn/LOGS: The logs files of the build.
 
-Notes: You cannot use the same working directory to build a dynamic and static.
+ARCHIVES and SOURCES are independent of the build type you choose.
 
-Notes: At the end of the script, meson may raise a error when it install the
-kiwix-server binary. This is a meson bug and it has been fixed here
-(https://github.com/mesonbuild/meson/pull/1240) but your meson version may
-not include this fix.
-However, the kiwix-server binary is valid and ready to use.
+If you want to install all those directories elsewhere, you can pass the
+`--working-dir` option:
 
-## Mac OSX Universal
+```
+./kiwix-build.py --working-dir <a_directory_somewhere>
+```
 
-The script has not been tested on Mac OSX.
+### Other target
 
-Please, if you have a Mac OSX, try to run the kiwix-build script and
-report errors or, better, fixes :)
+By default, kiwix-build will build kiwix-tools and all its dependencies.
+If you want to compile another target only (let's said kiwixlib), you can
+specify it:
 
-## Android and Windows
+```
+./kiwix-build Kiwixlib
+```
 
-Cross compilation and strange stuff are to come.
+### Other target platforms.
+
+By default, kiwix-build will build everything for the current (native) platform
+using dynamic linkage (hence the `native_dyn` of the BUILD_* directory).
+
+But you can select another target platforms using the option `target-platform`.
+For now, there is ten different platform supported :
+
+- native_dyn
+- native_static
+- win32_dyn
+- win32_static
+- android_arm
+- android_arm64
+- android_mips
+- android_mips64
+- android_x86
+- android_x86_64
+
+So, if you want to compile for win32 using static linkage:
+
+```
+./kiwix-build.py --target-platform win32_dyn
+```
+
+As said before, the ARCHIVES and SOURCES directories are common of the target
+platform. So, if you always work in the same working directory the sources will
+not be downloaded and prepared again.
+
+On android* platforms, only kiwixlib is supported. So you must ask to
+kiwix-build to compile only kiwixlib:
+
+```
+./kiwix-build.py --target-platform android_arm Kiwixlib
+```
+
+## Know bugs
+
+- The script has not been tested on Mac OSX.
+- Cross-compilation to arm (raspberry-PI) is missing.
+
+Help is welcome on those parts.
 
-If you wich to help, you're welcome.
 
 # CONTACT
 
-Email: kiwix-developer@lists.sourceforge.net
-
 IRC: #kiwix on irc.freenode.net  
 (I'm hidding myself under the starmad pseudo)
 

From eac3b791511989bacf6be894dba589e002a677e1 Mon Sep 17 00:00:00 2001
From: Matthieu Gautier <mgautier@kymeria.fr>
Date: Tue, 28 Feb 2017 10:25:45 +0100
Subject: [PATCH 25/25] Do not install unneeded package in
 travis/install_extra_deps.sh

---
 travis/install_extra_deps.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/travis/install_extra_deps.sh b/travis/install_extra_deps.sh
index 6f2540d..b888ae6 100755
--- a/travis/install_extra_deps.sh
+++ b/travis/install_extra_deps.sh
@@ -5,7 +5,7 @@ set -e
 orig_dir=$(pwd)
 
 sudo apt-get update -qq
-sudo apt-get install -qq uuid-dev libicu-dev libctpp2-dev automake libtool python3-pip zlib1g-dev lzma-dev libbz2-dev cmake
+sudo apt-get install -qq python3-pip
 pip3 install meson
 
 # ninja