Merge pull request #14 from kiwix/android

Android

.travis.yml

@@ -12,12 +12,18 @@ deploy:
   skip_cleanup: true
   script: travis/deploy.sh
   on:
-    condition: ( "$STATIC_BUILD" = "true" ) && ( "$TRAVIS_EVENT_TYPE" = "cron" )
+    condition: ( "$DEPLOY" = "true" ) && ( "$TRAVIS_EVENT_TYPE" = "cron" )
 env:
- - STATIC_BUILD=true BUILD_TARGET=native
+ - BUILD_OPTION="--target-platform=native_dyn"
- - STATIC_BUILD=true BUILD_TARGET=win32
+ - BUILD_OPTION="--target-platform=native_static" ARCHIVE_TYPE="--tar" DEPLOY=true
- - STATIC_BUILD=false BUILD_TARGET=native
+ - BUILD_OPTION="--target-platform=win32_dyn"
- - STATIC_BUILD=false BUILD_TARGET=win32
+ - BUILD_OPTION="--target-platform=win32_static" ARCHIVE_TYPE="--zip" DEPLOY=true
+ - BUILD_OPTION="--target-platform=android_arm Kiwixlib"
+ - BUILD_OPTION="--target-platform=android_arm64 Kiwixlib"
+ - BUILD_OPTION="--target-platform=android_mips Kiwixlib"
+ - BUILD_OPTION="--target-platform=android_mips64 Kiwixlib"
+ - BUILD_OPTION="--target-platform=android_x86 Kiwixlib"
+ - BUILD_OPTION="--target-platform=android_x86_64 Kiwixlib"
 notifications:
   irc:
     channels:

README.md

@@ -20,67 +20,121 @@ Take care, the paragraphs are about the *target platforms*. If you
 want to build Kiwix for Android on a GNU/Linux system, you should
 follow the instructions of the "Android" paragraph.
 
-## GNU/Linux
+## Preparing environment
-Install pre-requisties in your distro, eg, in Debian based:
 
-    sudo apt install git cmake meson python3-virtualenv virtualenvwrapper zlib1g-dev libicu-dev aria2 libtool
+You will need a recent version of meson (0.34) and ninja (1.6)
+If your distribution provide a recent enough versions for them, just install
+them with your package manager.
 
-### Dynamic
+Else you can install them manually
 
-Pretty simple once all dependencies are installed :
+### Meson
 
-The archives and sources will be copied in your current working dir so
+```
-I recommend you to create a working directory:
+pip install meson --user # Install Meson
+```
 
-    $ mkdir <MY_WORKING_DIR>
+(You may want to install meson in a virtualenv if you prefer)
-    $ cd <MY_WOKRING_DIR>
 
-Once ready, just run the script with the install dir as argument:
+### ninja
 
-    $ <right/path/to/>kiwix-build.py <INSTALL_DIR>
+```
+git clone git://github.com/ninja-build/ninja.git
+cd ninja
+git checkout release
+./configure.py --bootstrap
+cp ninja <a_directory_in_your_path>
+```
 
-The script will download and install all the needed dependencies and
+## Buildings
-kiwix related repository.
 
-At the end of the script you will found the binaries in <INSTALL_DIR>/bin.
+This is the simple one.
 
-As it is a dynamic linked binary, you will need to add the lib directory to
+```
-LD_LIBRARY_PATH :
+./kiwix-build.py
+```
 
-    $ LD_LIBRARY_PATH=<INSTALL_DIR>/lib <INSTALL_DIR>/bin/kiwix-serve
+It will compile everything.
+If you are using a supported platform (redhad or debian based) it will install
+missing packages using sudo.
+If you don't want to trust kiwix-build.py and give it the root right, just
+launch yourself the the printed command.
 
-### Static
+### Outputs
 
-To build a static binary simply add the --build_static option to the 
+Kiwix-build.py will create several directories:
-kiwix-build.py script :
 
-    $ kiwix-build.py --build_static <INSTALL_DIR>
+- ARCHIVES : All the downloaded archives go there.
+- SOURCES : All the sources (extracted from archives and patched) go there.
+- BUILD_native_dyn : All the build files go there.
+- BUILD_native_dyn/INSTALL : The installed files go there.
+- BUILD_native_dyn/LOGS: The logs files of the build.
 
-Notes: You cannot use the same working directory to build a dynamic and static.
+ARCHIVES and SOURCES are independent of the build type you choose.
 
-Notes: At the end of the script, meson may raise a error when it install the
+If you want to install all those directories elsewhere, you can pass the
-kiwix-server binary. This is a meson bug and it has been fixed here
+`--working-dir` option:
-(https://github.com/mesonbuild/meson/pull/1240) but your meson version may
-not include this fix.
-However, the kiwix-server binary is valid and ready to use.
 
-## Mac OSX Universal
+```
+./kiwix-build.py --working-dir <a_directory_somewhere>
+```
 
-The script has not been tested on Mac OSX.
+### Other target
 
-Please, if you have a Mac OSX, try to run the kiwix-build script and
+By default, kiwix-build will build kiwix-tools and all its dependencies.
-report errors or, better, fixes :)
+If you want to compile another target only (let's said kiwixlib), you can
+specify it:
 
-## Android and Windows
+```
+./kiwix-build Kiwixlib
+```
 
-Cross compilation and strange stuff are to come.
+### Other target platforms.
+
+By default, kiwix-build will build everything for the current (native) platform
+using dynamic linkage (hence the `native_dyn` of the BUILD_* directory).
+
+But you can select another target platforms using the option `target-platform`.
+For now, there is ten different platform supported :
+
+- native_dyn
+- native_static
+- win32_dyn
+- win32_static
+- android_arm
+- android_arm64
+- android_mips
+- android_mips64
+- android_x86
+- android_x86_64
+
+So, if you want to compile for win32 using static linkage:
+
+```
+./kiwix-build.py --target-platform win32_dyn
+```
+
+As said before, the ARCHIVES and SOURCES directories are common of the target
+platform. So, if you always work in the same working directory the sources will
+not be downloaded and prepared again.
+
+On android* platforms, only kiwixlib is supported. So you must ask to
+kiwix-build to compile only kiwixlib:
+
+```
+./kiwix-build.py --target-platform android_arm Kiwixlib
+```
+
+## Know bugs
+
+- The script has not been tested on Mac OSX.
+- Cross-compilation to arm (raspberry-PI) is missing.
+
+Help is welcome on those parts.
 
-If you wich to help, you're welcome.
 
 # CONTACT
 
-Email: kiwix-developer@lists.sourceforge.net
-
 IRC: #kiwix on irc.freenode.net  
 (I'm hidding myself under the starmad pseudo)
 

dependencies.py

@@ -0,0 +1,286 @@
+import shutil
+
+from dependency_utils import (
+    Dependency,
+    ReleaseDownload,
+    GitClone,
+    MakeBuilder,
+    CMakeBuilder,
+    MesonBuilder)
+
+from utils import Remotefile, pj, SkipCommand
+
+# *************************************
+# Missing dependencies
+# Is this ok to assume that those libs
+# exist in your "distri" (linux/mac) ?
+# If not, we need to compile them here
+# *************************************
+# aria2
+# Argtable
+# MSVirtual
+# Android
+# libiconv
+# gettext
+# *************************************
+
+
+class zlib(Dependency):
+    name = 'zlib'
+    version = '1.2.8'
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('zlib-1.2.8.tar.gz',
+                             '36658cb768a54c1d4dec43c3116c27ed893e88b02ecfcb44f2166f9c0b7f2a0d')
+        patches = ['zlib_std_libname.patch']
+
+    class Builder(MakeBuilder):
+        dynamic_configure_option = "--shared"
+        static_configure_option = "--static"
+
+        def _pre_build_script(self, context):
+            context.try_skip(self.build_path)
+            shutil.copytree(self.source_path, self.build_path)
+
+        @property
+        def all_configure_option(self):
+            return '--static' if self.buildEnv.platform_info.static else '--shared'
+
+        @property
+        def configure_option(self):
+            options = "-DINSTALL_PKGCONFIG_DIR={}".format(pj(self.buildEnv.install_dir, self.buildEnv.libprefix, 'pkgconfig'))
+            if self.buildEnv.platform_info.static:
+                options += " -DBUILD_SHARED_LIBS=false"
+            else:
+                options += " -DBUILD_SHARED_LIBS=true"
+            return options
+
+        def _configure(self, context):
+            if self.buildEnv.platform_info.build == 'win32':
+                raise SkipCommand()
+            return super()._configure(context)
+
+        @property
+        def make_option(self):
+            if self.buildEnv.platform_info.build == 'win32':
+                return "--makefile win32/Makefile.gcc PREFIX={host}- SHARED_MODE={static} INCLUDE_PATH={include_path} LIBRARY_PATH={library_path} BINARY_PATH={binary_path}".format(
+                    host='i686-w64-mingw32',
+                    static="0" if self.buildEnv.platform_info.static else "1",
+                    include_path=pj(self.buildEnv.install_dir, 'include'),
+                    library_path=pj(self.buildEnv.install_dir, self.buildEnv.libprefix),
+                    binary_path=pj(self.buildEnv.install_dir, 'bin'),
+                    )
+            return ""
+
+class lzma(Dependency):
+    name = 'lzma'
+    version = '5.0.4'
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('xz-5.0.4.tar.bz2',
+                             '5cd9b060d3a1ad396b3be52c9b9311046a1c369e6062aea752658c435629ce92')
+
+    class Builder(MakeBuilder):
+        @property
+        def configure_option(self):
+            return "--disable-assembler"
+
+class UUID(Dependency):
+    name = 'uuid'
+    version = "1.43.4"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('e2fsprogs-libs-1.43.4.tar.gz',
+                             'eed4516325768255c9745e7b82c9d7d0393abce302520a5b2cde693204b0e419',
+                             'https://www.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v1.43.4/e2fsprogs-libs-1.43.4.tar.gz')
+        extract_dir = 'e2fsprogs-libs-1.43.4'
+
+    class Builder(MakeBuilder):
+        configure_option = ("--enable-libuuid --disable-debugfs --disable-imager --disable-resizer --disable-defrag --enable-fsck"
+                            " --disable-uuidd")
+        configure_env = {'_format_CFLAGS': "{env.CFLAGS} -fPIC"}
+        static_configure_option = dynamic_configure_option = ""
+        make_target = 'libs'
+        make_install_target = 'install-libs'
+
+
+class Xapian(Dependency):
+    name = "xapian-core"
+    version = "1.4.2"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('xapian-core-1.4.2.tar.xz',
+                             'aec2c4352998127a2f2316218bf70f48cef0a466a87af3939f5f547c5246e1ce')
+        patches = ["xapian_pkgconfig.patch"]
+
+    class Builder(MakeBuilder):
+        configure_option = "--disable-sse --disable-backend-inmemory --disable-documentation"
+        configure_env = {'_format_LDFLAGS': "-L{buildEnv.install_dir}/{buildEnv.libprefix}",
+                         '_format_CXXFLAGS': "-I{buildEnv.install_dir}/include"}
+
+    @property
+    def dependencies(self):
+        deps = ['zlib', 'lzma']
+        if self.buildEnv.platform_info.build == 'win32':
+            return deps
+        return deps + ['UUID']
+
+
+class CTPP2(Dependency):
+    name = "ctpp2"
+    version = "2.8.3"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('ctpp2-2.8.3.tar.gz',
+                             'a83ffd07817adb575295ef40fbf759892512e5a63059c520f9062d9ab8fb42fc')
+        patches = ["ctpp2_include.patch",
+                   "ctpp2_no_src_modification.patch",
+                   "ctpp2_fix-static-libname.patch",
+                   "ctpp2_mingw32.patch",
+                   "ctpp2_dll_export_VMExecutable.patch",
+                   "ctpp2_win_install_lib_in_lib_dir.patch",
+                   "ctpp2_iconv_support.patch"]
+
+    class Builder(CMakeBuilder):
+        configure_option = "-DMD5_SUPPORT=OFF"
+
+
+class Pugixml(Dependency):
+    name = "pugixml"
+    version = "1.2"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('pugixml-1.2.tar.gz',
+                             '0f422dad86da0a2e56a37fb2a88376aae6e931f22cc8b956978460c9db06136b')
+        patches = ["pugixml_meson.patch"]
+
+    Builder = MesonBuilder
+
+
+class MicroHttpd(Dependency):
+    name = "libmicrohttpd"
+    version = "0.9.46"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('libmicrohttpd-0.9.46.tar.gz',
+                             '06dbd2654f390fa1e8196fe063fc1449a6c2ed65a38199a49bf29ad8a93b8979',
+                             'http://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.46.tar.gz')
+
+    class Builder(MakeBuilder):
+        configure_option = "--disable-https --without-libgcrypt --without-libcurl"
+
+
+class Icu(Dependency):
+    name = "icu4c"
+    version = "58_2"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('icu4c-58_2-src.tgz',
+                             '2b0a4410153a9b20de0e20c7d8b66049a72aef244b53683d0d7521371683da0c',
+                             'https://freefr.dl.sourceforge.net/project/icu/ICU4C/58.2/icu4c-58_2-src.tgz')
+        patches = ["icu4c_fix_static_lib_name_mingw.patch",
+                   "icu4c_android_elf64_st_info.patch"]
+        data = Remotefile('icudt56l.dat',
+                          'e23d85eee008f335fc49e8ef37b1bc2b222db105476111e3d16f0007d371cbca')
+
+        def _download_data(self, context):
+            self.buildEnv.download(self.data)
+
+        def _copy_data(self, context):
+            context.try_skip(self.extract_path)
+            shutil.copyfile(pj(self.buildEnv.archive_dir, self.data.name), pj(self.extract_path, 'source', 'data', 'in', self.data.name))
+
+        def prepare(self):
+            super().prepare()
+            self.command("download_data", self._download_data)
+            self.command("copy_data", self._copy_data)
+
+    class Builder(MakeBuilder):
+        subsource_dir = "source"
+        configure_option = "--disable-samples --disable-tests --disable-extras --disable-dyload"
+
+
+class Icu_native(Icu):
+    force_native_build = True
+
+    class Builder(Icu.Builder):
+        name = "icu_native"
+        @property
+        def build_path(self):
+            return super().build_path+"_native"
+
+        def _install(self, context):
+            raise SkipCommand()
+
+
+class Icu_cross_compile(Icu):
+    dependencies = ['Icu_native']
+
+    class Builder(Icu.Builder):
+        name = "icu_cross-compile"
+        @property
+        def configure_option(self):
+            Icu_native = self.buildEnv.targetsDict['Icu_native']
+            return super().configure_option + " --with-cross-build=" + Icu_native.builder.build_path
+
+
+class Zimlib(Dependency):
+    name = "zimlib"
+
+    class Source(GitClone):
+        #git_remote = "https://gerrit.wikimedia.org/r/p/openzim.git"
+        git_remote = "https://github.com/mgautierfr/openzim"
+        git_dir = "openzim"
+        git_ref = "meson"
+
+    class Builder(MesonBuilder):
+        subsource_dir = "zimlib"
+
+
+class Kiwixlib(Dependency):
+    name = "kiwix-lib"
+    dependencies = ['zlib', 'lzma']
+
+    @property
+    def dependencies(self):
+        base_dependencies = ["Xapian", "Pugixml", "Zimlib"]
+        if self.buildEnv.platform_info.build != 'android':
+            base_dependencies += ['CTPP2']
+        if self.buildEnv.platform_info.build != 'native':
+            return base_dependencies + ["Icu_cross_compile"]
+        else:
+            return base_dependencies + ["Icu"]
+
+    class Source(GitClone):
+        git_remote = "https://github.com/kiwix/kiwix-lib.git"
+        git_dir = "kiwix-lib"
+
+    class Builder(MesonBuilder):
+        @property
+        def configure_option(self):
+            base_option = "-Dctpp2-install-prefix={buildEnv.install_dir}"
+            if self.buildEnv.platform_info.build == 'android':
+                base_option += ' -Dandroid=true'
+            return base_option
+
+        @property
+        def library_type(self):
+            if self.buildEnv.platform_info.build == 'android':
+                return 'shared'
+            return super().library_type
+
+
+class KiwixTools(Dependency):
+    name = "kiwix-tools"
+    dependencies = ["Kiwixlib", "MicroHttpd", "zlib"]
+
+    class Source(GitClone):
+        git_remote = "https://github.com/kiwix/kiwix-tools.git"
+        git_dir = "kiwix-tools"
+
+    class Builder(MesonBuilder):
+        @property
+        def configure_option(self):
+            if self.buildEnv.platform_info.static:
+                return "-Dstatic-linkage=true"
+            return ""

dependency_utils.py

@@ -0,0 +1,304 @@
+import subprocess
+import os
+import shutil
+
+from utils import pj, Context, SkipCommand, extract_archive, Defaultdict, StopBuild
+
+SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
+
+
+class _MetaDependency(type):
+    def __new__(cls, name, bases, dct):
+        _class = type.__new__(cls, name, bases, dct)
+        if name != 'Dependency':
+            Dependency.all_deps[name] = _class
+        return _class
+
+
+class Dependency(metaclass=_MetaDependency):
+    all_deps = {}
+    dependencies = []
+    force_native_build = False
+    version = None
+
+    def __init__(self, buildEnv):
+        self.buildEnv = buildEnv
+        self.source = self.Source(self)
+        self.builder = self.Builder(self)
+        self.skip = False
+
+    @property
+    def full_name(self):
+        if self.version:
+            return "{}-{}".format(self.name, self.version)
+        return self.name
+
+    @property
+    def source_path(self):
+        return pj(self.buildEnv.source_dir, self.source.source_dir)
+
+    @property
+    def _log_dir(self):
+        return self.buildEnv.log_dir
+
+    def command(self, name, function, *args):
+        print("  {} {} : ".format(name, self.name), end="", flush=True)
+        log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
+        context = Context(name, log, self.force_native_build)
+        try:
+            ret = function(*args, context=context)
+            context._finalise()
+            print("OK")
+            return ret
+        except SkipCommand:
+            print("SKIP")
+        except subprocess.CalledProcessError:
+            print("ERROR")
+            try:
+                with open(log, 'r') as f:
+                    print(f.read())
+            except:
+                pass
+            raise StopBuild()
+        except:
+            print("ERROR")
+            raise
+
+
+class Source:
+    """Base Class to the real preparator
+       A source preparator must install source in the self.source_dir attribute
+       inside the buildEnv.source_dir."""
+    def __init__(self, target):
+        self.target = target
+        self.buildEnv = target.buildEnv
+
+    @property
+    def name(self):
+        return self.target.name
+
+    @property
+    def source_dir(self):
+        return self.target.full_name
+
+    def command(self, *args, **kwargs):
+        return self.target.command(*args, **kwargs)
+
+
+class ReleaseDownload(Source):
+    archive_top_dir = None
+
+    @property
+    def extract_path(self):
+        return pj(self.buildEnv.source_dir, self.source_dir)
+
+    def _download(self, context):
+        self.buildEnv.download(self.archive)
+
+    def _extract(self, context):
+        context.try_skip(self.extract_path)
+        if os.path.exists(self.extract_path):
+            shutil.rmtree(self.extract_path)
+        extract_archive(pj(self.buildEnv.archive_dir, self.archive.name),
+                        self.buildEnv.source_dir,
+                        topdir=self.archive_top_dir,
+                        name=self.source_dir)
+
+    def _patch(self, context):
+        context.try_skip(self.extract_path)
+        context.force_native_build = True
+        for p in self.patches:
+            with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input:
+                self.buildEnv.run_command("patch -p1", self.extract_path, context, input=patch_input)
+
+    def prepare(self):
+        self.command('download', self._download)
+        self.command('extract', self._extract)
+        if hasattr(self, 'patches'):
+            self.command('patch', self._patch)
+
+
+class GitClone(Source):
+    git_ref = "master"
+
+    @property
+    def source_dir(self):
+        return self.git_dir
+
+    @property
+    def git_path(self):
+        return pj(self.buildEnv.source_dir, self.git_dir)
+
+    def _git_clone(self, context):
+        context.force_native_build = True
+        if os.path.exists(self.git_path):
+            raise SkipCommand()
+        command = "git clone " + self.git_remote
+        self.buildEnv.run_command(command, self.buildEnv.source_dir, context)
+
+    def _git_update(self, context):
+        context.force_native_build = True
+        self.buildEnv.run_command("git fetch", self.git_path, context)
+        self.buildEnv.run_command("git checkout "+self.git_ref, self.git_path, context)
+
+    def prepare(self):
+        self.command('gitclone', self._git_clone)
+        self.command('gitupdate', self._git_update)
+
+
+class Builder:
+    subsource_dir = None
+
+    def __init__(self, target):
+        self.target = target
+        self.buildEnv = target.buildEnv
+
+    @property
+    def name(self):
+        return self.target.name
+
+    @property
+    def source_path(self):
+        base_source_path = self.target.source_path
+        if self.subsource_dir:
+            return pj(base_source_path, self.subsource_dir)
+        return base_source_path
+
+    @property
+    def build_path(self):
+        return pj(self.buildEnv.build_dir, self.target.full_name)
+
+    def command(self, *args, **kwargs):
+        return self.target.command(*args, **kwargs)
+
+    def build(self):
+        if hasattr(self, '_pre_build_script'):
+            self.command('pre_build_script', self._pre_build_script)
+        self.command('configure', self._configure)
+        self.command('compile', self._compile)
+        self.command('install', self._install)
+        if hasattr(self, '_post_build_script'):
+            self.command('post_build_script', self._post_build_script)
+
+
+class MakeBuilder(Builder):
+    configure_option = ""
+    dynamic_configure_option = "--enable-shared --disable-static"
+    static_configure_option = "--enable-static --disable-shared"
+    make_option = ""
+    install_option = ""
+    configure_script = "configure"
+    configure_env = None
+    make_target = ""
+    make_install_target = "install"
+
+    @property
+    def all_configure_option(self):
+        return "{} {} {}".format(
+            self.configure_option,
+            self.static_configure_option if self.buildEnv.platform_info.static else self.dynamic_configure_option,
+            self.buildEnv.configure_option if not self.target.force_native_build else "")
+
+    def _configure(self, context):
+        context.try_skip(self.build_path)
+        command = "{configure_script} {configure_option} --prefix {install_dir} --libdir {libdir}"
+        command = command.format(
+            configure_script=pj(self.source_path, self.configure_script),
+            configure_option=self.all_configure_option,
+            install_dir=self.buildEnv.install_dir,
+            libdir=pj(self.buildEnv.install_dir, self.buildEnv.libprefix)
+        )
+        env = Defaultdict(str, os.environ)
+        if self.buildEnv.platform_info.static:
+            env['CFLAGS'] = env['CFLAGS'] + ' -fPIC'
+        if self.configure_env:
+            for k in self.configure_env:
+                if k.startswith('_format_'):
+                    v = self.configure_env.pop(k)
+                    v = v.format(buildEnv=self.buildEnv, env=env)
+                    self.configure_env[k[8:]] = v
+            env.update(self.configure_env)
+        self.buildEnv.run_command(command, self.build_path, context, env=env)
+
+    def _compile(self, context):
+        context.try_skip(self.build_path)
+        command = "make -j4 {make_target} {make_option}".format(
+            make_target=self.make_target,
+            make_option=self.make_option
+        )
+        self.buildEnv.run_command(command, self.build_path, context)
+
+    def _install(self, context):
+        context.try_skip(self.build_path)
+        command = "make {make_install_target} {make_option}".format(
+            make_install_target=self.make_install_target,
+            make_option=self.make_option
+        )
+        self.buildEnv.run_command(command, self.build_path, context)
+
+
+class CMakeBuilder(MakeBuilder):
+    def _configure(self, context):
+        context.try_skip(self.build_path)
+        command = ("cmake {configure_option}"
+                   " -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON"
+                   " -DCMAKE_INSTALL_PREFIX={install_dir}"
+                   " -DCMAKE_INSTALL_LIBDIR={libdir}"
+                   " {source_path}"
+                   " {cross_option}")
+        command = command.format(
+            configure_option="{} {}".format(self.buildEnv.cmake_option, self.configure_option),
+            install_dir=self.buildEnv.install_dir,
+            libdir=self.buildEnv.libprefix,
+            source_path=self.source_path,
+            cross_option="-DCMAKE_TOOLCHAIN_FILE={}".format(self.buildEnv.cmake_crossfile) if self.buildEnv.cmake_crossfile else ""
+        )
+        env = Defaultdict(str, os.environ)
+        if self.buildEnv.platform_info.static:
+            env['CFLAGS'] = env['CFLAGS'] + ' -fPIC'
+        if self.configure_env:
+            for k in self.configure_env:
+                if k.startswith('_format_'):
+                    v = self.configure_env.pop(k)
+                    v = v.format(buildEnv=self.buildEnv, env=env)
+                    self.configure_env[k[8:]] = v
+            env.update(self.configure_env)
+        self.buildEnv.run_command(command, self.build_path, context, env=env, cross_path_only=True)
+
+
+class MesonBuilder(Builder):
+    configure_option = ""
+
+    @property
+    def library_type(self):
+        return 'static' if self.buildEnv.platform_info.static else 'shared'
+
+    def _configure(self, context):
+        context.try_skip(self.build_path)
+        if os.path.exists(self.build_path):
+            shutil.rmtree(self.build_path)
+        os.makedirs(self.build_path)
+        configure_option = self.configure_option.format(buildEnv=self.buildEnv)
+        command = ("{command} . {build_path}"
+                   " --default-library={library_type}"
+                   " {configure_option}"
+                   " --prefix={buildEnv.install_dir}"
+                   " --libdir={buildEnv.libprefix}"
+                   " {cross_option}")
+        command = command.format(
+            command=self.buildEnv.meson_command,
+            library_type=self.library_type,
+            configure_option=configure_option,
+            build_path=self.build_path,
+            buildEnv=self.buildEnv,
+            cross_option="--cross-file {}".format(self.buildEnv.meson_crossfile) if self.buildEnv.meson_crossfile else ""
+        )
+        self.buildEnv.run_command(command, self.source_path, context, cross_path_only=True)
+
+    def _compile(self, context):
+        command = "{} -v".format(self.buildEnv.ninja_command)
+        self.buildEnv.run_command(command, self.build_path, context)
+
+    def _install(self, context):
+        command = "{} -v install".format(self.buildEnv.ninja_command)
+        self.buildEnv.run_command(command, self.build_path, context)

patches/icu4c_android_elf64_st_info.patch

@@ -0,0 +1,13 @@
+diff -ur icu4c-58_2/source/tools/toolutil/pkg_genc.c icu4c-58_2.patched/source/tools/toolutil/pkg_genc.c
+--- icu4c-58_2/source/tools/toolutil/pkg_genc.c	2016-06-15 20:58:17.000000000 +0200
++++ icu4c-58_2.patched/source/tools/toolutil/pkg_genc.c	2017-02-27 10:23:39.985471339 +0100
+@@ -35,6 +35,9 @@
+ #       define EM_X86_64 62
+ #   endif
+ #   define ICU_ENTRY_OFFSET 0
++#   ifndef ELF64_ST_INFO
++#       define ELF64_ST_INFO(b,t)      (((b) << 4) + ((t) & 0xf))
++#   endif
+ #endif
+ 
+ #include <stdio.h>

patches/xapian_pkgconfig.patch

@@ -1,33 +1,44 @@
-diff -ur xapian-core-1.4.0/configure.ac xapian-core-1.4.0.patched/configure.ac
+diff -ur xapian-core-1.4.0/configure.ac xapian-core-1.4.0-patched/configure.ac
 --- xapian-core-1.4.0/configure.ac	2016-06-25 17:36:49.000000000 +0200
-+++ xapian-core-1.4.0.patched/configure.ac	2017-01-17 14:33:42.268536542 +0100
++++ xapian-core-1.4.0-patched/configure.ac	2017-02-22 17:45:57.066365636 +0100
-@@ -393,6 +393,7 @@
+@@ -393,22 +393,25 @@
  esac
  
  dnl We use timer_create() if available to implement a search time limit.
 +use_rt_lib=0
  SAVE_LIBS=$LIBS
- AC_SEARCH_LIBS([timer_create], [rt],
+-AC_SEARCH_LIBS([timer_create], [rt],
- 	       [
+-	       [
-@@ -403,12 +404,14 @@
++AC_SEARCH_LIBS([timer_create], [rt], [
+     AC_MSG_CHECKING([for timer_create() usability])
+-    AC_COMPILE_IFELSE([AC_LANG_PROGRAM(
++    AC_COMPILE_IFELSE(
++      [AC_LANG_PROGRAM(
+ [[#if defined _AIX || defined __GNU__
+ #error timer_create known not to work
  #endif]])],
- 	[AC_MSG_RESULT([yes])
+-	[AC_MSG_RESULT([yes])
- 	XAPIAN_LIBS="$LIBS $XAPIAN_LIBS"
+-	XAPIAN_LIBS="$LIBS $XAPIAN_LIBS"
 -	AC_DEFINE([HAVE_TIMER_CREATE], [1], [Define to 1 if you have the 'timer_create' function.])]
+-	,
+-	[AC_MSG_RESULT([no])
+-    ])
++	  [AC_MSG_RESULT([yes])
++	  XAPIAN_LIBS="$LIBS $XAPIAN_LIBS"
 +	  AC_DEFINE([HAVE_TIMER_CREATE], [1], [Define to 1 if you have the 'timer_create' function.])
-+        use_rt_lib=1]
++	  AS_IF([test "$ac_res" != "none required"], use_rt_lib=1)
- 	,
++      ],
- 	[AC_MSG_RESULT([no])
++	  [AC_MSG_RESULT([no])]
-     ])
++	)
  ])
  LIBS=$SAVE_LIBS
-+AM_CONDITIONAL([USE_RT_LIB], [test "$use_win32_uuid_api" = 1])
++AM_CONDITIONAL([USE_RT_LIB], [test "$use_rt_lib" = 1])
  
  dnl Used by tests/soaktest/soaktest.cc
  AC_CHECK_FUNCS([srandom random])
-diff -ur xapian-core-1.4.0/configure xapian-core-1.4.0.patched/configure
+diff -ur xapian-core-1.4.0/configure xapian-core-1.4.0-patched/configure
 --- xapian-core-1.4.0/configure	2016-06-25 17:39:25.000000000 +0200
-+++ xapian-core-1.4.0.patched/configure	2017-01-17 15:43:07.929290801 +0100
++++ xapian-core-1.4.0-patched/configure	2017-02-22 17:45:44.472585524 +0100
 @@ -671,6 +671,8 @@
  DOCUMENTATION_RULES_FALSE
  DOCUMENTATION_RULES_TRUE
@@ -45,15 +56,22 @@ diff -ur xapian-core-1.4.0/configure xapian-core-1.4.0.patched/configure
  SAVE_LIBS=$LIBS
  { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing timer_create" >&5
  $as_echo_n "checking for library containing timer_create... " >&6; }
-@@ -18324,6 +18327,7 @@
+@@ -18320,10 +18323,13 @@
+ if ac_fn_cxx_try_compile "$LINENO"; then :
+   { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+ $as_echo "yes" >&6; }
+-	XAPIAN_LIBS="$LIBS $XAPIAN_LIBS"
++	  XAPIAN_LIBS="$LIBS $XAPIAN_LIBS"
  
  $as_echo "#define HAVE_TIMER_CREATE 1" >>confdefs.h
  
++	  if test "$ac_res" != "none required"; then :
 +  use_rt_lib=1
++fi
  
  else
    { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
-@@ -18335,6 +18339,14 @@
+@@ -18335,6 +18341,14 @@
  fi
  
  LIBS=$SAVE_LIBS
@@ -68,7 +86,7 @@ diff -ur xapian-core-1.4.0/configure xapian-core-1.4.0.patched/configure
  
  for ac_func in srandom random
  do :
-@@ -20854,6 +20866,10 @@
+@@ -20854,6 +20868,10 @@
  Usually this means the macro was only invoked conditionally." "$LINENO" 5
  fi
  
@@ -79,9 +97,9 @@ diff -ur xapian-core-1.4.0/configure xapian-core-1.4.0.patched/configure
  if test -z "${DOCUMENTATION_RULES_TRUE}" && test -z "${DOCUMENTATION_RULES_FALSE}"; then
    as_fn_error $? "conditional \"DOCUMENTATION_RULES\" was never defined.
  Usually this means the macro was only invoked conditionally." "$LINENO" 5
-diff -ur xapian-core-1.4.0/pkgconfig/xapian-core.pc.in xapian-core-1.4.0.patched/pkgconfig/xapian-core.pc.in
+diff -ur xapian-core-1.4.0/pkgconfig/xapian-core.pc.in xapian-core-1.4.0-patched/pkgconfig/xapian-core.pc.in
---- xapian-core-1.4.0/pkgconfig/xapian-core.pc.in	2017-01-17 15:22:40.184786108 +0100
+--- xapian-core-1.4.0/pkgconfig/xapian-core.pc.in	2016-06-25 17:36:49.000000000 +0200
-+++ xapian-core-1.4.0.patched/pkgconfig/xapian-core.pc.in	2017-01-17 14:34:17.692972977 +0100
++++ xapian-core-1.4.0-patched/pkgconfig/xapian-core.pc.in	2017-02-22 17:09:12.488793901 +0100
 @@ -11,4 +11,6 @@
  URL: https://xapian.org/
  Version: @VERSION@
@@ -90,4 +108,3 @@ diff -ur xapian-core-1.4.0/pkgconfig/xapian-core.pc.in xapian-core-1.4.0.patched
 +@USE_RT_LIB_TRUE@Libs: @ldflags@ -L${libdir} -lxapian@LIBRARY_VERSION_SUFFIX@ -lrt
 +@USE_RT_LIB_FALSE@Libs: @ldflags@ -L${libdir} -lxapian@LIBRARY_VERSION_SUFFIX@
 +@USE_WIN32_UUID_API_FALSE@Requires: uuid
-

templates/cmake_android_cross_file.txt

@@ -0,0 +1,7 @@
+SET(CMAKE_SYSTEM_NAME {host_machine[system]})
+SET(CMAKE_ANDROID_STANDALONE_TOOLCHAIN {toolchain.builder.install_path})
+
+SET(CMAKE_C_COMPILER "{toolchain.binaries[CC]}")
+SET(CMAKE_CXX_COMPILER "{toolchain.binaries[CXX]}")
+
+SET(CMAKE_FIND_ROOT_PATH {toolchain.root_path})

templates/cmake_cross_file.txt

@@ -2,11 +2,11 @@ SET(CMAKE_SYSTEM_NAME Windows)
 SET(CMAKE_SYSTEM_PROCESSOR x86)
 
 # specify the cross compiler
-SET(CMAKE_C_COMPILER "{which:{binaries[c]}}")
+SET(CMAKE_C_COMPILER "{toolchain.binaries[CC]}")
-SET(CMAKE_CXX_COMPILER "{which:{binaries[cpp]}}")
+SET(CMAKE_CXX_COMPILER "{toolchain.binaries[CXX]}")
-SET(CMAKE_RC_COMPILER {which:{binaries[windres]}})
+SET(CMAKE_RC_COMPILER {toolchain.binaries[WINDRES]})
-SET(CMAKE_AR:FILEPATH {which:{binaries[ar]}})
+SET(CMAKE_AR:FILEPATH {toolchain.binaries[AR]})
-SET(CMAKE_RANLIB:FILEPATH {which:{binaries[ranlib]}})
+SET(CMAKE_RANLIB:FILEPATH {toolchain.binaries[RANLIB]})
 
 find_program(CCACHE_FOUND ccache)
 if(CCACHE_FOUND)
@@ -15,7 +15,7 @@ if(CCACHE_FOUND)
 endif(CCACHE_FOUND)
 
 # where is the target environment
-SET(CMAKE_FIND_ROOT_PATH {root_path})
+SET(CMAKE_FIND_ROOT_PATH {toolchain.root_path})
 
 # search for programs in the build host directories
 SET(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)

templates/meson_cross_file.txt

@@ -1,16 +1,16 @@
 [binaries]
 pkgconfig = 'pkg-config'
-c = '{which:{binaries[c]}}'
+c = '{toolchain.binaries[CC]}'
-ar = '{which:{binaries[ar]}}'
+ar = '{toolchain.binaries[AR]}'
-cpp = '{which:{binaries[cpp]}}'
+cpp = '{toolchain.binaries[CXX]}'
-strip = '{which:{binaries[strip]}}'
+strip = '{toolchain.binaries[STRIP]}'
 
 [properties]
-c_link_args = ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4']
+c_link_args = {extra_libs!r}
-cpp_link_args = ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4']
+cpp_link_args = {extra_libs!r}
 
 [host_machine]
-cpu_family = 'x86'
+cpu_family = '{host_machine[cpu_family]}'
-cpu = 'i586'
+cpu = '{host_machine[cpu]}'
-system = 'windows'
+system = '{host_machine[system]}'
-endian = 'little'
+endian = '{host_machine[endian]}'

travis/compile_all.sh

@@ -2,9 +2,4 @@
 
 set -e
 
-OPTION=""
+./kiwix-build.py $BUILD_OPTION
-if [ "${STATIC_BUILD}" = "true" ]; then
-    OPTION="--build-static"
-fi
-
-./kiwix-build.py --build-target=${BUILD_TARGET} ${OPTION}

travis/deploy.sh

@@ -6,14 +6,8 @@ SSH_KEY=travis/travisci_builder_id_key
 
 chmod 600 ${SSH_KEY}
 
-
+BASE_DIR="BUILD_*/INSTALL"
-BASE_DIR="BUILD_${BUILD_TARGET}_static/INSTALL"
+./kiwix-deploy.py ${BASE_DIR} ${ARCHIVE_TYPE} \
-if [ "${BUILD_TARGET}" = "win32" ]; then
-    ARCHIVE_OPTION="--zip"
-else
-    ARCHIVE_OPTION="--tar"
-fi
-./kiwix-deploy.py ${BASE_DIR} ${ARCHIVE_OPTION} \
     --deploy \
     --ssh_private_key=${SSH_KEY} \
     --server=nightlybot@download.kiwix.org \

travis/install_extra_deps.sh

@@ -5,7 +5,7 @@ set -e
 orig_dir=$(pwd)
 
 sudo apt-get update -qq
-sudo apt-get install -qq uuid-dev libicu-dev libctpp2-dev automake libtool python3-pip zlib1g-dev lzma-dev libbz2-dev cmake
+sudo apt-get install -qq python3-pip
 pip3 install meson
 
 # ninja

utils.py

@@ -0,0 +1,117 @@
+import os.path
+import hashlib
+import tarfile, zipfile
+import tempfile
+import os
+from collections import namedtuple, defaultdict
+
+pj = os.path.join
+
+
+class Defaultdict(defaultdict):
+    def __getattr__(self, name):
+        return self[name]
+
+
+def remove_duplicates(iterable, key_function=None):
+    seen = set()
+    if key_function is None:
+        key_function = lambda e: e
+    for elem in iterable:
+        key = key_function(elem)
+        if key in seen:
+            continue
+        seen.add(key)
+        yield elem
+
+
+def get_sha256(path):
+    sha256 = hashlib.sha256()
+    with open(path, 'br') as f:
+        sha256.update(f.read())
+    return sha256.hexdigest()
+
+
+class SkipCommand(Exception):
+    pass
+
+
+class StopBuild(Exception):
+    pass
+
+
+class Remotefile(namedtuple('Remotefile', ('name', 'sha256', 'url'))):
+    def __new__(cls, name, sha256, url=None):
+        return super().__new__(cls, name, sha256, url)
+
+
+class Context:
+    def __init__(self, command_name, log_file, force_native_build):
+        self.command_name = command_name
+        self.log_file = log_file
+        self.force_native_build = force_native_build
+        self.autoskip_file = None
+
+    def try_skip(self, path):
+        self.autoskip_file = pj(path, ".{}_ok".format(self.command_name))
+        if os.path.exists(self.autoskip_file):
+            raise SkipCommand()
+
+    def _finalise(self):
+        if self.autoskip_file is not None:
+            with open(self.autoskip_file, 'w'):
+                pass
+
+
+def extract_archive(archive_path, dest_dir, topdir=None, name=None):
+    is_zip_archive = archive_path.endswith('.zip')
+    try:
+        if is_zip_archive:
+            archive = zipfile.ZipFile(archive_path)
+            members = archive.infolist()
+            getname = lambda info : info.filename
+            isdir = lambda info: info.filename.endswith('/')
+        else:
+            archive = tarfile.open(archive_path)
+            members = archive.getmembers()
+            getname = lambda info : getattr(info, 'name')
+            isdir = lambda info: info.isdir()
+        if not topdir:
+            for d in (m for m in members if isdir(m)):
+                _name = getname(d)
+                if _name.endswith('/'):
+                    _name = _name[:-1]
+                if not os.path.dirname(_name):
+                    if topdir:
+                        # There is already a top dir.
+                        # Two topdirs in the same archive.
+                        # Extract all
+                        topdir = None
+                        break
+                    topdir = _name
+        if topdir:
+            members_to_extract = [m for m in members if getname(m).startswith(topdir+'/')]
+            os.makedirs(dest_dir, exist_ok=True)
+            with tempfile.TemporaryDirectory(prefix=os.path.basename(archive_path), dir=dest_dir) as tmpdir:
+                if is_zip_archive:
+                    _members_to_extract = [getname(m) for m in members_to_extract]
+                else:
+                    _members_to_extract = members_to_extract
+                archive.extractall(path=tmpdir, members=_members_to_extract)
+                if is_zip_archive:
+                    for member in members_to_extract:
+                        if isdir(member):
+                            continue
+                        perm = (member.external_attr >> 16) & 0x1FF
+                        os.chmod(pj(tmpdir, getname(member)), perm)
+                name = name or topdir
+                os.rename(pj(tmpdir, topdir), pj(dest_dir, name))
+        else:
+            if name:
+                dest_dir = pj(dest_dir, name)
+                os.makedirs(dest_dir)
+            archive.extractall(path=dest_dir)
+    finally:
+        if archive is not None:
+            archive.close()
+