Split the too long kiwix-build.py file into several smaller ones.

dependencies.py

@@ -0,0 +1,229 @@
+import shutil
+
+from dependency_utils import (
+    Dependency,
+    ReleaseDownload,
+    GitClone,
+    MakeBuilder,
+    CMakeBuilder,
+    MesonBuilder)
+
+from utils import Remotefile, pj, SkipCommand
+
+# *************************************
+# Missing dependencies
+# Is this ok to assume that those libs
+# exist in your "distri" (linux/mac) ?
+# If not, we need to compile them here
+# *************************************
+# Zlib
+# LZMA
+# aria2
+# Argtable
+# MSVirtual
+# Android
+# libiconv
+# gettext
+# *************************************
+
+
+class zlib(Dependency):
+    name = 'zlib'
+    version = '1.2.8'
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('zlib-1.2.8.tar.gz',
+                             '36658cb768a54c1d4dec43c3116c27ed893e88b02ecfcb44f2166f9c0b7f2a0d')
+        patches = ['zlib_std_libname.patch']
+
+    class Builder(CMakeBuilder):
+        @property
+        def configure_option(self):
+            return "-DINSTALL_PKGCONFIG_DIR={}".format(pj(self.buildEnv.install_dir, self.buildEnv.libprefix, 'pkgconfig'))
+
+
+class UUID(Dependency):
+    name = 'uuid'
+    version = "1.43.4"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('e2fsprogs-libs-1.43.4.tar.gz',
+                             'eed4516325768255c9745e7b82c9d7d0393abce302520a5b2cde693204b0e419',
+                             'https://www.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v1.43.4/e2fsprogs-libs-1.43.4.tar.gz')
+        extract_dir = 'e2fsprogs-libs-1.43.4'
+
+    class Builder(MakeBuilder):
+        configure_option = "--enable-libuuid"
+        configure_env = {'_format_CFLAGS': "{env.CFLAGS} -fPIC"}
+        make_target = 'libs'
+        make_install_target = 'install-libs'
+
+
+class Xapian(Dependency):
+    name = "xapian-core"
+    version = "1.4.2"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('xapian-core-1.4.2.tar.xz',
+                             'aec2c4352998127a2f2316218bf70f48cef0a466a87af3939f5f547c5246e1ce')
+        patches = ["xapian_pkgconfig.patch"]
+
+    class Builder(MakeBuilder):
+        configure_option = "--disable-sse --disable-backend-inmemory --disable-documentation"
+        dynamic_configure_option = "--enable-shared --disable-static"
+        static_configure_option = "--enable-static --disable-shared"
+        configure_env = {'_format_LDFLAGS': "-L{buildEnv.install_dir}/{buildEnv.libprefix}",
+                         '_format_CXXFLAGS': "-I{buildEnv.install_dir}/include"}
+
+    @property
+    def dependencies(self):
+        deps = ['zlib']
+        if self.buildEnv.build_target == 'win32':
+            return deps
+        return deps + ['UUID']
+
+
+class CTPP2(Dependency):
+    name = "ctpp2"
+    version = "2.8.3"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('ctpp2-2.8.3.tar.gz',
+                             'a83ffd07817adb575295ef40fbf759892512e5a63059c520f9062d9ab8fb42fc')
+        patches = ["ctpp2_include.patch",
+                   "ctpp2_no_src_modification.patch",
+                   "ctpp2_fix-static-libname.patch",
+                   "ctpp2_mingw32.patch",
+                   "ctpp2_dll_export_VMExecutable.patch",
+                   "ctpp2_win_install_lib_in_lib_dir.patch",
+                   "ctpp2_iconv_support.patch"]
+
+    class Builder(CMakeBuilder):
+        configure_option = "-DMD5_SUPPORT=OFF"
+
+
+class Pugixml(Dependency):
+    name = "pugixml"
+    version = "1.2"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('pugixml-1.2.tar.gz',
+                             '0f422dad86da0a2e56a37fb2a88376aae6e931f22cc8b956978460c9db06136b')
+        patches = ["pugixml_meson.patch"]
+
+    Builder = MesonBuilder
+
+
+class MicroHttpd(Dependency):
+    name = "libmicrohttpd"
+    version = "0.9.46"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('libmicrohttpd-0.9.46.tar.gz',
+                             '06dbd2654f390fa1e8196fe063fc1449a6c2ed65a38199a49bf29ad8a93b8979',
+                             'http://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.46.tar.gz')
+
+    class Builder(MakeBuilder):
+        configure_option = "--disable-https --without-libgcrypt --without-libcurl"
+        dynamic_configure_option = "--enable-shared --disable-static"
+        static_configure_option = "--enable-static --disable-shared"
+
+
+class Icu(Dependency):
+    name = "icu4c"
+    version = "56_1"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('icu4c-56_1-src.tgz',
+                             '3a64e9105c734dcf631c0b3ed60404531bce6c0f5a64bfe1a6402a4cc2314816')
+        patches = ["icu4c_fix_static_lib_name_mingw.patch"]
+        data = Remotefile('icudt56l.dat',
+                          'e23d85eee008f335fc49e8ef37b1bc2b222db105476111e3d16f0007d371cbca')
+
+        def _download_data(self, context):
+            self.buildEnv.download(self.data)
+
+        def _copy_data(self, context):
+            context.try_skip(self.extract_path)
+            shutil.copyfile(pj(self.buildEnv.archive_dir, self.data.name), pj(self.extract_path, 'source', 'data', 'in', self.data.name))
+
+        def prepare(self):
+            super().prepare()
+            self.command("download_data", self._download_data)
+            self.command("copy_data", self._copy_data)
+
+    class Builder(MakeBuilder):
+        subsource_dir = "source"
+        configure_option = "--disable-samples --disable-tests --disable-extras --disable-dyload"
+        dynamic_configure_option = "--enable-shared --disable-static"
+        static_configure_option = "--enable-static --disable-shared"
+
+
+class Icu_native(Icu):
+    force_native_build = True
+
+    class Builder(Icu.Builder):
+        @property
+        def build_path(self):
+            return super().build_path+"_native"
+
+        def _install(self, context):
+            raise SkipCommand()
+
+
+class Icu_cross_compile(Icu):
+    dependencies = ['Icu_native']
+
+    class Builder(Icu.Builder):
+        @property
+        def configure_option(self):
+            Icu_native = self.buildEnv.targetsDict['Icu_native']
+            return super().configure_option + " --with-cross-build=" + Icu_native.builder.build_path
+
+
+class Zimlib(Dependency):
+    name = "zimlib"
+
+    class Source(GitClone):
+        #git_remote = "https://gerrit.wikimedia.org/r/p/openzim.git"
+        git_remote = "https://github.com/mgautierfr/openzim"
+        git_dir = "openzim"
+        git_ref = "meson"
+
+    class Builder(MesonBuilder):
+        subsource_dir = "zimlib"
+
+
+class Kiwixlib(Dependency):
+    name = "kiwix-lib"
+    dependencies = ['zlib']
+
+    @property
+    def dependencies(self):
+        if self.buildEnv.build_target == 'win32':
+            return ["Xapian", "CTPP2", "Pugixml", "Icu_cross_compile", "Zimlib"]
+        return ["Xapian", "CTPP2", "Pugixml", "Icu", "Zimlib"]
+
+    class Source(GitClone):
+        git_remote = "https://github.com/kiwix/kiwix-lib.git"
+        git_dir = "kiwix-lib"
+
+    class Builder(MesonBuilder):
+        configure_option = "-Dctpp2-install-prefix={buildEnv.install_dir}"
+
+
+class KiwixTools(Dependency):
+    name = "kiwix-tools"
+    dependencies = ["Kiwixlib", "MicroHttpd", "zlib"]
+
+    class Source(GitClone):
+        git_remote = "https://github.com/kiwix/kiwix-tools.git"
+        git_dir = "kiwix-tools"
+
+    class Builder(MesonBuilder):
+        @property
+        def configure_option(self):
+            base_options = "-Dctpp2-install-prefix={buildEnv.install_dir}"
+            if self.buildEnv.build_static:
+                base_options += " -Dstatic-linkage=true"
+            return base_options

dependency_utils.py

@@ -0,0 +1,292 @@
+import subprocess
+import os
+import shutil
+
+from utils import pj, Context, SkipCommand, extract_archive, Defaultdict, StopBuild
+
+SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
+
+
+class _MetaDependency(type):
+    def __new__(cls, name, bases, dct):
+        _class = type.__new__(cls, name, bases, dct)
+        if name != 'Dependency':
+            Dependency.all_deps[name] = _class
+        return _class
+
+
+class Dependency(metaclass=_MetaDependency):
+    all_deps = {}
+    dependencies = []
+    force_native_build = False
+    version = None
+
+    def __init__(self, buildEnv):
+        self.buildEnv = buildEnv
+        self.source = self.Source(self)
+        self.builder = self.Builder(self)
+        self.skip = False
+
+    @property
+    def full_name(self):
+        if self.version:
+            return "{}-{}".format(self.name, self.version)
+        return self.name
+
+    @property
+    def source_path(self):
+        return pj(self.buildEnv.source_dir, self.source.source_dir)
+
+    @property
+    def _log_dir(self):
+        return self.buildEnv.log_dir
+
+    def command(self, name, function, *args):
+        print("  {} {} : ".format(name, self.name), end="", flush=True)
+        log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
+        context = Context(name, log)
+        try:
+            ret = function(*args, context=context)
+            context._finalise()
+            print("OK")
+            return ret
+        except SkipCommand:
+            print("SKIP")
+        except subprocess.CalledProcessError:
+            print("ERROR")
+            try:
+                with open(log, 'r') as f:
+                    print(f.read())
+            except:
+                pass
+            raise StopBuild()
+        except:
+            print("ERROR")
+            raise
+
+
+class Source:
+    """Base Class to the real preparator
+       A source preparator must install source in the self.source_dir attribute
+       inside the buildEnv.source_dir."""
+    def __init__(self, target):
+        self.target = target
+        self.buildEnv = target.buildEnv
+
+    @property
+    def name(self):
+        return self.target.name
+
+    @property
+    def source_dir(self):
+        return self.target.full_name
+
+    def command(self, *args, **kwargs):
+        return self.target.command(*args, **kwargs)
+
+
+class ReleaseDownload(Source):
+    archive_top_dir = None
+
+    @property
+    def extract_path(self):
+        return pj(self.buildEnv.source_dir, self.source_dir)
+
+    def _download(self, context):
+        self.buildEnv.download(self.archive)
+
+    def _extract(self, context):
+        context.try_skip(self.extract_path)
+        if os.path.exists(self.extract_path):
+            shutil.rmtree(self.extract_path)
+        extract_archive(pj(self.buildEnv.archive_dir, self.archive.name),
+                        self.buildEnv.source_dir,
+                        topdir=self.archive_top_dir,
+                        name=self.source_dir)
+
+    def _patch(self, context):
+        context.try_skip(self.extract_path)
+        for p in self.patches:
+            with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input:
+                self.buildEnv.run_command("patch -p1", self.extract_path, context, input=patch_input, allow_wrapper=False)
+
+    def prepare(self):
+        self.command('download', self._download)
+        self.command('extract', self._extract)
+        if hasattr(self, 'patches'):
+            self.command('patch', self._patch)
+
+
+class GitClone(Source):
+    git_ref = "master"
+
+    @property
+    def source_dir(self):
+        return self.git_dir
+
+    @property
+    def git_path(self):
+        return pj(self.buildEnv.source_dir, self.git_dir)
+
+    def _git_clone(self, context):
+        if os.path.exists(self.git_path):
+            raise SkipCommand()
+        command = "git clone " + self.git_remote
+        self.buildEnv.run_command(command, self.buildEnv.source_dir, context)
+
+    def _git_update(self, context):
+        self.buildEnv.run_command("git fetch", self.git_path, context)
+        self.buildEnv.run_command("git checkout "+self.git_ref, self.git_path, context)
+
+    def prepare(self):
+        self.command('gitclone', self._git_clone)
+        self.command('gitupdate', self._git_update)
+
+
+class Builder:
+    subsource_dir = None
+
+    def __init__(self, target):
+        self.target = target
+        self.buildEnv = target.buildEnv
+
+    @property
+    def name(self):
+        return self.target.name
+
+    @property
+    def source_path(self):
+        base_source_path = self.target.source_path
+        if self.subsource_dir:
+            return pj(base_source_path, self.subsource_dir)
+        return base_source_path
+
+    @property
+    def build_path(self):
+        return pj(self.buildEnv.build_dir, self.target.full_name)
+
+    def command(self, *args, **kwargs):
+        return self.target.command(*args, **kwargs)
+
+    def build(self):
+        self.command('configure', self._configure)
+        self.command('compile', self._compile)
+        self.command('install', self._install)
+
+
+class MakeBuilder(Builder):
+    configure_option = static_configure_option = dynamic_configure_option = ""
+    make_option = ""
+    install_option = ""
+    configure_script = "configure"
+    configure_env = None
+    make_target = ""
+    make_install_target = "install"
+
+    def _configure(self, context):
+        context.try_skip(self.build_path)
+        configure_option = "{} {} {}".format(
+            self.configure_option,
+            self.static_configure_option if self.buildEnv.build_static else self.dynamic_configure_option,
+            self.buildEnv.configure_option)
+        command = "{configure_script} {configure_option} --prefix {install_dir} --libdir {libdir}"
+        command = command.format(
+            configure_script=pj(self.source_path, self.configure_script),
+            configure_option=configure_option,
+            install_dir=self.buildEnv.install_dir,
+            libdir=pj(self.buildEnv.install_dir, self.buildEnv.libprefix)
+        )
+        env = Defaultdict(str, os.environ)
+        if self.buildEnv.build_static:
+            env['CFLAGS'] = env['CFLAGS'] + ' -fPIC'
+        if self.configure_env:
+            for k in self.configure_env:
+                if k.startswith('_format_'):
+                    v = self.configure_env.pop(k)
+                    v = v.format(buildEnv=self.buildEnv, env=env)
+                    self.configure_env[k[8:]] = v
+            env.update(self.configure_env)
+        self.buildEnv.run_command(command, self.build_path, context, env=env)
+
+    def _compile(self, context):
+        context.try_skip(self.build_path)
+        command = "make -j4 {make_target} {make_option}".format(
+            make_target=self.make_target,
+            make_option=self.make_option
+        )
+        self.buildEnv.run_command(command, self.build_path, context)
+
+    def _install(self, context):
+        context.try_skip(self.build_path)
+        command = "make {make_install_target} {make_option}".format(
+            make_install_target=self.make_install_target,
+            make_option=self.make_option
+        )
+        self.buildEnv.run_command(command, self.build_path, context)
+
+
+class CMakeBuilder(MakeBuilder):
+    def _configure(self, context):
+        context.try_skip(self.build_path)
+        command = ("cmake {configure_option}"
+                   " -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON"
+                   " -DCMAKE_INSTALL_PREFIX={install_dir}"
+                   " -DCMAKE_INSTALL_LIBDIR={libdir}"
+                   " {source_path}"
+                   " {cross_option}")
+        command = command.format(
+            configure_option="{} {}".format(self.buildEnv.cmake_option, self.configure_option),
+            install_dir=self.buildEnv.install_dir,
+            libdir=self.buildEnv.libprefix,
+            source_path=self.source_path,
+            cross_option="-DCMAKE_TOOLCHAIN_FILE={}".format(self.buildEnv.cmake_crossfile) if self.buildEnv.cmake_crossfile else ""
+        )
+        env = Defaultdict(str, os.environ)
+        if self.buildEnv.build_static:
+            env['CFLAGS'] = env['CFLAGS'] + ' -fPIC'
+        if self.configure_env:
+            for k in self.configure_env:
+                if k.startswith('_format_'):
+                    v = self.configure_env.pop(k)
+                    v = v.format(buildEnv=self.buildEnv, env=env)
+                    self.configure_env[k[8:]] = v
+            env.update(self.configure_env)
+        self.buildEnv.run_command(command, self.build_path, context, env=env, allow_wrapper=False)
+
+
+class MesonBuilder(Builder):
+    configure_option = ""
+
+    def _configure(self, context):
+        context.try_skip(self.build_path)
+        if os.path.exists(self.build_path):
+            shutil.rmtree(self.build_path)
+        os.makedirs(self.build_path)
+        if self.buildEnv.build_static:
+            library_type = 'static'
+        else:
+            library_type = 'shared'
+        configure_option = self.configure_option.format(buildEnv=self.buildEnv)
+        command = ("{command} . {build_path}"
+                   " --default-library={library_type}"
+                   " {configure_option}"
+                   " --prefix={buildEnv.install_dir}"
+                   " --libdir={buildEnv.libprefix}"
+                   " {cross_option}")
+        command = command.format(
+            command=self.buildEnv.meson_command,
+            library_type=library_type,
+            configure_option=configure_option,
+            build_path=self.build_path,
+            buildEnv=self.buildEnv,
+            cross_option="--cross-file {}".format(self.buildEnv.meson_crossfile) if self.buildEnv.meson_crossfile else ""
+        )
+        self.buildEnv.run_command(command, self.source_path, context, allow_wrapper=False)
+
+    def _compile(self, context):
+        command = "{} -v".format(self.buildEnv.ninja_command)
+        self.buildEnv.run_command(command, self.build_path, context, allow_wrapper=False)
+
+    def _install(self, context):
+        command = "{} -v install".format(self.buildEnv.ninja_command)
+        self.buildEnv.run_command(command, self.build_path, context, allow_wrapper=False)

kiwix-build.py

@@ -4,17 +4,18 @@ import os, sys, stat
 import argparse
 import ssl
 import urllib.request
-import tarfile
 import subprocess
-import hashlib
-import shutil
-import tempfile
-import configparser
 import platform
-from collections import defaultdict, namedtuple, OrderedDict
+from collections import OrderedDict
 
-
+from dependencies import Dependency
-pj = os.path.join
+from utils import (
+    pj,
+    remove_duplicates,
+    get_sha256,
+    StopBuild,
+    SkipCommand,
+    Defaultdict)
 
 REMOTE_PREFIX = 'http://download.kiwix.org/dev/'
 
@@ -123,10 +124,6 @@ PACKAGE_NAME_MAPPERS = {
     },
 }
 
-class Defaultdict(defaultdict):
-    def __getattr__(self, name):
-        return self[name]
-
 
 class Which():
     def __getattr__(self, name):
@@ -137,77 +134,6 @@ class Which():
     def __format__(self, format_spec):
         return getattr(self, format_spec)
 
-def remove_duplicates(iterable, key_function=None):
-    seen = set()
-    if key_function is None:
-        key_function = lambda e:e
-    for elem in iterable:
-        key = key_function(elem)
-        if key in seen:
-            continue
-        seen.add(key)
-        yield elem
-
-def get_sha256(path):
-    sha256 = hashlib.sha256()
-    with open(path, 'br') as f:
-        sha256.update(f.read())
-    return sha256.hexdigest()
-
-class SkipCommand(Exception):
-    pass
-
-class StopBuild(Exception):
-    pass
-
-class Remotefile(namedtuple('Remotefile', ('name', 'sha256', 'url'))):
-    def __new__(cls, name, sha256, url=None):
-        return super().__new__(cls, name, sha256, url)
-
-class Context:
-    def __init__(self, command_name, log_file):
-        self.command_name = command_name
-        self.log_file = log_file
-        self.autoskip_file = None
-
-    def try_skip(self, path):
-        self.autoskip_file = pj(path, ".{}_ok".format(self.command_name))
-        if os.path.exists(self.autoskip_file):
-            raise SkipCommand()
-
-    def _finalise(self):
-        if self.autoskip_file is not None:
-            with open(self.autoskip_file, 'w') as f: pass
-
-
-def extract_archive(archive_path, dest_dir, topdir=None, name=None):
-    with tarfile.open(archive_path) as archive:
-        members = archive.getmembers()
-        if not topdir:
-            for d in (m for m in members if m.isdir()):
-                if not os.path.dirname(d.name):
-                    if topdir:
-                        # There is already a top dir.
-                        # Two topdirs in the same archive.
-                        # Extract all
-                        topdir = None
-                        break
-                    topdir = d
-        else:
-            topdir = archive.getmember(topdir)
-        if topdir:
-            members_to_extract = [m for m in members if m.name.startswith(topdir.name+'/')]
-            os.makedirs(dest_dir, exist_ok=True)
-            with tempfile.TemporaryDirectory(prefix=os.path.basename(archive_path), dir=dest_dir) as tmpdir:
-                archive.extractall(path=tmpdir, members=members_to_extract)
-                name = name or topdir.name
-                os.rename(pj(tmpdir, topdir.name), pj(dest_dir, name))
-        else:
-            if name:
-                dest_dir = pj(dest_dir, name)
-                os.makedirs(dest_dir)
-            archive.extractall(path=dest_dir)
-
 
 class BuildEnv:
     build_targets = ['native', 'win32']
@@ -446,8 +372,7 @@ class BuildEnv:
                   " a {target} {build_type} version on a {host} host.".format(
                       target=self.build_target,
                       build_type='static' if self.options.build_static else 'dyn',
-                      host = self.distname
+                      host=self.distname))
-                 ))
             return
 
         packages_list = package_name_mapper.get('COMMON', [])
@@ -469,508 +394,8 @@ class BuildEnv:
         else:
             print("SKIP, No package to install.")
 
-        with open(autoskip_file, 'w') as f: pass
+        with open(autoskip_file, 'w'):
-
-################################################################################
-##### PROJECT
-################################################################################
-class _MetaDependency(type):
-    def __new__(cls, name, bases, dct):
-        _class = type.__new__(cls, name, bases, dct)
-        if name != 'Dependency':
-            Dependency.all_deps[name] = _class
-        return _class
-
-
-class Dependency(metaclass=_MetaDependency):
-    all_deps = {}
-    dependencies = []
-    force_native_build = False
-    version = None
-    def __init__(self, buildEnv):
-        self.buildEnv = buildEnv
-        self.source = self.Source(self)
-        self.builder = self.Builder(self)
-        self.skip = False
-
-    @property
-    def full_name(self):
-        if self.version:
-            return "{}-{}".format(self.name, self.version)
-        return self.name
-
-    @property
-    def source_path(self):
-        return pj(self.buildEnv.source_dir, self.source.source_dir)
-
-    @property
-    def _log_dir(self):
-        return self.buildEnv.log_dir
-
-    def command(self, name, function, *args):
-        print("  {} {} : ".format(name, self.name), end="", flush=True)
-        log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
-        context = Context(name, log)
-        try:
-            ret = function(*args, context=context)
-            context._finalise()
-            print("OK")
-            return ret
-        except SkipCommand:
-            print("SKIP")
-        except subprocess.CalledProcessError:
-            print("ERROR")
-            try:
-                with open(log, 'r') as f:
-                    print(f.read())
-            except:
             pass
-            raise StopBuild()
-        except:
-            print("ERROR")
-            raise
-
-
-class Source:
-    """Base Class to the real preparator
-       A source preparator must install source in the self.source_dir attribute
-       inside the buildEnv.source_dir."""
-    def __init__(self, target):
-        self.target = target
-        self.buildEnv = target.buildEnv
-
-    @property
-    def name(self):
-        return self.target.name
-
-    @property
-    def source_dir(self):
-        return self.target.full_name
-
-    def command(self, *args, **kwargs):
-        return self.target.command(*args, **kwargs)
-
-
-class ReleaseDownload(Source):
-    archive_top_dir = None
-    @property
-    def extract_path(self):
-        return pj(self.buildEnv.source_dir, self.source_dir)
-
-    def _download(self, context):
-        self.buildEnv.download(self.archive)
-
-    def _extract(self, context):
-        context.try_skip(self.extract_path)
-        if os.path.exists(self.extract_path):
-            shutil.rmtree(self.extract_path)
-        extract_archive(pj(self.buildEnv.archive_dir, self.archive.name),
-                           self.buildEnv.source_dir,
-                           topdir=self.archive_top_dir,
-                           name=self.source_dir)
-
-    def _patch(self, context):
-        context.try_skip(self.extract_path)
-        for p in self.patches:
-            with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input:
-                self.buildEnv.run_command("patch -p1", self.extract_path, context, input=patch_input, allow_wrapper=False)
-
-    def prepare(self):
-        self.command('download', self._download)
-        self.command('extract', self._extract)
-        if hasattr(self, 'patches'):
-            self.command('patch', self._patch)
-
-
-class GitClone(Source):
-    git_ref = "master"
-    @property
-    def source_dir(self):
-        return self.git_dir
-
-    @property
-    def git_path(self):
-        return pj(self.buildEnv.source_dir, self.git_dir)
-
-    def _git_clone(self, context):
-        if os.path.exists(self.git_path):
-            raise SkipCommand()
-        command = "git clone " + self.git_remote
-        self.buildEnv.run_command(command, self.buildEnv.source_dir, context)
-
-    def _git_update(self, context):
-        self.buildEnv.run_command("git fetch", self.git_path, context)
-        self.buildEnv.run_command("git checkout "+self.git_ref, self.git_path, context)
-
-    def prepare(self):
-        self.command('gitclone', self._git_clone)
-        self.command('gitupdate', self._git_update)
-
-
-class Builder:
-    subsource_dir = None
-    def __init__(self, target):
-        self.target = target
-        self.buildEnv = target.buildEnv
-
-    @property
-    def name(self):
-        return self.target.name
-
-    @property
-    def source_path(self):
-        base_source_path = self.target.source_path
-        if self.subsource_dir:
-            return pj(base_source_path, self.subsource_dir)
-        return base_source_path
-
-    @property
-    def build_path(self):
-        return pj(self.buildEnv.build_dir, self.target.full_name)
-
-    def command(self, *args, **kwargs):
-        return self.target.command(*args, **kwargs)
-
-    def build(self):
-        self.command('configure', self._configure)
-        self.command('compile', self._compile)
-        self.command('install', self._install)
-
-
-class MakeBuilder(Builder):
-    configure_option = static_configure_option = dynamic_configure_option = ""
-    make_option = ""
-    install_option = ""
-    configure_script = "configure"
-    configure_env = None
-    make_target = ""
-    make_install_target = "install"
-
-    def _configure(self, context):
-        context.try_skip(self.build_path)
-        configure_option = "{} {} {}".format(
-            self.configure_option,
-            self.static_configure_option if self.buildEnv.build_static else self.dynamic_configure_option,
-            self.buildEnv.configure_option)
-        command = "{configure_script} {configure_option} --prefix {install_dir} --libdir {libdir}"
-        command = command.format(
-            configure_script = pj(self.source_path, self.configure_script),
-            configure_option = configure_option,
-            install_dir = self.buildEnv.install_dir,
-            libdir = pj(self.buildEnv.install_dir, self.buildEnv.libprefix)
-        )
-        env = Defaultdict(str, os.environ)
-        if self.buildEnv.build_static:
-            env['CFLAGS'] = env['CFLAGS'] + ' -fPIC'
-        if self.configure_env:
-           for k in self.configure_env:
-               if k.startswith('_format_'):
-                   v = self.configure_env.pop(k)
-                   v = v.format(buildEnv=self.buildEnv, env=env)
-                   self.configure_env[k[8:]] = v
-           env.update(self.configure_env)
-        self.buildEnv.run_command(command, self.build_path, context, env=env)
-
-    def _compile(self, context):
-        context.try_skip(self.build_path)
-        command = "make -j4 {make_target} {make_option}".format(
-            make_target = self.make_target,
-            make_option = self.make_option
-        )
-        self.buildEnv.run_command(command, self.build_path, context)
-
-    def  _install(self, context):
-        context.try_skip(self.build_path)
-        command = "make {make_install_target} {make_option}".format(
-            make_install_target = self.make_install_target,
-            make_option = self.make_option
-        )
-        self.buildEnv.run_command(command, self.build_path, context)
-
-
-class CMakeBuilder(MakeBuilder):
-    def _configure(self, context):
-        context.try_skip(self.build_path)
-        command = ("cmake {configure_option}"
-                   " -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON"
-                   " -DCMAKE_INSTALL_PREFIX={install_dir}"
-                   " -DCMAKE_INSTALL_LIBDIR={libdir}"
-                   " {source_path}"
-                   " {cross_option}")
-        command = command.format(
-            configure_option = "{} {}".format(self.buildEnv.cmake_option, self.configure_option),
-            install_dir = self.buildEnv.install_dir,
-            libdir = self.buildEnv.libprefix,
-            source_path = self.source_path,
-            cross_option = "-DCMAKE_TOOLCHAIN_FILE={}".format(self.buildEnv.cmake_crossfile) if self.buildEnv.cmake_crossfile else ""
-        )
-        env = Defaultdict(str, os.environ)
-        if self.buildEnv.build_static:
-            env['CFLAGS'] = env['CFLAGS'] + ' -fPIC'
-        if self.configure_env:
-           for k in self.configure_env:
-               if k.startswith('_format_'):
-                   v = self.configure_env.pop(k)
-                   v = v.format(buildEnv=self.buildEnv, env=env)
-                   self.configure_env[k[8:]] = v
-           env.update(self.configure_env)
-        self.buildEnv.run_command(command, self.build_path, context, env=env, allow_wrapper=False)
-
-
-class MesonBuilder(Builder):
-    configure_option = ""
-
-    def _configure(self, context):
-        context.try_skip(self.build_path)
-        if os.path.exists(self.build_path):
-            shutil.rmtree(self.build_path)
-        os.makedirs(self.build_path)
-        if self.buildEnv.build_static:
-            library_type = 'static'
-        else:
-            library_type = 'shared'
-        configure_option = self.configure_option.format(buildEnv=self.buildEnv)
-        command = ("{command} . {build_path}"
-                   " --default-library={library_type}"
-                   " {configure_option}"
-                   " --prefix={buildEnv.install_dir}"
-                   " --libdir={buildEnv.libprefix}"
-                   " {cross_option}")
-        command = command.format(
-            command = self.buildEnv.meson_command,
-            library_type=library_type,
-            configure_option=configure_option,
-            build_path = self.build_path,
-            buildEnv=self.buildEnv,
-            cross_option = "--cross-file {}".format(self.buildEnv.meson_crossfile) if self.buildEnv.meson_crossfile else ""
-        )
-        self.buildEnv.run_command(command, self.source_path, context, allow_wrapper=False)
-
-    def _compile(self, context):
-        command = "{} -v".format(self.buildEnv.ninja_command)
-        self.buildEnv.run_command(command, self.build_path, context, allow_wrapper=False)
-
-    def _install(self, context):
-        command = "{} -v install".format(self.buildEnv.ninja_command)
-        self.buildEnv.run_command(command, self.build_path, context, allow_wrapper=False)
-
-
-# *************************************
-# Missing dependencies
-# Is this ok to assume that those libs
-# exist in your "distri" (linux/mac) ?
-# If not, we need to compile them here
-# *************************************
-# Zlib
-# LZMA
-# aria2
-# Argtable
-# MSVirtual
-# Android
-# libiconv
-# gettext
-# *************************************
-
-class zlib(Dependency):
-    name = 'zlib'
-    version = '1.2.8'
-
-    class Source(ReleaseDownload):
-        archive = Remotefile('zlib-1.2.8.tar.gz',
-                             '36658cb768a54c1d4dec43c3116c27ed893e88b02ecfcb44f2166f9c0b7f2a0d')
-        patches = ['zlib_std_libname.patch']
-
-    class Builder(CMakeBuilder):
-        @property
-        def configure_option(self):
-            return "-DINSTALL_PKGCONFIG_DIR={}".format(pj(self.buildEnv.install_dir, self.buildEnv.libprefix, 'pkgconfig'))
-
-class UUID(Dependency):
-    name = 'uuid'
-    version = "1.43.4"
-
-    class Source(ReleaseDownload):
-        archive = Remotefile('e2fsprogs-1.43.4.tar.gz',
-                             '1644db4fc58300c363ba1ab688cf9ca1e46157323aee1029f8255889be4bc856',
-                             'https://www.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v1.43.4/e2fsprogs-1.43.4.tar.gz')
-        extract_dir = 'e2fsprogs-1.43.4'
-
-    class Builder(MakeBuilder):
-        configure_option = "--enable-libuuid"
-        configure_env = {'_format_CFLAGS' : "{env.CFLAGS} -fPIC"}
-        make_target = 'libs'
-        make_install_target = 'install-libs'
-
-
-class Xapian(Dependency):
-    name = "xapian-core"
-    version = "1.4.2"
-
-    class Source(ReleaseDownload):
-        archive = Remotefile('xapian-core-1.4.2.tar.xz',
-                             'aec2c4352998127a2f2316218bf70f48cef0a466a87af3939f5f547c5246e1ce')
-        patches = ["xapian_pkgconfig.patch"]
-
-    class Builder(MakeBuilder):
-        configure_option = "--disable-sse --disable-backend-inmemory --disable-documentation"
-        dynamic_configure_option = "--enable-shared --disable-static"
-        static_configure_option = "--enable-static --disable-shared"
-        configure_env = {'_format_LDFLAGS' : "-L{buildEnv.install_dir}/{buildEnv.libprefix}",
-                         '_format_CXXFLAGS' : "-I{buildEnv.install_dir}/include"}
-
-    @property
-    def dependencies(self):
-        deps = ['zlib']
-        if self.buildEnv.build_target == 'win32':
-            return deps
-        return deps + ['UUID']
-
-
-class CTPP2(Dependency):
-    name = "ctpp2"
-    version = "2.8.3"
-
-    class Source(ReleaseDownload):
-         archive = Remotefile('ctpp2-2.8.3.tar.gz',
-                              'a83ffd07817adb575295ef40fbf759892512e5a63059c520f9062d9ab8fb42fc')
-         patches = ["ctpp2_include.patch",
-               "ctpp2_no_src_modification.patch",
-               "ctpp2_fix-static-libname.patch",
-               "ctpp2_mingw32.patch",
-               "ctpp2_dll_export_VMExecutable.patch",
-               "ctpp2_win_install_lib_in_lib_dir.patch",
-               "ctpp2_iconv_support.patch"]
-
-    class Builder(CMakeBuilder):
-        configure_option = "-DMD5_SUPPORT=OFF"
-
-
-class Pugixml(Dependency):
-    name = "pugixml"
-    version = "1.2"
-
-    class Source(ReleaseDownload):
-        archive = Remotefile('pugixml-1.2.tar.gz',
-                             '0f422dad86da0a2e56a37fb2a88376aae6e931f22cc8b956978460c9db06136b')
-        patches = ["pugixml_meson.patch"]
-
-    Builder = MesonBuilder
-
-
-class MicroHttpd(Dependency):
-    name = "libmicrohttpd"
-    version = "0.9.46"
-
-    class Source(ReleaseDownload):
-        archive = Remotefile('libmicrohttpd-0.9.46.tar.gz',
-                             '06dbd2654f390fa1e8196fe063fc1449a6c2ed65a38199a49bf29ad8a93b8979',
-                             'http://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.46.tar.gz')
-
-    class Builder(MakeBuilder):
-        configure_option = "--disable-https --without-libgcrypt --without-libcurl"
-        dynamic_configure_option = "--enable-shared --disable-static"
-        static_configure_option = "--enable-static --disable-shared"
-
-
-class Icu(Dependency):
-    name = "icu4c"
-    version = "56_1"
-
-    class Source(ReleaseDownload):
-        archive = Remotefile('icu4c-56_1-src.tgz',
-                             '3a64e9105c734dcf631c0b3ed60404531bce6c0f5a64bfe1a6402a4cc2314816'
-                            )
-        patches = ["icu4c_fix_static_lib_name_mingw.patch"]
-        data = Remotefile('icudt56l.dat',
-                          'e23d85eee008f335fc49e8ef37b1bc2b222db105476111e3d16f0007d371cbca')
-
-        def _download_data(self, context):
-            self.buildEnv.download(self.data)
-
-        def _copy_data(self, context):
-            context.try_skip(self.extract_path)
-            shutil.copyfile(pj(self.buildEnv.archive_dir, self.data.name), pj(self.extract_path, 'source', 'data', 'in', self.data.name))
-
-        def prepare(self):
-            super().prepare()
-            self.command("download_data", self._download_data)
-            self.command("copy_data", self._copy_data)
-
-    class Builder(MakeBuilder):
-        subsource_dir = "source"
-        configure_option = "--disable-samples --disable-tests --disable-extras --disable-dyload"
-        dynamic_configure_option = "--enable-shared --disable-static"
-        static_configure_option = "--enable-static --disable-shared"
-
-
-class Icu_native(Icu):
-    force_native_build = True
-
-    class Builder(Icu.Builder):
-        @property
-        def build_path(self):
-            return super().build_path+"_native"
-
-        def _install(self, context):
-            raise SkipCommand()
-
-
-class Icu_cross_compile(Icu):
-    dependencies = ['Icu_native']
-
-    class Builder(Icu.Builder):
-        @property
-        def configure_option(self):
-            Icu_native = self.buildEnv.targetsDict['Icu_native']
-            return super().configure_option + " --with-cross-build=" + Icu_native.builder.build_path
-
-
-class Zimlib(Dependency):
-    name = "zimlib"
-
-    class Source(GitClone):
-        #git_remote = "https://gerrit.wikimedia.org/r/p/openzim.git"
-        git_remote = "https://github.com/mgautierfr/openzim"
-        git_dir = "openzim"
-        git_ref = "meson"
-
-    class Builder(MesonBuilder):
-        subsource_dir = "zimlib"
-
-
-class Kiwixlib(Dependency):
-    name = "kiwix-lib"
-    dependencies = ['zlib']
-
-    @property
-    def dependencies(self):
-        if self.buildEnv.build_target == 'win32':
-            return ["Xapian", "CTPP2", "Pugixml", "Icu_cross_compile", "Zimlib"]
-        return ["Xapian", "CTPP2", "Pugixml", "Icu", "Zimlib"]
-
-    class Source(GitClone):
-        git_remote = "https://github.com/kiwix/kiwix-lib.git"
-        git_dir = "kiwix-lib"
-
-    class Builder(MesonBuilder):
-        configure_option = "-Dctpp2-install-prefix={buildEnv.install_dir}"
-
-
-class KiwixTools(Dependency):
-    name = "kiwix-tools"
-    dependencies = ["Kiwixlib", "MicroHttpd", "zlib"]
-
-    class Source(GitClone):
-        git_remote = "https://github.com/kiwix/kiwix-tools.git"
-        git_dir = "kiwix-tools"
-
-    class Builder(MesonBuilder):
-        @property
-        def configure_option(self):
-           base_options = "-Dctpp2-install-prefix={buildEnv.install_dir}"
-           if self.buildEnv.build_static:
-               base_options += " -Dstatic-linkage=true"
-           return base_options
 
 
 class Builder:
@@ -1036,6 +461,7 @@ class Builder:
         except StopBuild:
             sys.exit("Stopping build due to errors")
 
+
 def parse_args():
     parser = argparse.ArgumentParser()
     parser.add_argument('working_dir', default=".", nargs='?')
@@ -1049,6 +475,7 @@ def parse_args():
                         help="Skip SSL certificate verification during download")
     return parser.parse_args()
 
+
 if __name__ == "__main__":
     options = parse_args()
     options.working_dir = os.path.abspath(options.working_dir)

utils.py

@@ -0,0 +1,90 @@
+import os.path
+import hashlib
+import tarfile
+import tempfile
+from collections import namedtuple, defaultdict
+
+pj = os.path.join
+
+
+class Defaultdict(defaultdict):
+    def __getattr__(self, name):
+        return self[name]
+
+
+def remove_duplicates(iterable, key_function=None):
+    seen = set()
+    if key_function is None:
+        key_function = lambda e: e
+    for elem in iterable:
+        key = key_function(elem)
+        if key in seen:
+            continue
+        seen.add(key)
+        yield elem
+
+
+def get_sha256(path):
+    sha256 = hashlib.sha256()
+    with open(path, 'br') as f:
+        sha256.update(f.read())
+    return sha256.hexdigest()
+
+
+class SkipCommand(Exception):
+    pass
+
+
+class StopBuild(Exception):
+    pass
+
+
+class Remotefile(namedtuple('Remotefile', ('name', 'sha256', 'url'))):
+    def __new__(cls, name, sha256, url=None):
+        return super().__new__(cls, name, sha256, url)
+
+
+class Context:
+    def __init__(self, command_name, log_file):
+        self.command_name = command_name
+        self.log_file = log_file
+        self.autoskip_file = None
+
+    def try_skip(self, path):
+        self.autoskip_file = pj(path, ".{}_ok".format(self.command_name))
+        if os.path.exists(self.autoskip_file):
+            raise SkipCommand()
+
+    def _finalise(self):
+        if self.autoskip_file is not None:
+            with open(self.autoskip_file, 'w'):
+                pass
+
+
+def extract_archive(archive_path, dest_dir, topdir=None, name=None):
+    with tarfile.open(archive_path) as archive:
+        members = archive.getmembers()
+        if not topdir:
+            for d in (m for m in members if m.isdir()):
+                if not os.path.dirname(d.name):
+                    if topdir:
+                        # There is already a top dir.
+                        # Two topdirs in the same archive.
+                        # Extract all
+                        topdir = None
+                        break
+                    topdir = d
+        else:
+            topdir = archive.getmember(topdir)
+        if topdir:
+            members_to_extract = [m for m in members if m.name.startswith(topdir.name+'/')]
+            os.makedirs(dest_dir, exist_ok=True)
+            with tempfile.TemporaryDirectory(prefix=os.path.basename(archive_path), dir=dest_dir) as tmpdir:
+                archive.extractall(path=tmpdir, members=members_to_extract)
+                name = name or topdir.name
+                os.rename(pj(tmpdir, topdir.name), pj(dest_dir, name))
+        else:
+            if name:
+                dest_dir = pj(dest_dir, name)
+                os.makedirs(dest_dir)
+            archive.extractall(path=dest_dir)