From ffee068fd029afe0d4ceaecb582f537b8cf7e865 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 22 Feb 2017 10:44:01 +0100 Subject: [PATCH] Split the too long kiwix-build.py file into several smaller ones. --- dependencies.py | 229 +++++++++++++ dependency_utils.py | 292 +++++++++++++++++ kiwix-build.py | 767 ++++++-------------------------------------- utils.py | 90 ++++++ 4 files changed, 708 insertions(+), 670 deletions(-) create mode 100644 dependencies.py create mode 100644 dependency_utils.py create mode 100644 utils.py diff --git a/dependencies.py b/dependencies.py new file mode 100644 index 0000000..f630e71 --- /dev/null +++ b/dependencies.py @@ -0,0 +1,229 @@ +import shutil + +from dependency_utils import ( + Dependency, + ReleaseDownload, + GitClone, + MakeBuilder, + CMakeBuilder, + MesonBuilder) + +from utils import Remotefile, pj, SkipCommand + +# ************************************* +# Missing dependencies +# Is this ok to assume that those libs +# exist in your "distri" (linux/mac) ? +# If not, we need to compile them here +# ************************************* +# Zlib +# LZMA +# aria2 +# Argtable +# MSVirtual +# Android +# libiconv +# gettext +# ************************************* + + +class zlib(Dependency): + name = 'zlib' + version = '1.2.8' + + class Source(ReleaseDownload): + archive = Remotefile('zlib-1.2.8.tar.gz', + '36658cb768a54c1d4dec43c3116c27ed893e88b02ecfcb44f2166f9c0b7f2a0d') + patches = ['zlib_std_libname.patch'] + + class Builder(CMakeBuilder): + @property + def configure_option(self): + return "-DINSTALL_PKGCONFIG_DIR={}".format(pj(self.buildEnv.install_dir, self.buildEnv.libprefix, 'pkgconfig')) + + +class UUID(Dependency): + name = 'uuid' + version = "1.43.4" + + class Source(ReleaseDownload): + archive = Remotefile('e2fsprogs-libs-1.43.4.tar.gz', + 'eed4516325768255c9745e7b82c9d7d0393abce302520a5b2cde693204b0e419', + 'https://www.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v1.43.4/e2fsprogs-libs-1.43.4.tar.gz') + extract_dir = 'e2fsprogs-libs-1.43.4' + + class Builder(MakeBuilder): + configure_option = "--enable-libuuid" + configure_env = {'_format_CFLAGS': "{env.CFLAGS} -fPIC"} + make_target = 'libs' + make_install_target = 'install-libs' + + +class Xapian(Dependency): + name = "xapian-core" + version = "1.4.2" + + class Source(ReleaseDownload): + archive = Remotefile('xapian-core-1.4.2.tar.xz', + 'aec2c4352998127a2f2316218bf70f48cef0a466a87af3939f5f547c5246e1ce') + patches = ["xapian_pkgconfig.patch"] + + class Builder(MakeBuilder): + configure_option = "--disable-sse --disable-backend-inmemory --disable-documentation" + dynamic_configure_option = "--enable-shared --disable-static" + static_configure_option = "--enable-static --disable-shared" + configure_env = {'_format_LDFLAGS': "-L{buildEnv.install_dir}/{buildEnv.libprefix}", + '_format_CXXFLAGS': "-I{buildEnv.install_dir}/include"} + + @property + def dependencies(self): + deps = ['zlib'] + if self.buildEnv.build_target == 'win32': + return deps + return deps + ['UUID'] + + +class CTPP2(Dependency): + name = "ctpp2" + version = "2.8.3" + + class Source(ReleaseDownload): + archive = Remotefile('ctpp2-2.8.3.tar.gz', + 'a83ffd07817adb575295ef40fbf759892512e5a63059c520f9062d9ab8fb42fc') + patches = ["ctpp2_include.patch", + "ctpp2_no_src_modification.patch", + "ctpp2_fix-static-libname.patch", + "ctpp2_mingw32.patch", + "ctpp2_dll_export_VMExecutable.patch", + "ctpp2_win_install_lib_in_lib_dir.patch", + "ctpp2_iconv_support.patch"] + + class Builder(CMakeBuilder): + configure_option = "-DMD5_SUPPORT=OFF" + + +class Pugixml(Dependency): + name = "pugixml" + version = "1.2" + + class Source(ReleaseDownload): + archive = Remotefile('pugixml-1.2.tar.gz', + '0f422dad86da0a2e56a37fb2a88376aae6e931f22cc8b956978460c9db06136b') + patches = ["pugixml_meson.patch"] + + Builder = MesonBuilder + + +class MicroHttpd(Dependency): + name = "libmicrohttpd" + version = "0.9.46" + + class Source(ReleaseDownload): + archive = Remotefile('libmicrohttpd-0.9.46.tar.gz', + '06dbd2654f390fa1e8196fe063fc1449a6c2ed65a38199a49bf29ad8a93b8979', + 'http://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.46.tar.gz') + + class Builder(MakeBuilder): + configure_option = "--disable-https --without-libgcrypt --without-libcurl" + dynamic_configure_option = "--enable-shared --disable-static" + static_configure_option = "--enable-static --disable-shared" + + +class Icu(Dependency): + name = "icu4c" + version = "56_1" + + class Source(ReleaseDownload): + archive = Remotefile('icu4c-56_1-src.tgz', + '3a64e9105c734dcf631c0b3ed60404531bce6c0f5a64bfe1a6402a4cc2314816') + patches = ["icu4c_fix_static_lib_name_mingw.patch"] + data = Remotefile('icudt56l.dat', + 'e23d85eee008f335fc49e8ef37b1bc2b222db105476111e3d16f0007d371cbca') + + def _download_data(self, context): + self.buildEnv.download(self.data) + + def _copy_data(self, context): + context.try_skip(self.extract_path) + shutil.copyfile(pj(self.buildEnv.archive_dir, self.data.name), pj(self.extract_path, 'source', 'data', 'in', self.data.name)) + + def prepare(self): + super().prepare() + self.command("download_data", self._download_data) + self.command("copy_data", self._copy_data) + + class Builder(MakeBuilder): + subsource_dir = "source" + configure_option = "--disable-samples --disable-tests --disable-extras --disable-dyload" + dynamic_configure_option = "--enable-shared --disable-static" + static_configure_option = "--enable-static --disable-shared" + + +class Icu_native(Icu): + force_native_build = True + + class Builder(Icu.Builder): + @property + def build_path(self): + return super().build_path+"_native" + + def _install(self, context): + raise SkipCommand() + + +class Icu_cross_compile(Icu): + dependencies = ['Icu_native'] + + class Builder(Icu.Builder): + @property + def configure_option(self): + Icu_native = self.buildEnv.targetsDict['Icu_native'] + return super().configure_option + " --with-cross-build=" + Icu_native.builder.build_path + + +class Zimlib(Dependency): + name = "zimlib" + + class Source(GitClone): + #git_remote = "https://gerrit.wikimedia.org/r/p/openzim.git" + git_remote = "https://github.com/mgautierfr/openzim" + git_dir = "openzim" + git_ref = "meson" + + class Builder(MesonBuilder): + subsource_dir = "zimlib" + + +class Kiwixlib(Dependency): + name = "kiwix-lib" + dependencies = ['zlib'] + + @property + def dependencies(self): + if self.buildEnv.build_target == 'win32': + return ["Xapian", "CTPP2", "Pugixml", "Icu_cross_compile", "Zimlib"] + return ["Xapian", "CTPP2", "Pugixml", "Icu", "Zimlib"] + + class Source(GitClone): + git_remote = "https://github.com/kiwix/kiwix-lib.git" + git_dir = "kiwix-lib" + + class Builder(MesonBuilder): + configure_option = "-Dctpp2-install-prefix={buildEnv.install_dir}" + + +class KiwixTools(Dependency): + name = "kiwix-tools" + dependencies = ["Kiwixlib", "MicroHttpd", "zlib"] + + class Source(GitClone): + git_remote = "https://github.com/kiwix/kiwix-tools.git" + git_dir = "kiwix-tools" + + class Builder(MesonBuilder): + @property + def configure_option(self): + base_options = "-Dctpp2-install-prefix={buildEnv.install_dir}" + if self.buildEnv.build_static: + base_options += " -Dstatic-linkage=true" + return base_options diff --git a/dependency_utils.py b/dependency_utils.py new file mode 100644 index 0000000..20bfb89 --- /dev/null +++ b/dependency_utils.py @@ -0,0 +1,292 @@ +import subprocess +import os +import shutil + +from utils import pj, Context, SkipCommand, extract_archive, Defaultdict, StopBuild + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) + + +class _MetaDependency(type): + def __new__(cls, name, bases, dct): + _class = type.__new__(cls, name, bases, dct) + if name != 'Dependency': + Dependency.all_deps[name] = _class + return _class + + +class Dependency(metaclass=_MetaDependency): + all_deps = {} + dependencies = [] + force_native_build = False + version = None + + def __init__(self, buildEnv): + self.buildEnv = buildEnv + self.source = self.Source(self) + self.builder = self.Builder(self) + self.skip = False + + @property + def full_name(self): + if self.version: + return "{}-{}".format(self.name, self.version) + return self.name + + @property + def source_path(self): + return pj(self.buildEnv.source_dir, self.source.source_dir) + + @property + def _log_dir(self): + return self.buildEnv.log_dir + + def command(self, name, function, *args): + print(" {} {} : ".format(name, self.name), end="", flush=True) + log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name)) + context = Context(name, log) + try: + ret = function(*args, context=context) + context._finalise() + print("OK") + return ret + except SkipCommand: + print("SKIP") + except subprocess.CalledProcessError: + print("ERROR") + try: + with open(log, 'r') as f: + print(f.read()) + except: + pass + raise StopBuild() + except: + print("ERROR") + raise + + +class Source: + """Base Class to the real preparator + A source preparator must install source in the self.source_dir attribute + inside the buildEnv.source_dir.""" + def __init__(self, target): + self.target = target + self.buildEnv = target.buildEnv + + @property + def name(self): + return self.target.name + + @property + def source_dir(self): + return self.target.full_name + + def command(self, *args, **kwargs): + return self.target.command(*args, **kwargs) + + +class ReleaseDownload(Source): + archive_top_dir = None + + @property + def extract_path(self): + return pj(self.buildEnv.source_dir, self.source_dir) + + def _download(self, context): + self.buildEnv.download(self.archive) + + def _extract(self, context): + context.try_skip(self.extract_path) + if os.path.exists(self.extract_path): + shutil.rmtree(self.extract_path) + extract_archive(pj(self.buildEnv.archive_dir, self.archive.name), + self.buildEnv.source_dir, + topdir=self.archive_top_dir, + name=self.source_dir) + + def _patch(self, context): + context.try_skip(self.extract_path) + for p in self.patches: + with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input: + self.buildEnv.run_command("patch -p1", self.extract_path, context, input=patch_input, allow_wrapper=False) + + def prepare(self): + self.command('download', self._download) + self.command('extract', self._extract) + if hasattr(self, 'patches'): + self.command('patch', self._patch) + + +class GitClone(Source): + git_ref = "master" + + @property + def source_dir(self): + return self.git_dir + + @property + def git_path(self): + return pj(self.buildEnv.source_dir, self.git_dir) + + def _git_clone(self, context): + if os.path.exists(self.git_path): + raise SkipCommand() + command = "git clone " + self.git_remote + self.buildEnv.run_command(command, self.buildEnv.source_dir, context) + + def _git_update(self, context): + self.buildEnv.run_command("git fetch", self.git_path, context) + self.buildEnv.run_command("git checkout "+self.git_ref, self.git_path, context) + + def prepare(self): + self.command('gitclone', self._git_clone) + self.command('gitupdate', self._git_update) + + +class Builder: + subsource_dir = None + + def __init__(self, target): + self.target = target + self.buildEnv = target.buildEnv + + @property + def name(self): + return self.target.name + + @property + def source_path(self): + base_source_path = self.target.source_path + if self.subsource_dir: + return pj(base_source_path, self.subsource_dir) + return base_source_path + + @property + def build_path(self): + return pj(self.buildEnv.build_dir, self.target.full_name) + + def command(self, *args, **kwargs): + return self.target.command(*args, **kwargs) + + def build(self): + self.command('configure', self._configure) + self.command('compile', self._compile) + self.command('install', self._install) + + +class MakeBuilder(Builder): + configure_option = static_configure_option = dynamic_configure_option = "" + make_option = "" + install_option = "" + configure_script = "configure" + configure_env = None + make_target = "" + make_install_target = "install" + + def _configure(self, context): + context.try_skip(self.build_path) + configure_option = "{} {} {}".format( + self.configure_option, + self.static_configure_option if self.buildEnv.build_static else self.dynamic_configure_option, + self.buildEnv.configure_option) + command = "{configure_script} {configure_option} --prefix {install_dir} --libdir {libdir}" + command = command.format( + configure_script=pj(self.source_path, self.configure_script), + configure_option=configure_option, + install_dir=self.buildEnv.install_dir, + libdir=pj(self.buildEnv.install_dir, self.buildEnv.libprefix) + ) + env = Defaultdict(str, os.environ) + if self.buildEnv.build_static: + env['CFLAGS'] = env['CFLAGS'] + ' -fPIC' + if self.configure_env: + for k in self.configure_env: + if k.startswith('_format_'): + v = self.configure_env.pop(k) + v = v.format(buildEnv=self.buildEnv, env=env) + self.configure_env[k[8:]] = v + env.update(self.configure_env) + self.buildEnv.run_command(command, self.build_path, context, env=env) + + def _compile(self, context): + context.try_skip(self.build_path) + command = "make -j4 {make_target} {make_option}".format( + make_target=self.make_target, + make_option=self.make_option + ) + self.buildEnv.run_command(command, self.build_path, context) + + def _install(self, context): + context.try_skip(self.build_path) + command = "make {make_install_target} {make_option}".format( + make_install_target=self.make_install_target, + make_option=self.make_option + ) + self.buildEnv.run_command(command, self.build_path, context) + + +class CMakeBuilder(MakeBuilder): + def _configure(self, context): + context.try_skip(self.build_path) + command = ("cmake {configure_option}" + " -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON" + " -DCMAKE_INSTALL_PREFIX={install_dir}" + " -DCMAKE_INSTALL_LIBDIR={libdir}" + " {source_path}" + " {cross_option}") + command = command.format( + configure_option="{} {}".format(self.buildEnv.cmake_option, self.configure_option), + install_dir=self.buildEnv.install_dir, + libdir=self.buildEnv.libprefix, + source_path=self.source_path, + cross_option="-DCMAKE_TOOLCHAIN_FILE={}".format(self.buildEnv.cmake_crossfile) if self.buildEnv.cmake_crossfile else "" + ) + env = Defaultdict(str, os.environ) + if self.buildEnv.build_static: + env['CFLAGS'] = env['CFLAGS'] + ' -fPIC' + if self.configure_env: + for k in self.configure_env: + if k.startswith('_format_'): + v = self.configure_env.pop(k) + v = v.format(buildEnv=self.buildEnv, env=env) + self.configure_env[k[8:]] = v + env.update(self.configure_env) + self.buildEnv.run_command(command, self.build_path, context, env=env, allow_wrapper=False) + + +class MesonBuilder(Builder): + configure_option = "" + + def _configure(self, context): + context.try_skip(self.build_path) + if os.path.exists(self.build_path): + shutil.rmtree(self.build_path) + os.makedirs(self.build_path) + if self.buildEnv.build_static: + library_type = 'static' + else: + library_type = 'shared' + configure_option = self.configure_option.format(buildEnv=self.buildEnv) + command = ("{command} . {build_path}" + " --default-library={library_type}" + " {configure_option}" + " --prefix={buildEnv.install_dir}" + " --libdir={buildEnv.libprefix}" + " {cross_option}") + command = command.format( + command=self.buildEnv.meson_command, + library_type=library_type, + configure_option=configure_option, + build_path=self.build_path, + buildEnv=self.buildEnv, + cross_option="--cross-file {}".format(self.buildEnv.meson_crossfile) if self.buildEnv.meson_crossfile else "" + ) + self.buildEnv.run_command(command, self.source_path, context, allow_wrapper=False) + + def _compile(self, context): + command = "{} -v".format(self.buildEnv.ninja_command) + self.buildEnv.run_command(command, self.build_path, context, allow_wrapper=False) + + def _install(self, context): + command = "{} -v install".format(self.buildEnv.ninja_command) + self.buildEnv.run_command(command, self.build_path, context, allow_wrapper=False) diff --git a/kiwix-build.py b/kiwix-build.py index 9fbab10..6279401 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -4,71 +4,72 @@ import os, sys, stat import argparse import ssl import urllib.request -import tarfile import subprocess -import hashlib -import shutil -import tempfile -import configparser import platform -from collections import defaultdict, namedtuple, OrderedDict +from collections import OrderedDict - -pj = os.path.join +from dependencies import Dependency +from utils import ( + pj, + remove_duplicates, + get_sha256, + StopBuild, + SkipCommand, + Defaultdict) REMOTE_PREFIX = 'http://download.kiwix.org/dev/' SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) CROSS_ENV = { - 'fedora_win32' : { - 'root_path' : '/usr/i686-w64-mingw32/sys-root/mingw', - 'binaries' : { - 'c' : 'i686-w64-mingw32-gcc', - 'cpp' : 'i686-w64-mingw32-g++', - 'ar' : 'i686-w64-mingw32-ar', - 'strip' : 'i686-w64-mingw32-strip', - 'windres' : 'i686-w64-mingw32-windres', - 'ranlib' : 'i686-w64-mingw32-ranlib', - 'exe_wrapper' : 'wine' + 'fedora_win32': { + 'root_path': '/usr/i686-w64-mingw32/sys-root/mingw', + 'binaries': { + 'c': 'i686-w64-mingw32-gcc', + 'cpp': 'i686-w64-mingw32-g++', + 'ar': 'i686-w64-mingw32-ar', + 'strip': 'i686-w64-mingw32-strip', + 'windres': 'i686-w64-mingw32-windres', + 'ranlib': 'i686-w64-mingw32-ranlib', + 'exe_wrapper': 'wine' }, - 'properties' : { + 'properties': { 'c_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4'], 'cpp_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4'] }, - 'host_machine' : { - 'system' : 'windows', - 'cpu_family' : 'x86', - 'cpu' : 'i686', - 'endian' : 'little' + 'host_machine': { + 'system': 'windows', + 'cpu_family': 'x86', + 'cpu': 'i686', + 'endian': 'little' }, 'env': { - '_format_PKG_CONFIG_LIBDIR' : '{root_path}/lib/pkgconfig' + '_format_PKG_CONFIG_LIBDIR': '{root_path}/lib/pkgconfig' } }, - 'debian_win32' : { - 'root_path' : '/usr/i686-w64-mingw32/', - 'binaries' : { - 'c' : 'i686-w64-mingw32-gcc', - 'cpp' : 'i686-w64-mingw32-g++', - 'ar' : 'i686-w64-mingw32-ar', - 'strip' : 'i686-w64-mingw32-strip', - 'windres' : 'i686-w64-mingw32-windres', - 'ranlib' : 'i686-w64-mingw32-ranlib', - 'exe_wrapper' : 'wine' + 'debian_win32': { + 'root_path': '/usr/i686-w64-mingw32/', + 'binaries': { + 'c': 'i686-w64-mingw32-gcc', + 'cpp': 'i686-w64-mingw32-g++', + 'ar': 'i686-w64-mingw32-ar', + 'strip': 'i686-w64-mingw32-strip', + 'windres': 'i686-w64-mingw32-windres', + 'ranlib': 'i686-w64-mingw32-ranlib', + 'exe_wrapper': 'wine' }, - 'properties' : { + 'properties': { 'c_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4'], 'cpp_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4'] }, - 'host_machine' : { - 'system' : 'windows', - 'cpu_family' : 'x86', - 'cpu' : 'i686', - 'endian' : 'little' + 'host_machine': { + 'system': 'windows', + 'cpu_family': 'x86', + 'cpu': 'i686', + 'endian': 'little' }, 'env': { - '_format_PKG_CONFIG_LIBDIR' : '{root_path}/lib/pkgconfig' + '_format_PKG_CONFIG_LIBDIR': '{root_path}/lib/pkgconfig' } } } @@ -76,57 +77,53 @@ CROSS_ENV = { PACKAGE_NAME_MAPPERS = { 'fedora_native_dyn': { - 'COMMON' : ['gcc-c++', 'cmake', 'automake', 'ccache'], + 'COMMON': ['gcc-c++', 'cmake', 'automake', 'ccache'], 'uuid': ['libuuid-devel'], - 'xapian-core' : None, # Not the right version on fedora 25 - 'ctpp2' : None, - 'pugixml' : None, # ['pugixml-devel'] but package doesn't provide pkg-config file - 'libmicrohttpd' : ['libmicrohttpd-devel'], - 'zlib' : ['zlib-devel'], + 'xapian-core': None, # Not the right version on fedora 25 + 'ctpp2': None, + 'pugixml': None, # ['pugixml-devel'] but package doesn't provide pkg-config file + 'libmicrohttpd': ['libmicrohttpd-devel'], + 'zlib': ['zlib-devel'], 'icu4c': None, 'zimlib': None, }, - 'fedora_native_static' : { - 'COMMON' : ['gcc-c++', 'cmake', 'automake', 'glibc-static', 'libstdc++-static', 'ccache'], - 'zlib' : ['zlib-devel', 'zlib-static'] + 'fedora_native_static': { + 'COMMON': ['gcc-c++', 'cmake', 'automake', 'glibc-static', 'libstdc++-static', 'ccache'], + 'zlib': ['zlib-devel', 'zlib-static'] # Either there is no packages, or no static or too old }, - 'fedora_win32_dyn' : { - 'COMMON' : ['mingw32-gcc-c++', 'mingw32-bzip2', 'mingw32-win-iconv', 'mingw32-winpthreads', 'wine', 'ccache'], - 'zlib' : ['mingw32-zlib'], - 'libmicrohttpd' : ['mingw32-libmicrohttpd'], + 'fedora_win32_dyn': { + 'COMMON': ['mingw32-gcc-c++', 'mingw32-bzip2', 'mingw32-win-iconv', 'mingw32-winpthreads', 'wine', 'ccache'], + 'zlib': ['mingw32-zlib'], + 'libmicrohttpd': ['mingw32-libmicrohttpd'], }, - 'fedora_win32_static' : { - 'COMMON' : ['mingw32-gcc-c++', 'mingw32-bzip2-static', 'mingw32-win-iconv-static', 'mingw32-winpthreads-static', 'wine', 'ccache'], - 'zlib' : ['mingw32-zlib-static'], - 'libmicrohttpd' : None, # ['mingw32-libmicrohttpd-static'] packaging dependecy seems buggy, and some static lib are name libfoo.dll.a and - # gcc cannot found them. + 'fedora_win32_static': { + 'COMMON': ['mingw32-gcc-c++', 'mingw32-bzip2-static', 'mingw32-win-iconv-static', 'mingw32-winpthreads-static', 'wine', 'ccache'], + 'zlib': ['mingw32-zlib-static'], + 'libmicrohttpd': None, # ['mingw32-libmicrohttpd-static'] packaging dependecy seems buggy, and some static lib are name libfoo.dll.a and + # gcc cannot found them. }, - 'debian_native_dyn' : { - 'COMMON' : ['gcc', 'cmake', 'libbz2-dev', 'ccache'], - 'zlib' : ['zlib1g-dev'], - 'uuid' : ['uuid-dev'], + 'debian_native_dyn': { + 'COMMON': ['gcc', 'cmake', 'libbz2-dev', 'ccache'], + 'zlib': ['zlib1g-dev'], + 'uuid': ['uuid-dev'], 'ctpp2': ['libctpp2-dev'], - 'libmicrohttpd' : ['libmicrohttpd-dev', 'ccache'] + 'libmicrohttpd': ['libmicrohttpd-dev', 'ccache'] }, - 'debian_native_static' : { - 'COMMON' : ['gcc', 'cmake', 'libbz2-dev', 'ccache'], - 'zlib' : ['zlib1g-dev'], - 'uuid' : ['uuid-dev'], + 'debian_native_static': { + 'COMMON': ['gcc', 'cmake', 'libbz2-dev', 'ccache'], + 'zlib': ['zlib1g-dev'], + 'uuid': ['uuid-dev'], 'ctpp2': ['libctpp2-dev'], }, - 'debian_win32_dyn' : { - 'COMMON' : ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools', 'ccache'] + 'debian_win32_dyn': { + 'COMMON': ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools', 'ccache'] }, - 'debian_win32_static' : { - 'COMMON' : ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools', 'ccache'] + 'debian_win32_static': { + 'COMMON': ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools', 'ccache'] }, } -class Defaultdict(defaultdict): - def __getattr__(self, name): - return self[name] - class Which(): def __getattr__(self, name): @@ -137,77 +134,6 @@ class Which(): def __format__(self, format_spec): return getattr(self, format_spec) -def remove_duplicates(iterable, key_function=None): - seen = set() - if key_function is None: - key_function = lambda e:e - for elem in iterable: - key = key_function(elem) - if key in seen: - continue - seen.add(key) - yield elem - -def get_sha256(path): - sha256 = hashlib.sha256() - with open(path, 'br') as f: - sha256.update(f.read()) - return sha256.hexdigest() - -class SkipCommand(Exception): - pass - -class StopBuild(Exception): - pass - -class Remotefile(namedtuple('Remotefile', ('name', 'sha256', 'url'))): - def __new__(cls, name, sha256, url=None): - return super().__new__(cls, name, sha256, url) - -class Context: - def __init__(self, command_name, log_file): - self.command_name = command_name - self.log_file = log_file - self.autoskip_file = None - - def try_skip(self, path): - self.autoskip_file = pj(path, ".{}_ok".format(self.command_name)) - if os.path.exists(self.autoskip_file): - raise SkipCommand() - - def _finalise(self): - if self.autoskip_file is not None: - with open(self.autoskip_file, 'w') as f: pass - - -def extract_archive(archive_path, dest_dir, topdir=None, name=None): - with tarfile.open(archive_path) as archive: - members = archive.getmembers() - if not topdir: - for d in (m for m in members if m.isdir()): - if not os.path.dirname(d.name): - if topdir: - # There is already a top dir. - # Two topdirs in the same archive. - # Extract all - topdir = None - break - topdir = d - else: - topdir = archive.getmember(topdir) - if topdir: - members_to_extract = [m for m in members if m.name.startswith(topdir.name+'/')] - os.makedirs(dest_dir, exist_ok=True) - with tempfile.TemporaryDirectory(prefix=os.path.basename(archive_path), dir=dest_dir) as tmpdir: - archive.extractall(path=tmpdir, members=members_to_extract) - name = name or topdir.name - os.rename(pj(tmpdir, topdir.name), pj(dest_dir, name)) - else: - if name: - dest_dir = pj(dest_dir, name) - os.makedirs(dest_dir) - archive.extractall(path=dest_dir) - class BuildEnv: build_targets = ['native', 'win32'] @@ -282,16 +208,16 @@ class BuildEnv: def setup_win32(self): cross_name = "{host}_{target}".format( - host = self.distname, - target = self.build_target) + host=self.distname, + target=self.build_target) try: self.cross_env = CROSS_ENV[cross_name] except KeyError: sys.exit("ERROR : We don't know how to set env to compile" " a {target} version on a {host} host.".format( - target = self.build_target, - host = self.distname - )) + target=self.build_target, + host=self.distname + )) self.wrapper = self._gen_crossfile('bash_wrapper.sh') current_permissions = stat.S_IMODE(os.lstat(self.wrapper).st_mode) @@ -357,11 +283,11 @@ class BuildEnv: env['PKG_CONFIG_PATH'] = (env['PKG_CONFIG_PATH'] + ':' + pkgconfig_path if env['PKG_CONFIG_PATH'] else pkgconfig_path - ) + ) # Add ccache path for p in ('/usr/lib/ccache', '/usr/lib64/ccache'): if os.path.isdir(p): - ccache_path=[p] + ccache_path = [p] break else: ccache_path = [] @@ -373,7 +299,7 @@ class BuildEnv: env['LD_LIBRARY_PATH'] = (env['LD_LIBRARY_PATH'] + ':' + ld_library_path if env['LD_LIBRARY_PATH'] else ld_library_path - ) + ) env['CPPFLAGS'] = '-I'+pj(self.install_dir, 'include') env['LDFLAGS'] = '-L'+pj(self.install_dir, 'lib') return env @@ -388,7 +314,7 @@ class BuildEnv: if not self.options.verbose: log = open(context.log_file, 'w') print("run command '{}'".format(command), file=log) - print("env is :",file=log) + print("env is :", file=log) for k, v in env.items(): print(" {} : {!r}".format(k, v), file=log) @@ -436,18 +362,17 @@ class BuildEnv: elif self.distname in ('debian', 'Ubuntu'): package_installer = 'apt-get' mapper_name = "{host}_{target}_{build_type}".format( - host = self.distname, - target = self.build_target, - build_type = 'static' if self.options.build_static else 'dyn') + host=self.distname, + target=self.build_target, + build_type='static' if self.options.build_static else 'dyn') try: package_name_mapper = PACKAGE_NAME_MAPPERS[mapper_name] except KeyError: print("SKIP : We don't know which packages we must install to compile" " a {target} {build_type} version on a {host} host.".format( - target = self.build_target, - build_type = 'static' if self.options.build_static else 'dyn', - host = self.distname - )) + target=self.build_target, + build_type='static' if self.options.build_static else 'dyn', + host=self.distname)) return packages_list = package_name_mapper.get('COMMON', []) @@ -461,516 +386,16 @@ class BuildEnv: return if packages_list: command = "sudo {package_installer} install {packages_list}".format( - package_installer = package_installer, - packages_list = " ".join(packages_list) + package_installer=package_installer, + packages_list=" ".join(packages_list) ) print(command) subprocess.check_call(command, shell=True) else: print("SKIP, No package to install.") - with open(autoskip_file, 'w') as f: pass - -################################################################################ -##### PROJECT -################################################################################ -class _MetaDependency(type): - def __new__(cls, name, bases, dct): - _class = type.__new__(cls, name, bases, dct) - if name != 'Dependency': - Dependency.all_deps[name] = _class - return _class - - -class Dependency(metaclass=_MetaDependency): - all_deps = {} - dependencies = [] - force_native_build = False - version = None - def __init__(self, buildEnv): - self.buildEnv = buildEnv - self.source = self.Source(self) - self.builder = self.Builder(self) - self.skip = False - - @property - def full_name(self): - if self.version: - return "{}-{}".format(self.name, self.version) - return self.name - - @property - def source_path(self): - return pj(self.buildEnv.source_dir, self.source.source_dir) - - @property - def _log_dir(self): - return self.buildEnv.log_dir - - def command(self, name, function, *args): - print(" {} {} : ".format(name, self.name), end="", flush=True) - log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name)) - context = Context(name, log) - try: - ret = function(*args, context=context) - context._finalise() - print("OK") - return ret - except SkipCommand: - print("SKIP") - except subprocess.CalledProcessError: - print("ERROR") - try: - with open(log, 'r') as f: - print(f.read()) - except: - pass - raise StopBuild() - except: - print("ERROR") - raise - - -class Source: - """Base Class to the real preparator - A source preparator must install source in the self.source_dir attribute - inside the buildEnv.source_dir.""" - def __init__(self, target): - self.target = target - self.buildEnv = target.buildEnv - - @property - def name(self): - return self.target.name - - @property - def source_dir(self): - return self.target.full_name - - def command(self, *args, **kwargs): - return self.target.command(*args, **kwargs) - - -class ReleaseDownload(Source): - archive_top_dir = None - @property - def extract_path(self): - return pj(self.buildEnv.source_dir, self.source_dir) - - def _download(self, context): - self.buildEnv.download(self.archive) - - def _extract(self, context): - context.try_skip(self.extract_path) - if os.path.exists(self.extract_path): - shutil.rmtree(self.extract_path) - extract_archive(pj(self.buildEnv.archive_dir, self.archive.name), - self.buildEnv.source_dir, - topdir=self.archive_top_dir, - name=self.source_dir) - - def _patch(self, context): - context.try_skip(self.extract_path) - for p in self.patches: - with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input: - self.buildEnv.run_command("patch -p1", self.extract_path, context, input=patch_input, allow_wrapper=False) - - def prepare(self): - self.command('download', self._download) - self.command('extract', self._extract) - if hasattr(self, 'patches'): - self.command('patch', self._patch) - - -class GitClone(Source): - git_ref = "master" - @property - def source_dir(self): - return self.git_dir - - @property - def git_path(self): - return pj(self.buildEnv.source_dir, self.git_dir) - - def _git_clone(self, context): - if os.path.exists(self.git_path): - raise SkipCommand() - command = "git clone " + self.git_remote - self.buildEnv.run_command(command, self.buildEnv.source_dir, context) - - def _git_update(self, context): - self.buildEnv.run_command("git fetch", self.git_path, context) - self.buildEnv.run_command("git checkout "+self.git_ref, self.git_path, context) - - def prepare(self): - self.command('gitclone', self._git_clone) - self.command('gitupdate', self._git_update) - - -class Builder: - subsource_dir = None - def __init__(self, target): - self.target = target - self.buildEnv = target.buildEnv - - @property - def name(self): - return self.target.name - - @property - def source_path(self): - base_source_path = self.target.source_path - if self.subsource_dir: - return pj(base_source_path, self.subsource_dir) - return base_source_path - - @property - def build_path(self): - return pj(self.buildEnv.build_dir, self.target.full_name) - - def command(self, *args, **kwargs): - return self.target.command(*args, **kwargs) - - def build(self): - self.command('configure', self._configure) - self.command('compile', self._compile) - self.command('install', self._install) - - -class MakeBuilder(Builder): - configure_option = static_configure_option = dynamic_configure_option = "" - make_option = "" - install_option = "" - configure_script = "configure" - configure_env = None - make_target = "" - make_install_target = "install" - - def _configure(self, context): - context.try_skip(self.build_path) - configure_option = "{} {} {}".format( - self.configure_option, - self.static_configure_option if self.buildEnv.build_static else self.dynamic_configure_option, - self.buildEnv.configure_option) - command = "{configure_script} {configure_option} --prefix {install_dir} --libdir {libdir}" - command = command.format( - configure_script = pj(self.source_path, self.configure_script), - configure_option = configure_option, - install_dir = self.buildEnv.install_dir, - libdir = pj(self.buildEnv.install_dir, self.buildEnv.libprefix) - ) - env = Defaultdict(str, os.environ) - if self.buildEnv.build_static: - env['CFLAGS'] = env['CFLAGS'] + ' -fPIC' - if self.configure_env: - for k in self.configure_env: - if k.startswith('_format_'): - v = self.configure_env.pop(k) - v = v.format(buildEnv=self.buildEnv, env=env) - self.configure_env[k[8:]] = v - env.update(self.configure_env) - self.buildEnv.run_command(command, self.build_path, context, env=env) - - def _compile(self, context): - context.try_skip(self.build_path) - command = "make -j4 {make_target} {make_option}".format( - make_target = self.make_target, - make_option = self.make_option - ) - self.buildEnv.run_command(command, self.build_path, context) - - def _install(self, context): - context.try_skip(self.build_path) - command = "make {make_install_target} {make_option}".format( - make_install_target = self.make_install_target, - make_option = self.make_option - ) - self.buildEnv.run_command(command, self.build_path, context) - - -class CMakeBuilder(MakeBuilder): - def _configure(self, context): - context.try_skip(self.build_path) - command = ("cmake {configure_option}" - " -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON" - " -DCMAKE_INSTALL_PREFIX={install_dir}" - " -DCMAKE_INSTALL_LIBDIR={libdir}" - " {source_path}" - " {cross_option}") - command = command.format( - configure_option = "{} {}".format(self.buildEnv.cmake_option, self.configure_option), - install_dir = self.buildEnv.install_dir, - libdir = self.buildEnv.libprefix, - source_path = self.source_path, - cross_option = "-DCMAKE_TOOLCHAIN_FILE={}".format(self.buildEnv.cmake_crossfile) if self.buildEnv.cmake_crossfile else "" - ) - env = Defaultdict(str, os.environ) - if self.buildEnv.build_static: - env['CFLAGS'] = env['CFLAGS'] + ' -fPIC' - if self.configure_env: - for k in self.configure_env: - if k.startswith('_format_'): - v = self.configure_env.pop(k) - v = v.format(buildEnv=self.buildEnv, env=env) - self.configure_env[k[8:]] = v - env.update(self.configure_env) - self.buildEnv.run_command(command, self.build_path, context, env=env, allow_wrapper=False) - - -class MesonBuilder(Builder): - configure_option = "" - - def _configure(self, context): - context.try_skip(self.build_path) - if os.path.exists(self.build_path): - shutil.rmtree(self.build_path) - os.makedirs(self.build_path) - if self.buildEnv.build_static: - library_type = 'static' - else: - library_type = 'shared' - configure_option = self.configure_option.format(buildEnv=self.buildEnv) - command = ("{command} . {build_path}" - " --default-library={library_type}" - " {configure_option}" - " --prefix={buildEnv.install_dir}" - " --libdir={buildEnv.libprefix}" - " {cross_option}") - command = command.format( - command = self.buildEnv.meson_command, - library_type=library_type, - configure_option=configure_option, - build_path = self.build_path, - buildEnv=self.buildEnv, - cross_option = "--cross-file {}".format(self.buildEnv.meson_crossfile) if self.buildEnv.meson_crossfile else "" - ) - self.buildEnv.run_command(command, self.source_path, context, allow_wrapper=False) - - def _compile(self, context): - command = "{} -v".format(self.buildEnv.ninja_command) - self.buildEnv.run_command(command, self.build_path, context, allow_wrapper=False) - - def _install(self, context): - command = "{} -v install".format(self.buildEnv.ninja_command) - self.buildEnv.run_command(command, self.build_path, context, allow_wrapper=False) - - -# ************************************* -# Missing dependencies -# Is this ok to assume that those libs -# exist in your "distri" (linux/mac) ? -# If not, we need to compile them here -# ************************************* -# Zlib -# LZMA -# aria2 -# Argtable -# MSVirtual -# Android -# libiconv -# gettext -# ************************************* - -class zlib(Dependency): - name = 'zlib' - version = '1.2.8' - - class Source(ReleaseDownload): - archive = Remotefile('zlib-1.2.8.tar.gz', - '36658cb768a54c1d4dec43c3116c27ed893e88b02ecfcb44f2166f9c0b7f2a0d') - patches = ['zlib_std_libname.patch'] - - class Builder(CMakeBuilder): - @property - def configure_option(self): - return "-DINSTALL_PKGCONFIG_DIR={}".format(pj(self.buildEnv.install_dir, self.buildEnv.libprefix, 'pkgconfig')) - -class UUID(Dependency): - name = 'uuid' - version = "1.43.4" - - class Source(ReleaseDownload): - archive = Remotefile('e2fsprogs-1.43.4.tar.gz', - '1644db4fc58300c363ba1ab688cf9ca1e46157323aee1029f8255889be4bc856', - 'https://www.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v1.43.4/e2fsprogs-1.43.4.tar.gz') - extract_dir = 'e2fsprogs-1.43.4' - - class Builder(MakeBuilder): - configure_option = "--enable-libuuid" - configure_env = {'_format_CFLAGS' : "{env.CFLAGS} -fPIC"} - make_target = 'libs' - make_install_target = 'install-libs' - - -class Xapian(Dependency): - name = "xapian-core" - version = "1.4.2" - - class Source(ReleaseDownload): - archive = Remotefile('xapian-core-1.4.2.tar.xz', - 'aec2c4352998127a2f2316218bf70f48cef0a466a87af3939f5f547c5246e1ce') - patches = ["xapian_pkgconfig.patch"] - - class Builder(MakeBuilder): - configure_option = "--disable-sse --disable-backend-inmemory --disable-documentation" - dynamic_configure_option = "--enable-shared --disable-static" - static_configure_option = "--enable-static --disable-shared" - configure_env = {'_format_LDFLAGS' : "-L{buildEnv.install_dir}/{buildEnv.libprefix}", - '_format_CXXFLAGS' : "-I{buildEnv.install_dir}/include"} - - @property - def dependencies(self): - deps = ['zlib'] - if self.buildEnv.build_target == 'win32': - return deps - return deps + ['UUID'] - - -class CTPP2(Dependency): - name = "ctpp2" - version = "2.8.3" - - class Source(ReleaseDownload): - archive = Remotefile('ctpp2-2.8.3.tar.gz', - 'a83ffd07817adb575295ef40fbf759892512e5a63059c520f9062d9ab8fb42fc') - patches = ["ctpp2_include.patch", - "ctpp2_no_src_modification.patch", - "ctpp2_fix-static-libname.patch", - "ctpp2_mingw32.patch", - "ctpp2_dll_export_VMExecutable.patch", - "ctpp2_win_install_lib_in_lib_dir.patch", - "ctpp2_iconv_support.patch"] - - class Builder(CMakeBuilder): - configure_option = "-DMD5_SUPPORT=OFF" - - -class Pugixml(Dependency): - name = "pugixml" - version = "1.2" - - class Source(ReleaseDownload): - archive = Remotefile('pugixml-1.2.tar.gz', - '0f422dad86da0a2e56a37fb2a88376aae6e931f22cc8b956978460c9db06136b') - patches = ["pugixml_meson.patch"] - - Builder = MesonBuilder - - -class MicroHttpd(Dependency): - name = "libmicrohttpd" - version = "0.9.46" - - class Source(ReleaseDownload): - archive = Remotefile('libmicrohttpd-0.9.46.tar.gz', - '06dbd2654f390fa1e8196fe063fc1449a6c2ed65a38199a49bf29ad8a93b8979', - 'http://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.46.tar.gz') - - class Builder(MakeBuilder): - configure_option = "--disable-https --without-libgcrypt --without-libcurl" - dynamic_configure_option = "--enable-shared --disable-static" - static_configure_option = "--enable-static --disable-shared" - - -class Icu(Dependency): - name = "icu4c" - version = "56_1" - - class Source(ReleaseDownload): - archive = Remotefile('icu4c-56_1-src.tgz', - '3a64e9105c734dcf631c0b3ed60404531bce6c0f5a64bfe1a6402a4cc2314816' - ) - patches = ["icu4c_fix_static_lib_name_mingw.patch"] - data = Remotefile('icudt56l.dat', - 'e23d85eee008f335fc49e8ef37b1bc2b222db105476111e3d16f0007d371cbca') - - def _download_data(self, context): - self.buildEnv.download(self.data) - - def _copy_data(self, context): - context.try_skip(self.extract_path) - shutil.copyfile(pj(self.buildEnv.archive_dir, self.data.name), pj(self.extract_path, 'source', 'data', 'in', self.data.name)) - - def prepare(self): - super().prepare() - self.command("download_data", self._download_data) - self.command("copy_data", self._copy_data) - - class Builder(MakeBuilder): - subsource_dir = "source" - configure_option = "--disable-samples --disable-tests --disable-extras --disable-dyload" - dynamic_configure_option = "--enable-shared --disable-static" - static_configure_option = "--enable-static --disable-shared" - - -class Icu_native(Icu): - force_native_build = True - - class Builder(Icu.Builder): - @property - def build_path(self): - return super().build_path+"_native" - - def _install(self, context): - raise SkipCommand() - - -class Icu_cross_compile(Icu): - dependencies = ['Icu_native'] - - class Builder(Icu.Builder): - @property - def configure_option(self): - Icu_native = self.buildEnv.targetsDict['Icu_native'] - return super().configure_option + " --with-cross-build=" + Icu_native.builder.build_path - - -class Zimlib(Dependency): - name = "zimlib" - - class Source(GitClone): - #git_remote = "https://gerrit.wikimedia.org/r/p/openzim.git" - git_remote = "https://github.com/mgautierfr/openzim" - git_dir = "openzim" - git_ref = "meson" - - class Builder(MesonBuilder): - subsource_dir = "zimlib" - - -class Kiwixlib(Dependency): - name = "kiwix-lib" - dependencies = ['zlib'] - - @property - def dependencies(self): - if self.buildEnv.build_target == 'win32': - return ["Xapian", "CTPP2", "Pugixml", "Icu_cross_compile", "Zimlib"] - return ["Xapian", "CTPP2", "Pugixml", "Icu", "Zimlib"] - - class Source(GitClone): - git_remote = "https://github.com/kiwix/kiwix-lib.git" - git_dir = "kiwix-lib" - - class Builder(MesonBuilder): - configure_option = "-Dctpp2-install-prefix={buildEnv.install_dir}" - - -class KiwixTools(Dependency): - name = "kiwix-tools" - dependencies = ["Kiwixlib", "MicroHttpd", "zlib"] - - class Source(GitClone): - git_remote = "https://github.com/kiwix/kiwix-tools.git" - git_dir = "kiwix-tools" - - class Builder(MesonBuilder): - @property - def configure_option(self): - base_options = "-Dctpp2-install-prefix={buildEnv.install_dir}" - if self.buildEnv.build_static: - base_options += " -Dstatic-linkage=true" - return base_options + with open(autoskip_file, 'w'): + pass class Builder: @@ -989,7 +414,7 @@ class Builder: dependencies = list(remove_duplicates(dependencies)) for dep in dependencies: - self.targets[dep] = _targets[dep] + self.targets[dep] = _targets[dep] def add_targets(self, targetName, targets): if targetName in targets: @@ -1011,7 +436,7 @@ class Builder: def prepare_sources(self): sources = (dep.source for dep in self.targets.values() if not dep.skip) - sources = remove_duplicates(sources, lambda s:s.__class__) + sources = remove_duplicates(sources, lambda s: s.__class__) for source in sources: print("prepare sources {} :".format(source.name)) source.prepare() @@ -1036,6 +461,7 @@ class Builder: except StopBuild: sys.exit("Stopping build due to errors") + def parse_args(): parser = argparse.ArgumentParser() parser.add_argument('working_dir', default=".", nargs='?') @@ -1049,6 +475,7 @@ def parse_args(): help="Skip SSL certificate verification during download") return parser.parse_args() + if __name__ == "__main__": options = parse_args() options.working_dir = os.path.abspath(options.working_dir) diff --git a/utils.py b/utils.py new file mode 100644 index 0000000..9c1464b --- /dev/null +++ b/utils.py @@ -0,0 +1,90 @@ +import os.path +import hashlib +import tarfile +import tempfile +from collections import namedtuple, defaultdict + +pj = os.path.join + + +class Defaultdict(defaultdict): + def __getattr__(self, name): + return self[name] + + +def remove_duplicates(iterable, key_function=None): + seen = set() + if key_function is None: + key_function = lambda e: e + for elem in iterable: + key = key_function(elem) + if key in seen: + continue + seen.add(key) + yield elem + + +def get_sha256(path): + sha256 = hashlib.sha256() + with open(path, 'br') as f: + sha256.update(f.read()) + return sha256.hexdigest() + + +class SkipCommand(Exception): + pass + + +class StopBuild(Exception): + pass + + +class Remotefile(namedtuple('Remotefile', ('name', 'sha256', 'url'))): + def __new__(cls, name, sha256, url=None): + return super().__new__(cls, name, sha256, url) + + +class Context: + def __init__(self, command_name, log_file): + self.command_name = command_name + self.log_file = log_file + self.autoskip_file = None + + def try_skip(self, path): + self.autoskip_file = pj(path, ".{}_ok".format(self.command_name)) + if os.path.exists(self.autoskip_file): + raise SkipCommand() + + def _finalise(self): + if self.autoskip_file is not None: + with open(self.autoskip_file, 'w'): + pass + + +def extract_archive(archive_path, dest_dir, topdir=None, name=None): + with tarfile.open(archive_path) as archive: + members = archive.getmembers() + if not topdir: + for d in (m for m in members if m.isdir()): + if not os.path.dirname(d.name): + if topdir: + # There is already a top dir. + # Two topdirs in the same archive. + # Extract all + topdir = None + break + topdir = d + else: + topdir = archive.getmember(topdir) + if topdir: + members_to_extract = [m for m in members if m.name.startswith(topdir.name+'/')] + os.makedirs(dest_dir, exist_ok=True) + with tempfile.TemporaryDirectory(prefix=os.path.basename(archive_path), dir=dest_dir) as tmpdir: + archive.extractall(path=tmpdir, members=members_to_extract) + name = name or topdir.name + os.rename(pj(tmpdir, topdir.name), pj(dest_dir, name)) + else: + if name: + dest_dir = pj(dest_dir, name) + os.makedirs(dest_dir) + archive.extractall(path=dest_dir)