diff --git a/.travis.yml b/.travis.yml index 8f792ce..78778a4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,12 +12,18 @@ deploy: skip_cleanup: true script: travis/deploy.sh on: - condition: ( "$STATIC_BUILD" = "true" ) && ( "$TRAVIS_EVENT_TYPE" = "cron" ) + condition: ( "$DEPLOY" = "true" ) && ( "$TRAVIS_EVENT_TYPE" = "cron" ) env: - - STATIC_BUILD=true BUILD_TARGET=native - - STATIC_BUILD=true BUILD_TARGET=win32 - - STATIC_BUILD=false BUILD_TARGET=native - - STATIC_BUILD=false BUILD_TARGET=win32 + - BUILD_OPTION="--target-platform=native_dyn" + - BUILD_OPTION="--target-platform=native_static" ARCHIVE_TYPE="--tar" DEPLOY=true + - BUILD_OPTION="--target-platform=win32_dyn" + - BUILD_OPTION="--target-platform=win32_static" ARCHIVE_TYPE="--zip" DEPLOY=true + - BUILD_OPTION="--target-platform=android_arm Kiwixlib" + - BUILD_OPTION="--target-platform=android_arm64 Kiwixlib" + - BUILD_OPTION="--target-platform=android_mips Kiwixlib" + - BUILD_OPTION="--target-platform=android_mips64 Kiwixlib" + - BUILD_OPTION="--target-platform=android_x86 Kiwixlib" + - BUILD_OPTION="--target-platform=android_x86_64 Kiwixlib" notifications: irc: channels: diff --git a/README.md b/README.md index 50f01d9..e2f36e4 100644 --- a/README.md +++ b/README.md @@ -20,67 +20,121 @@ Take care, the paragraphs are about the *target platforms*. If you want to build Kiwix for Android on a GNU/Linux system, you should follow the instructions of the "Android" paragraph. -## GNU/Linux -Install pre-requisties in your distro, eg, in Debian based: +## Preparing environment - sudo apt install git cmake meson python3-virtualenv virtualenvwrapper zlib1g-dev libicu-dev aria2 libtool +You will need a recent version of meson (0.34) and ninja (1.6) +If your distribution provide a recent enough versions for them, just install +them with your package manager. -### Dynamic +Else you can install them manually -Pretty simple once all dependencies are installed : +### Meson -The archives and sources will be copied in your current working dir so -I recommend you to create a working directory: +``` +pip install meson --user # Install Meson +``` - $ mkdir - $ cd +(You may want to install meson in a virtualenv if you prefer) -Once ready, just run the script with the install dir as argument: +### ninja - $ kiwix-build.py +``` +git clone git://github.com/ninja-build/ninja.git +cd ninja +git checkout release +./configure.py --bootstrap +cp ninja +``` -The script will download and install all the needed dependencies and -kiwix related repository. +## Buildings -At the end of the script you will found the binaries in /bin. +This is the simple one. -As it is a dynamic linked binary, you will need to add the lib directory to -LD_LIBRARY_PATH : +``` +./kiwix-build.py +``` - $ LD_LIBRARY_PATH=/lib /bin/kiwix-serve +It will compile everything. +If you are using a supported platform (redhad or debian based) it will install +missing packages using sudo. +If you don't want to trust kiwix-build.py and give it the root right, just +launch yourself the the printed command. -### Static +### Outputs -To build a static binary simply add the --build_static option to the -kiwix-build.py script : +Kiwix-build.py will create several directories: - $ kiwix-build.py --build_static +- ARCHIVES : All the downloaded archives go there. +- SOURCES : All the sources (extracted from archives and patched) go there. +- BUILD_native_dyn : All the build files go there. +- BUILD_native_dyn/INSTALL : The installed files go there. +- BUILD_native_dyn/LOGS: The logs files of the build. -Notes: You cannot use the same working directory to build a dynamic and static. +ARCHIVES and SOURCES are independent of the build type you choose. -Notes: At the end of the script, meson may raise a error when it install the -kiwix-server binary. This is a meson bug and it has been fixed here -(https://github.com/mesonbuild/meson/pull/1240) but your meson version may -not include this fix. -However, the kiwix-server binary is valid and ready to use. +If you want to install all those directories elsewhere, you can pass the +`--working-dir` option: -## Mac OSX Universal +``` +./kiwix-build.py --working-dir +``` -The script has not been tested on Mac OSX. +### Other target -Please, if you have a Mac OSX, try to run the kiwix-build script and -report errors or, better, fixes :) +By default, kiwix-build will build kiwix-tools and all its dependencies. +If you want to compile another target only (let's said kiwixlib), you can +specify it: -## Android and Windows +``` +./kiwix-build Kiwixlib +``` -Cross compilation and strange stuff are to come. +### Other target platforms. + +By default, kiwix-build will build everything for the current (native) platform +using dynamic linkage (hence the `native_dyn` of the BUILD_* directory). + +But you can select another target platforms using the option `target-platform`. +For now, there is ten different platform supported : + +- native_dyn +- native_static +- win32_dyn +- win32_static +- android_arm +- android_arm64 +- android_mips +- android_mips64 +- android_x86 +- android_x86_64 + +So, if you want to compile for win32 using static linkage: + +``` +./kiwix-build.py --target-platform win32_dyn +``` + +As said before, the ARCHIVES and SOURCES directories are common of the target +platform. So, if you always work in the same working directory the sources will +not be downloaded and prepared again. + +On android* platforms, only kiwixlib is supported. So you must ask to +kiwix-build to compile only kiwixlib: + +``` +./kiwix-build.py --target-platform android_arm Kiwixlib +``` + +## Know bugs + +- The script has not been tested on Mac OSX. +- Cross-compilation to arm (raspberry-PI) is missing. + +Help is welcome on those parts. -If you wich to help, you're welcome. # CONTACT -Email: kiwix-developer@lists.sourceforge.net - IRC: #kiwix on irc.freenode.net (I'm hidding myself under the starmad pseudo) diff --git a/dependencies.py b/dependencies.py new file mode 100644 index 0000000..1120763 --- /dev/null +++ b/dependencies.py @@ -0,0 +1,286 @@ +import shutil + +from dependency_utils import ( + Dependency, + ReleaseDownload, + GitClone, + MakeBuilder, + CMakeBuilder, + MesonBuilder) + +from utils import Remotefile, pj, SkipCommand + +# ************************************* +# Missing dependencies +# Is this ok to assume that those libs +# exist in your "distri" (linux/mac) ? +# If not, we need to compile them here +# ************************************* +# aria2 +# Argtable +# MSVirtual +# Android +# libiconv +# gettext +# ************************************* + + +class zlib(Dependency): + name = 'zlib' + version = '1.2.8' + + class Source(ReleaseDownload): + archive = Remotefile('zlib-1.2.8.tar.gz', + '36658cb768a54c1d4dec43c3116c27ed893e88b02ecfcb44f2166f9c0b7f2a0d') + patches = ['zlib_std_libname.patch'] + + class Builder(MakeBuilder): + dynamic_configure_option = "--shared" + static_configure_option = "--static" + + def _pre_build_script(self, context): + context.try_skip(self.build_path) + shutil.copytree(self.source_path, self.build_path) + + @property + def all_configure_option(self): + return '--static' if self.buildEnv.platform_info.static else '--shared' + + @property + def configure_option(self): + options = "-DINSTALL_PKGCONFIG_DIR={}".format(pj(self.buildEnv.install_dir, self.buildEnv.libprefix, 'pkgconfig')) + if self.buildEnv.platform_info.static: + options += " -DBUILD_SHARED_LIBS=false" + else: + options += " -DBUILD_SHARED_LIBS=true" + return options + + def _configure(self, context): + if self.buildEnv.platform_info.build == 'win32': + raise SkipCommand() + return super()._configure(context) + + @property + def make_option(self): + if self.buildEnv.platform_info.build == 'win32': + return "--makefile win32/Makefile.gcc PREFIX={host}- SHARED_MODE={static} INCLUDE_PATH={include_path} LIBRARY_PATH={library_path} BINARY_PATH={binary_path}".format( + host='i686-w64-mingw32', + static="0" if self.buildEnv.platform_info.static else "1", + include_path=pj(self.buildEnv.install_dir, 'include'), + library_path=pj(self.buildEnv.install_dir, self.buildEnv.libprefix), + binary_path=pj(self.buildEnv.install_dir, 'bin'), + ) + return "" + +class lzma(Dependency): + name = 'lzma' + version = '5.0.4' + + class Source(ReleaseDownload): + archive = Remotefile('xz-5.0.4.tar.bz2', + '5cd9b060d3a1ad396b3be52c9b9311046a1c369e6062aea752658c435629ce92') + + class Builder(MakeBuilder): + @property + def configure_option(self): + return "--disable-assembler" + +class UUID(Dependency): + name = 'uuid' + version = "1.43.4" + + class Source(ReleaseDownload): + archive = Remotefile('e2fsprogs-libs-1.43.4.tar.gz', + 'eed4516325768255c9745e7b82c9d7d0393abce302520a5b2cde693204b0e419', + 'https://www.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v1.43.4/e2fsprogs-libs-1.43.4.tar.gz') + extract_dir = 'e2fsprogs-libs-1.43.4' + + class Builder(MakeBuilder): + configure_option = ("--enable-libuuid --disable-debugfs --disable-imager --disable-resizer --disable-defrag --enable-fsck" + " --disable-uuidd") + configure_env = {'_format_CFLAGS': "{env.CFLAGS} -fPIC"} + static_configure_option = dynamic_configure_option = "" + make_target = 'libs' + make_install_target = 'install-libs' + + +class Xapian(Dependency): + name = "xapian-core" + version = "1.4.2" + + class Source(ReleaseDownload): + archive = Remotefile('xapian-core-1.4.2.tar.xz', + 'aec2c4352998127a2f2316218bf70f48cef0a466a87af3939f5f547c5246e1ce') + patches = ["xapian_pkgconfig.patch"] + + class Builder(MakeBuilder): + configure_option = "--disable-sse --disable-backend-inmemory --disable-documentation" + configure_env = {'_format_LDFLAGS': "-L{buildEnv.install_dir}/{buildEnv.libprefix}", + '_format_CXXFLAGS': "-I{buildEnv.install_dir}/include"} + + @property + def dependencies(self): + deps = ['zlib', 'lzma'] + if self.buildEnv.platform_info.build == 'win32': + return deps + return deps + ['UUID'] + + +class CTPP2(Dependency): + name = "ctpp2" + version = "2.8.3" + + class Source(ReleaseDownload): + archive = Remotefile('ctpp2-2.8.3.tar.gz', + 'a83ffd07817adb575295ef40fbf759892512e5a63059c520f9062d9ab8fb42fc') + patches = ["ctpp2_include.patch", + "ctpp2_no_src_modification.patch", + "ctpp2_fix-static-libname.patch", + "ctpp2_mingw32.patch", + "ctpp2_dll_export_VMExecutable.patch", + "ctpp2_win_install_lib_in_lib_dir.patch", + "ctpp2_iconv_support.patch"] + + class Builder(CMakeBuilder): + configure_option = "-DMD5_SUPPORT=OFF" + + +class Pugixml(Dependency): + name = "pugixml" + version = "1.2" + + class Source(ReleaseDownload): + archive = Remotefile('pugixml-1.2.tar.gz', + '0f422dad86da0a2e56a37fb2a88376aae6e931f22cc8b956978460c9db06136b') + patches = ["pugixml_meson.patch"] + + Builder = MesonBuilder + + +class MicroHttpd(Dependency): + name = "libmicrohttpd" + version = "0.9.46" + + class Source(ReleaseDownload): + archive = Remotefile('libmicrohttpd-0.9.46.tar.gz', + '06dbd2654f390fa1e8196fe063fc1449a6c2ed65a38199a49bf29ad8a93b8979', + 'http://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.46.tar.gz') + + class Builder(MakeBuilder): + configure_option = "--disable-https --without-libgcrypt --without-libcurl" + + +class Icu(Dependency): + name = "icu4c" + version = "58_2" + + class Source(ReleaseDownload): + archive = Remotefile('icu4c-58_2-src.tgz', + '2b0a4410153a9b20de0e20c7d8b66049a72aef244b53683d0d7521371683da0c', + 'https://freefr.dl.sourceforge.net/project/icu/ICU4C/58.2/icu4c-58_2-src.tgz') + patches = ["icu4c_fix_static_lib_name_mingw.patch", + "icu4c_android_elf64_st_info.patch"] + data = Remotefile('icudt56l.dat', + 'e23d85eee008f335fc49e8ef37b1bc2b222db105476111e3d16f0007d371cbca') + + def _download_data(self, context): + self.buildEnv.download(self.data) + + def _copy_data(self, context): + context.try_skip(self.extract_path) + shutil.copyfile(pj(self.buildEnv.archive_dir, self.data.name), pj(self.extract_path, 'source', 'data', 'in', self.data.name)) + + def prepare(self): + super().prepare() + self.command("download_data", self._download_data) + self.command("copy_data", self._copy_data) + + class Builder(MakeBuilder): + subsource_dir = "source" + configure_option = "--disable-samples --disable-tests --disable-extras --disable-dyload" + + +class Icu_native(Icu): + force_native_build = True + + class Builder(Icu.Builder): + name = "icu_native" + @property + def build_path(self): + return super().build_path+"_native" + + def _install(self, context): + raise SkipCommand() + + +class Icu_cross_compile(Icu): + dependencies = ['Icu_native'] + + class Builder(Icu.Builder): + name = "icu_cross-compile" + @property + def configure_option(self): + Icu_native = self.buildEnv.targetsDict['Icu_native'] + return super().configure_option + " --with-cross-build=" + Icu_native.builder.build_path + + +class Zimlib(Dependency): + name = "zimlib" + + class Source(GitClone): + #git_remote = "https://gerrit.wikimedia.org/r/p/openzim.git" + git_remote = "https://github.com/mgautierfr/openzim" + git_dir = "openzim" + git_ref = "meson" + + class Builder(MesonBuilder): + subsource_dir = "zimlib" + + +class Kiwixlib(Dependency): + name = "kiwix-lib" + dependencies = ['zlib', 'lzma'] + + @property + def dependencies(self): + base_dependencies = ["Xapian", "Pugixml", "Zimlib"] + if self.buildEnv.platform_info.build != 'android': + base_dependencies += ['CTPP2'] + if self.buildEnv.platform_info.build != 'native': + return base_dependencies + ["Icu_cross_compile"] + else: + return base_dependencies + ["Icu"] + + class Source(GitClone): + git_remote = "https://github.com/kiwix/kiwix-lib.git" + git_dir = "kiwix-lib" + + class Builder(MesonBuilder): + @property + def configure_option(self): + base_option = "-Dctpp2-install-prefix={buildEnv.install_dir}" + if self.buildEnv.platform_info.build == 'android': + base_option += ' -Dandroid=true' + return base_option + + @property + def library_type(self): + if self.buildEnv.platform_info.build == 'android': + return 'shared' + return super().library_type + + +class KiwixTools(Dependency): + name = "kiwix-tools" + dependencies = ["Kiwixlib", "MicroHttpd", "zlib"] + + class Source(GitClone): + git_remote = "https://github.com/kiwix/kiwix-tools.git" + git_dir = "kiwix-tools" + + class Builder(MesonBuilder): + @property + def configure_option(self): + if self.buildEnv.platform_info.static: + return "-Dstatic-linkage=true" + return "" diff --git a/dependency_utils.py b/dependency_utils.py new file mode 100644 index 0000000..677d638 --- /dev/null +++ b/dependency_utils.py @@ -0,0 +1,304 @@ +import subprocess +import os +import shutil + +from utils import pj, Context, SkipCommand, extract_archive, Defaultdict, StopBuild + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) + + +class _MetaDependency(type): + def __new__(cls, name, bases, dct): + _class = type.__new__(cls, name, bases, dct) + if name != 'Dependency': + Dependency.all_deps[name] = _class + return _class + + +class Dependency(metaclass=_MetaDependency): + all_deps = {} + dependencies = [] + force_native_build = False + version = None + + def __init__(self, buildEnv): + self.buildEnv = buildEnv + self.source = self.Source(self) + self.builder = self.Builder(self) + self.skip = False + + @property + def full_name(self): + if self.version: + return "{}-{}".format(self.name, self.version) + return self.name + + @property + def source_path(self): + return pj(self.buildEnv.source_dir, self.source.source_dir) + + @property + def _log_dir(self): + return self.buildEnv.log_dir + + def command(self, name, function, *args): + print(" {} {} : ".format(name, self.name), end="", flush=True) + log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name)) + context = Context(name, log, self.force_native_build) + try: + ret = function(*args, context=context) + context._finalise() + print("OK") + return ret + except SkipCommand: + print("SKIP") + except subprocess.CalledProcessError: + print("ERROR") + try: + with open(log, 'r') as f: + print(f.read()) + except: + pass + raise StopBuild() + except: + print("ERROR") + raise + + +class Source: + """Base Class to the real preparator + A source preparator must install source in the self.source_dir attribute + inside the buildEnv.source_dir.""" + def __init__(self, target): + self.target = target + self.buildEnv = target.buildEnv + + @property + def name(self): + return self.target.name + + @property + def source_dir(self): + return self.target.full_name + + def command(self, *args, **kwargs): + return self.target.command(*args, **kwargs) + + +class ReleaseDownload(Source): + archive_top_dir = None + + @property + def extract_path(self): + return pj(self.buildEnv.source_dir, self.source_dir) + + def _download(self, context): + self.buildEnv.download(self.archive) + + def _extract(self, context): + context.try_skip(self.extract_path) + if os.path.exists(self.extract_path): + shutil.rmtree(self.extract_path) + extract_archive(pj(self.buildEnv.archive_dir, self.archive.name), + self.buildEnv.source_dir, + topdir=self.archive_top_dir, + name=self.source_dir) + + def _patch(self, context): + context.try_skip(self.extract_path) + context.force_native_build = True + for p in self.patches: + with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input: + self.buildEnv.run_command("patch -p1", self.extract_path, context, input=patch_input) + + def prepare(self): + self.command('download', self._download) + self.command('extract', self._extract) + if hasattr(self, 'patches'): + self.command('patch', self._patch) + + +class GitClone(Source): + git_ref = "master" + + @property + def source_dir(self): + return self.git_dir + + @property + def git_path(self): + return pj(self.buildEnv.source_dir, self.git_dir) + + def _git_clone(self, context): + context.force_native_build = True + if os.path.exists(self.git_path): + raise SkipCommand() + command = "git clone " + self.git_remote + self.buildEnv.run_command(command, self.buildEnv.source_dir, context) + + def _git_update(self, context): + context.force_native_build = True + self.buildEnv.run_command("git fetch", self.git_path, context) + self.buildEnv.run_command("git checkout "+self.git_ref, self.git_path, context) + + def prepare(self): + self.command('gitclone', self._git_clone) + self.command('gitupdate', self._git_update) + + +class Builder: + subsource_dir = None + + def __init__(self, target): + self.target = target + self.buildEnv = target.buildEnv + + @property + def name(self): + return self.target.name + + @property + def source_path(self): + base_source_path = self.target.source_path + if self.subsource_dir: + return pj(base_source_path, self.subsource_dir) + return base_source_path + + @property + def build_path(self): + return pj(self.buildEnv.build_dir, self.target.full_name) + + def command(self, *args, **kwargs): + return self.target.command(*args, **kwargs) + + def build(self): + if hasattr(self, '_pre_build_script'): + self.command('pre_build_script', self._pre_build_script) + self.command('configure', self._configure) + self.command('compile', self._compile) + self.command('install', self._install) + if hasattr(self, '_post_build_script'): + self.command('post_build_script', self._post_build_script) + + +class MakeBuilder(Builder): + configure_option = "" + dynamic_configure_option = "--enable-shared --disable-static" + static_configure_option = "--enable-static --disable-shared" + make_option = "" + install_option = "" + configure_script = "configure" + configure_env = None + make_target = "" + make_install_target = "install" + + @property + def all_configure_option(self): + return "{} {} {}".format( + self.configure_option, + self.static_configure_option if self.buildEnv.platform_info.static else self.dynamic_configure_option, + self.buildEnv.configure_option if not self.target.force_native_build else "") + + def _configure(self, context): + context.try_skip(self.build_path) + command = "{configure_script} {configure_option} --prefix {install_dir} --libdir {libdir}" + command = command.format( + configure_script=pj(self.source_path, self.configure_script), + configure_option=self.all_configure_option, + install_dir=self.buildEnv.install_dir, + libdir=pj(self.buildEnv.install_dir, self.buildEnv.libprefix) + ) + env = Defaultdict(str, os.environ) + if self.buildEnv.platform_info.static: + env['CFLAGS'] = env['CFLAGS'] + ' -fPIC' + if self.configure_env: + for k in self.configure_env: + if k.startswith('_format_'): + v = self.configure_env.pop(k) + v = v.format(buildEnv=self.buildEnv, env=env) + self.configure_env[k[8:]] = v + env.update(self.configure_env) + self.buildEnv.run_command(command, self.build_path, context, env=env) + + def _compile(self, context): + context.try_skip(self.build_path) + command = "make -j4 {make_target} {make_option}".format( + make_target=self.make_target, + make_option=self.make_option + ) + self.buildEnv.run_command(command, self.build_path, context) + + def _install(self, context): + context.try_skip(self.build_path) + command = "make {make_install_target} {make_option}".format( + make_install_target=self.make_install_target, + make_option=self.make_option + ) + self.buildEnv.run_command(command, self.build_path, context) + + +class CMakeBuilder(MakeBuilder): + def _configure(self, context): + context.try_skip(self.build_path) + command = ("cmake {configure_option}" + " -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON" + " -DCMAKE_INSTALL_PREFIX={install_dir}" + " -DCMAKE_INSTALL_LIBDIR={libdir}" + " {source_path}" + " {cross_option}") + command = command.format( + configure_option="{} {}".format(self.buildEnv.cmake_option, self.configure_option), + install_dir=self.buildEnv.install_dir, + libdir=self.buildEnv.libprefix, + source_path=self.source_path, + cross_option="-DCMAKE_TOOLCHAIN_FILE={}".format(self.buildEnv.cmake_crossfile) if self.buildEnv.cmake_crossfile else "" + ) + env = Defaultdict(str, os.environ) + if self.buildEnv.platform_info.static: + env['CFLAGS'] = env['CFLAGS'] + ' -fPIC' + if self.configure_env: + for k in self.configure_env: + if k.startswith('_format_'): + v = self.configure_env.pop(k) + v = v.format(buildEnv=self.buildEnv, env=env) + self.configure_env[k[8:]] = v + env.update(self.configure_env) + self.buildEnv.run_command(command, self.build_path, context, env=env, cross_path_only=True) + + +class MesonBuilder(Builder): + configure_option = "" + + @property + def library_type(self): + return 'static' if self.buildEnv.platform_info.static else 'shared' + + def _configure(self, context): + context.try_skip(self.build_path) + if os.path.exists(self.build_path): + shutil.rmtree(self.build_path) + os.makedirs(self.build_path) + configure_option = self.configure_option.format(buildEnv=self.buildEnv) + command = ("{command} . {build_path}" + " --default-library={library_type}" + " {configure_option}" + " --prefix={buildEnv.install_dir}" + " --libdir={buildEnv.libprefix}" + " {cross_option}") + command = command.format( + command=self.buildEnv.meson_command, + library_type=self.library_type, + configure_option=configure_option, + build_path=self.build_path, + buildEnv=self.buildEnv, + cross_option="--cross-file {}".format(self.buildEnv.meson_crossfile) if self.buildEnv.meson_crossfile else "" + ) + self.buildEnv.run_command(command, self.source_path, context, cross_path_only=True) + + def _compile(self, context): + command = "{} -v".format(self.buildEnv.ninja_command) + self.buildEnv.run_command(command, self.build_path, context) + + def _install(self, context): + command = "{} -v install".format(self.buildEnv.ninja_command) + self.buildEnv.run_command(command, self.build_path, context) diff --git a/kiwix-build.py b/kiwix-build.py index 9fbab10..5494179 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -4,71 +4,67 @@ import os, sys, stat import argparse import ssl import urllib.request -import tarfile import subprocess -import hashlib -import shutil -import tempfile -import configparser import platform -from collections import defaultdict, namedtuple, OrderedDict +from collections import OrderedDict - -pj = os.path.join +from dependencies import Dependency +from dependency_utils import ReleaseDownload, Builder +from utils import ( + pj, + remove_duplicates, + get_sha256, + StopBuild, + SkipCommand, + Defaultdict, + Remotefile, + Context) REMOTE_PREFIX = 'http://download.kiwix.org/dev/' SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) CROSS_ENV = { - 'fedora_win32' : { - 'root_path' : '/usr/i686-w64-mingw32/sys-root/mingw', - 'binaries' : { - 'c' : 'i686-w64-mingw32-gcc', - 'cpp' : 'i686-w64-mingw32-g++', - 'ar' : 'i686-w64-mingw32-ar', - 'strip' : 'i686-w64-mingw32-strip', - 'windres' : 'i686-w64-mingw32-windres', - 'ranlib' : 'i686-w64-mingw32-ranlib', - 'exe_wrapper' : 'wine' - }, - 'properties' : { - 'c_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4'], - 'cpp_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4'] - }, - 'host_machine' : { - 'system' : 'windows', - 'cpu_family' : 'x86', - 'cpu' : 'i686', - 'endian' : 'little' - }, - 'env': { - '_format_PKG_CONFIG_LIBDIR' : '{root_path}/lib/pkgconfig' + 'fedora_win32': { + 'toolchain_names': ['mingw32_toolchain'], + 'root_path': '/usr/i686-w64-mingw32/sys-root/mingw', + 'extra_libs': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4', '-lmsvcr90'], + 'host_machine': { + 'system': 'windows', + 'cpu_family': 'x86', + 'cpu': 'i686', + 'endian': 'little' } }, - 'debian_win32' : { - 'root_path' : '/usr/i686-w64-mingw32/', - 'binaries' : { - 'c' : 'i686-w64-mingw32-gcc', - 'cpp' : 'i686-w64-mingw32-g++', - 'ar' : 'i686-w64-mingw32-ar', - 'strip' : 'i686-w64-mingw32-strip', - 'windres' : 'i686-w64-mingw32-windres', - 'ranlib' : 'i686-w64-mingw32-ranlib', - 'exe_wrapper' : 'wine' - }, - 'properties' : { - 'c_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4'], - 'cpp_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4'] - }, - 'host_machine' : { - 'system' : 'windows', - 'cpu_family' : 'x86', - 'cpu' : 'i686', - 'endian' : 'little' - }, - 'env': { - '_format_PKG_CONFIG_LIBDIR' : '{root_path}/lib/pkgconfig' + 'fedora_android': { + 'toolchain_names': ['android_ndk'], + 'extra_libs': [], + 'host_machine': { + 'system': 'Android', + 'cpu_family': 'x86', + 'cpu': 'i686', + 'endian': 'little' + } + }, + 'debian_win32': { + 'toolchain_names': ['mingw32_toolchain'], + 'root_path': '/usr/i686-w64-mingw32/', + 'extra_libs': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4', '-lmsvcr90'], + 'host_machine': { + 'system': 'windows', + 'cpu_family': 'x86', + 'cpu': 'i686', + 'endian': 'little' + } + }, + 'debian_android': { + 'toolchain_names': ['android_ndk'], + 'extra_libs': [], + 'host_machine': { + 'system': 'Android', + 'cpu_family': 'x86', + 'cpu': 'i686', + 'endian': 'little' } } } @@ -76,157 +72,125 @@ CROSS_ENV = { PACKAGE_NAME_MAPPERS = { 'fedora_native_dyn': { - 'COMMON' : ['gcc-c++', 'cmake', 'automake', 'ccache'], + 'COMMON': ['gcc-c++', 'cmake', 'automake', 'ccache'], 'uuid': ['libuuid-devel'], - 'xapian-core' : None, # Not the right version on fedora 25 - 'ctpp2' : None, - 'pugixml' : None, # ['pugixml-devel'] but package doesn't provide pkg-config file - 'libmicrohttpd' : ['libmicrohttpd-devel'], - 'zlib' : ['zlib-devel'], + 'xapian-core': None, # Not the right version on fedora 25 + 'ctpp2': None, + 'pugixml': None, # ['pugixml-devel'] but package doesn't provide pkg-config file + 'libmicrohttpd': ['libmicrohttpd-devel'], + 'zlib': ['zlib-devel'], + 'lzma': ['xz-devel'], 'icu4c': None, 'zimlib': None, }, - 'fedora_native_static' : { - 'COMMON' : ['gcc-c++', 'cmake', 'automake', 'glibc-static', 'libstdc++-static', 'ccache'], - 'zlib' : ['zlib-devel', 'zlib-static'] + 'fedora_native_static': { + 'COMMON': ['gcc-c++', 'cmake', 'automake', 'glibc-static', 'libstdc++-static', 'ccache'], + 'zlib': ['zlib-devel', 'zlib-static'], + 'lzma': ['xz-devel', 'xz-static'] # Either there is no packages, or no static or too old }, - 'fedora_win32_dyn' : { - 'COMMON' : ['mingw32-gcc-c++', 'mingw32-bzip2', 'mingw32-win-iconv', 'mingw32-winpthreads', 'wine', 'ccache'], - 'zlib' : ['mingw32-zlib'], - 'libmicrohttpd' : ['mingw32-libmicrohttpd'], + 'fedora_win32_dyn': { + 'COMMON': ['mingw32-gcc-c++', 'mingw32-bzip2', 'mingw32-win-iconv', 'mingw32-winpthreads', 'wine', 'ccache'], + 'zlib': ['mingw32-zlib'], + 'lzma': ['mingw32-xz-libs'], + 'libmicrohttpd': ['mingw32-libmicrohttpd'], }, - 'fedora_win32_static' : { - 'COMMON' : ['mingw32-gcc-c++', 'mingw32-bzip2-static', 'mingw32-win-iconv-static', 'mingw32-winpthreads-static', 'wine', 'ccache'], - 'zlib' : ['mingw32-zlib-static'], - 'libmicrohttpd' : None, # ['mingw32-libmicrohttpd-static'] packaging dependecy seems buggy, and some static lib are name libfoo.dll.a and - # gcc cannot found them. + 'fedora_win32_static': { + 'COMMON': ['mingw32-gcc-c++', 'mingw32-bzip2-static', 'mingw32-win-iconv-static', 'mingw32-winpthreads-static', 'wine', 'ccache'], + 'zlib': ['mingw32-zlib-static'], + 'lzma': ['mingw32-xz-libs-static'], + 'libmicrohttpd': None, # ['mingw32-libmicrohttpd-static'] packaging dependecy seems buggy, and some static lib are name libfoo.dll.a and + # gcc cannot found them. }, - 'debian_native_dyn' : { - 'COMMON' : ['gcc', 'cmake', 'libbz2-dev', 'ccache'], - 'zlib' : ['zlib1g-dev'], - 'uuid' : ['uuid-dev'], + 'fedora_android': { + 'COMMON': ['gcc-c++', 'cmake', 'automake', 'ccache', 'java-1.8.0-openjdk-devel'] + }, + 'debian_native_dyn': { + 'COMMON': ['gcc', 'cmake', 'libbz2-dev', 'ccache'], + 'zlib': ['zlib1g-dev'], + 'uuid': ['uuid-dev'], 'ctpp2': ['libctpp2-dev'], - 'libmicrohttpd' : ['libmicrohttpd-dev', 'ccache'] + 'libmicrohttpd': ['libmicrohttpd-dev', 'ccache'] }, - 'debian_native_static' : { - 'COMMON' : ['gcc', 'cmake', 'libbz2-dev', 'ccache'], - 'zlib' : ['zlib1g-dev'], - 'uuid' : ['uuid-dev'], + 'debian_native_static': { + 'COMMON': ['gcc', 'cmake', 'libbz2-dev', 'ccache'], + 'zlib': ['zlib1g-dev'], + 'uuid': ['uuid-dev'], 'ctpp2': ['libctpp2-dev'], }, - 'debian_win32_dyn' : { - 'COMMON' : ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools', 'ccache'] + 'debian_win32_dyn': { + 'COMMON': ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools', 'ccache'] }, - 'debian_win32_static' : { - 'COMMON' : ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools', 'ccache'] + 'debian_win32_static': { + 'COMMON': ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools', 'ccache'] + }, + 'debian_android': { + 'COMMON': ['gcc', 'cmake', 'ccache'] }, } -class Defaultdict(defaultdict): - def __getattr__(self, name): - return self[name] + +def which(name): + command = "which {}".format(name) + output = subprocess.check_output(command, shell=True) + return output[:-1].decode() -class Which(): - def __getattr__(self, name): - command = "which {}".format(name) - output = subprocess.check_output(command, shell=True) - return output[:-1].decode() +class TargetInfo: + def __init__(self, build, static): + self.build = build + self.static = static - def __format__(self, format_spec): - return getattr(self, format_spec) - -def remove_duplicates(iterable, key_function=None): - seen = set() - if key_function is None: - key_function = lambda e:e - for elem in iterable: - key = key_function(elem) - if key in seen: - continue - seen.add(key) - yield elem - -def get_sha256(path): - sha256 = hashlib.sha256() - with open(path, 'br') as f: - sha256.update(f.read()) - return sha256.hexdigest() - -class SkipCommand(Exception): - pass - -class StopBuild(Exception): - pass - -class Remotefile(namedtuple('Remotefile', ('name', 'sha256', 'url'))): - def __new__(cls, name, sha256, url=None): - return super().__new__(cls, name, sha256, url) - -class Context: - def __init__(self, command_name, log_file): - self.command_name = command_name - self.log_file = log_file - self.autoskip_file = None - - def try_skip(self, path): - self.autoskip_file = pj(path, ".{}_ok".format(self.command_name)) - if os.path.exists(self.autoskip_file): - raise SkipCommand() - - def _finalise(self): - if self.autoskip_file is not None: - with open(self.autoskip_file, 'w') as f: pass + def __str__(self): + return "{}_{}".format(self.build, 'static' if self.static else 'dyn') -def extract_archive(archive_path, dest_dir, topdir=None, name=None): - with tarfile.open(archive_path) as archive: - members = archive.getmembers() - if not topdir: - for d in (m for m in members if m.isdir()): - if not os.path.dirname(d.name): - if topdir: - # There is already a top dir. - # Two topdirs in the same archive. - # Extract all - topdir = None - break - topdir = d - else: - topdir = archive.getmember(topdir) - if topdir: - members_to_extract = [m for m in members if m.name.startswith(topdir.name+'/')] - os.makedirs(dest_dir, exist_ok=True) - with tempfile.TemporaryDirectory(prefix=os.path.basename(archive_path), dir=dest_dir) as tmpdir: - archive.extractall(path=tmpdir, members=members_to_extract) - name = name or topdir.name - os.rename(pj(tmpdir, topdir.name), pj(dest_dir, name)) - else: - if name: - dest_dir = pj(dest_dir, name) - os.makedirs(dest_dir) - archive.extractall(path=dest_dir) +class AndroidTargetInfo(TargetInfo): + __arch_infos = { + 'arm' : ('arm-linux-androideabi'), + 'arm64': ('aarch64-linux-android'), + 'mips': ('mipsel-linux-android'), + 'mips64': ('mips64el-linux-android'), + 'x86': ('i686-linux-android'), + 'x86_64': ('x86_64-linux-android'), + } + + def __init__(self, arch): + super().__init__('android', True) + self.arch = arch + self.arch_full = self.__arch_infos[arch] + + def __str__(self): + return "android" class BuildEnv: - build_targets = ['native', 'win32'] + target_platforms = { + 'native_dyn': TargetInfo('native', False), + 'native_static': TargetInfo('native', True), + 'win32_dyn': TargetInfo('win32', False), + 'win32_static': TargetInfo('win32', True), + 'android_arm': AndroidTargetInfo('arm'), + 'android_arm64': AndroidTargetInfo('arm64'), + 'android_mips': AndroidTargetInfo('mips'), + 'android_mips64': AndroidTargetInfo('mips64'), + 'android_x86': AndroidTargetInfo('x86'), + 'android_x86_64': AndroidTargetInfo('x86_64'), + } def __init__(self, options, targetsDict): self.source_dir = pj(options.working_dir, "SOURCE") - build_dir = "BUILD_{target}_{libmod}".format( - target=options.build_target, - libmod='static' if options.build_static else 'dyn' - ) + build_dir = "BUILD_{}".format(options.target_platform) self.build_dir = pj(options.working_dir, build_dir) self.archive_dir = pj(options.working_dir, "ARCHIVE") - self.log_dir = pj(options.working_dir, 'LOGS') + self.log_dir = pj(self.build_dir, 'LOGS') self.install_dir = pj(self.build_dir, "INSTALL") - os.makedirs(self.source_dir, exist_ok=True) - os.makedirs(self.archive_dir, exist_ok=True) - os.makedirs(self.build_dir, exist_ok=True) - os.makedirs(self.log_dir, exist_ok=True) - os.makedirs(self.install_dir, exist_ok=True) + for d in (self.source_dir, + self.build_dir, + self.archive_dir, + self.log_dir, + self.install_dir): + os.makedirs(d, exist_ok=True) self.detect_platform() self.ninja_command = self._detect_ninja() if not self.ninja_command: @@ -234,6 +198,8 @@ class BuildEnv: self.meson_command = self._detect_meson() if not self.meson_command: sys.exit("ERROR: meson command not fount") + self.setup_build(options.target_platform) + self.setup_toolchains() self.options = options self.libprefix = options.libprefix or self._detect_libdir() self.targetsDict = targetsDict @@ -256,14 +222,32 @@ class BuildEnv: if self.distname == 'ubuntu': self.distname = 'debian' + def setup_build(self, target_platform): + self.platform_info = platform_info = self.target_platforms[target_platform] + if platform_info.build == 'native': + self.cross_env = {} + else: + cross_name = "{host}_{target}".format( + host = self.distname, + target = platform_info.build) + try: + self.cross_env = CROSS_ENV[cross_name] + except KeyError: + sys.exit("ERROR : We don't know how to set env to compile" + " a {target} version on a {host} host.".format( + target = platform_info.build, + host = self.distname + )) + + def setup_toolchains(self): + toolchain_names = self.cross_env.get('toolchain_names', []) + self.toolchains =[Toolchain.all_toolchains[toolchain_name](self) + for toolchain_name in toolchain_names] + def finalize_setup(self): - getattr(self, 'setup_{}'.format(self.build_target))() + getattr(self, 'setup_{}'.format(self.platform_info.build))() def setup_native(self): - self.cross_env = {} - self.wrapper = None - self.configure_option = "" - self.cmake_option = "" self.cmake_crossfile = None self.meson_crossfile = None @@ -273,7 +257,7 @@ class BuildEnv: with open(template_file, 'r') as f: template = f.read() content = template.format( - which=Which(), + toolchain=self.toolchains[0], **self.cross_env ) with open(crossfile, 'w') as outfile: @@ -281,26 +265,13 @@ class BuildEnv: return crossfile def setup_win32(self): - cross_name = "{host}_{target}".format( - host = self.distname, - target = self.build_target) - try: - self.cross_env = CROSS_ENV[cross_name] - except KeyError: - sys.exit("ERROR : We don't know how to set env to compile" - " a {target} version on a {host} host.".format( - target = self.build_target, - host = self.distname - )) - - self.wrapper = self._gen_crossfile('bash_wrapper.sh') - current_permissions = stat.S_IMODE(os.lstat(self.wrapper).st_mode) - os.chmod(self.wrapper, current_permissions | stat.S_IXUSR) - self.configure_option = "--host=i686-w64-mingw32" - self.cmake_option = "" self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') + def setup_android(self): + self.cmake_crossfile = self._gen_crossfile('cmake_android_cross_file.txt') + self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') + def __getattr__(self, name): return getattr(self.options, name) @@ -345,50 +316,73 @@ class BuildEnv: if retcode == 0: return n - def _set_env(self, env): + @property + def configure_option(self): + configure_options = [tlc.configure_option for tlc in self.toolchains] + return " ".join(configure_options) + + @property + def cmake_option(self): + cmake_options = [tlc.cmake_option for tlc in self.toolchains] + return " ".join(cmake_options) + + def _set_env(self, env, cross_compile_env, cross_compile_path): if env is None: env = Defaultdict(str, os.environ) - for k, v in self.cross_env.get('env', {}).items(): - if k.startswith('_format_'): - v = v.format(**self.cross_env) - k = k[8:] - env[k] = v + + bin_dirs = [] + if cross_compile_env: + for k, v in self.cross_env.get('env', {}).items(): + if k.startswith('_format_'): + v = v.format(**self.cross_env) + k = k[8:] + env[k] = v + for toolchain in self.toolchains: + toolchain.set_env(env) + if cross_compile_path: + for tlc in self.toolchains: + bin_dirs += tlc.get_bin_dir() + pkgconfig_path = pj(self.install_dir, self.libprefix, 'pkgconfig') - env['PKG_CONFIG_PATH'] = (env['PKG_CONFIG_PATH'] + ':' + pkgconfig_path - if env['PKG_CONFIG_PATH'] - else pkgconfig_path - ) + env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path]) + # Add ccache path for p in ('/usr/lib/ccache', '/usr/lib64/ccache'): if os.path.isdir(p): - ccache_path=[p] + ccache_path = [p] break else: ccache_path = [] - env['PATH'] = ':'.join([pj(self.install_dir, 'bin')] + ccache_path + [env['PATH']]) - ld_library_path = ':'.join([ - pj(self.install_dir, 'lib'), - pj(self.install_dir, 'lib64') - ]) - env['LD_LIBRARY_PATH'] = (env['LD_LIBRARY_PATH'] + ':' + ld_library_path - if env['LD_LIBRARY_PATH'] - else ld_library_path - ) - env['CPPFLAGS'] = '-I'+pj(self.install_dir, 'include') - env['LDFLAGS'] = '-L'+pj(self.install_dir, 'lib') + env['PATH'] = ':'.join(bin_dirs + + [pj(self.install_dir, 'bin')] + + ccache_path + + [env['PATH']]) + + env['LD_LIBRARY_PATH'] = ':'.join([env['LD_LIBRARY_PATH'], + pj(self.install_dir, 'lib'), + pj(self.install_dir, 'lib64') + ]) + + env['CPPFLAGS'] = " ".join(['-I'+pj(self.install_dir, 'include'), env['CPPFLAGS']]) + env['LDFLAGS'] = " ".join(['-L'+pj(self.install_dir, 'lib'), env['LDFLAGS']]) return env - def run_command(self, command, cwd, context, env=None, input=None, allow_wrapper=True): + def run_command(self, command, cwd, context, env=None, input=None, cross_path_only=False): os.makedirs(cwd, exist_ok=True) - if allow_wrapper and self.wrapper: - command = "{} {}".format(self.wrapper, command) - env = self._set_env(env) + cross_compile_env = True + cross_compile_path = True + if context.force_native_build: + cross_compile_env = False + cross_compile_path = False + if cross_path_only: + cross_compile_env = False + env = self._set_env(env, cross_compile_env, cross_compile_path) log = None try: if not self.options.verbose: log = open(context.log_file, 'w') print("run command '{}'".format(command), file=log) - print("env is :",file=log) + print("env is :", file=log) for k, v in env.items(): print(" {} : {!r}".format(k, v), file=log) @@ -432,22 +426,21 @@ class BuildEnv: def install_packages(self): autoskip_file = pj(self.build_dir, ".install_packages_ok") if self.distname in ('fedora', 'redhat', 'centos'): - package_installer = 'dnf' + package_installer = 'sudo dnf install {}' + package_checker = 'rpm -q --quiet {}' elif self.distname in ('debian', 'Ubuntu'): - package_installer = 'apt-get' - mapper_name = "{host}_{target}_{build_type}".format( - host = self.distname, - target = self.build_target, - build_type = 'static' if self.options.build_static else 'dyn') + package_installer = 'sudo apt-get install {}' + package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1' + mapper_name = "{host}_{target}".format( + host=self.distname, + target=self.platform_info) try: package_name_mapper = PACKAGE_NAME_MAPPERS[mapper_name] except KeyError: print("SKIP : We don't know which packages we must install to compile" " a {target} {build_type} version on a {host} host.".format( - target = self.build_target, - build_type = 'static' if self.options.build_static else 'dyn', - host = self.distname - )) + target=self.platform_info, + host=self.distname)) return packages_list = package_name_mapper.get('COMMON', []) @@ -459,45 +452,54 @@ class BuildEnv: if os.path.exists(autoskip_file): print("SKIP") return - if packages_list: - command = "sudo {package_installer} install {packages_list}".format( - package_installer = package_installer, - packages_list = " ".join(packages_list) - ) + + packages_to_install = [] + for package in packages_list: + print(" - {} : ".format(package), end="") + command = package_checker.format(package) + try: + subprocess.check_call(command, shell=True) + except subprocess.CalledProcessError: + print("NEEDED") + packages_to_install.append(package) + else: + print("SKIP") + + if packages_to_install: + command = package_installer.format(" ".join(packages_to_install)) print(command) subprocess.check_call(command, shell=True) else: print("SKIP, No package to install.") - with open(autoskip_file, 'w') as f: pass + with open(autoskip_file, 'w'): + pass -################################################################################ -##### PROJECT -################################################################################ -class _MetaDependency(type): +class _MetaToolchain(type): def __new__(cls, name, bases, dct): _class = type.__new__(cls, name, bases, dct) - if name != 'Dependency': - Dependency.all_deps[name] = _class + if name != 'Toolchain': + Toolchain.all_toolchains[name] = _class return _class -class Dependency(metaclass=_MetaDependency): - all_deps = {} - dependencies = [] - force_native_build = False - version = None +class Toolchain(metaclass=_MetaToolchain): + all_toolchains = {} + configure_option = "" + cmake_option = "" + Builder = None + Source = None + def __init__(self, buildEnv): self.buildEnv = buildEnv - self.source = self.Source(self) - self.builder = self.Builder(self) - self.skip = False + self.source = self.Source(self) if self.Source else None + self.builder = self.Builder(self) if self.Builder else None @property def full_name(self): - if self.version: - return "{}-{}".format(self.name, self.version) - return self.name + return "{name}-{version}".format( + name = self.name, + version = self.version) @property def source_path(self): @@ -507,10 +509,13 @@ class Dependency(metaclass=_MetaDependency): def _log_dir(self): return self.buildEnv.log_dir + def set_env(self, env): + pass + def command(self, name, function, *args): print(" {} {} : ".format(name, self.name), end="", flush=True) log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name)) - context = Context(name, log) + context = Context(name, log, True) try: ret = function(*args, context=context) context._finalise() @@ -531,474 +536,188 @@ class Dependency(metaclass=_MetaDependency): raise -class Source: - """Base Class to the real preparator - A source preparator must install source in the self.source_dir attribute - inside the buildEnv.source_dir.""" - def __init__(self, target): - self.target = target - self.buildEnv = target.buildEnv +class mingw32_toolchain(Toolchain): + name = 'mingw32' + arch_full = 'i686-w64-mingw32' @property - def name(self): - return self.target.name + def root_path(self): + return self.buildEnv.cross_env['root_path'] @property - def source_dir(self): - return self.target.full_name - - def command(self, *args, **kwargs): - return self.target.command(*args, **kwargs) - - -class ReleaseDownload(Source): - archive_top_dir = None - @property - def extract_path(self): - return pj(self.buildEnv.source_dir, self.source_dir) - - def _download(self, context): - self.buildEnv.download(self.archive) - - def _extract(self, context): - context.try_skip(self.extract_path) - if os.path.exists(self.extract_path): - shutil.rmtree(self.extract_path) - extract_archive(pj(self.buildEnv.archive_dir, self.archive.name), - self.buildEnv.source_dir, - topdir=self.archive_top_dir, - name=self.source_dir) - - def _patch(self, context): - context.try_skip(self.extract_path) - for p in self.patches: - with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input: - self.buildEnv.run_command("patch -p1", self.extract_path, context, input=patch_input, allow_wrapper=False) - - def prepare(self): - self.command('download', self._download) - self.command('extract', self._extract) - if hasattr(self, 'patches'): - self.command('patch', self._patch) - - -class GitClone(Source): - git_ref = "master" - @property - def source_dir(self): - return self.git_dir + def binaries(self): + return {k:which('{}-{}'.format(self.arch_full, v)) + for k, v in (('CC', 'gcc'), + ('CXX', 'g++'), + ('AR', 'ar'), + ('STRIP', 'strip'), + ('WINDRES', 'windres'), + ('RANLIB', 'ranlib')) + } @property - def git_path(self): - return pj(self.buildEnv.source_dir, self.git_dir) + def configure_option(self): + return '--host={}'.format(self.arch_full) - def _git_clone(self, context): - if os.path.exists(self.git_path): - raise SkipCommand() - command = "git clone " + self.git_remote - self.buildEnv.run_command(command, self.buildEnv.source_dir, context) + def get_bin_dir(self): + return [pj(self.root_path, 'bin')] - def _git_update(self, context): - self.buildEnv.run_command("git fetch", self.git_path, context) - self.buildEnv.run_command("git checkout "+self.git_ref, self.git_path, context) + def set_env(self, env): + for k, v in self.binaries.items(): + env[k] = v - def prepare(self): - self.command('gitclone', self._git_clone) - self.command('gitupdate', self._git_update) + env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') + env['CFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS'] + env['CXXFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS'] + env['LIBS'] = " ".join(self.buildEnv.cross_env['extra_libs']) + " " +env['LIBS'] + + +class android_ndk(Toolchain): + name = 'android-ndk' + version = 'r13b' + gccver = '4.9.x' + + @property + def api(self): + return '21' if self.arch in ('arm64', 'mips64', 'x86_64') else '14' + + @property + def platform(self): + return 'android-'+self.api + + @property + def arch(self): + return self.buildEnv.platform_info.arch + + @property + def arch_full(self): + return self.buildEnv.platform_info.arch_full + + @property + def toolchain(self): + return self.arch_full+"-4.9" + + @property + def root_path(self): + return pj(self.builder.install_path, 'sysroot') + + @property + def binaries(self): + binaries = ((k,'{}-{}'.format(self.arch_full, v)) + for k, v in (('CC', 'gcc'), + ('CXX', 'g++'), + ('AR', 'ar'), + ('STRIP', 'strip'), + ('WINDRES', 'windres'), + ('RANLIB', 'ranlib'), + ('LD', 'ld')) + ) + return {k:pj(self.builder.install_path, 'bin', v) + for k,v in binaries} + + @property + def configure_option(self): + return '--host={}'.format(self.arch_full) + + @property + def full_name(self): + return "{name}-{version}-{arch}-{api}".format( + name = self.name, + version = self.version, + arch = self.arch, + api = self.api) + + class Source(ReleaseDownload): + archive = Remotefile('android-ndk-r13b-linux-x86_64.zip', + '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c', + 'https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip') + + @property + def source_dir(self): + return "{}-{}".format( + self.target.name, + self.target.version) + + + class Builder(Builder): + + @property + def install_path(self): + return self.build_path + + def _build_platform(self, context): + context.try_skip(self.build_path) + script = pj(self.source_path, 'build/tools/make_standalone_toolchain.py') + current_permissions = stat.S_IMODE(os.lstat(script).st_mode) + os.chmod(script, current_permissions | stat.S_IXUSR) + command = '{script} --arch={arch} --api={api} --install-dir={install_dir} --force' + command = command.format( + script=script, + arch=self.target.arch, + api=self.target.api, + install_dir=self.install_path + ) + self.buildEnv.run_command(command, self.build_path, context) + + def _fix_permission_right(self, context): + context.try_skip(self.build_path) + bin_dirs = [pj(self.install_path, 'bin'), + pj(self.install_path, self.target.arch_full, 'bin'), + pj(self.install_path, 'libexec', 'gcc', self.target.arch_full, self.target.gccver) + ] + for root, dirs, files in os.walk(self.install_path): + if not root in bin_dirs: + continue + + for file_ in files: + file_path = pj(root, file_) + if os.path.islink(file_path): + continue + current_permissions = stat.S_IMODE(os.lstat(file_path).st_mode) + os.chmod(file_path, current_permissions | stat.S_IXUSR) + + def build(self): + self.command('build_platform', self._build_platform) + self.command('fix_permission_right', self._fix_permission_right) + + def get_bin_dir(self): + return [pj(self.builder.install_path, 'bin')] + + def set_env(self, env): + env['CC'] = self.binaries['CC'] + env['CXX'] = self.binaries['CXX'] + + env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') + env['CFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(self.root_path) + env['CFLAGS'] + env['CXXFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(self.root_path) + env['CXXFLAGS'] + env['LDFLAGS'] = ' -fPIE -pie --sysroot={} '.format(self.root_path) + env['LDFLAGS'] + #env['CFLAGS'] = ' -fPIC -D_FILE_OFFSET_BITS=64 -O3 '+env['CFLAGS'] + #env['CXXFLAGS'] = (' -D__OPTIMIZE__ -fno-strict-aliasing ' + # ' -DU_HAVE_NL_LANGINFO_CODESET=0 ' + # '-DU_STATIC_IMPLEMENTATION -O3 ' + # '-DU_HAVE_STD_STRING -DU_TIMEZONE=0 ')+env['CXXFLAGS'] + env['NDK_DEBUG'] = '0' class Builder: - subsource_dir = None - def __init__(self, target): - self.target = target - self.buildEnv = target.buildEnv - - @property - def name(self): - return self.target.name - - @property - def source_path(self): - base_source_path = self.target.source_path - if self.subsource_dir: - return pj(base_source_path, self.subsource_dir) - return base_source_path - - @property - def build_path(self): - return pj(self.buildEnv.build_dir, self.target.full_name) - - def command(self, *args, **kwargs): - return self.target.command(*args, **kwargs) - - def build(self): - self.command('configure', self._configure) - self.command('compile', self._compile) - self.command('install', self._install) - - -class MakeBuilder(Builder): - configure_option = static_configure_option = dynamic_configure_option = "" - make_option = "" - install_option = "" - configure_script = "configure" - configure_env = None - make_target = "" - make_install_target = "install" - - def _configure(self, context): - context.try_skip(self.build_path) - configure_option = "{} {} {}".format( - self.configure_option, - self.static_configure_option if self.buildEnv.build_static else self.dynamic_configure_option, - self.buildEnv.configure_option) - command = "{configure_script} {configure_option} --prefix {install_dir} --libdir {libdir}" - command = command.format( - configure_script = pj(self.source_path, self.configure_script), - configure_option = configure_option, - install_dir = self.buildEnv.install_dir, - libdir = pj(self.buildEnv.install_dir, self.buildEnv.libprefix) - ) - env = Defaultdict(str, os.environ) - if self.buildEnv.build_static: - env['CFLAGS'] = env['CFLAGS'] + ' -fPIC' - if self.configure_env: - for k in self.configure_env: - if k.startswith('_format_'): - v = self.configure_env.pop(k) - v = v.format(buildEnv=self.buildEnv, env=env) - self.configure_env[k[8:]] = v - env.update(self.configure_env) - self.buildEnv.run_command(command, self.build_path, context, env=env) - - def _compile(self, context): - context.try_skip(self.build_path) - command = "make -j4 {make_target} {make_option}".format( - make_target = self.make_target, - make_option = self.make_option - ) - self.buildEnv.run_command(command, self.build_path, context) - - def _install(self, context): - context.try_skip(self.build_path) - command = "make {make_install_target} {make_option}".format( - make_install_target = self.make_install_target, - make_option = self.make_option - ) - self.buildEnv.run_command(command, self.build_path, context) - - -class CMakeBuilder(MakeBuilder): - def _configure(self, context): - context.try_skip(self.build_path) - command = ("cmake {configure_option}" - " -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON" - " -DCMAKE_INSTALL_PREFIX={install_dir}" - " -DCMAKE_INSTALL_LIBDIR={libdir}" - " {source_path}" - " {cross_option}") - command = command.format( - configure_option = "{} {}".format(self.buildEnv.cmake_option, self.configure_option), - install_dir = self.buildEnv.install_dir, - libdir = self.buildEnv.libprefix, - source_path = self.source_path, - cross_option = "-DCMAKE_TOOLCHAIN_FILE={}".format(self.buildEnv.cmake_crossfile) if self.buildEnv.cmake_crossfile else "" - ) - env = Defaultdict(str, os.environ) - if self.buildEnv.build_static: - env['CFLAGS'] = env['CFLAGS'] + ' -fPIC' - if self.configure_env: - for k in self.configure_env: - if k.startswith('_format_'): - v = self.configure_env.pop(k) - v = v.format(buildEnv=self.buildEnv, env=env) - self.configure_env[k[8:]] = v - env.update(self.configure_env) - self.buildEnv.run_command(command, self.build_path, context, env=env, allow_wrapper=False) - - -class MesonBuilder(Builder): - configure_option = "" - - def _configure(self, context): - context.try_skip(self.build_path) - if os.path.exists(self.build_path): - shutil.rmtree(self.build_path) - os.makedirs(self.build_path) - if self.buildEnv.build_static: - library_type = 'static' - else: - library_type = 'shared' - configure_option = self.configure_option.format(buildEnv=self.buildEnv) - command = ("{command} . {build_path}" - " --default-library={library_type}" - " {configure_option}" - " --prefix={buildEnv.install_dir}" - " --libdir={buildEnv.libprefix}" - " {cross_option}") - command = command.format( - command = self.buildEnv.meson_command, - library_type=library_type, - configure_option=configure_option, - build_path = self.build_path, - buildEnv=self.buildEnv, - cross_option = "--cross-file {}".format(self.buildEnv.meson_crossfile) if self.buildEnv.meson_crossfile else "" - ) - self.buildEnv.run_command(command, self.source_path, context, allow_wrapper=False) - - def _compile(self, context): - command = "{} -v".format(self.buildEnv.ninja_command) - self.buildEnv.run_command(command, self.build_path, context, allow_wrapper=False) - - def _install(self, context): - command = "{} -v install".format(self.buildEnv.ninja_command) - self.buildEnv.run_command(command, self.build_path, context, allow_wrapper=False) - - -# ************************************* -# Missing dependencies -# Is this ok to assume that those libs -# exist in your "distri" (linux/mac) ? -# If not, we need to compile them here -# ************************************* -# Zlib -# LZMA -# aria2 -# Argtable -# MSVirtual -# Android -# libiconv -# gettext -# ************************************* - -class zlib(Dependency): - name = 'zlib' - version = '1.2.8' - - class Source(ReleaseDownload): - archive = Remotefile('zlib-1.2.8.tar.gz', - '36658cb768a54c1d4dec43c3116c27ed893e88b02ecfcb44f2166f9c0b7f2a0d') - patches = ['zlib_std_libname.patch'] - - class Builder(CMakeBuilder): - @property - def configure_option(self): - return "-DINSTALL_PKGCONFIG_DIR={}".format(pj(self.buildEnv.install_dir, self.buildEnv.libprefix, 'pkgconfig')) - -class UUID(Dependency): - name = 'uuid' - version = "1.43.4" - - class Source(ReleaseDownload): - archive = Remotefile('e2fsprogs-1.43.4.tar.gz', - '1644db4fc58300c363ba1ab688cf9ca1e46157323aee1029f8255889be4bc856', - 'https://www.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v1.43.4/e2fsprogs-1.43.4.tar.gz') - extract_dir = 'e2fsprogs-1.43.4' - - class Builder(MakeBuilder): - configure_option = "--enable-libuuid" - configure_env = {'_format_CFLAGS' : "{env.CFLAGS} -fPIC"} - make_target = 'libs' - make_install_target = 'install-libs' - - -class Xapian(Dependency): - name = "xapian-core" - version = "1.4.2" - - class Source(ReleaseDownload): - archive = Remotefile('xapian-core-1.4.2.tar.xz', - 'aec2c4352998127a2f2316218bf70f48cef0a466a87af3939f5f547c5246e1ce') - patches = ["xapian_pkgconfig.patch"] - - class Builder(MakeBuilder): - configure_option = "--disable-sse --disable-backend-inmemory --disable-documentation" - dynamic_configure_option = "--enable-shared --disable-static" - static_configure_option = "--enable-static --disable-shared" - configure_env = {'_format_LDFLAGS' : "-L{buildEnv.install_dir}/{buildEnv.libprefix}", - '_format_CXXFLAGS' : "-I{buildEnv.install_dir}/include"} - - @property - def dependencies(self): - deps = ['zlib'] - if self.buildEnv.build_target == 'win32': - return deps - return deps + ['UUID'] - - -class CTPP2(Dependency): - name = "ctpp2" - version = "2.8.3" - - class Source(ReleaseDownload): - archive = Remotefile('ctpp2-2.8.3.tar.gz', - 'a83ffd07817adb575295ef40fbf759892512e5a63059c520f9062d9ab8fb42fc') - patches = ["ctpp2_include.patch", - "ctpp2_no_src_modification.patch", - "ctpp2_fix-static-libname.patch", - "ctpp2_mingw32.patch", - "ctpp2_dll_export_VMExecutable.patch", - "ctpp2_win_install_lib_in_lib_dir.patch", - "ctpp2_iconv_support.patch"] - - class Builder(CMakeBuilder): - configure_option = "-DMD5_SUPPORT=OFF" - - -class Pugixml(Dependency): - name = "pugixml" - version = "1.2" - - class Source(ReleaseDownload): - archive = Remotefile('pugixml-1.2.tar.gz', - '0f422dad86da0a2e56a37fb2a88376aae6e931f22cc8b956978460c9db06136b') - patches = ["pugixml_meson.patch"] - - Builder = MesonBuilder - - -class MicroHttpd(Dependency): - name = "libmicrohttpd" - version = "0.9.46" - - class Source(ReleaseDownload): - archive = Remotefile('libmicrohttpd-0.9.46.tar.gz', - '06dbd2654f390fa1e8196fe063fc1449a6c2ed65a38199a49bf29ad8a93b8979', - 'http://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.46.tar.gz') - - class Builder(MakeBuilder): - configure_option = "--disable-https --without-libgcrypt --without-libcurl" - dynamic_configure_option = "--enable-shared --disable-static" - static_configure_option = "--enable-static --disable-shared" - - -class Icu(Dependency): - name = "icu4c" - version = "56_1" - - class Source(ReleaseDownload): - archive = Remotefile('icu4c-56_1-src.tgz', - '3a64e9105c734dcf631c0b3ed60404531bce6c0f5a64bfe1a6402a4cc2314816' - ) - patches = ["icu4c_fix_static_lib_name_mingw.patch"] - data = Remotefile('icudt56l.dat', - 'e23d85eee008f335fc49e8ef37b1bc2b222db105476111e3d16f0007d371cbca') - - def _download_data(self, context): - self.buildEnv.download(self.data) - - def _copy_data(self, context): - context.try_skip(self.extract_path) - shutil.copyfile(pj(self.buildEnv.archive_dir, self.data.name), pj(self.extract_path, 'source', 'data', 'in', self.data.name)) - - def prepare(self): - super().prepare() - self.command("download_data", self._download_data) - self.command("copy_data", self._copy_data) - - class Builder(MakeBuilder): - subsource_dir = "source" - configure_option = "--disable-samples --disable-tests --disable-extras --disable-dyload" - dynamic_configure_option = "--enable-shared --disable-static" - static_configure_option = "--enable-static --disable-shared" - - -class Icu_native(Icu): - force_native_build = True - - class Builder(Icu.Builder): - @property - def build_path(self): - return super().build_path+"_native" - - def _install(self, context): - raise SkipCommand() - - -class Icu_cross_compile(Icu): - dependencies = ['Icu_native'] - - class Builder(Icu.Builder): - @property - def configure_option(self): - Icu_native = self.buildEnv.targetsDict['Icu_native'] - return super().configure_option + " --with-cross-build=" + Icu_native.builder.build_path - - -class Zimlib(Dependency): - name = "zimlib" - - class Source(GitClone): - #git_remote = "https://gerrit.wikimedia.org/r/p/openzim.git" - git_remote = "https://github.com/mgautierfr/openzim" - git_dir = "openzim" - git_ref = "meson" - - class Builder(MesonBuilder): - subsource_dir = "zimlib" - - -class Kiwixlib(Dependency): - name = "kiwix-lib" - dependencies = ['zlib'] - - @property - def dependencies(self): - if self.buildEnv.build_target == 'win32': - return ["Xapian", "CTPP2", "Pugixml", "Icu_cross_compile", "Zimlib"] - return ["Xapian", "CTPP2", "Pugixml", "Icu", "Zimlib"] - - class Source(GitClone): - git_remote = "https://github.com/kiwix/kiwix-lib.git" - git_dir = "kiwix-lib" - - class Builder(MesonBuilder): - configure_option = "-Dctpp2-install-prefix={buildEnv.install_dir}" - - -class KiwixTools(Dependency): - name = "kiwix-tools" - dependencies = ["Kiwixlib", "MicroHttpd", "zlib"] - - class Source(GitClone): - git_remote = "https://github.com/kiwix/kiwix-tools.git" - git_dir = "kiwix-tools" - - class Builder(MesonBuilder): - @property - def configure_option(self): - base_options = "-Dctpp2-install-prefix={buildEnv.install_dir}" - if self.buildEnv.build_static: - base_options += " -Dstatic-linkage=true" - return base_options - - -class Builder: - def __init__(self, options, targetDef='KiwixTools'): + def __init__(self, options): self.targets = OrderedDict() self.buildEnv = buildEnv = BuildEnv(options, self.targets) - if buildEnv.build_target != 'native': - self.nativeBuildEnv = BuildEnv(options, self.targets) - self.nativeBuildEnv.build_target = 'native' - else: - self.nativeBuildEnv = buildEnv _targets = {} + targetDef = options.targets self.add_targets(targetDef, _targets) dependencies = self.order_dependencies(_targets, targetDef) dependencies = list(remove_duplicates(dependencies)) for dep in dependencies: - self.targets[dep] = _targets[dep] + self.targets[dep] = _targets[dep] def add_targets(self, targetName, targets): if targetName in targets: return targetClass = Dependency.all_deps[targetName] - if targetClass.force_native_build: - target = targetClass(self.nativeBuildEnv) - else: - target = targetClass(self.buildEnv) + target = targetClass(self.buildEnv) targets[targetName] = target for dep in target.dependencies: self.add_targets(dep, targets) @@ -1010,14 +729,24 @@ class Builder: yield targetName def prepare_sources(self): + toolchain_sources = (tlc.source for tlc in self.buildEnv.toolchains if tlc.source) + for toolchain_source in toolchain_sources: + print("prepare sources for toolchain {} :".format(toolchain_source.name)) + toolchain_source.prepare() + sources = (dep.source for dep in self.targets.values() if not dep.skip) - sources = remove_duplicates(sources, lambda s:s.__class__) + sources = remove_duplicates(sources, lambda s: s.__class__) for source in sources: print("prepare sources {} :".format(source.name)) source.prepare() def build(self): - builders = (dep.builder for dep in self.targets.values() if not dep.skip) + toolchain_builders = (tlc.builder for tlc in self.buildEnv.toolchains if tlc.builder) + for toolchain_builder in toolchain_builders: + print("build toolchain {} :".format(toolchain_builder.name)) + toolchain_builder.build() + + builders = (dep.builder for dep in self.targets.values() if (dep.builder and not dep.skip)) for builder in builders: print("build {} :".format(builder.name)) builder.build() @@ -1027,8 +756,6 @@ class Builder: print("[INSTALL PACKAGES]") self.buildEnv.install_packages() self.buildEnv.finalize_setup() - if self.nativeBuildEnv != self.buildEnv: - self.nativeBuildEnv.finalize_setup() print("[PREPARE]") self.prepare_sources() print("[BUILD]") @@ -1036,12 +763,13 @@ class Builder: except StopBuild: sys.exit("Stopping build due to errors") + def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument('working_dir', default=".", nargs='?') + parser.add_argument('targets', default='KiwixTools', nargs='?') + parser.add_argument('--working-dir', default=".") parser.add_argument('--libprefix', default=None) - parser.add_argument('--build-static', action="store_true") - parser.add_argument('--build-target', default="native", choices=BuildEnv.build_targets) + parser.add_argument('--target-platform', default="native_dyn", choices=BuildEnv.target_platforms) parser.add_argument('--verbose', '-v', action="store_true", help=("Print all logs on stdout instead of in specific" " log files per commands")) @@ -1049,6 +777,7 @@ def parse_args(): help="Skip SSL certificate verification during download") return parser.parse_args() + if __name__ == "__main__": options = parse_args() options.working_dir = os.path.abspath(options.working_dir) diff --git a/patches/icu4c_android_elf64_st_info.patch b/patches/icu4c_android_elf64_st_info.patch new file mode 100644 index 0000000..12e901f --- /dev/null +++ b/patches/icu4c_android_elf64_st_info.patch @@ -0,0 +1,13 @@ +diff -ur icu4c-58_2/source/tools/toolutil/pkg_genc.c icu4c-58_2.patched/source/tools/toolutil/pkg_genc.c +--- icu4c-58_2/source/tools/toolutil/pkg_genc.c 2016-06-15 20:58:17.000000000 +0200 ++++ icu4c-58_2.patched/source/tools/toolutil/pkg_genc.c 2017-02-27 10:23:39.985471339 +0100 +@@ -35,6 +35,9 @@ + # define EM_X86_64 62 + # endif + # define ICU_ENTRY_OFFSET 0 ++# ifndef ELF64_ST_INFO ++# define ELF64_ST_INFO(b,t) (((b) << 4) + ((t) & 0xf)) ++# endif + #endif + + #include diff --git a/patches/xapian_pkgconfig.patch b/patches/xapian_pkgconfig.patch index b2d4dc9..1e7c2dc 100644 --- a/patches/xapian_pkgconfig.patch +++ b/patches/xapian_pkgconfig.patch @@ -1,33 +1,44 @@ -diff -ur xapian-core-1.4.0/configure.ac xapian-core-1.4.0.patched/configure.ac +diff -ur xapian-core-1.4.0/configure.ac xapian-core-1.4.0-patched/configure.ac --- xapian-core-1.4.0/configure.ac 2016-06-25 17:36:49.000000000 +0200 -+++ xapian-core-1.4.0.patched/configure.ac 2017-01-17 14:33:42.268536542 +0100 -@@ -393,6 +393,7 @@ ++++ xapian-core-1.4.0-patched/configure.ac 2017-02-22 17:45:57.066365636 +0100 +@@ -393,22 +393,25 @@ esac dnl We use timer_create() if available to implement a search time limit. +use_rt_lib=0 SAVE_LIBS=$LIBS - AC_SEARCH_LIBS([timer_create], [rt], - [ -@@ -403,12 +404,14 @@ +-AC_SEARCH_LIBS([timer_create], [rt], +- [ ++AC_SEARCH_LIBS([timer_create], [rt], [ + AC_MSG_CHECKING([for timer_create() usability]) +- AC_COMPILE_IFELSE([AC_LANG_PROGRAM( ++ AC_COMPILE_IFELSE( ++ [AC_LANG_PROGRAM( + [[#if defined _AIX || defined __GNU__ + #error timer_create known not to work #endif]])], - [AC_MSG_RESULT([yes]) - XAPIAN_LIBS="$LIBS $XAPIAN_LIBS" +- [AC_MSG_RESULT([yes]) +- XAPIAN_LIBS="$LIBS $XAPIAN_LIBS" - AC_DEFINE([HAVE_TIMER_CREATE], [1], [Define to 1 if you have the 'timer_create' function.])] -+ AC_DEFINE([HAVE_TIMER_CREATE], [1], [Define to 1 if you have the 'timer_create' function.]) -+ use_rt_lib=1] - , - [AC_MSG_RESULT([no]) - ]) +- , +- [AC_MSG_RESULT([no]) +- ]) ++ [AC_MSG_RESULT([yes]) ++ XAPIAN_LIBS="$LIBS $XAPIAN_LIBS" ++ AC_DEFINE([HAVE_TIMER_CREATE], [1], [Define to 1 if you have the 'timer_create' function.]) ++ AS_IF([test "$ac_res" != "none required"], use_rt_lib=1) ++ ], ++ [AC_MSG_RESULT([no])] ++ ) ]) LIBS=$SAVE_LIBS -+AM_CONDITIONAL([USE_RT_LIB], [test "$use_win32_uuid_api" = 1]) ++AM_CONDITIONAL([USE_RT_LIB], [test "$use_rt_lib" = 1]) dnl Used by tests/soaktest/soaktest.cc AC_CHECK_FUNCS([srandom random]) -diff -ur xapian-core-1.4.0/configure xapian-core-1.4.0.patched/configure +diff -ur xapian-core-1.4.0/configure xapian-core-1.4.0-patched/configure --- xapian-core-1.4.0/configure 2016-06-25 17:39:25.000000000 +0200 -+++ xapian-core-1.4.0.patched/configure 2017-01-17 15:43:07.929290801 +0100 ++++ xapian-core-1.4.0-patched/configure 2017-02-22 17:45:44.472585524 +0100 @@ -671,6 +671,8 @@ DOCUMENTATION_RULES_FALSE DOCUMENTATION_RULES_TRUE @@ -45,15 +56,22 @@ diff -ur xapian-core-1.4.0/configure xapian-core-1.4.0.patched/configure SAVE_LIBS=$LIBS { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing timer_create" >&5 $as_echo_n "checking for library containing timer_create... " >&6; } -@@ -18324,6 +18327,7 @@ +@@ -18320,10 +18323,13 @@ + if ac_fn_cxx_try_compile "$LINENO"; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 + $as_echo "yes" >&6; } +- XAPIAN_LIBS="$LIBS $XAPIAN_LIBS" ++ XAPIAN_LIBS="$LIBS $XAPIAN_LIBS" $as_echo "#define HAVE_TIMER_CREATE 1" >>confdefs.h -+ use_rt_lib=1 ++ if test "$ac_res" != "none required"; then : ++ use_rt_lib=1 ++fi else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -@@ -18335,6 +18339,14 @@ +@@ -18335,6 +18341,14 @@ fi LIBS=$SAVE_LIBS @@ -68,7 +86,7 @@ diff -ur xapian-core-1.4.0/configure xapian-core-1.4.0.patched/configure for ac_func in srandom random do : -@@ -20854,6 +20866,10 @@ +@@ -20854,6 +20868,10 @@ Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi @@ -79,9 +97,9 @@ diff -ur xapian-core-1.4.0/configure xapian-core-1.4.0.patched/configure if test -z "${DOCUMENTATION_RULES_TRUE}" && test -z "${DOCUMENTATION_RULES_FALSE}"; then as_fn_error $? "conditional \"DOCUMENTATION_RULES\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 -diff -ur xapian-core-1.4.0/pkgconfig/xapian-core.pc.in xapian-core-1.4.0.patched/pkgconfig/xapian-core.pc.in ---- xapian-core-1.4.0/pkgconfig/xapian-core.pc.in 2017-01-17 15:22:40.184786108 +0100 -+++ xapian-core-1.4.0.patched/pkgconfig/xapian-core.pc.in 2017-01-17 14:34:17.692972977 +0100 +diff -ur xapian-core-1.4.0/pkgconfig/xapian-core.pc.in xapian-core-1.4.0-patched/pkgconfig/xapian-core.pc.in +--- xapian-core-1.4.0/pkgconfig/xapian-core.pc.in 2016-06-25 17:36:49.000000000 +0200 ++++ xapian-core-1.4.0-patched/pkgconfig/xapian-core.pc.in 2017-02-22 17:09:12.488793901 +0100 @@ -11,4 +11,6 @@ URL: https://xapian.org/ Version: @VERSION@ @@ -90,4 +108,3 @@ diff -ur xapian-core-1.4.0/pkgconfig/xapian-core.pc.in xapian-core-1.4.0.patched +@USE_RT_LIB_TRUE@Libs: @ldflags@ -L${libdir} -lxapian@LIBRARY_VERSION_SUFFIX@ -lrt +@USE_RT_LIB_FALSE@Libs: @ldflags@ -L${libdir} -lxapian@LIBRARY_VERSION_SUFFIX@ +@USE_WIN32_UUID_API_FALSE@Requires: uuid - diff --git a/templates/cmake_android_cross_file.txt b/templates/cmake_android_cross_file.txt new file mode 100644 index 0000000..7f80a34 --- /dev/null +++ b/templates/cmake_android_cross_file.txt @@ -0,0 +1,7 @@ +SET(CMAKE_SYSTEM_NAME {host_machine[system]}) +SET(CMAKE_ANDROID_STANDALONE_TOOLCHAIN {toolchain.builder.install_path}) + +SET(CMAKE_C_COMPILER "{toolchain.binaries[CC]}") +SET(CMAKE_CXX_COMPILER "{toolchain.binaries[CXX]}") + +SET(CMAKE_FIND_ROOT_PATH {toolchain.root_path}) diff --git a/templates/cmake_cross_file.txt b/templates/cmake_cross_file.txt index 8410a07..75a98a0 100644 --- a/templates/cmake_cross_file.txt +++ b/templates/cmake_cross_file.txt @@ -2,11 +2,11 @@ SET(CMAKE_SYSTEM_NAME Windows) SET(CMAKE_SYSTEM_PROCESSOR x86) # specify the cross compiler -SET(CMAKE_C_COMPILER "{which:{binaries[c]}}") -SET(CMAKE_CXX_COMPILER "{which:{binaries[cpp]}}") -SET(CMAKE_RC_COMPILER {which:{binaries[windres]}}) -SET(CMAKE_AR:FILEPATH {which:{binaries[ar]}}) -SET(CMAKE_RANLIB:FILEPATH {which:{binaries[ranlib]}}) +SET(CMAKE_C_COMPILER "{toolchain.binaries[CC]}") +SET(CMAKE_CXX_COMPILER "{toolchain.binaries[CXX]}") +SET(CMAKE_RC_COMPILER {toolchain.binaries[WINDRES]}) +SET(CMAKE_AR:FILEPATH {toolchain.binaries[AR]}) +SET(CMAKE_RANLIB:FILEPATH {toolchain.binaries[RANLIB]}) find_program(CCACHE_FOUND ccache) if(CCACHE_FOUND) @@ -15,7 +15,7 @@ if(CCACHE_FOUND) endif(CCACHE_FOUND) # where is the target environment -SET(CMAKE_FIND_ROOT_PATH {root_path}) +SET(CMAKE_FIND_ROOT_PATH {toolchain.root_path}) # search for programs in the build host directories SET(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) diff --git a/templates/meson_cross_file.txt b/templates/meson_cross_file.txt index edea054..6edddf4 100644 --- a/templates/meson_cross_file.txt +++ b/templates/meson_cross_file.txt @@ -1,16 +1,16 @@ [binaries] pkgconfig = 'pkg-config' -c = '{which:{binaries[c]}}' -ar = '{which:{binaries[ar]}}' -cpp = '{which:{binaries[cpp]}}' -strip = '{which:{binaries[strip]}}' +c = '{toolchain.binaries[CC]}' +ar = '{toolchain.binaries[AR]}' +cpp = '{toolchain.binaries[CXX]}' +strip = '{toolchain.binaries[STRIP]}' [properties] -c_link_args = ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4'] -cpp_link_args = ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4'] +c_link_args = {extra_libs!r} +cpp_link_args = {extra_libs!r} [host_machine] -cpu_family = 'x86' -cpu = 'i586' -system = 'windows' -endian = 'little' +cpu_family = '{host_machine[cpu_family]}' +cpu = '{host_machine[cpu]}' +system = '{host_machine[system]}' +endian = '{host_machine[endian]}' diff --git a/travis/compile_all.sh b/travis/compile_all.sh index 7358c69..7cc09b4 100755 --- a/travis/compile_all.sh +++ b/travis/compile_all.sh @@ -2,9 +2,4 @@ set -e -OPTION="" -if [ "${STATIC_BUILD}" = "true" ]; then - OPTION="--build-static" -fi - -./kiwix-build.py --build-target=${BUILD_TARGET} ${OPTION} +./kiwix-build.py $BUILD_OPTION diff --git a/travis/deploy.sh b/travis/deploy.sh index 997637f..c3dfa61 100755 --- a/travis/deploy.sh +++ b/travis/deploy.sh @@ -6,14 +6,8 @@ SSH_KEY=travis/travisci_builder_id_key chmod 600 ${SSH_KEY} - -BASE_DIR="BUILD_${BUILD_TARGET}_static/INSTALL" -if [ "${BUILD_TARGET}" = "win32" ]; then - ARCHIVE_OPTION="--zip" -else - ARCHIVE_OPTION="--tar" -fi -./kiwix-deploy.py ${BASE_DIR} ${ARCHIVE_OPTION} \ +BASE_DIR="BUILD_*/INSTALL" +./kiwix-deploy.py ${BASE_DIR} ${ARCHIVE_TYPE} \ --deploy \ --ssh_private_key=${SSH_KEY} \ --server=nightlybot@download.kiwix.org \ diff --git a/travis/install_extra_deps.sh b/travis/install_extra_deps.sh index 6f2540d..b888ae6 100755 --- a/travis/install_extra_deps.sh +++ b/travis/install_extra_deps.sh @@ -5,7 +5,7 @@ set -e orig_dir=$(pwd) sudo apt-get update -qq -sudo apt-get install -qq uuid-dev libicu-dev libctpp2-dev automake libtool python3-pip zlib1g-dev lzma-dev libbz2-dev cmake +sudo apt-get install -qq python3-pip pip3 install meson # ninja diff --git a/utils.py b/utils.py new file mode 100644 index 0000000..e3ef32c --- /dev/null +++ b/utils.py @@ -0,0 +1,117 @@ +import os.path +import hashlib +import tarfile, zipfile +import tempfile +import os +from collections import namedtuple, defaultdict + +pj = os.path.join + + +class Defaultdict(defaultdict): + def __getattr__(self, name): + return self[name] + + +def remove_duplicates(iterable, key_function=None): + seen = set() + if key_function is None: + key_function = lambda e: e + for elem in iterable: + key = key_function(elem) + if key in seen: + continue + seen.add(key) + yield elem + + +def get_sha256(path): + sha256 = hashlib.sha256() + with open(path, 'br') as f: + sha256.update(f.read()) + return sha256.hexdigest() + + +class SkipCommand(Exception): + pass + + +class StopBuild(Exception): + pass + + +class Remotefile(namedtuple('Remotefile', ('name', 'sha256', 'url'))): + def __new__(cls, name, sha256, url=None): + return super().__new__(cls, name, sha256, url) + + +class Context: + def __init__(self, command_name, log_file, force_native_build): + self.command_name = command_name + self.log_file = log_file + self.force_native_build = force_native_build + self.autoskip_file = None + + def try_skip(self, path): + self.autoskip_file = pj(path, ".{}_ok".format(self.command_name)) + if os.path.exists(self.autoskip_file): + raise SkipCommand() + + def _finalise(self): + if self.autoskip_file is not None: + with open(self.autoskip_file, 'w'): + pass + + +def extract_archive(archive_path, dest_dir, topdir=None, name=None): + is_zip_archive = archive_path.endswith('.zip') + try: + if is_zip_archive: + archive = zipfile.ZipFile(archive_path) + members = archive.infolist() + getname = lambda info : info.filename + isdir = lambda info: info.filename.endswith('/') + else: + archive = tarfile.open(archive_path) + members = archive.getmembers() + getname = lambda info : getattr(info, 'name') + isdir = lambda info: info.isdir() + if not topdir: + for d in (m for m in members if isdir(m)): + _name = getname(d) + if _name.endswith('/'): + _name = _name[:-1] + if not os.path.dirname(_name): + if topdir: + # There is already a top dir. + # Two topdirs in the same archive. + # Extract all + topdir = None + break + topdir = _name + if topdir: + members_to_extract = [m for m in members if getname(m).startswith(topdir+'/')] + os.makedirs(dest_dir, exist_ok=True) + with tempfile.TemporaryDirectory(prefix=os.path.basename(archive_path), dir=dest_dir) as tmpdir: + if is_zip_archive: + _members_to_extract = [getname(m) for m in members_to_extract] + else: + _members_to_extract = members_to_extract + archive.extractall(path=tmpdir, members=_members_to_extract) + if is_zip_archive: + for member in members_to_extract: + if isdir(member): + continue + perm = (member.external_attr >> 16) & 0x1FF + os.chmod(pj(tmpdir, getname(member)), perm) + name = name or topdir + os.rename(pj(tmpdir, topdir), pj(dest_dir, name)) + else: + if name: + dest_dir = pj(dest_dir, name) + os.makedirs(dest_dir) + archive.extractall(path=dest_dir) + finally: + if archive is not None: + archive.close() +