From 7cd02ce146bad292223473a4340788d73dce0e40 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 17 Jan 2017 11:10:58 +0100 Subject: [PATCH 01/14] Add a first handling of build_target. - No more target_arch but a more generic build-target. - The buildEnv can now have custom options depending of the target. Mixins will use those options. --- kiwix-build.py | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/kiwix-build.py b/kiwix-build.py index d45487f..72fba97 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -108,9 +108,15 @@ def extract_archive(archive_path, dest_dir, topdir=None, name=None): class BuildEnv: + build_targets = ['native'] + + _targets_env = { + 'native' : {} + } + def __init__(self, options): self.source_dir = pj(options.working_dir, "SOURCE") - self.build_dir = pj(options.working_dir, "BUILD") + self.build_dir = pj(options.working_dir, "BUILD_{target}".format(target=options.build_target)) self.archive_dir = pj(options.working_dir, "ARCHIVE") self.log_dir = pj(options.working_dir, 'LOGS') self.install_dir = pj(self.build_dir, "INSTALL") @@ -119,11 +125,22 @@ class BuildEnv: os.makedirs(self.build_dir, exist_ok=True) os.makedirs(self.log_dir, exist_ok=True) os.makedirs(self.install_dir, exist_ok=True) + self.setup_build_target(options.build_target) self.ninja_command = self._detect_ninja() self.meson_command = self._detect_meson() self.options = options self.libprefix = options.libprefix or self._detect_libdir() + def setup_build_target(self, build_target): + self.build_target = build_target + self.target_env = self._targets_env[self.build_target] + getattr(self, 'setup_{}'.format(self.build_target))() + + def setup_native(self): + self.wrapper = None + self.configure_option = "" + self.cmake_option = "" + self.meson_crossfile = None def __getattr__(self, name): return getattr(self.options, name) @@ -312,7 +329,7 @@ class MakeMixin: command = "{configure_script} {configure_option} --prefix {install_dir} --libdir {libdir}" command = command.format( configure_script = pj(self.source_path, self.configure_script), - configure_option = self.configure_option, + configure_option = "{} {}".format(self.configure_option, self.buildEnv.configure_option), install_dir = self.buildEnv.install_dir, libdir = pj(self.buildEnv.install_dir, self.buildEnv.libprefix) ) @@ -358,7 +375,7 @@ class CMakeMixin(MakeMixin): context.try_skip(self.build_path) command = "cmake {configure_option} -DCMAKE_INSTALL_PREFIX={install_dir} -DCMAKE_INSTALL_LIBDIR={libdir} {source_path}" command = command.format( - configure_option = self.configure_option, + configure_option = "{} {}".format(self.buildEnv.cmake_option, self.configure_option), install_dir = self.buildEnv.install_dir, libdir = self.buildEnv.libprefix, source_path = self.source_path @@ -399,12 +416,14 @@ class MesonMixin(MakeMixin): else: library_type = 'shared' configure_option = self.configure_option.format(buildEnv=self.buildEnv) - command = "{command} --default-library={library_type} {configure_option} . {build_path} --prefix={buildEnv.install_dir} --libdir={buildEnv.libprefix}".format( + command = "{command} --default-library={library_type} {configure_option} . {build_path} --prefix={buildEnv.install_dir} --libdir={buildEnv.libprefix} {cross_option}".format( command = self.buildEnv.meson_command, library_type=library_type, configure_option=configure_option, build_path = self.build_path, - buildEnv=self.buildEnv) + buildEnv=self.buildEnv, + cross_option = "--cross-file {}".format(self.buildEnv.meson_crossfile) if self.buildEnv.meson_crossfile else "" + ) self.buildEnv.run_command(command, self.source_path, context, env=env) @command("compile") @@ -567,8 +586,8 @@ def parse_args(): parser = argparse.ArgumentParser() parser.add_argument('working_dir', default=".", nargs='?') parser.add_argument('--libprefix', default=None) - parser.add_argument('--target_arch', default="x86_64") parser.add_argument('--build_static', action="store_true") + parser.add_argument('--build-target', default="native", choices=BuildEnv.build_targets) parser.add_argument('--verbose', '-v', action="store_true", help=("Print all logs on stdout instead of in specific" " log files per commands")) From e5bb2c65c2250a21cfa441efd8299ebc4983d4ec Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 17 Jan 2017 11:26:52 +0100 Subject: [PATCH 02/14] We now can run subprocess with a wrapper from cross-compiling. --- kiwix-build.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/kiwix-build.py b/kiwix-build.py index 72fba97..2c4e3e9 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -185,8 +185,10 @@ class BuildEnv: if retcode == 0: return n - def run_command(self, command, cwd, context, env=None, input=None): + def run_command(self, command, cwd, context, env=None, input=None, allow_wrapper=True): os.makedirs(cwd, exist_ok=True) + if allow_wrapper and self.wrapper: + command = "{} {}".format(self.wrapper, command) if env is None: env = dict(os.environ) log = None @@ -278,7 +280,7 @@ class ReleaseDownloadMixin: context.try_skip(self.extract_path) for p in self.patches: with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input: - self.buildEnv.run_command("patch -p1", self.extract_path, context, input=patch_input) + self.buildEnv.run_command("patch -p1", self.extract_path, context, input=patch_input, allow_wrapper=False) def prepare(self): self._download() @@ -424,19 +426,19 @@ class MesonMixin(MakeMixin): buildEnv=self.buildEnv, cross_option = "--cross-file {}".format(self.buildEnv.meson_crossfile) if self.buildEnv.meson_crossfile else "" ) - self.buildEnv.run_command(command, self.source_path, context, env=env) + self.buildEnv.run_command(command, self.source_path, context, env=env, allow_wrapper=False) @command("compile") def _compile(self, context): env = self._gen_env() command = "{} -v".format(self.buildEnv.ninja_command) - self.buildEnv.run_command(command, self.build_path, context, env=env) + self.buildEnv.run_command(command, self.build_path, context, env=env, allow_wrapper=False) @command("install") def _install(self, context): env = self._gen_env() command = "{} -v install".format(self.buildEnv.ninja_command) - self.buildEnv.run_command(command, self.build_path, context) + self.buildEnv.run_command(command, self.build_path, context, allow_wrapper=False) def build(self): self._configure() From 9670d690e4c6e6b04f1cb052cdcd63448a1c9264 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 17 Jan 2017 11:29:14 +0100 Subject: [PATCH 03/14] Add a setup of buildEnv to cross-compile from fedora to win32. --- kiwix-build.py | 55 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/kiwix-build.py b/kiwix-build.py index 2c4e3e9..c296435 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import os, sys +import os, sys, stat import argparse import urllib.request import tarfile @@ -8,8 +8,10 @@ import subprocess import hashlib import shutil import tempfile +import configparser from collections import defaultdict, namedtuple + pj = os.path.join REMOTE_PREFIX = 'http://download.kiwix.org/dev/' @@ -108,10 +110,11 @@ def extract_archive(archive_path, dest_dir, topdir=None, name=None): class BuildEnv: - build_targets = ['native'] + build_targets = ['native', 'win32'] _targets_env = { - 'native' : {} + 'native' : {}, + 'win32' : {'wrapper': 'mingw32-env'} } def __init__(self, options): @@ -141,6 +144,52 @@ class BuildEnv: self.configure_option = "" self.cmake_option = "" self.meson_crossfile = None + + def _get_rpm_mingw32(self, value): + command = "rpm --eval %{{mingw32_{}}}".format(value) + output = subprocess.check_output(command, shell=True) + return output[:-1].decode() + + def _gen_meson_crossfile(self): + self.meson_crossfile = pj(self.build_dir, 'cross_file.txt') + config = configparser.ConfigParser() + config['binaries'] = { + 'c' : repr(self._get_rpm_mingw32('cc')), + 'cpp' : repr(self._get_rpm_mingw32('cxx')), + 'ar' : repr(self._get_rpm_mingw32('ar')), + 'strip' : repr(self._get_rpm_mingw32('strip')), + 'pkgconfig' : repr(self._get_rpm_mingw32('pkg_config')), + 'exe_wrapper' : repr('wine') # A command used to run generated executables. + } + config['properties'] = { + 'c_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4'], + 'cpp_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4'] + } + config['host_machine'] = { + 'system' : repr('windows'), + 'cpu_family' : repr('x86'), + 'cpu' : repr('i586'), + 'endian' : repr('little') + } + with open(self.meson_crossfile, 'w') as configfile: + config.write(configfile) + + def setup_win32(self): + command = "rpm --eval %{mingw32_env}" + self.wrapper = pj(self.build_dir, 'mingw32-wrapper.sh') + with open(self.wrapper, 'w') as output: + output.write("#!/usr/bin/sh\n\n") + output.flush() + output.write(self._get_rpm_mingw32('env')) + output.write('\n\nexec "$@"\n') + output.flush() + current_permissions = stat.S_IMODE(os.lstat(self.wrapper).st_mode) + os.chmod(self.wrapper, current_permissions | stat.S_IXUSR) + self.configure_option = "--host=i686-w64-mingw32" + self.cmake_option = "-DCMAKE_SYSTEM_NAME=Windows" + + self._gen_meson_crossfile() + def __getattr__(self, name): return getattr(self.options, name) From 4089928ed6b2a793c3adb37d728f535f171f6949 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 17 Jan 2017 11:30:52 +0100 Subject: [PATCH 04/14] There is no UUID dependency when we are compiling to win32. --- kiwix-build.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/kiwix-build.py b/kiwix-build.py index c296435..47d2885 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -612,7 +612,19 @@ class KiwixTools(Dependency, GitCloneMixin, MesonMixin): class Builder: def __init__(self, buildEnv): self.buildEnv = buildEnv - self.dependencies = [UUID(buildEnv), + if buildEnv.build_target != 'native': + self.dependencies = [ + Xapian(buildEnv), + CTPP2(buildEnv), + Pugixml(buildEnv), + Zimlib(buildEnv), + MicroHttpd(buildEnv), + Icu(buildENv), + Kiwixlib(buildEnv), + KiwixTools(buildEnv) + ] + else: + self.dependencies = [UUID(buildEnv), Xapian(buildEnv), CTPP2(buildEnv), Pugixml(buildEnv), From a8c4bff709891f05d26b13ced8c9f944c4f36dc3 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 17 Jan 2017 11:35:26 +0100 Subject: [PATCH 05/14] Correctly build Icu when cross-compiling. When cross-compiling, icu need to be build two times : - Once in 'native' mode. - Once cross-compiling. This is needed as the cross-compiling need to launch executable that are build in the first step. Icu dependency become a bit more complex as the second compilation needs to reference the first compilation and that configure options and buildEnv change. --- kiwix-build.py | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/kiwix-build.py b/kiwix-build.py index 47d2885..384654a 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -76,6 +76,7 @@ def command(name): except: print("ERROR") raise + wrapper._wrapped = function return wrapper return decorator @@ -567,10 +568,27 @@ class Icu(Dependency, ReleaseDownloadMixin, MakeMixin): ) data = Remotefile('icudt56l.dat', 'e23d85eee008f335fc49e8ef37b1bc2b222db105476111e3d16f0007d371cbca') - configure_option = "Linux --disable-samples --disable-tests --disable-extras --enable-static --disable-dyload" - configure_script = "runConfigureICU" + configure_option = "--disable-samples --disable-tests --disable-extras --enable-static --disable-dyload" subsource_dir = "source" + def __init__(self, buildEnv, cross_compile_process=False, cross_build=None): + Dependency.__init__(self, buildEnv) + self.cross_compile_process = cross_compile_process + self.cross_build = cross_build + + @property + def build_path(self): + if self.cross_compile_process and not self.cross_build: + return pj(self.buildEnv.build_dir, self.source_dir+"_native") + return pj(self.buildEnv.build_dir, self.source_dir) + + @property + def configure_option(self): + default_configure_option = "--disable-samples --disable-tests --disable-extras --enable-static --disable-dyload" + if self.cross_build: + return default_configure_option + " --with-cross-build=" + self.cross_build.build_path + return default_configure_option + @command("download_data") def _download_data(self, context): self.buildEnv.download(self.data) @@ -580,6 +598,12 @@ class Icu(Dependency, ReleaseDownloadMixin, MakeMixin): context.try_skip(self.source_path) shutil.copyfile(pj(self.buildEnv.archive_dir, self.data.name), pj(self.source_path, 'data', 'in', self.data.name)) + @command("install") + def _install(self, context): + if self.cross_compile_process and not self.cross_build: + raise SkipCommand() + return super()._install._wrapped(self, context) + def prepare(self): super().prepare() self._download_data() @@ -613,13 +637,17 @@ class Builder: def __init__(self, buildEnv): self.buildEnv = buildEnv if buildEnv.build_target != 'native': + subBuildEnv = BuildEnv(buildEnv.options) + subBuildEnv.setup_build_target('native') + nativeICU = Icu(subBuildEnv, True) self.dependencies = [ Xapian(buildEnv), CTPP2(buildEnv), Pugixml(buildEnv), Zimlib(buildEnv), MicroHttpd(buildEnv), - Icu(buildENv), + nativeICU, + Icu(buildEnv, True, nativeICU), Kiwixlib(buildEnv), KiwixTools(buildEnv) ] From ffe1c2ae5dc0fb1236639782fe3fabde57d3d580 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 17 Jan 2017 11:39:36 +0100 Subject: [PATCH 06/14] Use the last version of microhttpd (0.9.46) Else, older version of microhttpd needs plibc and it seems far more complex to cross-compile it (I'm not sure it is still maintained...) --- kiwix-build.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/kiwix-build.py b/kiwix-build.py index 384654a..d871ecc 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -554,10 +554,11 @@ class Pugixml(Dependency, ReleaseDownloadMixin, MesonMixin): class MicroHttpd(Dependency, ReleaseDownloadMixin, MakeMixin): name = "libmicrohttpd" - version = "0.9.19" - archive = Remotefile('libmicrohttpd-0.9.19.tar.gz', - 'dc418c7a595196f09d2f573212a0d794404fa4ac5311fc9588c1e7ad7a90fae6') configure_option = "--enable-shared --enable-static --disable-https --without-libgcrypt --without-libcurl" + version = "0.9.46" + archive = Remotefile('libmicrohttpd-0.9.46.tar.gz', + '06dbd2654f390fa1e8196fe063fc1449a6c2ed65a38199a49bf29ad8a93b8979', + 'http://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.46.tar.gz') class Icu(Dependency, ReleaseDownloadMixin, MakeMixin): From 013c1d36cfab68c1a894c3d81b2f07253647b000 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 17 Jan 2017 11:42:01 +0100 Subject: [PATCH 07/14] CMake now generate verbose makefile. This is better for debugging. --- kiwix-build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kiwix-build.py b/kiwix-build.py index d871ecc..59af2f6 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -425,7 +425,7 @@ class CMakeMixin(MakeMixin): @command("configure") def _configure(self, context): context.try_skip(self.build_path) - command = "cmake {configure_option} -DCMAKE_INSTALL_PREFIX={install_dir} -DCMAKE_INSTALL_LIBDIR={libdir} {source_path}" + command = "cmake {configure_option} -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -DCMAKE_INSTALL_PREFIX={install_dir} -DCMAKE_INSTALL_LIBDIR={libdir} {source_path}" command = command.format( configure_option = "{} {}".format(self.buildEnv.cmake_option, self.configure_option), install_dir = self.buildEnv.install_dir, From 597fd8dbe153e2ff31580b9f54b8ee4ccbf49388 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 17 Jan 2017 11:42:27 +0100 Subject: [PATCH 08/14] Rename option build_static to build-static. --- kiwix-build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kiwix-build.py b/kiwix-build.py index 59af2f6..320a182 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -678,7 +678,7 @@ def parse_args(): parser = argparse.ArgumentParser() parser.add_argument('working_dir', default=".", nargs='?') parser.add_argument('--libprefix', default=None) - parser.add_argument('--build_static', action="store_true") + parser.add_argument('--build-static', action="store_true") parser.add_argument('--build-target', default="native", choices=BuildEnv.build_targets) parser.add_argument('--verbose', '-v', action="store_true", help=("Print all logs on stdout instead of in specific" From ca53171bdc2ef48f3d94d282395c67a685b39c39 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 17 Jan 2017 12:02:21 +0100 Subject: [PATCH 09/14] Add patches to cross-compile ctpp2 correctly. --- kiwix-build.py | 7 +- patches/ctpp2_dll_export_VMExecutable.patch | 12 +++ patches/ctpp2_mingw32.patch | 85 +++++++++++++++++++ .../ctpp2_win_install_lib_in_lib_dir.patch | 12 +++ 4 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 patches/ctpp2_dll_export_VMExecutable.patch create mode 100644 patches/ctpp2_mingw32.patch create mode 100644 patches/ctpp2_win_install_lib_in_lib_dir.patch diff --git a/kiwix-build.py b/kiwix-build.py index 320a182..b8fb0b6 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -541,7 +541,12 @@ class CTPP2(Dependency, ReleaseDownloadMixin, CMakeMixin): archive = Remotefile('ctpp2-2.8.3.tar.gz', 'a83ffd07817adb575295ef40fbf759892512e5a63059c520f9062d9ab8fb42fc') configure_option = "-DMD5_SUPPORT=OFF" - patches = ["ctpp2_include.patch", "ctpp2_no_src_modification.patch", "ctpp2_fix-static-libname.patch"] + patches = ["ctpp2_include.patch", + "ctpp2_no_src_modification.patch", + "ctpp2_fix-static-libname.patch", + "ctpp2_mingw32.patch", + "ctpp2_dll_export_VMExecutable.patch", + "ctpp2_win_install_lib_in_lib_dir.patch"] class Pugixml(Dependency, ReleaseDownloadMixin, MesonMixin): diff --git a/patches/ctpp2_dll_export_VMExecutable.patch b/patches/ctpp2_dll_export_VMExecutable.patch new file mode 100644 index 0000000..e3a941e --- /dev/null +++ b/patches/ctpp2_dll_export_VMExecutable.patch @@ -0,0 +1,12 @@ +diff -ur ctpp2-2.8.3/include/CTPP2VMExecutable.hpp ctpp2-2.8.3.patched/include/CTPP2VMExecutable.hpp +--- ctpp2-2.8.3/include/CTPP2VMExecutable.hpp 2012-08-02 09:22:44.000000000 +0200 ++++ ctpp2-2.8.3.patched/include/CTPP2VMExecutable.hpp 2017-01-17 11:47:08.332691919 +0100 +@@ -52,7 +52,7 @@ + @struct VMExecutable CTPP2VMExecutable.hpp + @brief CTPP Executable file + */ +-struct VMExecutable ++struct CTPP2DECL VMExecutable + { + /** CTPP magic number */ + UCHAR_8 magic[4]; // 'CTPP' xor 0xFFFFFFFF diff --git a/patches/ctpp2_mingw32.patch b/patches/ctpp2_mingw32.patch new file mode 100644 index 0000000..ca62dd5 --- /dev/null +++ b/patches/ctpp2_mingw32.patch @@ -0,0 +1,85 @@ +diff -ur ctpp2-2.8.3/include/CTPP2SourceLoader.hpp ctpp2-2.8.3.patched/include/CTPP2SourceLoader.hpp +--- ctpp2-2.8.3/include/CTPP2SourceLoader.hpp 2017-01-09 17:00:57.542353340 +0100 ++++ ctpp2-2.8.3.patched/include/CTPP2SourceLoader.hpp 2017-01-09 16:54:04.467316656 +0100 +@@ -33,7 +33,7 @@ + #define _CTPP2_SOURCE_LOADER_HPP__ 1 + + #ifdef _WIN32 +-#include ++#include + #else + #include + #endif +diff -ur ctpp2-2.8.3/src/CTPP2DTOA.cpp ctpp2-2.8.3.patched/src/CTPP2DTOA.cpp +--- ctpp2-2.8.3/src/CTPP2DTOA.cpp 2012-11-11 00:50:01.000000000 +0100 ++++ ctpp2-2.8.3.patched/src/CTPP2DTOA.cpp 2017-01-09 16:53:13.184181977 +0100 +@@ -158,8 +158,8 @@ + #include + #include + +-#ifdef _MSC_VER +- #include ++#if defined(_MSC_VER) || defined(__MINGW32__) ++ #include + #ifndef BIG_ENDIAN + #define BIG_ENDIAN BIGENDIAN + #endif +diff -ur ctpp2-2.8.3/src/CTPP2StringIconvOutputCollector.cpp ctpp2-2.8.3.patched/src/CTPP2StringIconvOutputCollector.cpp +--- ctpp2-2.8.3/src/CTPP2StringIconvOutputCollector.cpp 2012-08-02 09:22:44.000000000 +0200 ++++ ctpp2-2.8.3.patched/src/CTPP2StringIconvOutputCollector.cpp 2017-01-09 16:56:11.162179732 +0100 +@@ -85,7 +85,7 @@ + size_t iDstLength = CTPP_ESCAPE_BUFFER_LEN; + + char aDstData[CTPP_ESCAPE_BUFFER_LEN]; +-#if defined(linux) || defined(__APPLE__) ++#if defined(linux) || defined(__APPLE__) || defined(__MINGW32__) + char * aSrcData = (char *)vData; + #else + const char * aSrcData = (const char *)vData; +diff -ur ctpp2-2.8.3/src/functions/FnHostname.cpp ctpp2-2.8.3.patched/src/functions/FnHostname.cpp +--- ctpp2-2.8.3/src/functions/FnHostname.cpp 2012-11-10 21:40:36.000000000 +0100 ++++ ctpp2-2.8.3.patched/src/functions/FnHostname.cpp 2017-01-09 16:58:53.987435623 +0100 +@@ -34,8 +34,8 @@ + #include "CTPP2Logger.hpp" + #include "FnHostname.hpp" + +-#ifdef _MSC_VER +-#include ++#if defined(_MSC_VER) || defined(__MINGW32__) ++#include + #else + #include + #endif +diff -ur ctpp2-2.8.3/src/functions/FnIconv.cpp ctpp2-2.8.3.patched/src/functions/FnIconv.cpp +--- ctpp2-2.8.3/src/functions/FnIconv.cpp 2012-08-02 09:22:44.000000000 +0200 ++++ ctpp2-2.8.3.patched/src/functions/FnIconv.cpp 2017-01-09 16:58:28.948857601 +0100 +@@ -173,7 +173,7 @@ + size_t iDstLength = CTPP_ESCAPE_BUFFER_LEN; + + char aDstData[CTPP_ESCAPE_BUFFER_LEN]; +-#if defined(linux) || defined(__APPLE__) ++#if defined(linux) || defined(__APPLE__) || defined(__MINGW32__) + char * aSrcData = (char *)sWhat.data(); + #else + const char * aSrcData = (const char *)sWhat.data(); +diff -ur ctpp2-2.8.3/src/functions/FnRandom.cpp ctpp2-2.8.3.patched/src/functions/FnRandom.cpp +--- ctpp2-2.8.3/src/functions/FnRandom.cpp 2012-11-10 21:36:30.000000000 +0100 ++++ ctpp2-2.8.3.patched/src/functions/FnRandom.cpp 2017-01-09 16:59:59.879325141 +0100 +@@ -37,7 +37,7 @@ + #include + #include + +-#ifdef _MSC_VER ++#if defined(_MSC_VER) || defined(__MINGW32__) + #define random() rand() + #define INT_64(x) (INT_64)(x) + #define srandomdev() srand( (unsigned)time(NULL) ); +@@ -51,7 +51,7 @@ + // + FnRandom::FnRandom() + { +-#if defined(__FreeBSD__) || defined(_MSC_VER) ++#if defined(__FreeBSD__) || defined(_MSC_VER) || defined(__MINGW32__) + srandomdev(); + #else + srandom(time(NULL)); diff --git a/patches/ctpp2_win_install_lib_in_lib_dir.patch b/patches/ctpp2_win_install_lib_in_lib_dir.patch new file mode 100644 index 0000000..611eaf1 --- /dev/null +++ b/patches/ctpp2_win_install_lib_in_lib_dir.patch @@ -0,0 +1,12 @@ +diff -ur ctpp2-2.8.3/CMakeLists.txt ctpp2-2.8.3.patched/CMakeLists.txt +--- ctpp2-2.8.3/CMakeLists.txt 2017-01-17 10:09:06.259494234 +0100 ++++ ctpp2-2.8.3.patched/CMakeLists.txt 2017-01-17 11:48:18.427522886 +0100 +@@ -785,7 +785,7 @@ + + INSTALL(TARGETS ctpp2-static DESTINATION lib${LIB_SUFFIX}) + +- INSTALL(TARGETS ctpp2 DESTINATION .) ++ INSTALL(TARGETS ctpp2 DESTINATION lib${LIB_SUFFIX}) + + FILE(WRITE ctpp2-config.cmd ${CTPP_COMPILER_HELPER}) + From a9343f23b447843b2ad201288a0ec6f710a4a3ee Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 17 Jan 2017 15:30:58 +0100 Subject: [PATCH 10/14] Update xapian_pkgconfig patch. --- patches/xapian_pkgconfig.patch | 95 +++++++++++++++++++++++++++++++--- 1 file changed, 89 insertions(+), 6 deletions(-) diff --git a/patches/xapian_pkgconfig.patch b/patches/xapian_pkgconfig.patch index 6aacfd5..b2d4dc9 100644 --- a/patches/xapian_pkgconfig.patch +++ b/patches/xapian_pkgconfig.patch @@ -1,10 +1,93 @@ -diff -ru xapian-core-1.4.0/pkgconfig/xapian-core.pc.in xapian-core-1.4.0_patched/pkgconfig/xapian-core.pc.in ---- xapian-core-1.4.0/pkgconfig/xapian-core.pc.in 2016-06-25 17:36:49.000000000 +0200 -+++ xapian-core-1.4.0_patched/pkgconfig/xapian-core.pc.in 2016-12-19 20:45:25.716329832 +0100 -@@ -11,4 +11,5 @@ +diff -ur xapian-core-1.4.0/configure.ac xapian-core-1.4.0.patched/configure.ac +--- xapian-core-1.4.0/configure.ac 2016-06-25 17:36:49.000000000 +0200 ++++ xapian-core-1.4.0.patched/configure.ac 2017-01-17 14:33:42.268536542 +0100 +@@ -393,6 +393,7 @@ + esac + + dnl We use timer_create() if available to implement a search time limit. ++use_rt_lib=0 + SAVE_LIBS=$LIBS + AC_SEARCH_LIBS([timer_create], [rt], + [ +@@ -403,12 +404,14 @@ + #endif]])], + [AC_MSG_RESULT([yes]) + XAPIAN_LIBS="$LIBS $XAPIAN_LIBS" +- AC_DEFINE([HAVE_TIMER_CREATE], [1], [Define to 1 if you have the 'timer_create' function.])] ++ AC_DEFINE([HAVE_TIMER_CREATE], [1], [Define to 1 if you have the 'timer_create' function.]) ++ use_rt_lib=1] + , + [AC_MSG_RESULT([no]) + ]) + ]) + LIBS=$SAVE_LIBS ++AM_CONDITIONAL([USE_RT_LIB], [test "$use_win32_uuid_api" = 1]) + + dnl Used by tests/soaktest/soaktest.cc + AC_CHECK_FUNCS([srandom random]) +diff -ur xapian-core-1.4.0/configure xapian-core-1.4.0.patched/configure +--- xapian-core-1.4.0/configure 2016-06-25 17:39:25.000000000 +0200 ++++ xapian-core-1.4.0.patched/configure 2017-01-17 15:43:07.929290801 +0100 +@@ -671,6 +671,8 @@ + DOCUMENTATION_RULES_FALSE + DOCUMENTATION_RULES_TRUE + PERL ++USE_RT_LIB_FALSE ++USE_RT_LIB_TRUE + ldflags + XAPIAN_LIBS + XAPIAN_LDFLAGS +@@ -18247,6 +18249,7 @@ + ;; + esac + ++use_rt_lib=0 + SAVE_LIBS=$LIBS + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing timer_create" >&5 + $as_echo_n "checking for library containing timer_create... " >&6; } +@@ -18324,6 +18327,7 @@ + + $as_echo "#define HAVE_TIMER_CREATE 1" >>confdefs.h + ++ use_rt_lib=1 + + else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +@@ -18335,6 +18339,14 @@ + fi + + LIBS=$SAVE_LIBS ++ if test "$use_rt_lib" = 1; then ++ USE_RT_LIB_TRUE= ++ USE_RT_LIB_FALSE='#' ++else ++ USE_RT_LIB_TRUE='#' ++ USE_RT_LIB_FALSE= ++fi ++ + + for ac_func in srandom random + do : +@@ -20854,6 +20866,10 @@ + Usually this means the macro was only invoked conditionally." "$LINENO" 5 + fi + ++if test -z "${USE_RT_LIB_TRUE}" && test -z "${USE_RT_LIB_FALSE}"; then ++ as_fn_error $? "conditional \"USE_RT_LIB\" was never defined. ++Usually this means the macro was only invoked conditionally." "$LINENO" 5 ++fi + if test -z "${DOCUMENTATION_RULES_TRUE}" && test -z "${DOCUMENTATION_RULES_FALSE}"; then + as_fn_error $? "conditional \"DOCUMENTATION_RULES\" was never defined. + Usually this means the macro was only invoked conditionally." "$LINENO" 5 +diff -ur xapian-core-1.4.0/pkgconfig/xapian-core.pc.in xapian-core-1.4.0.patched/pkgconfig/xapian-core.pc.in +--- xapian-core-1.4.0/pkgconfig/xapian-core.pc.in 2017-01-17 15:22:40.184786108 +0100 ++++ xapian-core-1.4.0.patched/pkgconfig/xapian-core.pc.in 2017-01-17 14:34:17.692972977 +0100 +@@ -11,4 +11,6 @@ URL: https://xapian.org/ Version: @VERSION@ Cflags: -I${includedir} @abi_affecting_cxxflags@ -Libs: @ldflags@ -L${libdir} -lxapian@LIBRARY_VERSION_SUFFIX@ -+Libs: @ldflags@ -L${libdir} -lxapian@LIBRARY_VERSION_SUFFIX@ -lrt -+Requires: uuid ++@USE_RT_LIB_TRUE@Libs: @ldflags@ -L${libdir} -lxapian@LIBRARY_VERSION_SUFFIX@ -lrt ++@USE_RT_LIB_FALSE@Libs: @ldflags@ -L${libdir} -lxapian@LIBRARY_VERSION_SUFFIX@ ++@USE_WIN32_UUID_API_FALSE@Requires: uuid + From 912bd34637e603f9348f02e699fe2764f2ca0bfc Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 17 Jan 2017 15:32:15 +0100 Subject: [PATCH 11/14] Patch icu : Do not use a specific name for static libs. --- kiwix-build.py | 1 + patches/icu4c_fix_static_lib_name_mingw.patch | 28 +++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 patches/icu4c_fix_static_lib_name_mingw.patch diff --git a/kiwix-build.py b/kiwix-build.py index b8fb0b6..cf54d05 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -574,6 +574,7 @@ class Icu(Dependency, ReleaseDownloadMixin, MakeMixin): ) data = Remotefile('icudt56l.dat', 'e23d85eee008f335fc49e8ef37b1bc2b222db105476111e3d16f0007d371cbca') + patches = ["icu4c_fix_static_lib_name_mingw.patch"] configure_option = "--disable-samples --disable-tests --disable-extras --enable-static --disable-dyload" subsource_dir = "source" diff --git a/patches/icu4c_fix_static_lib_name_mingw.patch b/patches/icu4c_fix_static_lib_name_mingw.patch new file mode 100644 index 0000000..3c94e76 --- /dev/null +++ b/patches/icu4c_fix_static_lib_name_mingw.patch @@ -0,0 +1,28 @@ +diff -ur icu4c-56_1/source/config/mh-mingw icu4c-56_1.patched/source/config/mh-mingw +--- icu4c-56_1/source/config/mh-mingw 2015-10-08 05:54:32.000000000 +0200 ++++ icu4c-56_1.patched/source/config/mh-mingw 2017-01-17 15:21:45.344657127 +0100 +@@ -65,6 +65,10 @@ + SO_TARGET_VERSION_SUFFIX = + endif + ++## Remove shared library 's' ++STATIC_PREFIX_WHEN_USED = ++STATIC_PREFIX = ++ + # Static library prefix and file extension + LIBSICU = lib$(LIBPREFIX)$(STATIC_PREFIX)$(ICUPREFIX) + A = a +diff -ur icu4c-56_1/source/config/mh-mingw64 icu4c-56_1.patched/source/config/mh-mingw64 +--- icu4c-56_1/source/config/mh-mingw64 2015-10-08 05:54:30.000000000 +0200 ++++ icu4c-56_1.patched/source/config/mh-mingw64 2017-01-17 14:25:14.072632426 +0100 +@@ -65,6 +65,10 @@ + SO_TARGET_VERSION_SUFFIX = + endif + ++## Remove shared library 's' ++STATIC_PREFIX_WHEN_USED = ++STATIC_PREFIX = ++ + # Static library prefix and file extension + LIBSICU = lib$(LIBPREFIX)$(STATIC_PREFIX)$(ICUPREFIX) + A = a From f287cf8b19af3d4e823fdebbb40c9df71eeac485 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 17 Jan 2017 15:33:17 +0100 Subject: [PATCH 12/14] Build dir also depends of if we are building static libs or not. --- kiwix-build.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kiwix-build.py b/kiwix-build.py index cf54d05..288e8fc 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -120,7 +120,11 @@ class BuildEnv: def __init__(self, options): self.source_dir = pj(options.working_dir, "SOURCE") - self.build_dir = pj(options.working_dir, "BUILD_{target}".format(target=options.build_target)) + build_dir = "BUILD_{target}_{libmod}".format( + target=options.build_target, + libmod='static' if options.build_static else 'dyn' + ) + self.build_dir = pj(options.working_dir, build_dir) self.archive_dir = pj(options.working_dir, "ARCHIVE") self.log_dir = pj(options.working_dir, 'LOGS') self.install_dir = pj(self.build_dir, "INSTALL") From 5d710fc774fabd55afb496a21273f4227aec7d69 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 17 Jan 2017 15:37:42 +0100 Subject: [PATCH 13/14] Do not build static AND shared libs when compiling ICU. ICU seems to be buggy when generating pkg_config file if we compile static and shared libs with mingw target. As icu mingw target definition seems a bit complex it is simpler to just compile the libs we want. As side effect, it reduces compilation time a bit. --- kiwix-build.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/kiwix-build.py b/kiwix-build.py index 288e8fc..1d8dd89 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -579,7 +579,6 @@ class Icu(Dependency, ReleaseDownloadMixin, MakeMixin): data = Remotefile('icudt56l.dat', 'e23d85eee008f335fc49e8ef37b1bc2b222db105476111e3d16f0007d371cbca') patches = ["icu4c_fix_static_lib_name_mingw.patch"] - configure_option = "--disable-samples --disable-tests --disable-extras --enable-static --disable-dyload" subsource_dir = "source" def __init__(self, buildEnv, cross_compile_process=False, cross_build=None): @@ -595,7 +594,11 @@ class Icu(Dependency, ReleaseDownloadMixin, MakeMixin): @property def configure_option(self): - default_configure_option = "--disable-samples --disable-tests --disable-extras --enable-static --disable-dyload" + default_configure_option = "--disable-samples --disable-tests --disable-extras --disable-dyload" + if self.buildEnv.build_static: + default_configure_option += " --enable-static --disable-shared" + else: + default_configure_option += " --enable-shared --enable-shared" if self.cross_build: return default_configure_option + " --with-cross-build=" + self.cross_build.build_path return default_configure_option From 585f8ae37de5b874434e8e06291f48062964bc1a Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 17 Jan 2017 15:39:31 +0100 Subject: [PATCH 14/14] Do not compile xapian documentation. --- kiwix-build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kiwix-build.py b/kiwix-build.py index 1d8dd89..e66f2a7 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -533,7 +533,8 @@ class Xapian(Dependency, ReleaseDownloadMixin, MakeMixin): version = "1.4.0" archive = Remotefile('xapian-core-1.4.0.tar.xz', '10584f57112aa5e9c0e8a89e251aecbf7c582097638bfee79c1fe39a8b6a6477') - configure_option = "--enable-shared --enable-static --disable-sse --disable-backend-inmemory" + configure_option = ("--enable-shared --enable-static --disable-sse " + "--disable-backend-inmemory --disable-documentation") patches = ["xapian_pkgconfig.patch"] configure_env = {'_format_LDFLAGS' : "-L{buildEnv.install_dir}/{buildEnv.libprefix}", '_format_CXXFLAGS' : "-I{buildEnv.install_dir}/include"}