From 1b2b3efb544354aeca32419c3e9b215176135874 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 4 Feb 2019 10:35:15 +0100 Subject: [PATCH 1/6] Allow the platform itself to set on which platform we need to build the dep For some complex build, we may want to build a dependency on another platform that the target platform. --- kiwixbuild/builder.py | 5 +---- kiwixbuild/platforms/base.py | 7 +++++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/kiwixbuild/builder.py b/kiwixbuild/builder.py index 9df8762..8a5dbba 100644 --- a/kiwixbuild/builder.py +++ b/kiwixbuild/builder.py @@ -76,10 +76,7 @@ class Builder: targetPlatform = PlatformInfo.get_platform(targetPlatformName) for dep in target.get_dependencies(targetPlatform, True): - if isinstance(dep, tuple): - depPlatform, depName = dep - else: - depPlatform, depName = targetPlatformName, dep + depPlatform, depName = targetPlatform.get_fully_qualified_dep(dep) if (depPlatform, depName) in targets: yield from self.order_dependencies((depPlatform, depName), targets) yield ('source', targetName) diff --git a/kiwixbuild/platforms/base.py b/kiwixbuild/platforms/base.py index 4252195..8e7704a 100644 --- a/kiwixbuild/platforms/base.py +++ b/kiwixbuild/platforms/base.py @@ -65,6 +65,13 @@ class PlatformInfo(metaclass=_MetaPlatform): depPlatform.add_targets(depName, targets) return [(self.name, targetName)] + def get_fully_qualified_dep(self, dep): + if isinstance(dep, tuple): + return dep + else: + return self.name, dep + + def get_cross_config(self): return {} From c9210bb0e08de3677c5cc58714f05c17d8ddfabd Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 4 Feb 2019 11:04:19 +0100 Subject: [PATCH 2/6] Allow kiwix-build to build shared libs linked statically with deps. We need to be able to build libzim as shared lib while using all other dependencies statically (to not have libxapian.so, ... to distribute) This add a new platform (static=False) that make all dependencies being build in a static platform. --- kiwixbuild/platforms/native.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/kiwixbuild/platforms/native.py b/kiwixbuild/platforms/native.py index 069d9f2..3ceb53f 100644 --- a/kiwixbuild/platforms/native.py +++ b/kiwixbuild/platforms/native.py @@ -1,5 +1,8 @@ from .base import PlatformInfo +from kiwixbuild.utils import pj +from kiwixbuild._global import option + class NativePlatformInfo(PlatformInfo): build = 'native' @@ -14,3 +17,33 @@ class NativeStatic(NativePlatformInfo): name = 'native_static' static = True compatible_hosts = ['fedora', 'debian'] + +class NativeMixed(NativePlatformInfo): + name = 'native_mixed' + static = False + compatible_hosts = ['fedora', 'debian'] + + def add_targets(self, targetName, targets): + print(targetName) + if option('target') == targetName: + return super().add_targets(targetName, targets) + else: + static_platform = self.get_platform('native_static', targets) + return static_platform.add_targets(targetName, targets) + + def get_fully_qualified_dep(self, dep): + if isinstance(dep, tuple): + return dep + if option('target') == dep: + return 'native_mixed', dep + return 'native_static', dep + + def set_env(self, env): + super().set_env(env) + static_platform = self.get_platform('native_static') + static_buildEnv = static_platform.buildEnv + static_install_dir = static_buildEnv.install_dir + env['CPPFLAGS'] = " ".join(['-I'+pj(static_install_dir, 'include'), env['CPPFLAGS']]) + env['PATH'] = ':'.join([pj(static_install_dir, 'bin')] + [env['PATH']]) + pkgconfig_path = pj(static_install_dir, static_buildEnv.libprefix, 'pkgconfig') + env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path]) From 466dc8b64afd43103cded8d6433d0bea699a2dda Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 4 Feb 2019 11:05:16 +0100 Subject: [PATCH 3/6] Fix `-fPIC` CXXFLAGS. --- kiwixbuild/dependencies/base.py | 2 ++ kiwixbuild/dependencies/xapian.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index feb8d03..bc008a2 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -312,6 +312,7 @@ class MakeBuilder(Builder): env = Defaultdict(str, os.environ) if self.buildEnv.platformInfo.static: env['CFLAGS'] = env['CFLAGS'] + ' -fPIC' + env['CXXFLAGS'] = env['CXXFLAGS'] + ' -fPIC' if self.configure_env: for k in self.configure_env: if k.startswith('_format_'): @@ -367,6 +368,7 @@ class CMakeBuilder(MakeBuilder): env = Defaultdict(str, os.environ) if self.buildEnv.platformInfo.static: env['CFLAGS'] = env['CFLAGS'] + ' -fPIC' + env['CXXFLAGS'] = env['CXXFLAGS'] + ' -fPIC' if self.configure_env: for k in self.configure_env: if k.startswith('_format_'): diff --git a/kiwixbuild/dependencies/xapian.py b/kiwixbuild/dependencies/xapian.py index 5816645..c26307c 100644 --- a/kiwixbuild/dependencies/xapian.py +++ b/kiwixbuild/dependencies/xapian.py @@ -18,8 +18,8 @@ class Xapian(Dependency): class Builder(MakeBuilder): configure_option = "--disable-sse --disable-backend-chert --disable-backend-inmemory --disable-documentation" - configure_env = {'_format_LDFLAGS': "-L{buildEnv.install_dir}/{buildEnv.libprefix}", - '_format_CXXFLAGS': "-I{buildEnv.install_dir}/include"} + configure_env = {'_format_LDFLAGS': "{env.LDFLAGS} -L{buildEnv.install_dir}/{buildEnv.libprefix}", + '_format_CXXFLAGS': "{env.CXXFLAGS} -I{buildEnv.install_dir}/include"} @classmethod def get_dependencies(cls, platformInfo, allDeps): From 4eb95fd4cce6b0ec4ae672a2847539e22722ffb4 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 4 Feb 2019 11:06:55 +0100 Subject: [PATCH 4/6] Pass correct option `-Dstatic_linkage` to libzim in native_mixed platform. In native_mixed, we want to build a shared lib but we need to compile with the static lib of the dependencies. --- kiwixbuild/dependencies/libzim.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kiwixbuild/dependencies/libzim.py b/kiwixbuild/dependencies/libzim.py index 6113dce..fdcff3a 100644 --- a/kiwixbuild/dependencies/libzim.py +++ b/kiwixbuild/dependencies/libzim.py @@ -2,6 +2,7 @@ from .base import ( Dependency, GitClone, MesonBuilder) +from kiwixbuild._global import option class Libzim(Dependency): name = "libzim" @@ -21,4 +22,6 @@ class Libzim(Dependency): return "-DUSE_BUFFER_HEADER=false" if platformInfo.build == 'iOS': return "-Db_bitcode=true" + if platformInfo.name == 'native_mixed' and option('target') == 'libzim': + return "-Dstatic-linkage=true" return "" From 0ad24e96560c0b4e202fef199591688a3700e77a Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 4 Feb 2019 11:25:29 +0100 Subject: [PATCH 5/6] Do not use system's static zlib. The system's static zlib is compiled without `-fPIC` but we need it to build the mixed libzim. --- kiwixbuild/packages.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/kiwixbuild/packages.py b/kiwixbuild/packages.py index 8c6c3ec..6a11de7 100644 --- a/kiwixbuild/packages.py +++ b/kiwixbuild/packages.py @@ -30,7 +30,6 @@ PACKAGE_NAME_MAPPERS = { }, 'fedora_native_static': { 'COMMON': _fedora_common + ['glibc-static', 'libstdc++-static'], - 'zlib': ['zlib-devel', 'zlib-static'], 'lzma': ['xz-devel', 'xz-static'] # Either there is no packages, or no static or too old }, @@ -73,8 +72,6 @@ PACKAGE_NAME_MAPPERS = { }, 'debian_native_static': { 'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'], - 'zlib': ['zlib1g-dev'], - 'uuid': ['uuid-dev'], }, 'debian_i586_dyn': { 'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib'], From 68902b564216ba454b2af5e11fa040253c489dae Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 4 Feb 2019 11:34:09 +0100 Subject: [PATCH 6/6] Build libzim in native_mixed platform on travis. --- .travis.yml | 1 + travis/compile_all.py | 23 ++++++++++++++++------- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index f080519..1bf1064 100644 --- a/.travis.yml +++ b/.travis.yml @@ -68,6 +68,7 @@ env: - PLATFORM="native_dyn" - PLATFORM="armhf_dyn" - PLATFORM="armhf_static" + - PLATFORM="native_mixed" addons: ssh_known_hosts: - download.kiwix.org diff --git a/travis/compile_all.py b/travis/compile_all.py index 25a4143..487f947 100755 --- a/travis/compile_all.py +++ b/travis/compile_all.py @@ -40,9 +40,10 @@ else: # We have build everything. Now create archives for public deployement. BINARIES = { - 'kiwix-tools': ('kiwix-manage', 'kiwix-read', 'kiwix-search', 'kiwix-serve'), - 'zim-tools': ('zimbench', 'zimcheck', 'zimdump', 'zimsearch', 'zimdiff', 'zimpatch', 'zimsplit'), - 'zimwriterfs': ('zimwriterfs',) + 'kiwix-tools': ('bin', ('kiwix-manage', 'kiwix-read', 'kiwix-search', 'kiwix-serve')), + 'zim-tools': ('bin', ('zimbench', 'zimcheck', 'zimdump', 'zimsearch', 'zimdiff', 'zimpatch', 'zimsplit')), + 'zimwriterfs': ('bin', ('zimwriterfs',)), + 'libzim': ('lib/x86_64-linux-gnu', ('libzim.so',)) } _date = date.today().isoformat() @@ -135,8 +136,8 @@ def create_desktop_image(): def make_archive(project, platform): - file_to_archives = BINARIES[project] - base_bin_dir = BASE_DIR/'INSTALL'/'bin' + base_dir, file_to_archives = BINARIES[project] + base_dir = BASE_DIR/'INSTALL'/base_dir if make_release: postfix = main_project_versions[project] @@ -167,11 +168,11 @@ def make_archive(project, platform): if platform == "win-i686": file_to_archives = ['{}.exe'.format(f) for f in file_to_archives] open_archive = lambda a : zipfile.ZipFile(str(a), 'w', compression=zipfile.ZIP_DEFLATED) - archive_add = lambda a, f : a.write(str(base_bin_dir/f), arcname=str(f)) + archive_add = lambda a, f : a.write(str(base_dir/f), arcname=str(f)) archive_ext = ".zip" else: open_archive = lambda a : tarfile.open(str(a), 'w:gz') - archive_add = lambda a, f : a.add(str(base_bin_dir/f), arcname="{}/{}".format(archive_name, str(f))) + archive_add = lambda a, f : a.add(str(base_dir/f), arcname="{}/{}".format(archive_name, str(f))) archive_ext = ".tar.gz" @@ -186,6 +187,8 @@ def make_deps_archive(target, full=False): TRAVIS_OS_NAME, PLATFORM, target) print_message("Create archive {}.", archive_name) files_to_archive = [BASE_DIR/'INSTALL'] + if PLATFORM == 'native_mixed': + files_to_archive += [HOME/'BUILD_native_static'/'INSTALL'] if PLATFORM.startswith('android'): files_to_archive.append(HOME/'BUILD_neutral'/'INSTALL') if PLATFORM == 'android': @@ -297,6 +300,8 @@ if environ['TRAVIS_EVENT_TYPE'] != 'cron' and not make_release: TARGETS = ('kiwix-lib', 'zim-tools', 'zimwriterfs') elif PLATFORM == 'native_dyn' and KIWIX_DESKTOP_ONLY: TARGETS = ('kiwix-desktop', ) + elif PLATFORM == 'native_mixed': + TARGETS = ('libzim', ) else: TARGETS = ('kiwix-tools', 'zim-tools', 'zimwriterfs') elif PLATFORM == 'flatpak': @@ -327,6 +332,8 @@ elif PLATFORM.startswith('native_'): else: if PLATFORM == 'native_dyn' and KIWIX_DESKTOP_ONLY: TARGETS = ('kiwix-desktop', ) + elif PLATFORM == 'native_mixed': + TARGETS = ('libzim', ) else: TARGETS = ('libzim', 'zimwriterfs', 'zim-tools', 'kiwix-lib', 'kiwix-tools') elif PLATFORM == 'flatpak': @@ -379,6 +386,8 @@ if make_release and PLATFORM == 'native_dyn': in_file = BASE_DIR/full_target_name/'{}.tar.gz'.format(full_target_name) if in_file.exists(): shutil.copy(str(in_file), str(out_dir/target)) +elif PLATFORM == 'native_mixed': + make_archive('libzim', 'linux-x86_64') elif PLATFORM == 'native_static': for target in ('kiwix-tools', 'zim-tools', 'zimwriterfs'): make_archive(target, 'linux-x86_64')