From c4295b164486594ca037e185c5b30370e40c7428 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 19 Feb 2020 15:19:41 +0100 Subject: [PATCH 1/7] Do not set a specific compiler for ios --- kiwixbuild/platforms/ios.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/kiwixbuild/platforms/ios.py b/kiwixbuild/platforms/ios.py index df08374..7c94a27 100644 --- a/kiwixbuild/platforms/ios.py +++ b/kiwixbuild/platforms/ios.py @@ -71,11 +71,6 @@ class iOSPlatformInfo(PlatformInfo): def configure_option(self): return '--host={}'.format(self.arch_full) - def set_compiler(self, env): - for k,v in self.binaries.items(): - env[k] = v - - class iOSArmv7(iOSPlatformInfo): name = 'iOS_armv7' arch = cpu = 'armv7' From 2c14c1a705a9417193b322dbd95b5fa4472c32cf Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 19 Feb 2020 16:31:45 +0100 Subject: [PATCH 2/7] Build base for all iOS architectures. --- .github/workflows/base.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml index afef18a..017b141 100644 --- a/.github/workflows/base.yml +++ b/.github/workflows/base.yml @@ -139,6 +139,10 @@ jobs: matrix: target: - native_dyn + - iOS_arm64 + - iOS_i386 + - iOS_x86_64 + - iOS_armv7 runs-on: macos-latest env: SSH_KEY: /tmp/id_rsa From 7d742f807ffdf075288fd3698cb4e21742657baa Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 19 Feb 2020 16:33:18 +0100 Subject: [PATCH 3/7] Fix setting of environment using env from dependency. If `self.configure_env` is a (computed) property, the previous code never format the value of the env. --- kiwixbuild/dependencies/base.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index 1d86a73..cf69055 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -323,13 +323,14 @@ class MakeBuilder(Builder): if self.buildEnv.platformInfo.static: env['CFLAGS'] = env['CFLAGS'] + ' -fPIC' env['CXXFLAGS'] = env['CXXFLAGS'] + ' -fPIC' - if self.configure_env: - for k in list(self.configure_env): + dep_conf_env = self.configure_env + if dep_conf_env: + for k in list(dep_conf_env): if k.startswith('_format_'): - v = self.configure_env.pop(k) + v = dep_conf_env.pop(k) v = v.format(buildEnv=self.buildEnv, env=env) - self.configure_env[k[8:]] = v - env.update(self.configure_env) + dep_conf_env[k[8:]] = v + env.update(dep_conf_env) run_command(command, self.build_path, context, buildEnv=self.buildEnv, env=env) def _compile(self, context): From 36366b2dd30055922174f191632b09ebdcbd6b6f Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 19 Feb 2020 17:19:17 +0100 Subject: [PATCH 4/7] Better define of value in ios platform. No real change, just better code to define it. --- kiwixbuild/platforms/ios.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/kiwixbuild/platforms/ios.py b/kiwixbuild/platforms/ios.py index 7c94a27..75f1070 100644 --- a/kiwixbuild/platforms/ios.py +++ b/kiwixbuild/platforms/ios.py @@ -9,6 +9,7 @@ class iOSPlatformInfo(PlatformInfo): build = 'iOS' static = True compatible_hosts = ['Darwin'] + min_iphoneos_version = '9.0' def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -17,7 +18,7 @@ class iOSPlatformInfo(PlatformInfo): @property def root_path(self): if self._root_path is None: - command = "xcodebuild -version -sdk {} | grep -E '^Path' | sed 's/Path: //'".format(self.sdk_name) + command = "xcrun --sdk {} --show-sdk-path".format(self.sdk_name) self._root_path = subprocess.check_output(command, shell=True)[:-1].decode() return self._root_path @@ -34,8 +35,8 @@ class iOSPlatformInfo(PlatformInfo): 'root_path': self.root_path, 'binaries': self.binaries, 'exe_wrapper_def': '', - 'extra_libs': ['-fembed-bitcode', '-isysroot', self.root_path, '-arch', self.arch, '-miphoneos-version-min=9.0', '-stdlib=libc++'], - 'extra_cflags': ['-fembed-bitcode', '-isysroot', self.root_path, '-arch', self.arch, '-miphoneos-version-min=9.0', '-stdlib=libc++', '-I{}'.format(pj(self.buildEnv.install_dir, 'include'))], + 'extra_libs': ['-fembed-bitcode', '-isysroot', self.root_path, '-arch', self.arch, '-miphoneos-version-min={}'.format(self.min_iphoneos_version), '-stdlib=libc++'], + 'extra_cflags': ['-fembed-bitcode', '-isysroot', self.root_path, '-arch', self.arch, '-miphoneos-version-min={}'.format(self.min_iphoneos_version), '-stdlib=libc++', '-I{}'.format(pj(self.buildEnv.install_dir, 'include'))], 'host_machine': { 'system': 'Darwin', 'lsystem': 'darwin', @@ -47,7 +48,7 @@ class iOSPlatformInfo(PlatformInfo): } def set_env(self, env): - env['CFLAGS'] = " -fembed-bitcode -isysroot {SDKROOT} -arch {arch} -miphoneos-version-min=9.0 ".format(SDKROOT=self.root_path, arch=self.arch) + env['CFLAGS'] + env['CFLAGS'] = " -fembed-bitcode -isysroot {SDKROOT} -arch {arch} -miphoneos-version-min={min_iphoneos_version} ".format(SDKROOT=self.root_path, min_iphoneos_version=self.min_iphoneos_version, arch=self.arch) + env['CFLAGS'] env['CXXFLAGS'] = env['CFLAGS'] + " -stdlib=libc++ -std=c++11 "+env['CXXFLAGS'] env['LDFLAGS'] = " -arch {arch} -isysroot {SDKROOT} ".format(SDKROOT=self.root_path, arch=self.arch) env['MACOSX_DEPLOYMENT_TARGET'] = "10.10" @@ -60,10 +61,10 @@ class iOSPlatformInfo(PlatformInfo): return { 'CC': xrun_find('clang'), 'CXX': xrun_find('clang++'), - 'AR': '/usr/bin/ar', - 'STRIP': '/usr/bin/strip', - 'RANLIB': '/usr/bin/ranlib', - 'LD': '/usr/bin/ld', + 'AR': xrun_find('ar'), + 'STRIP': xrun_find('strip'), + 'RANLIB': xrun_find('ranlib'), + 'LD': xrun_find('ld'), 'PKGCONFIG': 'pkg-config', } From e27ede80ccd307d20fb033a971572504fd4acbd4 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 20 Feb 2020 11:28:11 +0100 Subject: [PATCH 5/7] Use correct arch_full name. --- kiwixbuild/platforms/ios.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kiwixbuild/platforms/ios.py b/kiwixbuild/platforms/ios.py index 75f1070..a70c623 100644 --- a/kiwixbuild/platforms/ios.py +++ b/kiwixbuild/platforms/ios.py @@ -75,13 +75,13 @@ class iOSPlatformInfo(PlatformInfo): class iOSArmv7(iOSPlatformInfo): name = 'iOS_armv7' arch = cpu = 'armv7' - arch_full = 'arm-apple-darwin' + arch_full = 'armv7-apple-darwin' sdk_name = 'iphoneos' class iOSArm64(iOSPlatformInfo): name = 'iOS_arm64' arch = cpu = 'arm64' - arch_full = 'aarch64-apple-darwin' + arch_full = 'arm-apple-darwin' sdk_name = 'iphoneos' class iOSi386(iOSPlatformInfo): From 7b6c79482afb0210d5bb7688ac37af6817a54361 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 20 Feb 2020 16:37:42 +0100 Subject: [PATCH 6/7] Make the dependency responsible to set the compilation env. Instead of having the run_command function setting the env from the buildEnv, this is the dependency that create the env and then pass it to the run_command function. This way, each dependency will be able to set a specific env. --- kiwixbuild/buildenv.py | 33 +++------- kiwixbuild/dependencies/android_ndk.py | 4 +- kiwixbuild/dependencies/base.py | 85 ++++++++++++------------- kiwixbuild/dependencies/flatpak.py | 6 +- kiwixbuild/dependencies/libmagic.py | 4 +- kiwixbuild/flatpak_builder.py | 4 +- kiwixbuild/platforms/android.py | 23 ++++--- kiwixbuild/platforms/armhf.py | 11 +++- kiwixbuild/platforms/base.py | 18 ++++-- kiwixbuild/platforms/flatpak.py | 5 +- kiwixbuild/platforms/i586.py | 3 +- kiwixbuild/platforms/ios.py | 12 +++- kiwixbuild/platforms/native.py | 12 +++- kiwixbuild/platforms/win32.py | 4 +- kiwixbuild/platforms/win64.py | 88 ++++++++++++++++++++++++++ kiwixbuild/utils.py | 17 ++--- kiwixbuild/versions.py | 2 +- 17 files changed, 213 insertions(+), 118 deletions(-) create mode 100644 kiwixbuild/platforms/win64.py diff --git a/kiwixbuild/buildenv.py b/kiwixbuild/buildenv.py index 3b9fc41..73207e8 100644 --- a/kiwixbuild/buildenv.py +++ b/kiwixbuild/buildenv.py @@ -115,23 +115,12 @@ class BuildEnv: return 'lib64' return 'lib' - def _set_env(self, env, cross_compile_env, cross_compile_compiler, cross_compile_path): - if env is None: - env = Defaultdict(str, os.environ) - + def get_env(self, *, cross_comp_flags, cross_compilers, cross_path): + env = self.platformInfo.get_env() pkgconfig_path = pj(self.install_dir, self.libprefix, 'pkgconfig') env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path]) - # Add ccache path - for p in ('/usr/lib/ccache', '/usr/lib64/ccache'): - if os.path.isdir(p): - ccache_path = [p] - break - else: - ccache_path = [] - env['PATH'] = ':'.join([pj(self.install_dir, 'bin')] + - ccache_path + - [env['PATH']]) + env['PATH'] = ':'.join([pj(self.install_dir, 'bin'), env['PATH']]) env['LD_LIBRARY_PATH'] = ':'.join([env['LD_LIBRARY_PATH'], pj(self.install_dir, 'lib'), @@ -147,16 +136,10 @@ class BuildEnv: '-L'+pj(self.install_dir, self.libprefix), env['LDFLAGS']]) - if cross_compile_env: - for k, v in self.cross_config.get('env', {}).items(): - if k.startswith('_format_'): - v = v.format(**self.cross_config) - k = k[8:] - env[k] = v - self.platformInfo.set_env(env) - if cross_compile_compiler: + if cross_comp_flags: + self.platformInfo.set_comp_flags(env) + if cross_compilers: self.platformInfo.set_compiler(env) - if cross_compile_path: - bin_dirs = self.platformInfo.get_bind_dir() - env['PATH'] = ':'.join(bin_dirs + [env['PATH']]) + if cross_path: + env['PATH'] = ':'.join(self.platformInfo.get_bin_dir() + [env['PATH']]) return env diff --git a/kiwixbuild/dependencies/android_ndk.py b/kiwixbuild/dependencies/android_ndk.py index 4fbd239..57e0792 100644 --- a/kiwixbuild/dependencies/android_ndk.py +++ b/kiwixbuild/dependencies/android_ndk.py @@ -52,8 +52,8 @@ class android_ndk(Dependency): api=self.api, install_dir=self.install_path ) - context.force_native_build = True - run_command(command, self.build_path, context, buildEnv=self.buildEnv) + env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False) + run_command(command, self.build_path, context, env=env) def _fix_permission_right(self, context): context.try_skip(self.build_path) diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index cf69055..6581983 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -63,7 +63,6 @@ class Source: def _patch(self, context): context.try_skip(self.source_path) - context.force_native_build = True for p in self.patches: with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input: run_command("patch -p1", self.source_path, context, input=patch_input.read()) @@ -312,6 +311,18 @@ class MakeBuilder(Builder): ) return option + def set_configure_env(self, env): + dep_conf_env = self.configure_env + if not dep_conf_env: + return + for k in list(dep_conf_env): + if k.startswith('_format_'): + v = dep_conf_env.pop(k) + v = v.format(buildEnv=self.buildEnv, env=env) + dep_conf_env[k[8:]] = v + env.update(dep_conf_env) + + def _configure(self, context): context.try_skip(self.build_path) command = "{configure_script} {configure_option}" @@ -319,19 +330,9 @@ class MakeBuilder(Builder): configure_script=pj(self.source_path, self.configure_script), configure_option=self.all_configure_option ) - env = Defaultdict(str, os.environ) - if self.buildEnv.platformInfo.static: - env['CFLAGS'] = env['CFLAGS'] + ' -fPIC' - env['CXXFLAGS'] = env['CXXFLAGS'] + ' -fPIC' - dep_conf_env = self.configure_env - if dep_conf_env: - for k in list(dep_conf_env): - if k.startswith('_format_'): - v = dep_conf_env.pop(k) - v = v.format(buildEnv=self.buildEnv, env=env) - dep_conf_env[k[8:]] = v - env.update(dep_conf_env) - run_command(command, self.build_path, context, buildEnv=self.buildEnv, env=env) + env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) + self.set_configure_env(env) + run_command(command, self.build_path, context, env=env) def _compile(self, context): context.try_skip(self.build_path) @@ -339,7 +340,8 @@ class MakeBuilder(Builder): make_target=self.make_target, make_option=self.make_option ) - run_command(command, self.build_path, context, buildEnv=self.buildEnv) + env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) + run_command(command, self.build_path, context, env=env) def _install(self, context): context.try_skip(self.build_path) @@ -347,12 +349,14 @@ class MakeBuilder(Builder): make_install_target=self.make_install_target, make_option=self.make_option ) - run_command(command, self.build_path, context, buildEnv=self.buildEnv) + env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) + run_command(command, self.build_path, context, env=env) def _make_dist(self, context): context.try_skip(self.build_path) command = "make dist" - run_command(command, self.build_path, context, buildEnv=self.buildEnv) + env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) + run_command(command, self.build_path, context, env=env) class CMakeBuilder(MakeBuilder): @@ -376,18 +380,9 @@ class CMakeBuilder(MakeBuilder): source_path=self.source_path, cross_option=cross_option ) - env = Defaultdict(str, os.environ) - if self.buildEnv.platformInfo.static: - env['CFLAGS'] = env['CFLAGS'] + ' -fPIC' - env['CXXFLAGS'] = env['CXXFLAGS'] + ' -fPIC' - if self.configure_env: - for k in self.configure_env: - if k.startswith('_format_'): - v = self.configure_env.pop(k) - v = v.format(buildEnv=self.buildEnv, env=env) - self.configure_env[k[8:]] = v - env.update(self.configure_env) - run_command(command, self.build_path, context, env=env, buildEnv=self.buildEnv, cross_env_only=True) + env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=False, cross_path=True) + self.set_configure_env(env) + run_command(command, self.build_path, context, env=env) def set_flatpak_buildsystem(self, module): super().set_flatpak_buildsystem( module) @@ -420,15 +415,9 @@ class QMakeBuilder(MakeBuilder): source_path=self.source_path, cross_option=cross_option ) - run_command(command, self.build_path, context, buildEnv=self.buildEnv, cross_env_only=True) - - def _install(self, context): - context.try_skip(self.build_path) - command = "make {make_install_target} {make_option}".format( - make_install_target=self.make_install_target, - make_option=self.make_option - ) - run_command(command, self.build_path, context, buildEnv=self.buildEnv) + env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=False, cross_path=True) + self.set_configure_env(env) + run_command(command, self.build_path, context, env=env) def _make_dist(self, context): command = "git archive -o {build_dir}/{name}.tar.gz --prefix={name}/ HEAD" @@ -436,7 +425,7 @@ class QMakeBuilder(MakeBuilder): build_dir = self.build_path, name = self.target.full_name() ) - run_command(command, self.source_path, context, buildEnv=self.buildEnv) + run_command(command, self.source_path, context) @@ -484,11 +473,13 @@ class MesonBuilder(Builder): buildEnv=self.buildEnv, cross_option=cross_option ) - run_command(command, self.source_path, context, buildEnv=self.buildEnv, cross_env_only=True) + env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) + run_command(command, self.source_path, context, env=env) def _compile(self, context): command = "{} -v".format(neutralEnv('ninja_command')) - run_command(command, self.build_path, context, buildEnv=self.buildEnv) + env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) + run_command(command, self.build_path, context, env=env) def _test(self, context): if ( self.buildEnv.platformInfo.build == 'android' @@ -497,15 +488,18 @@ class MesonBuilder(Builder): ): raise SkipCommand() command = "{} --verbose {}".format(neutralEnv('mesontest_command'), self.test_option) - run_command(command, self.build_path, context, buildEnv=self.buildEnv) + env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) + run_command(command, self.build_path, context, env=env) def _install(self, context): command = "{} -v install".format(neutralEnv('ninja_command')) - run_command(command, self.build_path, context, buildEnv=self.buildEnv) + env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) + run_command(command, self.build_path, context, env=env) def _make_dist(self, context): command = "{} -v dist".format(neutralEnv('ninja_command')) - run_command(command, self.build_path, context, buildEnv=self.buildEnv) + env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) + run_command(command, self.build_path, context, env=env) class GradleBuilder(Builder): @@ -530,4 +524,5 @@ class GradleBuilder(Builder): command = command.format( gradle_target=self.gradle_target, gradle_option=self.gradle_option) - run_command(command, self.build_path, context, buildEnv=self.buildEnv) + env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) + run_command(command, self.build_path, context, env=env) diff --git a/kiwixbuild/dependencies/flatpak.py b/kiwixbuild/dependencies/flatpak.py index 3e32a98..b64fb5b 100644 --- a/kiwixbuild/dependencies/flatpak.py +++ b/kiwixbuild/dependencies/flatpak.py @@ -18,7 +18,8 @@ class org_kde(Dependency): remote_name = 'flathub', remote_url = 'https://flathub.org/repo/flathub.flatpakrepo' ) - run_command(command, self.buildEnv.build_dir, context, buildEnv=self.buildEnv) + env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False) + run_command(command, self.buildEnv.build_dir, context, env=env) def _install_sdk(self, context): command = "flatpak --user install -y {remote_name} {name}.Sdk//{version} {name}.Platform//{version}" @@ -27,7 +28,8 @@ class org_kde(Dependency): name = self.target.name, version = self.target.version() ) - run_command(command, self.buildEnv.build_dir, context, buildEnv=self.buildEnv) + env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False) + run_command(command, self.buildEnv.build_dir, context, env=env) def build(self): self.command('setup_remote', self._setup_remote) diff --git a/kiwixbuild/dependencies/libmagic.py b/kiwixbuild/dependencies/libmagic.py index fb5d438..4d269c3 100644 --- a/kiwixbuild/dependencies/libmagic.py +++ b/kiwixbuild/dependencies/libmagic.py @@ -36,7 +36,7 @@ class LibMagic(Dependency): make_target=self.make_target, make_option=self.make_option ) + env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) libmagic_native_builder = get_target_step('libmagic', 'native_static') - env = Defaultdict(str, os.environ) env['PATH'] = ':'.join([pj(libmagic_native_builder.build_path, 'src'), env['PATH']]) - run_command(command, self.build_path, context, buildEnv=self.buildEnv, env=env) + run_command(command, self.build_path, context, env=env) diff --git a/kiwixbuild/flatpak_builder.py b/kiwixbuild/flatpak_builder.py index d8c1f93..f0c4a59 100644 --- a/kiwixbuild/flatpak_builder.py +++ b/kiwixbuild/flatpak_builder.py @@ -224,7 +224,7 @@ class FlatpakBuilder: command = "flatpak-builder --user --ccache --force-clean --repo=repo builddir {id}.json" command = command.format(id = MANIFEST['app-id']) try: - run_command(command, self.platform.buildEnv.build_dir, context, self.platform.buildEnv) + run_command(command, self.platform.buildEnv.build_dir, context, env=self.platform.get_env()) context._finalise() except subprocess.CalledProcessError: with open(log, 'r') as f: @@ -237,7 +237,7 @@ class FlatpakBuilder: command = "flatpak build-bundle repo {id}.flatpak {id}" command = command.format(id = MANIFEST['app-id']) try: - run_command(command, self.platform.buildEnv.build_dir, context, self.platform.buildEnv) + run_command(command, self.platform.buildEnv.build_dir, context, env=self.platform.get_env()) context._finalise() except subprocess.CalledProcessError: with open(log, 'r') as f: diff --git a/kiwixbuild/platforms/android.py b/kiwixbuild/platforms/android.py index 668a2ab..4029667 100644 --- a/kiwixbuild/platforms/android.py +++ b/kiwixbuild/platforms/android.py @@ -56,24 +56,25 @@ class AndroidPlatformInfo(PlatformInfo): 'cpu': self.cpu, 'endian': 'little', 'abi': self.abi - }, + } } + def get_env(self): + env = super().get_env() + root_path = pj(self.install_path, 'sysroot') + env['PKG_CONFIG_LIBDIR'] = pj(root_path, 'lib', 'pkgconfig') + env['NDK_DEBUG'] = '0' + return env + def get_bin_dir(self): return [pj(self.install_path, 'bin')] - def set_env(self, env): + def set_comp_flags(self, env): + super().set_comp_flags(env) root_path = pj(self.install_path, 'sysroot') - env['PKG_CONFIG_LIBDIR'] = pj(root_path, 'lib', 'pkgconfig') env['CFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(root_path) + env['CFLAGS'] env['CXXFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(root_path) + env['CXXFLAGS'] env['LDFLAGS'] = '--sysroot={} '.format(root_path) + env['LDFLAGS'] - #env['CFLAGS'] = ' -fPIC -D_FILE_OFFSET_BITS=64 -O3 '+env['CFLAGS'] - #env['CXXFLAGS'] = (' -D__OPTIMIZE__ -fno-strict-aliasing ' - # ' -DU_HAVE_NL_LANGINFO_CODESET=0 ' - # '-DU_STATIC_IMPLEMENTATION -O3 ' - # '-DU_HAVE_STD_STRING -DU_TIMEZONE=0 ')+env['CXXFLAGS'] - env['NDK_DEBUG'] = '0' def set_compiler(self, env): binaries = self.binaries() @@ -140,5 +141,7 @@ class Android(MetaPlatformInfo): def sdk_builder(self): return get_target_step('android-sdk', 'neutral') - def set_env(self, env): + def get_env(self): + env = super().get_env() env['ANDROID_HOME'] = self.sdk_builder.install_path + return env diff --git a/kiwixbuild/platforms/armhf.py b/kiwixbuild/platforms/armhf.py index 6e0497d..0da7528 100644 --- a/kiwixbuild/platforms/armhf.py +++ b/kiwixbuild/platforms/armhf.py @@ -70,16 +70,21 @@ class ArmhfPlatformInfo(PlatformInfo): def get_bin_dir(self): return [pj(self.root_path, 'bin')] - def set_env(self, env): + def get_env(self): + env = super().get_env() env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') - env['CFLAGS'] = " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS'] - env['CXXFLAGS'] = " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS'] env['QEMU_LD_PREFIX'] = pj(self.root_path, "arm-linux-gnueabihf", "libc") env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format( ':'.join([ pj(self.root_path, self.arch_full, "lib"), env['LD_LIBRARY_PATH'] ])) + return env + + def set_comp_flags(self, env): + super().set_comp_flags(env) + env['CFLAGS'] = " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS'] + env['CXXFLAGS'] = " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS'] def set_compiler(self, env): for k, v in self.binaries.items(): diff --git a/kiwixbuild/platforms/base.py b/kiwixbuild/platforms/base.py index 952d74b..782fc5e 100644 --- a/kiwixbuild/platforms/base.py +++ b/kiwixbuild/platforms/base.py @@ -3,7 +3,7 @@ import os, sys import subprocess from kiwixbuild.dependencies import Dependency -from kiwixbuild.utils import pj, remove_duplicates +from kiwixbuild.utils import pj, remove_duplicates, DefaultEnv from kiwixbuild.buildenv import BuildEnv from kiwixbuild._global import neutralEnv, option, target_steps @@ -75,15 +75,25 @@ class PlatformInfo(metaclass=_MetaPlatform): def get_cross_config(self): return {} - def set_env(self, env): - pass - def get_bind_dir(self): + def get_env(self): + return DefaultEnv() + + + def get_bin_dir(self): return [] + def set_compiler(self, env): pass + + def set_comp_flags(self, env): + if self.static: + env['CFLAGS'] = env['CFLAGS'] + ' -fPIC' + env['CXXFLAGS'] = env['CXXFLAGS'] + ' -fPIC' + + def _gen_crossfile(self, name): crossfile = pj(self.buildEnv.build_dir, name) template_file = pj(TEMPLATES_DIR, name) diff --git a/kiwixbuild/platforms/flatpak.py b/kiwixbuild/platforms/flatpak.py index 8f3ff89..81ad37a 100644 --- a/kiwixbuild/platforms/flatpak.py +++ b/kiwixbuild/platforms/flatpak.py @@ -1,6 +1,5 @@ from .base import PlatformInfo from kiwixbuild._global import option, neutralEnv -from kiwixbuild.utils import run_command class FlatpakPlatformInfo(PlatformInfo): name = 'flatpak' @@ -12,6 +11,8 @@ class FlatpakPlatformInfo(PlatformInfo): def __str__(self): return "flatpak" - def set_env(self, env): + def get_env(self): + env = super().get_env() env['FLATPAK_USER_DIR'] = self.buildEnv.build_dir + return env diff --git a/kiwixbuild/platforms/i586.py b/kiwixbuild/platforms/i586.py index e090001..5dba476 100644 --- a/kiwixbuild/platforms/i586.py +++ b/kiwixbuild/platforms/i586.py @@ -41,7 +41,8 @@ class I586PlatformInfo(PlatformInfo): ('PKGCONFIG', 'pkg-config')) } - def set_env(self, env): + def set_comp_flags(self, env): + super().set_comp_flags(env) env['CFLAGS'] = "-m32 -march=i586 -mno-sse "+env['CFLAGS'] env['CXXFLAGS'] = "-m32 -march=i586 -mno-sse "+env['CXXFLAGS'] env['LDFLAGS'] = "-m32 -march=i586 -mno-sse "+env['LDFLAGS'] diff --git a/kiwixbuild/platforms/ios.py b/kiwixbuild/platforms/ios.py index a70c623..8dc217b 100644 --- a/kiwixbuild/platforms/ios.py +++ b/kiwixbuild/platforms/ios.py @@ -44,14 +44,20 @@ class iOSPlatformInfo(PlatformInfo): 'cpu': self.cpu, 'endian': '', 'abi': '' - }, + } } - def set_env(self, env): + def get_env(self): + env = super().get_env() + env['MACOSX_DEPLOYMENT_TARGET'] = '10.10' + return env + + def set_comp_flags(self, env): + super().set_comp_flags(env) env['CFLAGS'] = " -fembed-bitcode -isysroot {SDKROOT} -arch {arch} -miphoneos-version-min={min_iphoneos_version} ".format(SDKROOT=self.root_path, min_iphoneos_version=self.min_iphoneos_version, arch=self.arch) + env['CFLAGS'] env['CXXFLAGS'] = env['CFLAGS'] + " -stdlib=libc++ -std=c++11 "+env['CXXFLAGS'] env['LDFLAGS'] = " -arch {arch} -isysroot {SDKROOT} ".format(SDKROOT=self.root_path, arch=self.arch) - env['MACOSX_DEPLOYMENT_TARGET'] = "10.10" + def get_bin_dir(self): return [pj(self.root_path, 'bin')] diff --git a/kiwixbuild/platforms/native.py b/kiwixbuild/platforms/native.py index 3ceb53f..539b9dd 100644 --- a/kiwixbuild/platforms/native.py +++ b/kiwixbuild/platforms/native.py @@ -38,12 +38,18 @@ class NativeMixed(NativePlatformInfo): return 'native_mixed', dep return 'native_static', dep - def set_env(self, env): - super().set_env(env) + def get_env(self): + env = super().get_env() static_platform = self.get_platform('native_static') static_buildEnv = static_platform.buildEnv static_install_dir = static_buildEnv.install_dir - env['CPPFLAGS'] = " ".join(['-I'+pj(static_install_dir, 'include'), env['CPPFLAGS']]) env['PATH'] = ':'.join([pj(static_install_dir, 'bin')] + [env['PATH']]) pkgconfig_path = pj(static_install_dir, static_buildEnv.libprefix, 'pkgconfig') env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path]) + return env + + def set_comp_flags(self, env): + super().set_comp_flags(env) + static_platform = self.get_platform('native_static') + static_install_dir = static_platform.buildEnv.install_dir + env['CPPFLAGS'] = " ".join(['-I'+pj(static_install_dir, 'include'), env['CPPFLAGS']]) diff --git a/kiwixbuild/platforms/win32.py b/kiwixbuild/platforms/win32.py index 17b37c4..444e8cd 100644 --- a/kiwixbuild/platforms/win32.py +++ b/kiwixbuild/platforms/win32.py @@ -73,9 +73,11 @@ class Win32PlatformInfo(PlatformInfo): def get_bin_dir(self): return [pj(self.root_path, 'bin')] - def set_env(self, env): + def get_env(self): + env = super().get_env() env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') env['LIBS'] = " ".join(self.extra_libs) + " " +env['LIBS'] + return env class Win32Dyn(Win32PlatformInfo): name = 'win32_dyn' diff --git a/kiwixbuild/platforms/win64.py b/kiwixbuild/platforms/win64.py new file mode 100644 index 0000000..62c0669 --- /dev/null +++ b/kiwixbuild/platforms/win64.py @@ -0,0 +1,88 @@ +import subprocess + +from .base import PlatformInfo +from kiwixbuild.utils import which, pj +from kiwixbuild._global import neutralEnv + + +class Win64PlatformInfo(PlatformInfo): + extra_libs = ['-lmingw32', '-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4', '-lmsvcr100', '-liphlpapi', '-lshell32', '-lkernel32'] + build = 'win64' + compatible_hosts = ['fedora', 'debian'] + arch_full = 'x86_64-w64-mingw32' + + def get_cross_config(self): + return { + 'exe_wrapper_def': self.exe_wrapper_def, + 'binaries': self.binaries, + 'root_path': self.root_path, + 'extra_libs': self.extra_libs, + 'extra_cflags': ['-DWIN32'], + 'host_machine': { + 'system': 'Windows', + 'lsystem': 'windows', + 'cpu_family': 'x86_64', + 'cpu': 'x86_64', + 'endian': 'little', + 'abi': '' + } + } + + def finalize_setup(self): + super().finalize_setup() + self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') + self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') + + @property + def root_path(self): + root_paths = { + 'fedora': '/usr/x86_64-w64-mingw32/sys-root/mingw', + 'debian': '/usr/x86_64-w64-mingw32' + } + return root_paths[neutralEnv('distname')] + + @property + def binaries(self): + return {k:which('{}-{}'.format(self.arch_full, v)) + for k, v in (('CC', 'gcc'), + ('CXX', 'g++'), + ('AR', 'ar'), + ('STRIP', 'strip'), + ('WINDRES', 'windres'), + ('RANLIB', 'ranlib'), + ('PKGCONFIG', 'pkg-config')) + } + + @property + def exe_wrapper_def(self): + try: + which('wine') + except subprocess.CalledProcessError: + return "" + else: + return "exe_wrapper = 'wine'" + + @property + def configure_option(self): + return '--host={}'.format(self.arch_full) + + def set_compiler(self, env): + for k, v in self.binaries.items(): + env[k] = v + + def get_bin_dir(self): + return [pj(self.root_path, 'bin')] + + def get_env(self): + env = super().get_env() + env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') + env['LIBS'] = " ".join(self.extra_libs) + " " +env['LIBS'] + return env + +class Win64Dyn(Win64PlatformInfo): + name = 'win64_dyn' + static = False + +class Win64Static(Win64PlatformInfo): + name = 'win64_static' + static = True diff --git a/kiwixbuild/utils.py b/kiwixbuild/utils.py index 9cbd3f3..2405a21 100644 --- a/kiwixbuild/utils.py +++ b/kiwixbuild/utils.py @@ -34,6 +34,10 @@ class Defaultdict(defaultdict): return self[name] +def DefaultEnv(): + return Defaultdict(str, os.environ) + + def remove_duplicates(iterable, key_function=None): seen = set() if key_function is None: @@ -224,21 +228,10 @@ def extract_archive(archive_path, dest_dir, topdir=None, name=None): archive.close() -def run_command(command, cwd, context, buildEnv=None, env=None, input=None, cross_env_only=False): +def run_command(command, cwd, context, *, env=None, input=None): os.makedirs(cwd, exist_ok=True) if env is None: env = Defaultdict(str, os.environ) - if buildEnv is not None: - cross_compile_env = True - cross_compile_compiler = True - cross_compile_path = True - if context.force_native_build: - cross_compile_env = False - cross_compile_compiler = False - cross_compile_path = False - if cross_env_only: - cross_compile_compiler = False - env = buildEnv._set_env(env, cross_compile_env, cross_compile_compiler, cross_compile_path) log = None try: if not option('verbose'): diff --git a/kiwixbuild/versions.py b/kiwixbuild/versions.py index 7bc6e35..1b62da3 100644 --- a/kiwixbuild/versions.py +++ b/kiwixbuild/versions.py @@ -42,7 +42,7 @@ release_versions = { # This is the "version" of the whole base_deps_versions dict. # Change this when you change base_deps_versions. -base_deps_meta_version = '60' +base_deps_meta_version = '61' base_deps_versions = { 'zlib' : '1.2.8', From 661eb95df016e1d44e1e528cc19c04d08e04e072 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Fri, 21 Feb 2020 14:33:48 +0100 Subject: [PATCH 7/7] =?UTF-8?q?Use=20a=20specific=20cross=5Ffile=20on=20iO?= =?UTF-8?q?S=C2=A0cross=20compilation.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We need to set the property `needs_exe_wrapper` to true for x86_64 cross compilation. Else, meson will try to detect if we could run the cross compiled executable, and because it is the same arch, it will assume we can. So it will try to sanity check and execute the cross compiled binary. Then fails. --- kiwixbuild/platforms/ios.py | 2 +- kiwixbuild/templates/meson_ios_cross_file.txt | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 kiwixbuild/templates/meson_ios_cross_file.txt diff --git a/kiwixbuild/platforms/ios.py b/kiwixbuild/platforms/ios.py index 8dc217b..2fc379b 100644 --- a/kiwixbuild/platforms/ios.py +++ b/kiwixbuild/platforms/ios.py @@ -28,7 +28,7 @@ class iOSPlatformInfo(PlatformInfo): def finalize_setup(self): super().finalize_setup() self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_ios_cross_file.txt') - self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') + self.buildEnv.meson_crossfile = self._gen_crossfile('meson_ios_cross_file.txt') def get_cross_config(self): return { diff --git a/kiwixbuild/templates/meson_ios_cross_file.txt b/kiwixbuild/templates/meson_ios_cross_file.txt new file mode 100644 index 0000000..e7f417f --- /dev/null +++ b/kiwixbuild/templates/meson_ios_cross_file.txt @@ -0,0 +1,21 @@ +[binaries] +pkgconfig = '{binaries[PKGCONFIG]}' +c = '{binaries[CC]}' +ar = '{binaries[AR]}' +cpp = '{binaries[CXX]}' +strip = '{binaries[STRIP]}' +ranlib = '{binaries[RANLIB]}' +{exe_wrapper_def} + +[properties] +c_link_args = {extra_libs!r} +cpp_link_args = {extra_libs!r} +c_args = {extra_cflags!r} +cpp_args = {extra_cflags!r} +needs_exe_wrapper = true + +[host_machine] +cpu_family = '{host_machine[cpu_family]}' +cpu = '{host_machine[cpu]}' +system = '{host_machine[lsystem]}' +endian = '{host_machine[endian]}'