diff --git a/.travis.yml b/.travis.yml index 51c3c86..c3f6c76 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,6 +54,7 @@ env: - PLATFORM="armhf_static" - PLATFORM="i586_dyn" - PLATFORM="i586_static" + - PLATFORM="android" - PLATFORM="android_arm" - PLATFORM="android_arm64" - PLATFORM="android_x86" @@ -66,6 +67,7 @@ matrix: - env: PLATFORM="android_mips64" - env: PLATFORM="android_x86" - env: PLATFORM="android_x86_64" + - env: PLATFORM="android" include: - env: PLATFORM="android_mips" if: type!=cron AND type!=pull_request diff --git a/README.md b/README.md index b7011ad..4361e28 100644 --- a/README.md +++ b/README.md @@ -71,8 +71,10 @@ By default, `kiwix-build` will build `kiwix-tools` . ## Target platform -By default, `kiwix-build` will build everything for the current (native) -platform using dynamic linkage (`native_dyn`). +If no target platform is specified, a default one will be infered from +the specified target : +- `kiwix-android` will be build using the platform `android` +- Other targets will be build using the platform `native_dyn` But you can select another target platform using the option `--target-platform`. For now, there is ten different supported @@ -82,6 +84,7 @@ platforms : - native_static - win32_dyn - win32_static +- android - android_arm - android_arm64 - android_mips @@ -95,11 +98,58 @@ So, if you want to compile `kiwix-tools` for win32 using static linkage: $ kiwix-build --target-platform win32_dyn ``` -Or, android apk for android_arm : +## Android + +Android apk (kiwix-android) is a bit a special case. +`kiwix-android` itself is architecture independent (it is written in +java) but it use `kiwix-lib` who is architecture dependent. + +When building `kiwix-lib`, you should directly use the +target-platform `android_`: + ``` -$ kiwix-build --target-platform android_arm kiwix-android +$ kiwix-build kiwix-android --target-platform android_arm ``` +But, `kiwix-android` apk can also be multi arch (ie, it includes +`kiwix-lib` for several architectures). To do so, you must ask to build +`kiwix-android` using the `android` platform: + +``` +$ kiwix-build --target-platform android kiwix-android +$ kiwix-build kiwix-android # because `android` platform is the default `kiwix-android` +``` + +By default, when using platform `android`, `kiwix-lib` will be build for +all architectures. This can be change by using the option `--android-arch` : + +``` +$ kiwix-build kiwix-android # apk for all architectures +$ kiwix-build kiwix-android --android-arch arm # apk for arm architectures (equivalent to `kiwix-android --target-platform android_arm`) +$ kiwix-build kiwix-anrdoid --android-arch arm --android-arch arm64 # apk for arm and arm64 architectures +``` + +## IOS + +When building for ios, we may want to compile a "fat library", a library +for several architectures. + +To do so, you should directly use the target-platfrom `ios_multi`. +As for `android`, `kiwix-build` will build the library several times +(once for each platform) and then create the fat library. + +``` +$ kiwix-build --target-platform iOS_multi kiwix-lib +``` + +You can specify the supported architectures with the option `--ios-arch`: + +``` +$ kiwix-build --target-platform iOS_multi kiwix-lib # all architetures +$ kiwix-build --target-platform iOS_multi --ios-arch arm --ios-arch arm64 # arm and arm64 arch only +``` + + # Outputs Kiwix-build.py will create several directories: diff --git a/kiwixbuild/__init__.py b/kiwixbuild/__init__.py index d84be62..41855f9 100644 --- a/kiwixbuild/__init__.py +++ b/kiwixbuild/__init__.py @@ -6,15 +6,15 @@ import argparse from .dependencies import Dependency from .platforms import PlatformInfo from .builder import Builder -from .utils import setup_print_progress +from . import _global def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument('targets', default='kiwix-tools', nargs='?', metavar='TARGET', + parser.add_argument('target', default='kiwix-tools', nargs='?', metavar='TARGET', choices=Dependency.all_deps.keys()) parser.add_argument('--working-dir', default=".") parser.add_argument('--libprefix', default=None) - parser.add_argument('--target-platform', default="native_dyn", choices=PlatformInfo.all_platforms) + parser.add_argument('--target-platform', choices=PlatformInfo.all_platforms) parser.add_argument('--verbose', '-v', action="store_true", help=("Print all logs on stdout instead of in specific" " log files per commands")) @@ -23,7 +23,7 @@ def parse_args(): parser.add_argument('--skip-source-prepare', action='store_true', help="Skip the source download part") parser.add_argument('--build-deps-only', action='store_true', - help="Build only the dependencies of the specified targets.") + help="Build only the dependencies of the specified target.") parser.add_argument('--build-nodeps', action='store_true', help="Build only the target, not its dependencies.") parser.add_argument('--make-dist', action='store_true', @@ -35,8 +35,16 @@ def parse_args(): help="Skip SSL certificate verification during download") subgroup.add_argument('--clean-at-end', action='store_true', help="Clean all intermediate files after the (successfull) build") - subgroup.add_argument('--force-install-packages', action='store_true', - help="Allways check for needed packages before compiling") + subgroup.add_argument('--dont-install-packages', action='store_true', + help="Do not try to install packages before compiling") + subgroup.add_argument('--android-arch', action='append', + help=("Specify the architecture to build for android application/libraries.\n" + "Can be specified several times to build for several architectures.\n" + "If not specified, all architectures will be build.")) + subgroup.add_argument('--ios-arch', action='append', + help=("Specify the architecture to build for ios application/libraries.\n" + "Can be specified several times to build for several architectures.\n" + "If not specified, all architectures will be build.")) subgroup = parser.add_argument_group('custom app', description="Android custom app specific options") subgroup.add_argument('--android-custom-app', @@ -47,7 +55,7 @@ def parse_args(): help="The size of the zim file.") options = parser.parse_args() - if options.targets == 'kiwix-android-custom': + if options.target == 'kiwix-android-custom': err = False if not options.android_custom_app: print("You need to specify ANDROID_CUSTOM_APP if you " @@ -59,12 +67,19 @@ def parse_args(): err = True if err: sys.exit(1) + if not options.android_arch: + options.android_arch = ['arm', 'arm64', 'mips', 'mips64', 'x86', 'x86_64'] + if not options.ios_arch: + options.ios_arch = ['armv7', 'arm64', 'i386', 'x86_64'] + return options def main(): options = parse_args() options.working_dir = os.path.abspath(options.working_dir) - setup_print_progress(options.show_progress) - builder = Builder(options) + _global.set_options(options) + neutralEnv = buildenv.PlatformNeutralEnv() + _global.set_neutralEnv(neutralEnv) + builder = Builder() builder.run() diff --git a/kiwixbuild/_global.py b/kiwixbuild/_global.py new file mode 100644 index 0000000..f513eda --- /dev/null +++ b/kiwixbuild/_global.py @@ -0,0 +1,32 @@ +from collections import OrderedDict as _OrderedDict + +_neutralEnv = None +_options = None +_target_steps = _OrderedDict() + +def set_neutralEnv(env): + global _neutralEnv + _neutralEnv = env + +def neutralEnv(what): + return getattr(_neutralEnv, what) + +def set_options(options): + global _options + _options = options + +def option(what): + return getattr(_options, what) + +def add_target_step(key, what): + _target_steps[key] = what + +def get_target_step(key, default_context=None): + try: + context, target = key + except ValueError: + context, target = default_context, key + return _target_steps[(context, target)] + +def target_steps(): + return _target_steps diff --git a/kiwixbuild/buildenv.py b/kiwixbuild/buildenv.py index c742519..51fc163 100644 --- a/kiwixbuild/buildenv.py +++ b/kiwixbuild/buildenv.py @@ -3,30 +3,22 @@ import os, sys, shutil import subprocess import platform -from .platforms import PlatformInfo -from .toolchains import Toolchain -from .packages import PACKAGE_NAME_MAPPERS -from .utils import ( - pj, - download_remote, - Defaultdict) - -SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +from .utils import pj, download_remote, Defaultdict +from ._global import neutralEnv, option class PlatformNeutralEnv: - def __init__(self, options): - self.options = options - self.source_dir = pj(options.working_dir, "SOURCE") - self.archive_dir = pj(options.working_dir, "ARCHIVE") - self.toolchain_dir = pj(options.working_dir, "TOOLCHAINS") + def __init__(self): + self.working_dir = option('working_dir') + self.source_dir = pj(self.working_dir, "SOURCE") + self.archive_dir = pj(self.working_dir, "ARCHIVE") + self.toolchain_dir = pj(self.working_dir, "TOOLCHAINS") self.log_dir = pj(self.working_dir, 'LOGS') for d in (self.source_dir, self.archive_dir, self.toolchain_dir, self.log_dir): os.makedirs(d, exist_ok=True) - self.toolchains = {} self.detect_platform() self.ninja_command = self._detect_ninja() if not self.ninja_command: @@ -36,33 +28,6 @@ class PlatformNeutralEnv: sys.exit("ERROR: meson command not fount") self.mesontest_command = "{} test".format(self.meson_command) - def run_command(self, command, cwd, context, env=None, input=None): - os.makedirs(cwd, exist_ok=True) - if env is None: - env = Defaultdict(str, os.environ) - log = None - try: - if not self.options.verbose: - log = open(context.log_file, 'w') - print("run command '{}'".format(command), file=log) - print("current directory is '{}'".format(cwd), file=log) - print("env is :", file=log) - for k, v in env.items(): - print(" {} : {!r}".format(k, v), file=log) - - kwargs = dict() - if input: - kwargs['stdin'] = subprocess.PIPE - process = subprocess.Popen(command, shell=True, cwd=cwd, env=env, stdout=log or sys.stdout, stderr=subprocess.STDOUT, **kwargs) - if input: - process.communicate(input.encode()) - retcode = process.wait() - if retcode: - raise subprocess.CalledProcessError(retcode, command) - finally: - if log: - log.close() - def detect_platform(self): _platform = platform.system() self.distname = _platform @@ -80,7 +45,7 @@ class PlatformNeutralEnv: def download(self, what, where=None): where = where or self.archive_dir - download_remote(what, where, not self.options.no_cert_check) + download_remote(what, where) def _detect_ninja(self): for n in ['ninja', 'ninja-build']: @@ -93,12 +58,6 @@ class PlatformNeutralEnv: if retcode == 0: return n - def add_toolchain(self, toolchain_name): - if toolchain_name not in self.toolchains: - ToolchainClass = Toolchain.all_toolchains[toolchain_name] - self.toolchains[toolchain_name] = ToolchainClass(self) - return self.toolchains[toolchain_name] - def _detect_meson(self): for n in ['meson.py', 'meson']: try: @@ -110,25 +69,22 @@ class PlatformNeutralEnv: if retcode == 0: return n - def __getattr__(self, name): - return getattr(self.options, name) - class BuildEnv: - def __init__(self, options, neutralEnv, targetsDict): - build_dir = "BUILD_{}".format(options.target_platform) - self.neutralEnv = neutralEnv - self.build_dir = pj(options.working_dir, build_dir) + def __init__(self, platformInfo): + build_dir = "BUILD_{}".format(platformInfo.name) + self.platformInfo = platformInfo + self.build_dir = pj(option('working_dir'), build_dir) self.install_dir = pj(self.build_dir, "INSTALL") + self.toolchain_dir = pj(self.build_dir, "TOOLCHAINS") + self.log_dir = pj(self.build_dir, 'LOGS') for d in (self.build_dir, - self.install_dir): + self.install_dir, + self.toolchain_dir, + self.log_dir): os.makedirs(d, exist_ok=True) - self.setup_build(options.target_platform) - self.setup_toolchains() - self.options = options - self.libprefix = options.libprefix or self._detect_libdir() - self.targetsDict = targetsDict + self.libprefix = option('libprefix') or self._detect_libdir() def clean_intermediate_directories(self): for subdir in os.listdir(self.build_dir): @@ -140,70 +96,6 @@ class BuildEnv: else: os.remove(subpath) - def setup_build(self, target_platform): - self.platform_info = PlatformInfo.all_platforms[target_platform] - if self.distname not in self.platform_info.compatible_hosts: - print(('ERROR: The target {} cannot be build on host {}.\n' - 'Select another target platform, or change your host system.' - ).format(target_platform, self.distname)) - sys.exit(-1) - self.cross_config = self.platform_info.get_cross_config() - - def setup_toolchains(self): - toolchain_names = self.platform_info.toolchains - self.toolchains = [] - for toolchain_name in toolchain_names: - ToolchainClass = Toolchain.all_toolchains[toolchain_name] - if ToolchainClass.neutral: - self.toolchains.append( - self.neutralEnv.add_toolchain(toolchain_name) - ) - else: - self.toolchains.append(ToolchainClass(self)) - - def finalize_setup(self): - getattr(self, 'setup_{}'.format(self.platform_info.build))() - - def setup_native(self): - self.cmake_crossfile = None - self.meson_crossfile = None - - def _gen_crossfile(self, name): - crossfile = pj(self.build_dir, name) - template_file = pj(SCRIPT_DIR, 'templates', name) - with open(template_file, 'r') as f: - template = f.read() - content = template.format( - toolchain=self.toolchains[0], - **self.cross_config - ) - with open(crossfile, 'w') as outfile: - outfile.write(content) - return crossfile - - def setup_win32(self): - self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') - self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') - - def setup_android(self): - self.cmake_crossfile = self._gen_crossfile('cmake_android_cross_file.txt') - self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') - - def setup_armhf(self): - self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') - self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') - - def setup_iOS(self): - self.cmake_crossfile = self._gen_crossfile('cmake_ios_cross_file.txt') - self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') - - def setup_i586(self): - self.cmake_crossfile = self._gen_crossfile('cmake_i586_cross_file.txt') - self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') - - def __getattr__(self, name): - return getattr(self.neutralEnv, name) - def _is_debianlike(self): return os.path.isfile('/etc/debian_version') @@ -223,16 +115,6 @@ class BuildEnv: return 'lib64' return 'lib' - @property - def configure_option(self): - configure_options = [tlc.configure_option for tlc in self.toolchains] - return " ".join(configure_options) - - @property - def cmake_option(self): - cmake_options = [tlc.cmake_option for tlc in self.toolchains] - return " ".join(cmake_options) - def _set_env(self, env, cross_compile_env, cross_compile_compiler, cross_compile_path): if env is None: env = Defaultdict(str, os.environ) @@ -267,110 +149,10 @@ class BuildEnv: v = v.format(**self.cross_config) k = k[8:] env[k] = v - for toolchain in self.toolchains: - toolchain.set_env(env) - self.platform_info.set_env(env) + self.platformInfo.set_env(env) if cross_compile_compiler: - for toolchain in self.toolchains: - toolchain.set_compiler(env) + self.platformInfo.set_compiler(env) if cross_compile_path: - bin_dirs = [] - for tlc in self.toolchains: - bin_dirs += tlc.get_bin_dir() - bin_dirs += self.platform_info.get_bind_dir() + bin_dirs = self.platformInfo.get_bind_dir() env['PATH'] = ':'.join(bin_dirs + [env['PATH']]) return env - - def run_command(self, command, cwd, context, env=None, input=None, cross_env_only=False): - os.makedirs(cwd, exist_ok=True) - cross_compile_env = True - cross_compile_compiler = True - cross_compile_path = True - if context.force_native_build: - cross_compile_env = False - cross_compile_compiler = False - cross_compile_path = False - if cross_env_only: - cross_compile_compiler = False - env = self._set_env(env, cross_compile_env, cross_compile_compiler, cross_compile_path) - log = None - try: - if not self.options.verbose: - log = open(context.log_file, 'w') - print("run command '{}'".format(command), file=log) - print("current directory is '{}'".format(cwd), file=log) - print("env is :", file=log) - for k, v in env.items(): - print(" {} : {!r}".format(k, v), file=log) - - kwargs = dict() - if input: - kwargs['stdin'] = subprocess.PIPE - process = subprocess.Popen(command, shell=True, cwd=cwd, env=env, stdout=log or sys.stdout, stderr=subprocess.STDOUT, **kwargs) - if input: - process.communicate(input.encode()) - retcode = process.wait() - if retcode: - raise subprocess.CalledProcessError(retcode, command) - finally: - if log: - log.close() - - def install_packages(self): - autoskip_file = pj(self.build_dir, ".install_packages_ok") - if self.distname in ('fedora', 'redhat', 'centos'): - package_installer = 'sudo dnf install {}' - package_checker = 'rpm -q --quiet {}' - elif self.distname in ('debian', 'Ubuntu'): - package_installer = 'sudo apt-get install {}' - package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1' - elif self.distname == 'Darwin': - package_installer = 'brew install {}' - package_checker = 'brew list -1 | grep -q {}' - mapper_name = "{host}_{target}".format( - host=self.distname, - target=self.platform_info) - try: - package_name_mapper = PACKAGE_NAME_MAPPERS[mapper_name] - except KeyError: - print("SKIP : We don't know which packages we must install to compile" - " a {target} {build_type} version on a {host} host.".format( - target=self.platform_info, - host=self.distname)) - return - - packages_list = package_name_mapper.get('COMMON', []) - for dep in self.targetsDict.values(): - packages = package_name_mapper.get(dep.name) - if packages: - packages_list += packages - dep.skip = True - for dep in self.targetsDict.values(): - packages = getattr(dep, 'extra_packages', []) - for package in packages: - packages_list += package_name_mapper.get(package, []) - if not self.options.force_install_packages and os.path.exists(autoskip_file): - print("SKIP") - return - - packages_to_install = [] - for package in packages_list: - print(" - {} : ".format(package), end="") - command = package_checker.format(package) - try: - subprocess.check_call(command, shell=True) - except subprocess.CalledProcessError: - print("NEEDED") - packages_to_install.append(package) - else: - print("SKIP") - - if packages_to_install: - command = package_installer.format(" ".join(packages_to_install)) - print(command) - subprocess.check_call(command, shell=True) - else: - print("SKIP, No package to install.") - - with open(autoskip_file, 'w'): - pass diff --git a/kiwixbuild/builder.py b/kiwixbuild/builder.py index 92aab5a..4ee7dd5 100644 --- a/kiwixbuild/builder.py +++ b/kiwixbuild/builder.py @@ -3,93 +3,194 @@ import sys from collections import OrderedDict from .buildenv import * +from .platforms import PlatformInfo from .utils import remove_duplicates, StopBuild from .dependencies import Dependency +from .packages import PACKAGE_NAME_MAPPERS +from ._global import ( + neutralEnv, option, + add_target_step, get_target_step, target_steps) +from . import _global class Builder: - def __init__(self, options): - self.options = options - self.targets = OrderedDict() - self.neutralEnv = PlatformNeutralEnv(options) - self.buildEnv = BuildEnv(options, self.neutralEnv, self.targets) + def __init__(self): + self._targets = {} + PlatformInfo.get_platform('neutral', self._targets) - _targets = {} - targetDef = options.targets - self.add_targets(targetDef, _targets) - dependencies = self.order_dependencies(_targets, targetDef) - dependencies = list(remove_duplicates(dependencies)) + target_platform = option('target_platform') + if not target_platform: + if option('target') == 'kiwix-android': + target_platform = 'android' + else: + target_platform = 'native_dyn' + platform = PlatformInfo.get_platform(target_platform, self._targets) + if neutralEnv('distname') not in platform.compatible_hosts: + print(('ERROR: The target platform {} cannot be build on host {}.\n' + 'Select another target platform or change your host system.' + ).format(platform.name, neutralEnv('distname'))) + self.targetDefs = platform.add_targets(option('target'), self._targets) - if options.build_nodeps: - self.targets[targetDef] = _targets[targetDef] + def finalize_target_steps(self): + steps = [] + for targetDef in self.targetDefs: + steps += self.order_steps(targetDef) + steps = list(remove_duplicates(steps)) + + if option('build_nodeps'): + add_target_step(targetDef, self._targets[targetDef]) else: - for dep in dependencies: - if self.options.build_deps_only and dep == targetDef: + for dep in steps: + if option('build_deps_only') and dep == targetDef: continue - self.targets[dep] = _targets[dep] + add_target_step(dep, self._targets[dep]) + self.instanciate_steps() - def add_targets(self, targetName, targets): - if targetName in targets: + def order_steps(self, targetDef): + for pltName in PlatformInfo.all_running_platforms: + plt = PlatformInfo.all_platforms[pltName] + for tlcName in plt.toolchain_names: + tlc = Dependency.all_deps[tlcName] + yield('source', tlcName) + yield('neutral' if tlc.neutral else pltName, tlcName) + _targets =dict(self._targets) + yield from self.order_dependencies(targetDef, _targets) + + def order_dependencies(self, targetDef, targets): + targetPlatformName, targetName = targetDef + if targetPlatformName == 'source': + # Do not try to order sources, they will be added as dep by the + # build step two lines later. + return + try: + target = targets.pop(targetDef) + except KeyError: return - targetClass = Dependency.all_deps[targetName] - target = targetClass(self.neutralEnv, self.buildEnv) - targets[targetName] = target - for dep in target.dependencies: - self.add_targets(dep, targets) - def order_dependencies(self, _targets, targetName): - target = _targets[targetName] - for depName in target.dependencies: - yield from self.order_dependencies(_targets, depName) - yield targetName + targetPlatform = PlatformInfo.get_platform(targetPlatformName) + for dep in target.get_dependencies(targetPlatform, True): + try: + depPlatform, depName = dep + except ValueError: + depPlatform, depName = targetPlatformName, dep + if (depPlatform, depName) in targets: + yield from self.order_dependencies((depPlatform, depName), targets) + yield ('source', targetName) + yield targetDef + + def instanciate_steps(self): + for stepDef in list(target_steps()): + stepPlatform, stepName = stepDef + stepClass = Dependency.all_deps[stepName] + if stepPlatform == 'source': + source = get_target_step(stepDef)(stepClass) + add_target_step(stepDef, source) + else: + source = get_target_step(stepName, 'source') + env = PlatformInfo.get_platform(stepPlatform).buildEnv + builder = get_target_step(stepDef)(stepClass, source, env) + add_target_step(stepDef, builder) def prepare_sources(self): - if self.options.skip_source_prepare: + if option('skip_source_prepare'): print("SKIP") return - toolchain_sources = (tlc.source for tlc in self.buildEnv.toolchains if tlc.source) - for toolchain_source in toolchain_sources: - print("prepare sources for toolchain {} :".format(toolchain_source.name)) - toolchain_source.prepare() - - sources = (dep.source for dep in self.targets.values() if not dep.skip) - sources = remove_duplicates(sources, lambda s: s.__class__) - for source in sources: - print("prepare sources {} :".format(source.name)) + sourceDefs = remove_duplicates(tDef for tDef in target_steps() if tDef[0]=='source') + for sourceDef in sourceDefs: + print("prepare sources {} :".format(sourceDef[1])) + source = get_target_step(sourceDef) source.prepare() def build(self): - toolchain_builders = (tlc.builder for tlc in self.buildEnv.toolchains if tlc.builder) - for toolchain_builder in toolchain_builders: - print("build toolchain {} :".format(toolchain_builder.name)) - toolchain_builder.build() - - builders = (dep.builder for dep in self.targets.values() if (dep.builder and not dep.skip)) - for builder in builders: - if self.options.make_dist and builder.name == self.options.targets: + builderDefs = (tDef for tDef in target_steps() if tDef[0] != 'source') + for builderDef in builderDefs: + builder = get_target_step(builderDef) + if option('make_dist') and builderName == option('target'): + print("make dist {} ({}):".format(builder.name, builderDef[0])) + builder.make_dist() continue - print("build {} :".format(builder.name)) + print("build {} ({}):".format(builder.name, builderDef[0])) + add_target_step(builderDef, builder) builder.build() - if self.options.make_dist: - dep = self.targets[self.options.targets] - builder = dep.builder - print("make dist {}:".format(builder.name)) - builder.make_dist() + def _get_packages(self): + packages_list = [] + for platform in PlatformInfo.all_running_platforms.values(): + mapper_name = "{host}_{target}".format( + host=neutralEnv('distname'), + target=platform) + package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {}) + packages_list += package_name_mapper.get('COMMON', []) + + to_drop = [] + for builderDef in self._targets: + platformName, builderName = builderDef + mapper_name = "{host}_{target}".format( + host=neutralEnv('distname'), + target=platformName) + package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {}) + packages = package_name_mapper.get(builderName) + if packages: + packages_list += packages + to_drop.append(builderDef) + for dep in to_drop: + del self._targets[dep] + return packages_list + + def install_packages(self): + packages_to_have = self._get_packages() + packages_to_have = remove_duplicates(packages_to_have) + + distname = neutralEnv('distname') + if distname in ('fedora', 'redhat', 'centos'): + package_installer = 'sudo dnf install {}' + package_checker = 'rpm -q --quiet {}' + elif distname in ('debian', 'Ubuntu'): + package_installer = 'sudo apt-get install {}' + package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1' + elif distname == 'Darwin': + package_installer = 'brew install {}' + package_checker = 'brew list -1 | grep -q {}' + + packages_to_install = [] + for package in packages_to_have: + print(" - {} : ".format(package), end="") + command = package_checker.format(package) + try: + subprocess.check_call(command, shell=True) + except subprocess.CalledProcessError: + print("NEEDED") + packages_to_install.append(package) + else: + print("SKIP") + + if packages_to_install: + command = package_installer.format(" ".join(packages_to_install)) + print(command) + subprocess.check_call(command, shell=True) + else: + print("SKIP, No package to install.") def run(self): try: print("[INSTALL PACKAGES]") - self.buildEnv.install_packages() - self.buildEnv.finalize_setup() + if option('dont_install_packages'): + print("SKIP") + else: + self.install_packages() + self.finalize_target_steps() + print("[SETUP PLATFORMS]") + for platform in PlatformInfo.all_running_platforms.values(): + platform.finalize_setup() print("[PREPARE]") self.prepare_sources() print("[BUILD]") self.build() # No error, clean intermediate file at end of build if needed. print("[CLEAN]") - if self.buildEnv.options.clean_at_end: - self.buildEnv.clean_intermediate_directories() + if option('clean_at_end'): + for platform in PlatformInfo.all_running_platforms.values(): + platform.clean_intermediate_directories() else: print("SKIP") except StopBuild: diff --git a/kiwixbuild/dependencies/__init__.py b/kiwixbuild/dependencies/__init__.py index 8007287..5850450 100644 --- a/kiwixbuild/dependencies/__init__.py +++ b/kiwixbuild/dependencies/__init__.py @@ -2,10 +2,14 @@ from .base import * from . import ( all_dependencies, + android_ndk, + android_sdk, + armhf, ctpp2, gradle, gumbo, icu4c, + ios_fat_lib, kiwix_android, kiwix_custom_app, kiwix_lib, diff --git a/kiwixbuild/dependencies/all_dependencies.py b/kiwixbuild/dependencies/all_dependencies.py index 34845fd..22883eb 100644 --- a/kiwixbuild/dependencies/all_dependencies.py +++ b/kiwixbuild/dependencies/all_dependencies.py @@ -3,26 +3,22 @@ from .base import ( NoopSource, NoopBuilder) +from kiwixbuild._global import neutralEnv + class AllBaseDependencies(Dependency): name = "alldependencies" - @property - def dependencies(self): - base_deps = ['zlib', 'lzma', 'xapian-core', 'gumbo', 'pugixml', 'libmicrohttpd', 'libaria2'] - if self.buildEnv.platform_info.build != 'native': - base_deps += ["icu4c_cross-compile"] - if self.buildEnv.platform_info.build != 'win32': - base_deps += ["libmagic_cross-compile"] - else: - base_deps += ["icu4c", "libmagic"] - if ( self.buildEnv.platform_info.build != 'android' - and self.buildEnv.distname != 'Darwin'): - base_deps += ['ctpp2c', 'ctpp2'] - if self.buildEnv.platform_info.build == 'android': - base_deps += ['Gradle'] - - return base_deps - - Source = NoopSource - Builder = NoopBuilder + class Builder(NoopBuilder): + @classmethod + def get_dependencies(cls, platformInfo, allDeps): + base_deps = ['zlib', 'lzma', 'xapian-core', 'pugixml', 'libaria2', 'icu4c'] + # zimwriterfs + if platformInfo.build not in ('android', 'win32'): + base_deps += ['libmagic', 'gumbo'] + # kiwix-tools + if (platformInfo.build != 'android' and + neutralEnv('distname') != 'Darwin'): + base_deps += ['libmicrohttpd', 'ctpp2c', 'ctpp2'] + + return base_deps diff --git a/kiwixbuild/dependencies/android_ndk.py b/kiwixbuild/dependencies/android_ndk.py new file mode 100644 index 0000000..fa2b00e --- /dev/null +++ b/kiwixbuild/dependencies/android_ndk.py @@ -0,0 +1,77 @@ +import os + +from .base import Dependency, ReleaseDownload, Builder +from kiwixbuild.utils import Remotefile, add_execution_right, run_command + +pj = os.path.join + +class android_ndk(Dependency): + neutral = False + name = 'android-ndk' + gccver = '4.9.x' + + class Source(ReleaseDownload): + archive = Remotefile('android-ndk-r13b-linux-x86_64.zip', + '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c', + 'https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip') + + @property + def source_dir(self): + return self.target.full_name() + + + class Builder(Builder): + @property + def install_path(self): + return self.build_path + + @property + def api(self): + return '21' if self.arch in ('arm64', 'mips64', 'x86_64') else '14' + + @property + def platform(self): + return 'android-'+self.api + + @property + def arch(self): + return self.buildEnv.platformInfo.arch + + @property + def arch_full(self): + return self.buildEnv.platformInfo.arch_full + + def _build_platform(self, context): + context.try_skip(self.build_path) + script = pj(self.source_path, 'build/tools/make_standalone_toolchain.py') + add_execution_right(script) + command = '{script} --arch={arch} --api={api} --install-dir={install_dir} --force' + command = command.format( + script=script, + arch=self.arch, + api=self.api, + install_dir=self.install_path + ) + context.force_native_build = True + run_command(command, self.build_path, context, buildEnv=self.buildEnv) + + def _fix_permission_right(self, context): + context.try_skip(self.build_path) + bin_dirs = [pj(self.install_path, 'bin'), + pj(self.install_path, self.arch_full, 'bin'), + pj(self.install_path, 'libexec', 'gcc', self.arch_full, self.target.gccver) + ] + for root, dirs, files in os.walk(self.install_path): + if not root in bin_dirs: + continue + + for file_ in files: + file_path = pj(root, file_) + if os.path.islink(file_path): + continue + add_execution_right(file_path) + + def build(self): + self.command('build_platform', self._build_platform) + self.command('fix_permission_right', self._fix_permission_right) + diff --git a/kiwixbuild/toolchains/android_sdk.py b/kiwixbuild/dependencies/android_sdk.py similarity index 83% rename from kiwixbuild/toolchains/android_sdk.py rename to kiwixbuild/dependencies/android_sdk.py index 7eb8934..d09f433 100644 --- a/kiwixbuild/toolchains/android_sdk.py +++ b/kiwixbuild/dependencies/android_sdk.py @@ -1,15 +1,14 @@ import os import shutil -from .base_toolchain import Toolchain -from kiwixbuild.dependencies import ReleaseDownload, Builder -from kiwixbuild.utils import Remotefile +from .base import Dependency, ReleaseDownload, Builder +from kiwixbuild.utils import Remotefile, run_command pj = os.path.join -class android_sdk(Toolchain): +class android_sdk(Dependency): + neutral = True name = 'android-sdk' - version = 'r25.2.3' class Source(ReleaseDownload): archive = Remotefile('tools_r25.2.3-linux.zip', @@ -20,7 +19,7 @@ class android_sdk(Toolchain): @property def install_path(self): - return pj(self.buildEnv.toolchain_dir, self.target.full_name) + return pj(self.buildEnv.toolchain_dir, self.target.full_name()) def _build_platform(self, context): context.try_skip(self.install_path) @@ -38,7 +37,7 @@ class android_sdk(Toolchain): # - 8 : Android SDK Build-tools, revision 24.0.1 # - 34 : SDK Platform Android 7.0, API 24, revision 2 # - 162 : Android Support Repository, revision 44 - self.buildEnv.run_command(command, self.install_path, context, input="y\n") + run_command(command, self.install_path, context, input="y\n") def _fix_licenses(self, context): context.try_skip(self.install_path) @@ -49,6 +48,3 @@ class android_sdk(Toolchain): def build(self): self.command('build_platform', self._build_platform) self.command('fix_licenses', self._fix_licenses) - - def set_env(self, env): - env['ANDROID_HOME'] = self.builder.install_path diff --git a/kiwixbuild/dependencies/armhf.py b/kiwixbuild/dependencies/armhf.py new file mode 100644 index 0000000..4ba5200 --- /dev/null +++ b/kiwixbuild/dependencies/armhf.py @@ -0,0 +1,11 @@ +from .base import Dependency, GitClone, NoopBuilder + +class armhf_toolchain(Dependency): + neutral = True + name = 'armhf' + + class Source(GitClone): + git_remote = "https://github.com/raspberrypi/tools" + git_dir = "raspberrypi-tools" + + Builder = NoopBuilder diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index a1410bf..13d6de5 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -2,8 +2,9 @@ import subprocess import os import shutil -from kiwixbuild.utils import pj, Context, SkipCommand, extract_archive, Defaultdict, StopBuild +from kiwixbuild.utils import pj, Context, SkipCommand, extract_archive, Defaultdict, StopBuild, run_command from kiwixbuild.versions import main_project_versions, base_deps_versions +from kiwixbuild._global import neutralEnv, option SCRIPT_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) @@ -19,38 +20,53 @@ class _MetaDependency(type): class Dependency(metaclass=_MetaDependency): all_deps = {} - dependencies = [] force_native_build = False - def __init__(self, neutralEnv, buildEnv): - self.neutralEnv = neutralEnv - self.buildEnv = buildEnv - self.source = self.Source(self) - self.builder = self.Builder(self) - self.skip = False + @classmethod + def version(cls): + return base_deps_versions.get(cls.name, None) + + @classmethod + def full_name(cls): + if cls.version(): + return "{}-{}".format(cls.name, cls.version()) + return cls.name + + +class Source: + """Base Class to the real preparator + A source preparator must install source in the self.source_dir attribute + inside the neutralEnv.source_dir.""" + def __init__(self, target): + self.target = target @property - def version(self): - return base_deps_versions.get(self.name, None) + def name(self): + return self.target.name @property - def full_name(self): - if self.version: - return "{}-{}".format(self.name, self.version) - return self.name + def source_dir(self): + return self.target.full_name() @property def source_path(self): - return pj(self.neutralEnv.source_dir, self.source.source_dir) + return pj(neutralEnv('source_dir'), self.source_dir) @property def _log_dir(self): - return self.buildEnv.log_dir + return neutralEnv('log_dir') + + def _patch(self, context): + context.try_skip(self.source_path) + context.force_native_build = True + for p in self.patches: + with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input: + run_command("patch -p1", self.source_path, context, input=patch_input.read()) def command(self, name, function, *args): print(" {} {} : ".format(name, self.name), end="", flush=True) log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name)) - context = Context(name, log, self.force_native_build) + context = Context(name, log, True) try: ret = function(*args, context=context) context._finalise() @@ -71,34 +87,6 @@ class Dependency(metaclass=_MetaDependency): raise -class Source: - """Base Class to the real preparator - A source preparator must install source in the self.source_dir attribute - inside the neutralEnv.source_dir.""" - def __init__(self, target): - self.target = target - self.neutralEnv = target.neutralEnv - - @property - def name(self): - return self.target.name - - @property - def source_dir(self): - return self.target.full_name - - def _patch(self, context): - source_path = pj(self.neutralEnv.source_dir, self.source_dir) - context.try_skip(source_path) - context.force_native_build = True - for p in self.patches: - with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input: - self.neutralEnv.run_command("patch -p1", source_path, context, input=patch_input.read()) - - def command(self, *args, **kwargs): - return self.target.command(*args, **kwargs) - - class NoopSource(Source): def prepare(self): pass @@ -109,18 +97,18 @@ class ReleaseDownload(Source): @property def extract_path(self): - return pj(self.neutralEnv.source_dir, self.source_dir) + return pj(neutralEnv('source_dir'), self.source_dir) def _download(self, context): - context.try_skip(self.neutralEnv.archive_dir, self.name) - self.neutralEnv.download(self.archive) + context.try_skip(neutralEnv('archive_dir'), self.name) + neutralEnv('download')(self.archive) def _extract(self, context): context.try_skip(self.extract_path) if os.path.exists(self.extract_path): shutil.rmtree(self.extract_path) - extract_archive(pj(self.neutralEnv.archive_dir, self.archive.name), - self.neutralEnv.source_dir, + extract_archive(pj(neutralEnv('archive_dir'), self.archive.name), + neutralEnv('source_dir'), topdir=self.archive_top_dir, name=self.source_dir) @@ -142,18 +130,18 @@ class GitClone(Source): @property def source_dir(self): - if self.neutralEnv.make_release: + if option('make_release'): return "{}_release".format(self.git_dir) else: return self.git_dir @property def git_path(self): - return pj(self.neutralEnv.source_dir, self.source_dir) + return pj(neutralEnv('source_dir'), self.source_dir) @property def git_ref(self): - if self.neutralEnv.make_release: + if option('make_release'): return self.release_git_ref else: return self.base_git_ref @@ -163,13 +151,13 @@ class GitClone(Source): raise SkipCommand() command = "git clone --depth=1 --branch {} {} {}".format( self.git_ref, self.git_remote, self.source_dir) - self.neutralEnv.run_command(command, self.neutralEnv.source_dir, context) + run_command(command, neutralEnv('source_dir'), context) def _git_update(self, context): command = "git fetch origin {}".format( self.git_ref) - self.neutralEnv.run_command(command, self.git_path, context) - self.neutralEnv.run_command("git checkout "+self.git_ref, self.git_path, context) + run_command(command, self.git_path, context) + run_command("git checkout "+self.git_ref, self.git_path, context) def prepare(self): self.command('gitclone', self._git_clone) @@ -185,17 +173,17 @@ class SvnClone(Source): @property def svn_path(self): - return pj(self.neutralEnv.source_dir, self.svn_dir) + return pj(neutralEnv('source_dir'), self.svn_dir) def _svn_checkout(self, context): if os.path.exists(self.svn_path): raise SkipCommand() command = "svn checkout {} {}".format(self.svn_remote, self.svn_dir) - self.neutralEnv.run_command(command, self.neutralEnv.source_dir, context) + run_command(command, neutralEnv('source_dir'), context) def _svn_update(self, context): context.try_skip(self.svn_path) - self.neutralEnv.run_command("svn update", self.svn_path, context) + run_command("svn update", self.svn_path, context) def prepare(self): self.command('svncheckout', self._svn_checkout) @@ -206,10 +194,16 @@ class SvnClone(Source): class Builder: subsource_dir = None + dependencies = [] - def __init__(self, target): + def __init__(self, target, source, buildEnv): self.target = target - self.buildEnv = target.buildEnv + self.source = source + self.buildEnv = buildEnv + + @classmethod + def get_dependencies(cls, platformInfo, allDeps): + return cls.dependencies @property def name(self): @@ -217,17 +211,41 @@ class Builder: @property def source_path(self): - base_source_path = self.target.source_path + base_source_path = self.source.source_path if self.subsource_dir: return pj(base_source_path, self.subsource_dir) return base_source_path @property def build_path(self): - return pj(self.buildEnv.build_dir, self.target.full_name) + return pj(self.buildEnv.build_dir, self.target.full_name()) - def command(self, *args, **kwargs): - return self.target.command(*args, **kwargs) + @property + def _log_dir(self): + return self.buildEnv.log_dir + + def command(self, name, function, *args): + print(" {} {} : ".format(name, self.name), end="", flush=True) + log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name)) + context = Context(name, log, self.target.force_native_build) + try: + ret = function(*args, context=context) + context._finalise() + print("OK") + return ret + except SkipCommand: + print("SKIP") + except subprocess.CalledProcessError: + print("ERROR") + try: + with open(log, 'r') as f: + print(f.read()) + except: + pass + raise StopBuild() + except: + print("ERROR") + raise def build(self): if hasattr(self, '_pre_build_script'): @@ -271,8 +289,8 @@ class MakeBuilder(Builder): def all_configure_option(self): option = self.configure_option_template.format( dep_options=self.configure_option, - static_option=self.static_configure_option if self.buildEnv.platform_info.static else self.dynamic_configure_option, - env_option=self.buildEnv.configure_option if not self.target.force_native_build else "", + static_option=self.static_configure_option if self.buildEnv.platformInfo.static else self.dynamic_configure_option, + env_option=self.buildEnv.platformInfo.configure_option if not self.target.force_native_build else "", install_dir=self.buildEnv.install_dir, libdir=pj(self.buildEnv.install_dir, self.buildEnv.libprefix) ) @@ -286,7 +304,7 @@ class MakeBuilder(Builder): configure_option=self.all_configure_option ) env = Defaultdict(str, os.environ) - if self.buildEnv.platform_info.static: + if self.buildEnv.platformInfo.static: env['CFLAGS'] = env['CFLAGS'] + ' -fPIC' if self.configure_env: for k in self.configure_env: @@ -295,7 +313,7 @@ class MakeBuilder(Builder): v = v.format(buildEnv=self.buildEnv, env=env) self.configure_env[k[8:]] = v env.update(self.configure_env) - self.buildEnv.run_command(command, self.build_path, context, env=env) + run_command(command, self.build_path, context, buildEnv=self.buildEnv, env=env) def _compile(self, context): context.try_skip(self.build_path) @@ -303,7 +321,7 @@ class MakeBuilder(Builder): make_target=self.make_target, make_option=self.make_option ) - self.buildEnv.run_command(command, self.build_path, context) + run_command(command, self.build_path, context, buildEnv=self.buildEnv) def _install(self, context): context.try_skip(self.build_path) @@ -311,12 +329,12 @@ class MakeBuilder(Builder): make_install_target=self.make_install_target, make_option=self.make_option ) - self.buildEnv.run_command(command, self.build_path, context) + run_command(command, self.build_path, context, buildEnv=self.buildEnv) def _make_dist(self, context): context.try_skip(self.build_path) command = "make dist" - self.buildEnv.run_command(command, self.build_path, context) + run_command(command, self.build_path, context, buildEnv=self.buildEnv) class CMakeBuilder(MakeBuilder): @@ -332,14 +350,14 @@ class CMakeBuilder(MakeBuilder): " {source_path}" " {cross_option}") command = command.format( - configure_option="{} {}".format(self.buildEnv.cmake_option, self.configure_option), + configure_option=self.configure_option, install_dir=self.buildEnv.install_dir, libdir=self.buildEnv.libprefix, source_path=self.source_path, cross_option=cross_option ) env = Defaultdict(str, os.environ) - if self.buildEnv.platform_info.static: + if self.buildEnv.platformInfo.static: env['CFLAGS'] = env['CFLAGS'] + ' -fPIC' if self.configure_env: for k in self.configure_env: @@ -348,22 +366,16 @@ class CMakeBuilder(MakeBuilder): v = v.format(buildEnv=self.buildEnv, env=env) self.configure_env[k[8:]] = v env.update(self.configure_env) - self.buildEnv.run_command(command, self.build_path, context, env=env, cross_env_only=True) + run_command(command, self.build_path, context, env=env, buildEnv=self.buildEnv, cross_env_only=True) class MesonBuilder(Builder): configure_option = "" test_option = "" - def __init__(self, target): - super().__init__(target) - self.meson_command = self.buildEnv.neutralEnv.meson_command - self.mesontest_command = self.buildEnv.neutralEnv.mesontest_command - self.ninja_command = self.buildEnv.neutralEnv.ninja_command - @property def library_type(self): - return 'static' if self.buildEnv.platform_info.static else 'shared' + return 'static' if self.buildEnv.platformInfo.static else 'shared' def _configure(self, context): context.try_skip(self.build_path) @@ -382,40 +394,40 @@ class MesonBuilder(Builder): " --libdir={buildEnv.libprefix}" " {cross_option}") command = command.format( - command=self.meson_command, + command=neutralEnv('meson_command'), library_type=self.library_type, configure_option=configure_option, build_path=self.build_path, buildEnv=self.buildEnv, cross_option=cross_option ) - self.buildEnv.run_command(command, self.source_path, context, cross_env_only=True) + run_command(command, self.source_path, context, buildEnv=self.buildEnv, cross_env_only=True) def _compile(self, context): - command = "{} -v".format(self.ninja_command) - self.buildEnv.run_command(command, self.build_path, context) + command = "{} -v".format(neutralEnv('ninja_command')) + run_command(command, self.build_path, context, buildEnv=self.buildEnv) def _test(self, context): - if ( self.buildEnv.platform_info.build == 'android' - or (self.buildEnv.platform_info.build != 'native' - and not self.buildEnv.platform_info.static) + if ( self.buildEnv.platformInfo.build == 'android' + or (self.buildEnv.platformInfo.build != 'native' + and not self.buildEnv.platformInfo.static) ): raise SkipCommand() - command = "{} --verbose {}".format(self.mesontest_command, self.test_option) - self.buildEnv.run_command(command, self.build_path, context) + command = "{} --verbose {}".format(neutralEnv('mesontest_command'), self.test_option) + run_command(command, self.build_path, context, buildEnv=self.buildEnv) def _install(self, context): - command = "{} -v install".format(self.ninja_command) - self.buildEnv.run_command(command, self.build_path, context) + command = "{} -v install".format(neutralEnv('ninja_command')) + run_command(command, self.build_path, context, buildEnv=self.buildEnv) def _make_dist(self, context): - command = "{} -v dist".format(self.ninja_command) - self.buildEnv.run_command(command, self.build_path, context) + command = "{} -v dist".format(neutralEnv('ninja_command')) + run_command(command, self.build_path, context, buildEnv=self.buildEnv) class GradleBuilder(Builder): gradle_target = "build" - gradle_option = "-i" + gradle_option = "-i --no-daemon --build-cache" def build(self): self.command('configure', self._configure) @@ -435,4 +447,4 @@ class GradleBuilder(Builder): command = command.format( gradle_target=self.gradle_target, gradle_option=self.gradle_option) - self.buildEnv.run_command(command, self.build_path, context) + run_command(command, self.build_path, context, buildEnv=self.buildEnv) diff --git a/kiwixbuild/dependencies/ctpp2.py b/kiwixbuild/dependencies/ctpp2.py index 4e1410f..4b05b03 100644 --- a/kiwixbuild/dependencies/ctpp2.py +++ b/kiwixbuild/dependencies/ctpp2.py @@ -3,7 +3,7 @@ from .base import ( ReleaseDownload, CMakeBuilder) -from kiwixbuild.utils import Remotefile, pj +from kiwixbuild.utils import Remotefile, pj, run_command class CTPP2(Dependency): name = "ctpp2" @@ -51,4 +51,4 @@ class CTPP2C(CTPP2): ctpp2c=pj(self.build_path, 'ctpp2c'), install_dir=pj(self.buildEnv.install_dir, 'bin') ) - self.buildEnv.run_command(command, self.build_path, context) + run_command(command, self.build_path, context, buildEnv=self.buildEnv) diff --git a/kiwixbuild/dependencies/gradle.py b/kiwixbuild/dependencies/gradle.py index 42ed104..e62c464 100644 --- a/kiwixbuild/dependencies/gradle.py +++ b/kiwixbuild/dependencies/gradle.py @@ -6,7 +6,8 @@ from .base import ( from kiwixbuild.utils import Remotefile, pj, copy_tree, add_execution_right class Gradle(Dependency): - name = "Gradle" + neutral = True + name = "gradle" class Source(ReleaseDownload): archive = Remotefile('gradle-4.6-bin.zip', @@ -14,14 +15,18 @@ class Gradle(Dependency): 'https://services.gradle.org/distributions/gradle-4.6-bin.zip') class Builder(BaseBuilder): + @property + def install_path(self): + return self.buildEnv.install_dir + def build(self): self.command('install', self._install) def _install(self, context): copy_tree( pj(self.source_path, "bin"), - pj(self.buildEnv.install_dir, "bin"), + pj(self.install_path, "bin"), post_copy_function = add_execution_right) copy_tree( pj(self.source_path, "lib"), - pj(self.buildEnv.install_dir, "lib")) + pj(self.install_path, "lib")) diff --git a/kiwixbuild/dependencies/gumbo.py b/kiwixbuild/dependencies/gumbo.py index 3d9cbe1..078f0fd 100644 --- a/kiwixbuild/dependencies/gumbo.py +++ b/kiwixbuild/dependencies/gumbo.py @@ -4,7 +4,7 @@ from .base import ( MakeBuilder ) -from kiwixbuild.utils import Remotefile +from kiwixbuild.utils import Remotefile, run_command class Gumbo(Dependency): @@ -18,6 +18,6 @@ class Gumbo(Dependency): def _post_prepare_script(self, context): context.try_skip(self.extract_path) command = "./autogen.sh" - self.buildEnv.run_command(command, self.extract_path, context) + run_command(command, self.extract_path, context) Builder = MakeBuilder diff --git a/kiwixbuild/dependencies/icu4c.py b/kiwixbuild/dependencies/icu4c.py index 84ce484..14dbf8e 100644 --- a/kiwixbuild/dependencies/icu4c.py +++ b/kiwixbuild/dependencies/icu4c.py @@ -5,7 +5,7 @@ from .base import ( ) from kiwixbuild.utils import SkipCommand - +from kiwixbuild._global import get_target_step class Icu(Dependency): name = "icu4c" @@ -25,33 +25,21 @@ class Icu(Dependency): class Builder(MakeBuilder): subsource_dir = "source" + @classmethod + def get_dependencies(cls, platformInfo, allDeps): + plt = 'native_static' if platformInfo.static else 'native_dyn' + return [(plt, 'icu4c')] + @property def configure_option(self): options = "--disable-samples --disable-tests --disable-extras --disable-dyload --enable-rpath" - if self.buildEnv.platform_info.build == 'android': + platformInfo = self.buildEnv.platformInfo + if platformInfo.build != 'native': + icu_native_builder = get_target_step( + 'icu4c', + 'native_static' if platformInfo.static else 'native_dyn') + options += " --with-cross-build={} --disable-tools".format( + icu_native_builder.build_path) + if platformInfo.build == 'android': options += " --with-data-packaging=archive" return options - - -class Icu_native(Icu): - name = "icu4c_native" - force_native_build = True - - class Builder(Icu.Builder): - @property - def build_path(self): - return super().build_path+"_native" - - def _install(self, context): - raise SkipCommand() - - -class Icu_cross_compile(Icu): - name = "icu4c_cross-compile" - dependencies = ['icu4c_native'] - - class Builder(Icu.Builder): - @property - def configure_option(self): - icu_native_dep = self.buildEnv.targetsDict['icu4c_native'] - return super().configure_option + " --with-cross-build={} --disable-tools".format(icu_native_dep.builder.build_path) diff --git a/kiwixbuild/dependencies/ios_fat_lib.py b/kiwixbuild/dependencies/ios_fat_lib.py new file mode 100644 index 0000000..fddf107 --- /dev/null +++ b/kiwixbuild/dependencies/ios_fat_lib.py @@ -0,0 +1,52 @@ +import os + +from kiwixbuild.platforms import PlatformInfo +from kiwixbuild.utils import pj, copy_tree, run_command +from kiwixbuild._global import option +from .base import ( + Dependency, + NoopSource, + Builder as BaseBuilder) + + + +class IOSFatLib(Dependency): + name = "_ios_fat_lib" + + Source = NoopSource + + class Builder(BaseBuilder): + + @classmethod + def get_dependencies(self, platfomInfo, alldeps): + base_target = option('target') + return [('iOS_{}'.format(arch), base_target) for arch in option('ios_arch')] + + def _copy_headers(self, context): + plt = PlatformInfo.get_platform('iOS_{}'.format(option('ios_arch')[0])) + include_src = pj(plt.buildEnv.install_dir, 'include') + include_dst = pj(self.buildEnv.install_dir, 'include') + copy_tree(include_src, include_dst) + + def _merge_libs(self, context): + lib_dirs = [] + for arch in option('ios_arch'): + plt = PlatformInfo.get_platform('iOS_{}'.format(arch)) + lib_dirs.append(pj(plt.buildEnv.install_dir, 'lib')) + libs = [] + for f in os.listdir(lib_dirs[0]): + if os.path.islink(pj(lib_dirs[0], f)): + continue + if f.endswith('.a') or f.endswith('.dylib'): + libs.append(f) + os.makedirs(pj(self.buildEnv.install_dir, 'lib'), exist_ok=True) + command_tmp = "lipo -create {input} -output {output}" + for l in libs: + command = command_tmp.format( + input=" ".join(pj(d, l) for d in lib_dirs), + output=pj(self.buildEnv.install_dir, 'lib', l)) + run_command(command, self.buildEnv.install_dir, context) + + def build(self): + self.command('copy_headers', self._copy_headers) + self.command('merge_libs', self._merge_libs) diff --git a/kiwixbuild/dependencies/kiwix_android.py b/kiwixbuild/dependencies/kiwix_android.py index 9273a01..4a2de5b 100644 --- a/kiwixbuild/dependencies/kiwix_android.py +++ b/kiwixbuild/dependencies/kiwix_android.py @@ -5,19 +5,30 @@ from .base import ( GitClone, GradleBuilder) -from kiwixbuild.utils import pj +from kiwixbuild.utils import pj, copy_tree +from kiwixbuild._global import option, get_target_step class KiwixAndroid(Dependency): name = "kiwix-android" - dependencies = ["Gradle", "kiwix-lib"] class Source(GitClone): git_remote = "https://github.com/kiwix/kiwix-android" git_dir = "kiwix-android" class Builder(GradleBuilder): + dependencies = ["kiwix-lib"] + + @classmethod + def get_dependencies(cls, platformInfo, allDeps): + if not allDeps: + return super().get_dependencies(platformInfo, allDeps) + else: + deps = [('android_{}'.format(arch), 'kiwix-lib') + for arch in option('android_arch')] + return deps + def build(self): - if self.buildEnv.options.targets == 'kiwix-android-custom': + if option('target') == 'kiwix-android-custom': print("SKIP") else: super().build() @@ -29,12 +40,25 @@ class KiwixAndroid(Dependency): shutil.rmtree(pj(self.build_path, 'kiwixlib', 'src', 'main')) except FileNotFoundError: pass - shutil.copytree(pj(self.buildEnv.install_dir, 'kiwix-lib'), - pj(self.build_path, 'kiwixlib', 'src', 'main')) + for arch in option('android_arch'): + try: + kiwix_builder = get_target_step('kiwix-lib', 'android_{}'.format(arch)) + except KeyError: + pass + else: + copy_tree(pj(kiwix_builder.buildEnv.install_dir, 'kiwix-lib'), + pj(self.build_path, 'kiwixlib', 'src', 'main')) os.makedirs( pj(self.build_path, 'app', 'src', 'main', 'assets', 'icu'), exist_ok=True) - shutil.copy2(pj(self.buildEnv.install_dir, 'share', 'icu', '58.2', - 'icudt58l.dat'), - pj(self.build_path, 'app', 'src', 'main', 'assets', - 'icu', 'icudt58l.dat')) + for arch in option('android_arch'): + try: + kiwix_builder = get_target_step('kiwix-lib', 'android_{}'.format(arch)) + except KeyError: + pass + else: + shutil.copy2(pj(kiwix_builder.buildEnv.install_dir, 'share', 'icu', '58.2', + 'icudt58l.dat'), + pj(self.build_path, 'app', 'src', 'main', 'assets', + 'icu', 'icudt58l.dat')) + break diff --git a/kiwixbuild/dependencies/kiwix_custom_app.py b/kiwixbuild/dependencies/kiwix_custom_app.py index 75d4058..9422301 100644 --- a/kiwixbuild/dependencies/kiwix_custom_app.py +++ b/kiwixbuild/dependencies/kiwix_custom_app.py @@ -7,20 +7,22 @@ from .base import ( GradleBuilder) from kiwixbuild.utils import Remotefile, pj, SkipCommand +from kiwixbuild._global import option, get_target_step class KiwixCustomApp(Dependency): name = "kiwix-android-custom" - dependencies = ["kiwix-android", "kiwix-lib"] def __init__(self, buildEnv): super().__init__(buildEnv) - self.custom_name = buildEnv.options.android_custom_app + self.custom_name = option('android_custom_app') class Source(GitClone): git_remote = "https://github.com/kiwix/kiwix-android-custom" git_dir = "kiwix-android-custom" class Builder(GradleBuilder): + dependencies = ["kiwix-android", "kiwix-lib"] + @property def gradle_target(self): return "assemble{}".format(self.target.custom_name) @@ -49,7 +51,7 @@ class KiwixCustomApp(Dependency): def _get_zim_size(self): try: - zim_size = self.buildEnv.options.zim_file_size + zim_size = option('zim_file_size') except AttributeError: with open(pj(self.source_path, self.target.custom_name, 'info.json')) as f: app_info = json.load(f) @@ -62,7 +64,7 @@ class KiwixCustomApp(Dependency): self.command('compile', self._compile) def _download_zim(self, context): - zim_url = self.buildEnv.options.zim_file_url + zim_url = option('zim_file_url') if zim_url is None: raise SkipCommand() with open(pj(self.source_path, self.target.custom_name, 'info.json')) as f: @@ -77,9 +79,9 @@ class KiwixCustomApp(Dependency): def _configure(self, context): # Copy kiwix-android in build dir. - kiwix_android_dep = self.buildEnv.targetsDict['kiwix-android'] + kiwix_android_source = get_target_step('kiwix-android', 'source') if not os.path.exists(self.build_path): - shutil.copytree(kiwix_android_dep.source_path, self.build_path) + shutil.copytree(kiwix_android_source.source_path, self.build_path) # Copy kiwix-lib application in build dir try: diff --git a/kiwixbuild/dependencies/kiwix_lib.py b/kiwixbuild/dependencies/kiwix_lib.py index 6d18a34..a72e5fe 100644 --- a/kiwixbuild/dependencies/kiwix_lib.py +++ b/kiwixbuild/dependencies/kiwix_lib.py @@ -2,35 +2,34 @@ from .base import ( Dependency, GitClone, MesonBuilder) +from kiwixbuild._global import neutralEnv class Kiwixlib(Dependency): name = "kiwix-lib" - @property - def dependencies(self): - base_dependencies = ["pugixml", "libzim", "zlib", "lzma", "libaria2"] - if ( self.buildEnv.platform_info.build != 'android' - and self.buildEnv.distname != 'Darwin'): - base_dependencies += ['ctpp2c', 'ctpp2'] - if self.buildEnv.platform_info.build != 'native': - return base_dependencies + ["icu4c_cross-compile"] - else: - return base_dependencies + ["icu4c"] - class Source(GitClone): git_remote = "https://github.com/kiwix/kiwix-lib.git" git_dir = "kiwix-lib" class Builder(MesonBuilder): + @classmethod + def get_dependencies(cls, platformInfo, allDeps): + base_dependencies = ["pugixml", "libzim", "zlib", "lzma", "libaria2", "icu4c"] + if (platformInfo.build != 'android' and + neutralEnv('distname') != 'Darwin'): + base_dependencies += ['ctpp2c', 'ctpp2'] + return base_dependencies + + @property def configure_option(self): base_option = "-Dctpp2-install-prefix={buildEnv.install_dir}" - if self.buildEnv.platform_info.build == 'android': + if self.buildEnv.platformInfo.build == 'android': base_option += ' -Dandroid=true' return base_option @property def library_type(self): - if self.buildEnv.platform_info.build == 'android': + if self.buildEnv.platformInfo.build == 'android': return 'shared' return super().library_type diff --git a/kiwixbuild/dependencies/kiwix_tools.py b/kiwixbuild/dependencies/kiwix_tools.py index 3ff33c0..cfdce90 100644 --- a/kiwixbuild/dependencies/kiwix_tools.py +++ b/kiwixbuild/dependencies/kiwix_tools.py @@ -5,15 +5,16 @@ from .base import ( class KiwixTools(Dependency): name = "kiwix-tools" - dependencies = ["kiwix-lib", "libmicrohttpd", "zlib"] class Source(GitClone): git_remote = "https://github.com/kiwix/kiwix-tools.git" git_dir = "kiwix-tools" class Builder(MesonBuilder): + dependencies = ["kiwix-lib", "libmicrohttpd", "zlib"] + @property def configure_option(self): - if self.buildEnv.platform_info.static: + if self.buildEnv.platformInfo.static: return "-Dstatic-linkage=true" return "" diff --git a/kiwixbuild/dependencies/libaria2.py b/kiwixbuild/dependencies/libaria2.py index 90f1567..e0a5d38 100644 --- a/kiwixbuild/dependencies/libaria2.py +++ b/kiwixbuild/dependencies/libaria2.py @@ -4,11 +4,10 @@ from .base import ( MakeBuilder ) -from kiwixbuild.utils import Remotefile +from kiwixbuild.utils import Remotefile, run_command class Aria2(Dependency): name = "libaria2" - dependencies = ['zlib'] class Source(ReleaseDownload): archive = Remotefile('libaria2-1.33.1.tar.gz', @@ -20,7 +19,8 @@ class Aria2(Dependency): def _post_prepare_script(self, context): context.try_skip(self.extract_path) command = "autoreconf -i" - self.neutralEnv.run_command(command, self.extract_path, context) + run_command(command, self.extract_path, context) class Builder(MakeBuilder): + dependencies = ['zlib'] configure_option = "--enable-libaria2 --disable-ssl --disable-bittorent --disable-metalink --without-sqlite3 --without-libxml2 --without-libexpat" diff --git a/kiwixbuild/dependencies/libmagic.py b/kiwixbuild/dependencies/libmagic.py index b7691a6..f25e9da 100644 --- a/kiwixbuild/dependencies/libmagic.py +++ b/kiwixbuild/dependencies/libmagic.py @@ -6,9 +6,10 @@ from .base import ( MakeBuilder, ) -from kiwixbuild.utils import Remotefile, pj, Defaultdict, SkipCommand +from kiwixbuild.utils import Remotefile, pj, Defaultdict, SkipCommand, run_command +from kiwixbuild._global import get_target_step -class LibMagicBase(Dependency): +class LibMagic(Dependency): name = "libmagic" class Source(ReleaseDownload): @@ -18,36 +19,24 @@ class LibMagicBase(Dependency): '1c52c8c3d271cd898d5511c36a68059cda94036111ab293f01f83c3525b737c6', 'https://fossies.org/linux/misc/file-5.33.tar.gz') - Builder = MakeBuilder + class Builder(MakeBuilder): + @classmethod + def get_dependencies(cls, platformInfo, allDeps): + if platformInfo.build != 'native': + return [('native_static', 'libmagic')] + return [] -class LibMagic_native(LibMagicBase): - name = "libmagic_native" - force_native_build = True - - class Builder(LibMagicBase.Builder): - static_configure_option = dynamic_configure_option = "--disable-shared --enable-static" - - @property - def build_path(self): - return super().build_path+"_native" - - def _install(self, context): - raise SkipCommand() - - -class LibMagic_cross_compile(LibMagicBase): - name = "libmagic_cross-compile" - dependencies = ['libmagic_native'] - - class Builder(LibMagicBase.Builder): def _compile(self, context): + platformInfo = self.buildEnv.platformInfo + if platformInfo.build == 'native': + return super()._compile(context) context.try_skip(self.build_path) command = "make -j4 {make_target} {make_option}".format( make_target=self.make_target, make_option=self.make_option ) - libmagic_native_dep = self.buildEnv.targetsDict['libmagic_native'] + libmagic_native_builder = get_target_step('libmagic', 'native_static') env = Defaultdict(str, os.environ) - env['PATH'] = ':'.join([pj(libmagic_native_dep.builder.build_path, 'src'), env['PATH']]) - self.buildEnv.run_command(command, self.build_path, context, env=env) + env['PATH'] = ':'.join([pj(libmagic_native_builder.build_path, 'src'), env['PATH']]) + run_command(command, self.build_path, context, buildEnv=self.buildEnv, env=env) diff --git a/kiwixbuild/dependencies/libzim.py b/kiwixbuild/dependencies/libzim.py index eb46df8..aa01108 100644 --- a/kiwixbuild/dependencies/libzim.py +++ b/kiwixbuild/dependencies/libzim.py @@ -6,17 +6,10 @@ from .base import ( class Libzim(Dependency): name = "libzim" - @property - def dependencies(self): - base_dependencies = ['zlib', 'lzma', 'xapian-core'] - if self.buildEnv.platform_info.build != 'native': - return base_dependencies + ["icu4c_cross-compile"] - else: - return base_dependencies + ["icu4c"] - class Source(GitClone): git_remote = "https://github.com/openzim/libzim.git" git_dir = "libzim" class Builder(MesonBuilder): test_option = "-t 8" + dependencies = ['zlib', 'lzma', 'xapian-core', 'icu4c'] diff --git a/kiwixbuild/dependencies/xapian.py b/kiwixbuild/dependencies/xapian.py index 13f9b4b..3d76cbc 100644 --- a/kiwixbuild/dependencies/xapian.py +++ b/kiwixbuild/dependencies/xapian.py @@ -5,6 +5,7 @@ from .base import ( ) from kiwixbuild.utils import Remotefile +from kiwixbuild._global import neutralEnv class Xapian(Dependency): @@ -19,10 +20,10 @@ class Xapian(Dependency): configure_env = {'_format_LDFLAGS': "-L{buildEnv.install_dir}/{buildEnv.libprefix}", '_format_CXXFLAGS': "-I{buildEnv.install_dir}/include"} - @property - def dependencies(self): - deps = ['zlib', 'lzma'] - if (self.buildEnv.platform_info.build == 'win32' - or self.buildEnv.distname == 'Darwin'): - return deps - return deps + ['uuid'] + @classmethod + def get_dependencies(cls, platformInfo, allDeps): + deps = ['zlib', 'lzma'] + if (platformInfo.build == 'win32' + or neutralEnv('distname') == 'Darwin'): + return deps + return deps + ['uuid'] diff --git a/kiwixbuild/dependencies/zim_tools.py b/kiwixbuild/dependencies/zim_tools.py index 0634390..46332e0 100644 --- a/kiwixbuild/dependencies/zim_tools.py +++ b/kiwixbuild/dependencies/zim_tools.py @@ -5,15 +5,16 @@ from .base import ( class ZimTools(Dependency): name = "zim-tools" - dependencies = ['libzim'] class Source(GitClone): git_remote = "https://github.com/openzim/zim-tools.git" git_dir = "zim-tools" class Builder(MesonBuilder): + dependencies = ['libzim'] + @property def configure_option(self): - if self.buildEnv.platform_info.static: + if self.buildEnv.platformInfo.static: return "-Dstatic-linkage=true" return "" diff --git a/kiwixbuild/dependencies/zimwriterfs.py b/kiwixbuild/dependencies/zimwriterfs.py index 075ef8e..3ac977c 100644 --- a/kiwixbuild/dependencies/zimwriterfs.py +++ b/kiwixbuild/dependencies/zimwriterfs.py @@ -6,23 +6,17 @@ from .base import ( class Zimwriterfs(Dependency): name = "zimwriterfs" - @property - def dependencies(self): - base_dependencies = ['libzim', 'zlib', 'xapian-core', 'gumbo'] - if self.buildEnv.platform_info.build != 'native': - return base_dependencies + ["icu4c_cross-compile", "libmagic_cross-compile"] - else: - return base_dependencies + ["icu4c", "libmagic"] - class Source(GitClone): git_remote = "https://github.com/openzim/zimwriterfs.git" git_dir = "zimwriterfs" release_git_ref = "1.1" class Builder(MesonBuilder): + dependencies = ['libzim', 'zlib', 'xapian-core', 'gumbo', 'icu4c', 'libmagic'] + @property def configure_option(self): base_option = "-Dmagic-install-prefix={buildEnv.install_dir}" - if self.buildEnv.platform_info.static: + if self.buildEnv.platformInfo.static: base_option += " -Dstatic-linkage=true" return base_option diff --git a/kiwixbuild/dependencies/zlib.py b/kiwixbuild/dependencies/zlib.py index 821eee6..babc294 100644 --- a/kiwixbuild/dependencies/zlib.py +++ b/kiwixbuild/dependencies/zlib.py @@ -28,16 +28,16 @@ class zlib(Dependency): def _configure(self, context): - if self.buildEnv.platform_info.build == 'win32': + if self.buildEnv.platformInfo.build == 'win32': raise SkipCommand() return super()._configure(context) @property def make_option(self): - if self.buildEnv.platform_info.build == 'win32': + if self.buildEnv.platformInfo.build == 'win32': return "--makefile win32/Makefile.gcc PREFIX={host}- SHARED_MODE={static} INCLUDE_PATH={include_path} LIBRARY_PATH={library_path} BINARY_PATH={binary_path}".format( host='i686-w64-mingw32', - static="0" if self.buildEnv.platform_info.static else "1", + static="0" if self.buildEnv.platformInfo.static else "1", include_path=pj(self.buildEnv.install_dir, 'include'), library_path=pj(self.buildEnv.install_dir, self.buildEnv.libprefix), binary_path=pj(self.buildEnv.install_dir, 'bin'), diff --git a/kiwixbuild/platforms/__init__.py b/kiwixbuild/platforms/__init__.py index 5975eca..4b63719 100644 --- a/kiwixbuild/platforms/__init__.py +++ b/kiwixbuild/platforms/__init__.py @@ -7,5 +7,6 @@ from . import ( i586, ios, native, + neutral, win32 ) diff --git a/kiwixbuild/platforms/android.py b/kiwixbuild/platforms/android.py index 6caa326..ce06276 100644 --- a/kiwixbuild/platforms/android.py +++ b/kiwixbuild/platforms/android.py @@ -1,27 +1,41 @@ -from .base import PlatformInfo +from .base import PlatformInfo, MetaPlatformInfo +from kiwixbuild.utils import pj +from kiwixbuild._global import get_target_step, option class AndroidPlatformInfo(PlatformInfo): - __arch_infos = { - 'arm' : ('arm-linux-androideabi', 'arm', 'armeabi'), - 'arm64': ('aarch64-linux-android', 'aarch64', 'arm64-v8a'), - 'mips': ('mipsel-linux-android', 'mipsel', 'mips'), - 'mips64': ('mips64el-linux-android', 'mips64el', 'mips64'), - 'x86': ('i686-linux-android', 'i686', 'x86'), - 'x86_64': ('x86_64-linux-android', 'x86_64', 'x86_64'), - } - - def __init__(self, name, arch): - super().__init__(name, 'android', True, ['android_ndk', 'android_sdk'], - hosts=['fedora', 'debian']) - self.arch = arch - self.arch_full, self.cpu, self.abi = self.__arch_infos[arch] + build = 'android' + static = True + toolchain_names = ['android-ndk'] + compatible_hosts = ['fedora', 'debian'] def __str__(self): return "android" + def binaries(self, install_path): + binaries = ((k,'{}-{}'.format(self.arch_full, v)) + for k, v in (('CC', 'gcc'), + ('CXX', 'g++'), + ('AR', 'ar'), + ('STRIP', 'strip'), + ('WINDRES', 'windres'), + ('RANLIB', 'ranlib'), + ('LD', 'ld')) + ) + return {k:pj(install_path, 'bin', v) + for k,v in binaries} + + @property + def ndk_builder(self): + return get_target_step('android-ndk', self.name) + def get_cross_config(self): + install_path = self.ndk_builder.install_path return { + 'exec_wrapper_def': '', + 'install_path': install_path, + 'binaries': self.binaries(install_path), + 'root_path': pj(install_path, 'sysroot'), 'extra_libs': [], 'extra_cflags': [], 'host_machine': { @@ -34,11 +48,99 @@ class AndroidPlatformInfo(PlatformInfo): }, } + def get_bin_dir(self): + return [pj(self.ndk_builder.install_path, 'bin')] + + def set_env(self, env): + root_path = pj(self.ndk_builder.install_path, 'sysroot') + env['PKG_CONFIG_LIBDIR'] = pj(root_path, 'lib', 'pkgconfig') + env['CFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(root_path) + env['CFLAGS'] + env['CXXFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(root_path) + env['CXXFLAGS'] + env['LDFLAGS'] = '--sysroot={} '.format(root_path) + env['LDFLAGS'] + #env['CFLAGS'] = ' -fPIC -D_FILE_OFFSET_BITS=64 -O3 '+env['CFLAGS'] + #env['CXXFLAGS'] = (' -D__OPTIMIZE__ -fno-strict-aliasing ' + # ' -DU_HAVE_NL_LANGINFO_CODESET=0 ' + # '-DU_STATIC_IMPLEMENTATION -O3 ' + # '-DU_HAVE_STD_STRING -DU_TIMEZONE=0 ')+env['CXXFLAGS'] + env['NDK_DEBUG'] = '0' + + def set_compiler(self, env): + binaries = self.binaries(self.ndk_builder.install_path) + env['CC'] = binaries['CC'] + env['CXX'] = binaries['CXX'] + + @property + def configure_option(self): + return '--host={}'.format(self.arch_full) + + def finalize_setup(self): + super().finalize_setup() + self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_android_cross_file.txt') + self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') -AndroidPlatformInfo('android_arm', 'arm') -AndroidPlatformInfo('android_arm64', 'arm64') -AndroidPlatformInfo('android_mips', 'mips') -AndroidPlatformInfo('android_mips64', 'mips64') -AndroidPlatformInfo('android_x86', 'x86') -AndroidPlatformInfo('android_x86_64', 'x86_64') +class AndroidArm(AndroidPlatformInfo): + name = 'android_arm' + arch = cpu = 'arm' + arch_full = 'arm-linux-androideabi' + abi = 'armeabi' + +class AndroidArm(AndroidPlatformInfo): + name = 'android_arm64' + arch = 'arm64' + arch_full = 'aarch64-linux-android' + cpu = 'aarch64' + abi = 'arm64-v8a' + +class AndroidArm(AndroidPlatformInfo): + name = 'android_mips' + arch = abi = 'mips' + arch_full = 'mipsel-linux-android' + cpu = 'mipsel' + +class AndroidArm(AndroidPlatformInfo): + name = 'android_mips64' + arch = abi = 'mips64' + arch_full = 'mips64el-linux-android' + cpu = 'mips64el' + +class AndroidArm(AndroidPlatformInfo): + name = 'android_x86' + arch = abi = 'x86' + arch_full = 'i686-linux-android' + cpu = 'i686' + +class AndroidArm(AndroidPlatformInfo): + name = 'android_x86_64' + arch = cpu = abi = 'x86_64' + arch_full = 'x86_64-linux-android' + +class Android(MetaPlatformInfo): + name = "android" + toolchain_names = ['android-sdk', 'gradle'] + compatible_hosts = ['fedora', 'debian'] + + @property + def subPlatformNames(self): + return ['android_{}'.format(arch) for arch in option('android_arch')] + + def add_targets(self, targetName, targets): + if targetName != 'kiwix-android': + return super().add_targets(targetName, targets) + else: + return AndroidPlatformInfo.add_targets(self, targetName, targets) + + def __str__(self): + return self.name + + @property + def sdk_builder(self): + return get_target_step('android-sdk', 'neutral') + + @property + def gradle_builder(self): + return get_target_step('gradle', 'neutral') + + def set_env(self, env): + env['ANDROID_HOME'] = self.sdk_builder.install_path + env['PATH'] = ':'.join([pj(self.gradle_builder.install_path, 'bin'), env['PATH']]) diff --git a/kiwixbuild/platforms/armhf.py b/kiwixbuild/platforms/armhf.py index 742d3cd..354af4b 100644 --- a/kiwixbuild/platforms/armhf.py +++ b/kiwixbuild/platforms/armhf.py @@ -1,12 +1,20 @@ from .base import PlatformInfo +from kiwixbuild.utils import pj +from kiwixbuild._global import get_target_step + class ArmhfPlatformInfo(PlatformInfo): - def __init__(self, name, static): - super().__init__(name, 'armhf', static, ['armhf_toolchain'], ['fedora', 'debian']) + build = 'armhf' + arch_full = 'arm-linux-gnueabihf' + toolchain_names = ['armhf'] + compatible_hosts = ['fedora', 'debian'] def get_cross_config(self): return { + 'binaries': self.binaries, + 'exec_wrapper_def': '', + 'root_path': self.root_path, 'extra_libs': [], 'extra_cflags': [], 'host_machine': { @@ -19,6 +27,70 @@ class ArmhfPlatformInfo(PlatformInfo): } } + @property + def tlc_source(self): + return get_target_step('armhf', 'source') -ArmhfPlatformInfo('armhf_dyn', False) -ArmhfPlatformInfo('armhf_static', True) + @property + def root_path(self): + return pj(self.tlc_source.source_path, + 'arm-bcm2708', + 'gcc-linaro-{}-raspbian-x64'.format(self.arch_full)) + + @property + def binaries(self): + binaries = ((k,'{}-{}'.format(self.arch_full, v)) + for k, v in (('CC', 'gcc'), + ('CXX', 'g++'), + ('AR', 'ar'), + ('STRIP', 'strip'), + ('WINDRES', 'windres'), + ('RANLIB', 'ranlib'), + ('LD', 'ld')) + ) + return {k:pj(self.root_path, 'bin', v) + for k,v in binaries} + + @property + def exec_wrapper_def(self): + try: + which('qemu-arm') + except subprocess.CalledProcessError: + return "" + else: + return "exec_wrapper = 'qemu-arm'" + + @property + def configure_option(self): + return '--host={}'.format(self.arch_full) + + def get_bin_dir(self): + return [pj(self.root_path, 'bin')] + + def set_env(self, env): + env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') + env['CFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS'] + env['CXXFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS'] + env['QEMU_LD_PREFIX'] = pj(self.root_path, "arm-linux-gnueabihf", "libc") + env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format( + ':'.join([ + pj(self.root_path, self.arch_full, "lib"), + env['LD_LIBRARY_PATH'] + ])) + + def set_compiler(self, env): + env['CC'] = self.binaries['CC'] + env['CXX'] = self.binaries['CXX'] + + def finalize_setup(self): + super().finalize_setup() + self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') + self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') + +class ArmhfDyn(ArmhfPlatformInfo): + name = 'armhf_dyn' + static = False + +class ArmhfStatic(ArmhfPlatformInfo): + name = 'armhf_static' + static = True diff --git a/kiwixbuild/platforms/base.py b/kiwixbuild/platforms/base.py index 6ddd37b..71edc13 100644 --- a/kiwixbuild/platforms/base.py +++ b/kiwixbuild/platforms/base.py @@ -1,21 +1,111 @@ +import os, sys +import subprocess + +from kiwixbuild.dependencies import Dependency +from kiwixbuild.utils import pj, remove_duplicates +from kiwixbuild.buildenv import BuildEnv +from kiwixbuild._global import neutralEnv, option, target_steps + +_SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +TEMPLATES_DIR = pj(os.path.dirname(_SCRIPT_DIR), 'templates') + +class _MetaPlatform(type): + def __new__(cls, name, bases, dct): + _class = type.__new__(cls, name, bases, dct) + if name not in ('PlatformInfo', 'MetaPlatformInfo') and 'name' in dct: + dep_name = dct['name'] + PlatformInfo.all_platforms[dep_name] = _class + return _class + - -class PlatformInfo: +class PlatformInfo(metaclass=_MetaPlatform): all_platforms = {} + all_running_platforms = {} + toolchain_names = [] + configure_option = "" - def __init__(self, name, build, static, toolchains, hosts=None): - self.all_platforms[name] = self - self.build = build - self.static = static - self.toolchains = toolchains - self.compatible_hosts = hosts + @classmethod + def get_platform(cls, name, targets=None): + if name not in cls.all_running_platforms: + if targets is None: + print("Should not got there.") + print(cls.all_running_platforms) + raise KeyError(name) + cls.all_running_platforms[name] = cls.all_platforms[name](targets) + return cls.all_running_platforms[name] + + def __init__(self, targets): + self.all_running_platforms[self.name] = self + self.buildEnv = BuildEnv(self) + self.setup_toolchains(targets) def __str__(self): return "{}_{}".format(self.build, 'static' if self.static else 'dyn') + def setup_toolchains(self, targets): + for tlc_name in self.toolchain_names: + ToolchainClass = Dependency.all_deps[tlc_name] + targets[('source', tlc_name)] = ToolchainClass.Source + plt_name = 'neutral' if ToolchainClass.neutral else self.name + targets[(plt_name, tlc_name)] = ToolchainClass.Builder + + def add_targets(self, targetName, targets): + if (self.name, targetName) in targets: + return [] + targetClass = Dependency.all_deps[targetName] + targets[('source', targetName)] = targetClass.Source + targets[(self.name, targetName)] = targetClass.Builder + for dep in targetClass.Builder.get_dependencies(self, False): + try: + depPlatformName, depName = dep + except ValueError: + depPlatformName, depName = self.name, dep + depPlatform = self.get_platform(depPlatformName, targets) + depPlatform.add_targets(depName, targets) + return [(self.name, targetName)] + + def get_cross_config(self): + return {} + def set_env(self, env): pass def get_bind_dir(self): return [] + + def set_compiler(self, env): + pass + + def _gen_crossfile(self, name): + crossfile = pj(self.buildEnv.build_dir, name) + template_file = pj(TEMPLATES_DIR, name) + with open(template_file, 'r') as f: + template = f.read() + content = template.format( + **self.get_cross_config() + ) + with open(crossfile, 'w') as outfile: + outfile.write(content) + return crossfile + + def finalize_setup(self): + self.buildEnv.cross_config = self.get_cross_config() + self.buildEnv.meson_crossfile = None + self.buildEnv.cmake_crossfile = None + + def clean_intermediate_directories(self): + self.buildEnv.clean_intermediate_directories() + + + +class MetaPlatformInfo(PlatformInfo): + subPlatformNames = [] + + def add_targets(self, targetName, targets): + targetDefs = [] + for platformName in self.subPlatformNames: + print("radd {}".format(platformName)) + platform = self.get_platform(platformName, targets) + targetDefs += platform.add_targets(targetName, targets) + return targetDefs diff --git a/kiwixbuild/platforms/i586.py b/kiwixbuild/platforms/i586.py index 262c97d..b52dd9e 100644 --- a/kiwixbuild/platforms/i586.py +++ b/kiwixbuild/platforms/i586.py @@ -1,13 +1,18 @@ +import os from .base import PlatformInfo +from kiwixbuild.utils import which class I586PlatformInfo(PlatformInfo): - def __init__(self, name, static): - super().__init__(name, 'i586', static, ['linux_i586_toolchain'], ['fedora', 'debian']) + build = 'i586' + arch_full = 'i586-linux-gnu' + compatible_hosts = ['fedora', 'debian'] def get_cross_config(self): return { + 'binaries': self.binaries, + 'exec_wrapper_def': '', 'extra_libs': ['-m32', '-march=i586', '-mno-sse'], 'extra_cflags': ['-m32', '-march=i586', '-mno-sse'], 'host_machine': { @@ -20,6 +25,39 @@ class I586PlatformInfo(PlatformInfo): } } + @property + def configure_option(self): + return '--host={}'.format(self.arch_full) -I586PlatformInfo('i586_dyn', False) -I586PlatformInfo('i586_static', True) + @property + def binaries(self): + return {k:which(v) + for k, v in (('CC', os.environ.get('CC', 'gcc')), + ('CXX', os.environ.get('CXX', 'g++')), + ('AR', 'ar'), + ('STRIP', 'strip'), + ('RANLIB', 'ranlib'), + ('LD', 'ld')) + } + + def set_env(self, env): + env['CFLAGS'] = "-m32 -march=i586 -mno-sse "+env['CFLAGS'] + env['CXXFLAGS'] = "-m32 -march=i586 -mno-sse "+env['CXXFLAGS'] + env['LDFLAGS'] = "-m32 -march=i586 -mno-sse "+env['LDFLAGS'] + + def get_bin_dir(self): + return [] + + + def finalize_setup(self): + super().finalize_setup() + self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_i586_cross_file.txt') + self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') + +class I586Dyn(I586PlatformInfo): + name = 'i586_dyn' + static = False + +class I586Static(I586PlatformInfo): + name = 'i586_static' + static = True diff --git a/kiwixbuild/platforms/ios.py b/kiwixbuild/platforms/ios.py index 86a9b55..1459741 100644 --- a/kiwixbuild/platforms/ios.py +++ b/kiwixbuild/platforms/ios.py @@ -1,22 +1,17 @@ -from .base import PlatformInfo import subprocess - +from .base import PlatformInfo, MetaPlatformInfo +from kiwixbuild.utils import pj, xrun_find +from kiwixbuild._global import option class iOSPlatformInfo(PlatformInfo): - __arch_infos = { - 'armv7': ('arm-apple-darwin', 'armv7', 'iphoneos'), - 'arm64': ('arm-apple-darwin', 'arm64', 'iphoneos'), - 'i386': ('', 'i386', 'iphonesimulator'), - 'x86_64': ('', 'x86_64', 'iphonesimulator'), - } + build = 'iOS' + static = True + compatible_hosts = ['Darwin'] - def __init__(self, name, arch): - super().__init__(name, 'iOS', True, ['iOS_sdk'], - hosts=['Darwin']) - self.arch = arch - self.arch_full, self.cpu, self.sdk_name = self.__arch_infos[arch] + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self._root_path = None @property @@ -29,9 +24,16 @@ class iOSPlatformInfo(PlatformInfo): def __str__(self): return "iOS" + def finalize_setup(self): + super().finalize_setup() + self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_ios_cross_file.txt') + self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') + def get_cross_config(self): return { 'root_path': self.root_path, + 'binaries': self.binaries, + 'exec_wrapper_def': '', 'extra_libs': ['-fembed-bitcode', '-isysroot', self.root_path, '-arch', self.arch, '-miphoneos-version-min=9.0', '-stdlib=libc++'], 'extra_cflags': ['-fembed-bitcode', '-isysroot', self.root_path, '-arch', self.arch, '-miphoneos-version-min=9.0', '-stdlib=libc++'], 'host_machine': { @@ -53,7 +55,61 @@ class iOSPlatformInfo(PlatformInfo): def get_bin_dir(self): return [pj(self.root_path, 'bin')] -iOSPlatformInfo('iOS_armv7', 'armv7') -iOSPlatformInfo('iOS_arm64', 'arm64') -iOSPlatformInfo('iOS_i386', 'i386') -iOSPlatformInfo('iOS_x86_64', 'x86_64') + @property + def binaries(self): + return { + 'CC': xrun_find('clang'), + 'CXX': xrun_find('clang++'), + 'AR': '/usr/bin/ar', + 'STRIP': '/usr/bin/strip', + 'RANLIB': '/usr/bin/ranlib', + 'LD': '/usr/bin/ld', + } + + @property + def configure_option(self): + return '--host={}'.format(self.arch_full) + + def set_compiler(self, env): + env['CC'] = self.binaries['CC'] + env['CXX'] = self.binaries['CXX'] + + +class iOSArmv7(iOSPlatformInfo): + name = 'iOS_armv7' + arch = cpu = 'armv7' + arch_full = 'arm-apple-darwin' + sdk_name = 'iphoneos' + +class iOSArm64(iOSPlatformInfo): + name = 'iOS_arm64' + arch = cpu = 'arm64' + arch_full = 'aarch64-apple-darwin' + sdk_name = 'iphoneos' + +class iOSi386(iOSPlatformInfo): + name = 'iOS_i386' + arch = cpu = 'i386' + arch_full = 'i386-apple-darwin' + sdk_name = 'iphonesimulator' + +class iOSx64(iOSPlatformInfo): + name = 'iOS_x86_64' + arch = cpu = 'x86_64' + arch_full = 'x86_64-apple-darwin' + sdk_name = 'iphonesimulator' + +class IOS(MetaPlatformInfo): + name = "iOS_multi" + compatible_hosts = ['Darwin'] + + @property + def subPlatformNames(self): + return ['iOS_{}'.format(arch) for arch in option('ios_arch')] + + def add_targets(self, targetName, targets): + super().add_targets(targetName, targets) + return PlatformInfo.add_targets(self, '_ios_fat_lib', targets) + + def __str__(self): + return self.name diff --git a/kiwixbuild/platforms/native.py b/kiwixbuild/platforms/native.py index 9a23b10..069d9f2 100644 --- a/kiwixbuild/platforms/native.py +++ b/kiwixbuild/platforms/native.py @@ -2,12 +2,15 @@ from .base import PlatformInfo class NativePlatformInfo(PlatformInfo): - def __init__(self, name, static, hosts): - super().__init__(name, 'native', static, [], hosts) - - def get_cross_config(self): - return {} + build = 'native' -NativePlatformInfo('native_dyn', False, ['fedora', 'debian', 'Darwin']) -NativePlatformInfo('native_static', True, ['fedora', 'debian']) +class NativeDyn(NativePlatformInfo): + name = 'native_dyn' + static = False + compatible_hosts = ['fedora', 'debian', 'Darwin'] + +class NativeStatic(NativePlatformInfo): + name = 'native_static' + static = True + compatible_hosts = ['fedora', 'debian'] diff --git a/kiwixbuild/platforms/neutral.py b/kiwixbuild/platforms/neutral.py new file mode 100644 index 0000000..bbe339c --- /dev/null +++ b/kiwixbuild/platforms/neutral.py @@ -0,0 +1,9 @@ +from .base import PlatformInfo + +class NeutralPlatformInfo(PlatformInfo): + name = 'neutral' + static = '' + compatible_hosts = ['fedora', 'debian', 'Darwin'] + + def __str__(self): + return "neutral" diff --git a/kiwixbuild/platforms/win32.py b/kiwixbuild/platforms/win32.py index 5d6b643..81ec903 100644 --- a/kiwixbuild/platforms/win32.py +++ b/kiwixbuild/platforms/win32.py @@ -1,13 +1,21 @@ +import subprocess + from .base import PlatformInfo +from kiwixbuild.utils import which, pj +from kiwixbuild._global import neutralEnv class Win32PlatformInfo(PlatformInfo): extra_libs = ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4', '-lmsvcr90', '-liphlpapi'] - def __init__(self, name, static): - super().__init__(name, 'win32', static, ['mingw32_toolchain'], ['fedora', 'debian']) + build = 'win32' + compatible_hosts = ['fedora', 'debian'] + arch_full = 'i686-w64-mingw32' def get_cross_config(self): return { + 'exec_wrapper_def': self.exec_wrapper_def, + 'binaries': self.binaries, + 'root_path': self.root_path, 'extra_libs': self.extra_libs, 'extra_cflags': ['-DWIN32'], 'host_machine': { @@ -20,5 +28,58 @@ class Win32PlatformInfo(PlatformInfo): } } -Win32PlatformInfo('win32_dyn', False) -Win32PlatformInfo('win32_static', True) + def finalize_setup(self): + super().finalize_setup() + self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') + self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') + + @property + def root_path(self): + root_paths = { + 'fedora': '/usr/i686-w64-mingw32/sys-root/mingw', + 'debian': '/usr/i686-w64-mingw32' + } + return root_paths[neutralEnv('distname')] + + @property + def binaries(self): + return {k:which('{}-{}'.format(self.arch_full, v)) + for k, v in (('CC', 'gcc'), + ('CXX', 'g++'), + ('AR', 'ar'), + ('STRIP', 'strip'), + ('WINDRES', 'windres'), + ('RANLIB', 'ranlib')) + } + + @property + def exec_wrapper_def(self): + try: + which('wine') + except subprocess.CalledProcessError: + return "" + else: + return "exec_wrapper = 'wine'" + + @property + def configure_option(self): + return '--host={}'.format(self.arch_full) + + def set_compiler(self, env): + for k, v in self.binaries.items(): + env[k] = v + + def get_bin_dir(self): + return [pj(self.root_path, 'bin')] + + def set_env(self, env): + env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') + env['LIBS'] = " ".join(self.extra_libs) + " " +env['LIBS'] + +class Win32Dyn(Win32PlatformInfo): + name = 'win32_dyn' + static = False + +class Win32Static(Win32PlatformInfo): + name = 'win32_static' + static = True diff --git a/kiwixbuild/templates/cmake_android_cross_file.txt b/kiwixbuild/templates/cmake_android_cross_file.txt index 7f80a34..09f121e 100644 --- a/kiwixbuild/templates/cmake_android_cross_file.txt +++ b/kiwixbuild/templates/cmake_android_cross_file.txt @@ -1,7 +1,8 @@ SET(CMAKE_SYSTEM_NAME {host_machine[system]}) -SET(CMAKE_ANDROID_STANDALONE_TOOLCHAIN {toolchain.builder.install_path}) -SET(CMAKE_C_COMPILER "{toolchain.binaries[CC]}") -SET(CMAKE_CXX_COMPILER "{toolchain.binaries[CXX]}") +SET(CMAKE_ANDROID_STANDALONE_TOOLCHAIN {install_path}) -SET(CMAKE_FIND_ROOT_PATH {toolchain.root_path}) +SET(CMAKE_C_COMPILER "{binaries[CC]}") +SET(CMAKE_CXX_COMPILER "{binaries[CXX]}") + +SET(CMAKE_FIND_ROOT_PATH {root_path}) diff --git a/kiwixbuild/templates/cmake_cross_file.txt b/kiwixbuild/templates/cmake_cross_file.txt index 70184cf..f236854 100644 --- a/kiwixbuild/templates/cmake_cross_file.txt +++ b/kiwixbuild/templates/cmake_cross_file.txt @@ -2,11 +2,11 @@ SET(CMAKE_SYSTEM_NAME {host_machine[system]}) SET(CMAKE_SYSTEM_PROCESSOR {host_machine[cpu_family]}) # specify the cross compiler -SET(CMAKE_C_COMPILER "{toolchain.binaries[CC]}") -SET(CMAKE_CXX_COMPILER "{toolchain.binaries[CXX]}") -SET(CMAKE_RC_COMPILER {toolchain.binaries[WINDRES]}) -SET(CMAKE_AR:FILEPATH {toolchain.binaries[AR]}) -SET(CMAKE_RANLIB:FILEPATH {toolchain.binaries[RANLIB]}) +SET(CMAKE_C_COMPILER "{binaries[CC]}") +SET(CMAKE_CXX_COMPILER "{binaries[CXX]}") +SET(CMAKE_RC_COMPILER {binaries[WINDRES]}) +SET(CMAKE_AR:FILEPATH {binaries[AR]}) +SET(CMAKE_RANLIB:FILEPATH {binaries[RANLIB]}) find_program(CCACHE_FOUND ccache) if(CCACHE_FOUND) @@ -15,7 +15,7 @@ if(CCACHE_FOUND) endif(CCACHE_FOUND) # where is the target environment -SET(CMAKE_FIND_ROOT_PATH {toolchain.root_path}) +SET(CMAKE_FIND_ROOT_PATH {root_path}) # search for programs in the build host directories SET(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) diff --git a/kiwixbuild/templates/cmake_i586_cross_file.txt b/kiwixbuild/templates/cmake_i586_cross_file.txt index 03d186a..037d783 100644 --- a/kiwixbuild/templates/cmake_i586_cross_file.txt +++ b/kiwixbuild/templates/cmake_i586_cross_file.txt @@ -2,13 +2,13 @@ SET(CMAKE_SYSTEM_NAME {host_machine[system]}) SET(CMAKE_SYSTEM_PROCESSOR {host_machine[cpu_family]}) # specify the cross compiler -SET(CMAKE_C_COMPILER "{toolchain.binaries[CC]}") -SET(CMAKE_CXX_COMPILER "{toolchain.binaries[CXX]}") +SET(CMAKE_C_COMPILER "{binaries[CC]}") +SET(CMAKE_CXX_COMPILER "{binaries[CXX]}") SET(C_FLAGS "-m32 -march=i586") SET(CXX_FLAGS "-m32 -march=i586") SET(CMAKE_LD_FLAGS "-m32 -march=i586") -SET(CMAKE_AR:FILEPATH {toolchain.binaries[AR]}) -SET(CMAKE_RANLIB:FILEPATH {toolchain.binaries[RANLIB]}) +SET(CMAKE_AR:FILEPATH {binaries[AR]}) +SET(CMAKE_RANLIB:FILEPATH {binaries[RANLIB]}) find_program(CCACHE_FOUND ccache) if(CCACHE_FOUND) diff --git a/kiwixbuild/templates/cmake_ios_cross_file.txt b/kiwixbuild/templates/cmake_ios_cross_file.txt index 7ceaa46..41ccd60 100644 --- a/kiwixbuild/templates/cmake_ios_cross_file.txt +++ b/kiwixbuild/templates/cmake_ios_cross_file.txt @@ -1,7 +1,7 @@ SET(CMAKE_SYSTEM_NAME {host_machine[system]}) -SET(CMAKE_C_COMPILER "{toolchain.binaries[CC]}") -SET(CMAKE_CXX_COMPILER "{toolchain.binaries[CXX]}") +SET(CMAKE_C_COMPILER "{binaries[CC]}") +SET(CMAKE_CXX_COMPILER "{binaries[CXX]}") SET(CMAKE_FIND_ROOT_PATH {root_path}) diff --git a/kiwixbuild/templates/meson_android_cross_file.txt b/kiwixbuild/templates/meson_android_cross_file.txt index d0710b4..2d2f6d6 100644 --- a/kiwixbuild/templates/meson_android_cross_file.txt +++ b/kiwixbuild/templates/meson_android_cross_file.txt @@ -1,9 +1,9 @@ [binaries] pkgconfig = 'pkg-config' -c = '{toolchain.binaries[CC]}' -ar = '{toolchain.binaries[AR]}' -cpp = '{toolchain.binaries[CXX]}' -strip = '{toolchain.binaries[STRIP]}' +c = '{binaries[CC]}' +ar = '{binaries[AR]}' +cpp = '{binaries[CXX]}' +strip = '{binaries[STRIP]}' [properties] c_link_args = {extra_libs!r} diff --git a/kiwixbuild/templates/meson_cross_file.txt b/kiwixbuild/templates/meson_cross_file.txt index 069b2ee..eef32f1 100644 --- a/kiwixbuild/templates/meson_cross_file.txt +++ b/kiwixbuild/templates/meson_cross_file.txt @@ -1,10 +1,10 @@ [binaries] pkgconfig = 'pkg-config' -c = '{toolchain.binaries[CC]}' -ar = '{toolchain.binaries[AR]}' -cpp = '{toolchain.binaries[CXX]}' -strip = '{toolchain.binaries[STRIP]}' -{toolchain.exec_wrapper_def} +c = '{binaries[CC]}' +ar = '{binaries[AR]}' +cpp = '{binaries[CXX]}' +strip = '{binaries[STRIP]}' +{exec_wrapper_def} [properties] c_link_args = {extra_libs!r} diff --git a/kiwixbuild/toolchains/__init__.py b/kiwixbuild/toolchains/__init__.py deleted file mode 100644 index d0ca8b1..0000000 --- a/kiwixbuild/toolchains/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ - -from . import android_ndk, android_sdk, armhf, ios, mingw32, linux_i586 - -from .base_toolchain import Toolchain diff --git a/kiwixbuild/toolchains/android_ndk.py b/kiwixbuild/toolchains/android_ndk.py deleted file mode 100644 index 8a3f3d9..0000000 --- a/kiwixbuild/toolchains/android_ndk.py +++ /dev/null @@ -1,134 +0,0 @@ -import os - -from .base_toolchain import Toolchain -from kiwixbuild.dependencies import ReleaseDownload, Builder -from kiwixbuild.utils import Remotefile, add_execution_right - -pj = os.path.join - -class android_ndk(Toolchain): - neutral = False - name = 'android-ndk' - version = 'r13b' - gccver = '4.9.x' - - @property - def api(self): - return '21' if self.arch in ('arm64', 'mips64', 'x86_64') else '14' - - @property - def platform(self): - return 'android-'+self.api - - @property - def arch(self): - return self.buildEnv.platform_info.arch - - @property - def arch_full(self): - return self.buildEnv.platform_info.arch_full - - @property - def toolchain(self): - return self.arch_full+"-4.9" - - @property - def root_path(self): - return pj(self.builder.install_path, 'sysroot') - - @property - def binaries(self): - binaries = ((k,'{}-{}'.format(self.arch_full, v)) - for k, v in (('CC', 'gcc'), - ('CXX', 'g++'), - ('AR', 'ar'), - ('STRIP', 'strip'), - ('WINDRES', 'windres'), - ('RANLIB', 'ranlib'), - ('LD', 'ld')) - ) - return {k:pj(self.builder.install_path, 'bin', v) - for k,v in binaries} - - @property - def configure_option(self): - return '--host={}'.format(self.arch_full) - - @property - def full_name(self): - return "{name}-{version}-{arch}-{api}".format( - name = self.name, - version = self.version, - arch = self.arch, - api = self.api) - - class Source(ReleaseDownload): - archive = Remotefile('android-ndk-r13b-linux-x86_64.zip', - '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c', - 'https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip') - - @property - def source_dir(self): - return "{}-{}".format( - self.target.name, - self.target.version) - - - class Builder(Builder): - - @property - def install_path(self): - return self.build_path - - def _build_platform(self, context): - context.try_skip(self.build_path) - script = pj(self.source_path, 'build/tools/make_standalone_toolchain.py') - add_execution_right(script) - command = '{script} --arch={arch} --api={api} --install-dir={install_dir} --force' - command = command.format( - script=script, - arch=self.target.arch, - api=self.target.api, - install_dir=self.install_path - ) - self.buildEnv.run_command(command, self.build_path, context) - - def _fix_permission_right(self, context): - context.try_skip(self.build_path) - bin_dirs = [pj(self.install_path, 'bin'), - pj(self.install_path, self.target.arch_full, 'bin'), - pj(self.install_path, 'libexec', 'gcc', self.target.arch_full, self.target.gccver) - ] - for root, dirs, files in os.walk(self.install_path): - if not root in bin_dirs: - continue - - for file_ in files: - file_path = pj(root, file_) - if os.path.islink(file_path): - continue - add_execution_right(file_path) - - def build(self): - self.command('build_platform', self._build_platform) - self.command('fix_permission_right', self._fix_permission_right) - - def get_bin_dir(self): - return [pj(self.builder.install_path, 'bin')] - - def set_env(self, env): - env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') - env['CFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(self.root_path) + env['CFLAGS'] - env['CXXFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(self.root_path) + env['CXXFLAGS'] - env['LDFLAGS'] = '--sysroot={} '.format(self.root_path) + env['LDFLAGS'] - #env['CFLAGS'] = ' -fPIC -D_FILE_OFFSET_BITS=64 -O3 '+env['CFLAGS'] - #env['CXXFLAGS'] = (' -D__OPTIMIZE__ -fno-strict-aliasing ' - # ' -DU_HAVE_NL_LANGINFO_CODESET=0 ' - # '-DU_STATIC_IMPLEMENTATION -O3 ' - # '-DU_HAVE_STD_STRING -DU_TIMEZONE=0 ')+env['CXXFLAGS'] - env['NDK_DEBUG'] = '0' - - def set_compiler(self, env): - env['CC'] = self.binaries['CC'] - env['CXX'] = self.binaries['CXX'] - diff --git a/kiwixbuild/toolchains/armhf.py b/kiwixbuild/toolchains/armhf.py deleted file mode 100644 index 1975665..0000000 --- a/kiwixbuild/toolchains/armhf.py +++ /dev/null @@ -1,64 +0,0 @@ -import os -import subprocess - -from .base_toolchain import Toolchain -from kiwixbuild.dependencies import GitClone -from kiwixbuild.utils import which -pj = os.path.join - -class armhf_toolchain(Toolchain): - name = 'armhf' - arch_full = 'arm-linux-gnueabihf' - - class Source(GitClone): - git_remote = "https://github.com/raspberrypi/tools" - git_dir = "raspberrypi-tools" - - @property - def root_path(self): - return pj(self.source_path, 'arm-bcm2708', 'gcc-linaro-arm-linux-gnueabihf-raspbian-x64') - - @property - def binaries(self): - binaries = ((k,'{}-{}'.format(self.arch_full, v)) - for k, v in (('CC', 'gcc'), - ('CXX', 'g++'), - ('AR', 'ar'), - ('STRIP', 'strip'), - ('WINDRES', 'windres'), - ('RANLIB', 'ranlib'), - ('LD', 'ld')) - ) - return {k:pj(self.root_path, 'bin', v) - for k,v in binaries} - - @property - def exec_wrapper_def(self): - try: - which('qemu-arm') - except subprocess.CalledProcessError: - return "" - else: - return "exec_wrapper = 'qemu-arm'" - - @property - def configure_option(self): - return '--host={}'.format(self.arch_full) - - def get_bin_dir(self): - return [pj(self.root_path, 'bin')] - - def set_env(self, env): - env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') - env['CFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS'] - env['CXXFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS'] - env['QEMU_LD_PREFIX'] = pj(self.root_path, "arm-linux-gnueabihf", "libc") - env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format( - ':'.join([ - pj(self.root_path, "arm-linux-gnueabihf", "lib"), - env['LD_LIBRARY_PATH'] - ])) - - def set_compiler(self, env): - env['CC'] = self.binaries['CC'] - env['CXX'] = self.binaries['CXX'] diff --git a/kiwixbuild/toolchains/base_toolchain.py b/kiwixbuild/toolchains/base_toolchain.py deleted file mode 100644 index e5040cd..0000000 --- a/kiwixbuild/toolchains/base_toolchain.py +++ /dev/null @@ -1,75 +0,0 @@ -import os -import subprocess - -pj = os.path.join - -from kiwixbuild.utils import Context, SkipCommand, StopBuild - -class _MetaToolchain(type): - def __new__(cls, name, bases, dct): - _class = type.__new__(cls, name, bases, dct) - if name != 'Toolchain': - Toolchain.all_toolchains[name] = _class - return _class - - -class Toolchain(metaclass=_MetaToolchain): - neutral = True - all_toolchains = {} - configure_option = "" - cmake_option = "" - exec_wrapper_def = "" - Builder = None - Source = None - - def __init__(self, neutralEnv): - self.neutralEnv = neutralEnv - self.buildEnv = neutralEnv - self.source = self.Source(self) if self.Source else None - self.builder = self.Builder(self) if self.Builder else None - - @property - def full_name(self): - return "{name}-{version}".format( - name = self.name, - version = self.version) - - @property - def source_path(self): - return pj(self.neutralEnv.source_dir, self.source.source_dir) - - @property - def _log_dir(self): - return self.neutralEnv.log_dir - - def set_env(self, env): - pass - - def set_compiler(self, env): - pass - - def get_bin_dir(self): - return [] - - def command(self, name, function, *args): - print(" {} {} : ".format(name, self.name), end="", flush=True) - log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name)) - context = Context(name, log, True) - try: - ret = function(*args, context=context) - context._finalise() - print("OK") - return ret - except SkipCommand: - print("SKIP") - except subprocess.CalledProcessError: - print("ERROR") - try: - with open(log, 'r') as f: - print(f.read()) - except: - pass - raise StopBuild() - except: - print("ERROR") - raise diff --git a/kiwixbuild/toolchains/ios.py b/kiwixbuild/toolchains/ios.py deleted file mode 100644 index f1dd4be..0000000 --- a/kiwixbuild/toolchains/ios.py +++ /dev/null @@ -1,23 +0,0 @@ -from .base_toolchain import Toolchain - -from kiwixbuild.utils import pj, xrun_find - -class iOS_sdk(Toolchain): - @property - def binaries(self): - return { - 'CC': xrun_find('clang'), - 'CXX': xrun_find('clang++'), - 'AR': '/usr/bin/ar', - 'STRIP': '/usr/bin/strip', - 'RANLIB': '/usr/bin/ranlib', - 'LD': '/usr/bin/ld', - } - - @property - def configure_option(self): - return '--host=arm-apple-darwin' - - def set_compiler(self, env): - env['CC'] = self.binaries['CC'] - env['CXX'] = self.binaries['CXX'] diff --git a/kiwixbuild/toolchains/linux_i586.py b/kiwixbuild/toolchains/linux_i586.py deleted file mode 100644 index ce89dd8..0000000 --- a/kiwixbuild/toolchains/linux_i586.py +++ /dev/null @@ -1,37 +0,0 @@ -import os - -from .base_toolchain import Toolchain -from kiwixbuild.dependencies import GitClone -from kiwixbuild.utils import which -pj = os.path.join - -class linux_i586_toolchain(Toolchain): - name = 'linux_i586' - arch_full = 'i586-linux-gnu' - - @property - def configure_option(self): - return '--host={}'.format(self.arch_full) - - @property - def binaries(self): - return {k:which(v) - for k, v in (('CC', os.environ.get('CC', 'gcc')), - ('CXX', os.environ.get('CXX', 'g++')), - ('AR', 'ar'), - ('STRIP', 'strip'), - ('RANLIB', 'ranlib'), - ('LD', 'ld')) - } - - @property - def configure_option(self): - return '--host={}'.format(self.arch_full) - - def set_env(self, env): - env['CFLAGS'] = "-m32 -march=i586 -mno-sse "+env['CFLAGS'] - env['CXXFLAGS'] = "-m32 -march=i586 -mno-sse "+env['CXXFLAGS'] - env['LDFLAGS'] = "-m32 -march=i586 -mno-sse "+env['LDFLAGS'] - - def get_bin_dir(self): - return [] diff --git a/kiwixbuild/toolchains/mingw32.py b/kiwixbuild/toolchains/mingw32.py deleted file mode 100644 index 7da1283..0000000 --- a/kiwixbuild/toolchains/mingw32.py +++ /dev/null @@ -1,55 +0,0 @@ -import os -import subprocess - -from .base_toolchain import Toolchain -from kiwixbuild.utils import which - -pj = os.path.join - -class mingw32_toolchain(Toolchain): - name = 'mingw32' - arch_full = 'i686-w64-mingw32' - extra_libs = ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4', '-lmsvcr90', '-liphlpapi'] - - @property - def root_path(self): - root_paths = { - 'fedora': '/usr/i686-w64-mingw32/sys-root/mingw', - 'debian': '/usr/i686-w64-mingw32' - } - return root_paths[self.neutralEnv.distname] - - @property - def binaries(self): - return {k:which('{}-{}'.format(self.arch_full, v)) - for k, v in (('CC', 'gcc'), - ('CXX', 'g++'), - ('AR', 'ar'), - ('STRIP', 'strip'), - ('WINDRES', 'windres'), - ('RANLIB', 'ranlib')) - } - - @property - def exec_wrapper_def(self): - try: - which('wine') - except subprocess.CalledProcessError: - return "" - else: - return "exec_wrapper = 'wine'" - - @property - def configure_option(self): - return '--host={}'.format(self.arch_full) - - def set_compiler(self, env): - for k, v in self.binaries.items(): - env[k] = v - - def get_bin_dir(self): - return [pj(self.root_path, 'bin')] - - def set_env(self, env): - env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') - env['LIBS'] = " ".join(self.extra_libs) + " " +env['LIBS'] diff --git a/kiwixbuild/utils.py b/kiwixbuild/utils.py index 34d6ae4..b76dc5a 100644 --- a/kiwixbuild/utils.py +++ b/kiwixbuild/utils.py @@ -10,9 +10,10 @@ import ssl import subprocess from collections import namedtuple, defaultdict +from kiwixbuild._global import neutralEnv, option + pj = os.path.join -g_print_progress = True REMOTE_PREFIX = 'http://download.kiwix.org/dev/' @@ -28,11 +29,6 @@ def xrun_find(name): return output[:-1].decode() -def setup_print_progress(print_progress): - global g_print_progress - g_print_progress = print_progress - - class Defaultdict(defaultdict): def __getattr__(self, name): return self[name] @@ -67,7 +63,7 @@ def get_sha256(path): def print_progress(progress): - if g_print_progress: + if option('show_progress'): text = "{}\033[{}D".format(progress, len(progress)) print(text, end="") @@ -90,7 +86,7 @@ def copy_tree(src, dst, post_copy_function=None): post_copy_function(dstfile) -def download_remote(what, where, check_certificate=True): +def download_remote(what, where): file_path = pj(where, what.name) file_url = what.url or (REMOTE_PREFIX + what.name) if os.path.exists(file_path): @@ -98,7 +94,7 @@ def download_remote(what, where, check_certificate=True): raise SkipCommand() os.remove(file_path) - if not check_certificate: + if option('no_cert_check'): context = ssl.create_default_context() context.check_hostname = False context.verify_mode = ssl.CERT_NONE @@ -221,3 +217,41 @@ def extract_archive(archive_path, dest_dir, topdir=None, name=None): if archive is not None: archive.close() + +def run_command(command, cwd, context, buildEnv=None, env=None, input=None, cross_env_only=False): + os.makedirs(cwd, exist_ok=True) + if env is None: + env = Defaultdict(str, os.environ) + if buildEnv is not None: + cross_compile_env = True + cross_compile_compiler = True + cross_compile_path = True + if context.force_native_build: + cross_compile_env = False + cross_compile_compiler = False + cross_compile_path = False + if cross_env_only: + cross_compile_compiler = False + env = buildEnv._set_env(env, cross_compile_env, cross_compile_compiler, cross_compile_path) + log = None + try: + if not option('verbose'): + log = open(context.log_file, 'w') + print("run command '{}'".format(command), file=log) + print("current directory is '{}'".format(cwd), file=log) + print("env is :", file=log) + for k, v in env.items(): + print(" {} : {!r}".format(k, v), file=log) + + kwargs = dict() + if input: + kwargs['stdin'] = subprocess.PIPE + process = subprocess.Popen(command, shell=True, cwd=cwd, env=env, stdout=log or sys.stdout, stderr=subprocess.STDOUT, **kwargs) + if input: + process.communicate(input.encode()) + retcode = process.wait() + if retcode: + raise subprocess.CalledProcessError(retcode, command) + finally: + if log: + log.close() diff --git a/kiwixbuild/versions.py b/kiwixbuild/versions.py index e2253ec..d018329 100644 --- a/kiwixbuild/versions.py +++ b/kiwixbuild/versions.py @@ -10,7 +10,7 @@ main_project_versions = { # This is the "version" of the whole base_deps_versions dict. # Change this when you change base_deps_versions. -base_deps_meta_version = '3' +base_deps_meta_version = '5' base_deps_versions = { @@ -23,7 +23,9 @@ base_deps_versions = { 'libmicrohttpd' : '0.9.46', 'gumbo' : '0.10.1', 'icu4c' : '58.2', - 'Gradle' : '4.6', + 'gradle' : '4.6', 'libaria2' : '1.33.1', - 'libmagic' : '5.33' + 'libmagic' : '5.33', + 'android-sdk' : 'r25.2.3', + 'android-ndk' : 'r13b' } diff --git a/travis/compile_all.py b/travis/compile_all.py index 265d446..d4716ad 100755 --- a/travis/compile_all.py +++ b/travis/compile_all.py @@ -66,8 +66,14 @@ def run_kiwix_build(target, platform, build_deps_only=False, make_release=False, command = ['kiwix-build'] command.append(target) command.append('--hide-progress') - command.append('--force-install-packages') - command.extend(['--target-platform', platform]) + if target == 'kiwix-android' and platform.startswith('android_'): + command.extend(['--target-platform', 'android', '--android-arch', platform[8:]]) + elif platform == 'android': + command.extend(['--target-platform', 'android']) + for arch in ('arm', 'arm64', 'x86', 'x86_64'): + command.extend(['--android-arch', arch]) + else: + command.extend(['--target-platform', platform]) if build_deps_only: command.append('--build-deps-only') if make_release: @@ -121,12 +127,11 @@ def make_archive(project, platform): def make_deps_archive(target, full=False): - (BASE_DIR/'.install_packages_ok').unlink() - archive_name = "deps_{}_{}_{}.tar.gz".format( TRAVIS_OS_NAME, PLATFORM, target) files_to_archive = [BASE_DIR/'INSTALL'] - files_to_archive += BASE_DIR.glob('**/android-ndk*') + files_to_archive += HOME.glob('BUILD_*/android-ndk*') + files_to_archive += HOME.glob('BUILD_*/android-sdk*') if (BASE_DIR/'meson_cross_file.txt').exists(): files_to_archive.append(BASE_DIR/'meson_cross_file.txt') @@ -138,17 +143,19 @@ def make_deps_archive(target, full=False): if full: files_to_archive += ARCHIVE_DIR.glob(".*_ok") files_to_archive += BASE_DIR.glob('*/.*_ok') + files_to_archive += (HOME/"BUILD_native_dyn").glob('*/.*_ok') + files_to_archive += (HOME/"BUILD_native_static").glob('*/.*_ok') + files_to_archive += HOME.glob('BUILD_android*/.*_ok') files_to_archive += SOURCE_DIR.glob('*/.*_ok') files_to_archive += [SOURCE_DIR/'pugixml-{}'.format( base_deps_versions['pugixml'])] - files_to_archive += [BASE_DIR/'pugixml-{}'.format( - base_deps_versions['pugixml'])] - if (TOOLCHAINS_DIR).exists(): - files_to_archive.append(TOOLCHAINS_DIR) + files_to_archive += HOME.glob('BUILD_*/pugixml-{}'.format( + base_deps_versions['pugixml'])) + files_to_archive += HOME.glob('**/TOOLCHAINS') relative_path = HOME with tarfile.open(str(relative_path/archive_name), 'w:gz') as tar: - for name in files_to_archive: + for name in set(files_to_archive): tar.add(str(name), arcname=str(name.relative_to(relative_path))) return relative_path/archive_name @@ -281,10 +288,10 @@ elif PLATFORM == 'armhf_static': make_archive('kiwix-tools', 'linux-armhf') elif PLATFORM == 'i586_static': make_archive('kiwix-tools', 'linux-i586') -elif PLATFORM.startswith('android_') and 'kiwix-android' in TARGETS: +elif PLATFORM.startswith('android') and 'kiwix-android' in TARGETS: APK_NAME = "kiwix-{}".format(PLATFORM) - source_debug_dir = BASE_DIR/'kiwix-android'/'app'/'build'/'outputs'/'apk'/'kiwix'/'debug' - source_release_dir = BASE_DIR/'kiwix-android'/'app'/'build'/'outputs'/'apk'/'kiwix'/'release' + source_debug_dir = HOME/'BUILD_android'/'kiwix-android'/'app'/'build'/'outputs'/'apk'/'kiwix'/'debug' + source_release_dir = HOME/'BUILD_android'/'kiwix-android'/'app'/'build'/'outputs'/'apk'/'kiwix'/'release' shutil.copy(str(source_debug_dir/'app-kiwix-debug.apk'), str(NIGHTLY_KIWIX_ARCHIVES_DIR/"{}-debug.apk".format(APK_NAME))) shutil.copy(str(source_release_dir/'app-kiwix-release-unsigned.apk'),