From b950feb893427d38e18b4389d2533c30fc3186bd Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 22 May 2018 16:36:54 +0200 Subject: [PATCH] Move Builder and BuildEnv in specific module. Introduce also a "NeutralEnv", a build environment independent of the targeted platform. All `Source` now build using the neutralEnv. Most of toolchains are also using neutralEnv except android_ndk who is specific to a platform. As toolchain are neutral, platform specific environment variables are now set by the platformInfo directly instead of the toolchain. --- kiwixbuild/__init__.py | 527 +----------------------- kiwixbuild/buildenv.py | 376 +++++++++++++++++ kiwixbuild/builder.py | 97 +++++ kiwixbuild/dependencies/base.py | 61 +-- kiwixbuild/dependencies/libaria2.py | 2 +- kiwixbuild/packages.py | 103 +++++ kiwixbuild/platforms/base.py | 6 + kiwixbuild/platforms/ios.py | 9 + kiwixbuild/platforms/win32.py | 12 +- kiwixbuild/toolchains/android_ndk.py | 1 + kiwixbuild/toolchains/android_sdk.py | 3 - kiwixbuild/toolchains/armhf.py | 6 +- kiwixbuild/toolchains/base_toolchain.py | 13 +- kiwixbuild/toolchains/ios.py | 14 - kiwixbuild/toolchains/mingw32.py | 8 - 15 files changed, 649 insertions(+), 589 deletions(-) create mode 100644 kiwixbuild/buildenv.py create mode 100644 kiwixbuild/builder.py create mode 100644 kiwixbuild/packages.py diff --git a/kiwixbuild/__init__.py b/kiwixbuild/__init__.py index 2c778a4..d84be62 100644 --- a/kiwixbuild/__init__.py +++ b/kiwixbuild/__init__.py @@ -1,533 +1,12 @@ #!/usr/bin/env python3 -import os, sys, shutil +import os, sys import argparse -import ssl -import urllib.request -import subprocess -import platform -from collections import OrderedDict -from .toolchains import Toolchain from .dependencies import Dependency from .platforms import PlatformInfo -from .utils import ( - pj, - remove_duplicates, - add_execution_right, - get_sha256, - print_progress, - setup_print_progress, - download_remote, - StopBuild, - SkipCommand, - Defaultdict, - Remotefile, - Context) - -REMOTE_PREFIX = 'http://download.kiwix.org/dev/' - -SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) - - -_fedora_common = ['automake', 'libtool', 'cmake', 'git', 'subversion', 'ccache', 'pkgconfig', 'gcc-c++', 'gettext-devel'] -_debian_common = ['automake', 'libtool', 'cmake', 'git', 'subversion', 'ccache', 'pkg-config', 'gcc', 'autopoint'] -PACKAGE_NAME_MAPPERS = { - 'fedora_native_dyn': { - 'COMMON': _fedora_common, - 'uuid': ['libuuid-devel'], - 'xapian-core': None, # Not the right version on fedora 25 - 'ctpp2': None, - 'pugixml': None, # ['pugixml-devel'] but package doesn't provide pkg-config file - 'libmicrohttpd': ['libmicrohttpd-devel'], - 'zlib': ['zlib-devel'], - 'lzma': ['xz-devel'], - 'icu4c': None, - 'zimlib': None, - 'file' : ['file-devel'], - 'gumbo' : ['gumbo-parser-devel'], - }, - 'fedora_native_static': { - 'COMMON': _fedora_common + ['glibc-static', 'libstdc++-static'], - 'zlib': ['zlib-devel', 'zlib-static'], - 'lzma': ['xz-devel', 'xz-static'] - # Either there is no packages, or no static or too old - }, - 'fedora_i586_dyn': { - 'COMMON': _fedora_common + ['glibc-devel.i686', 'libstdc++-devel.i686'], - }, - 'fedora_i586_static': { - 'COMMON': _fedora_common + ['glibc-devel.i686'], - }, - 'fedora_win32_dyn': { - 'COMMON': _fedora_common + ['mingw32-gcc-c++', 'mingw32-bzip2', 'mingw32-win-iconv', 'mingw32-winpthreads', 'wine'], - 'zlib': ['mingw32-zlib'], - 'lzma': ['mingw32-xz-libs'], - 'libmicrohttpd': ['mingw32-libmicrohttpd'], - }, - 'fedora_win32_static': { - 'COMMON': _fedora_common + ['mingw32-gcc-c++', 'mingw32-bzip2-static', 'mingw32-win-iconv-static', 'mingw32-winpthreads-static', 'wine'], - 'zlib': ['mingw32-zlib-static'], - 'lzma': ['mingw32-xz-libs-static'], - 'libmicrohttpd': None, # ['mingw32-libmicrohttpd-static'] packaging dependecy seems buggy, and some static lib are name libfoo.dll.a and - # gcc cannot found them. - }, - 'fedora_armhf_static': { - 'COMMON': _fedora_common - }, - 'fedora_armhf_dyn': { - 'COMMON': _fedora_common - }, - 'fedora_android': { - 'COMMON': _fedora_common + ['java-1.8.0-openjdk-devel'] - }, - 'debian_native_dyn': { - 'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'], - 'zlib': ['zlib1g-dev'], - 'uuid': ['uuid-dev'], - 'ctpp2': ['libctpp2-dev'], - 'ctpp2c': ['ctpp2-utils'], - 'libmicrohttpd': ['libmicrohttpd-dev', 'ccache'] - }, - 'debian_native_static': { - 'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'], - 'zlib': ['zlib1g-dev'], - 'uuid': ['uuid-dev'], - 'ctpp2': ['libctpp2-dev'], - 'ctpp2c': ['ctpp2-utils'], - }, - 'debian_i586_dyn': { - 'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib'], - }, - 'debian_i586_static': { - 'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib'], - }, - 'debian_win32_dyn': { - 'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'], - 'ctpp2c': ['ctpp2-utils'], - }, - 'debian_win32_static': { - 'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'], - 'ctpp2c': ['ctpp2-utils'], - }, - 'debian_armhf_static': { - 'COMMON': _debian_common, - 'ctpp2c': ['ctpp2-utils'], - }, - 'debian_armhf_dyn': { - 'COMMON': _debian_common, - 'ctpp2c': ['ctpp2-utils'], - }, - 'debian_android': { - 'COMMON': _debian_common + ['default-jdk'], - 'ctpp2c': ['ctpp2-utils'], - }, - 'Darwin_native_dyn': { - 'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'], - 'file': ['libmagic'] - }, - 'Darwin_iOS': { - 'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'], - 'file': ['libmagic'] - }, -} - -class BuildEnv: - def __init__(self, options, targetsDict): - self.source_dir = pj(options.working_dir, "SOURCE") - build_dir = "BUILD_{}".format(options.target_platform) - self.build_dir = pj(options.working_dir, build_dir) - self.archive_dir = pj(options.working_dir, "ARCHIVE") - self.toolchain_dir = pj(options.working_dir, "TOOLCHAINS") - self.log_dir = pj(self.build_dir, 'LOGS') - self.install_dir = pj(self.build_dir, "INSTALL") - for d in (self.source_dir, - self.build_dir, - self.archive_dir, - self.toolchain_dir, - self.log_dir, - self.install_dir): - os.makedirs(d, exist_ok=True) - self.detect_platform() - self.ninja_command = self._detect_ninja() - if not self.ninja_command: - sys.exit("ERROR: ninja command not found") - self.meson_command = self._detect_meson() - if not self.meson_command: - sys.exit("ERROR: meson command not fount") - self.mesontest_command = "meson test" - self.setup_build(options.target_platform) - self.setup_toolchains() - self.options = options - self.libprefix = options.libprefix or self._detect_libdir() - self.targetsDict = targetsDict - - def clean_intermediate_directories(self): - for subdir in os.listdir(self.build_dir): - subpath = pj(self.build_dir, subdir) - if subpath == self.install_dir: - continue - if os.path.isdir(subpath): - shutil.rmtree(subpath) - else: - os.remove(subpath) - - def detect_platform(self): - _platform = platform.system() - self.distname = _platform - if _platform == 'Windows': - print('ERROR: kiwix-build is not intented to run on Windows platform.\n' - 'It should probably not work, but well, you still can have a try.') - cont = input('Do you want to continue ? [y/N]') - if cont.lower() != 'y': - sys.exit(0) - if _platform == 'Linux': - self.distname, _, _ = platform.linux_distribution() - self.distname = self.distname.lower() - if self.distname == 'ubuntu': - self.distname = 'debian' - - def setup_build(self, target_platform): - self.platform_info = PlatformInfo.all_platforms[target_platform] - if self.distname not in self.platform_info.compatible_hosts: - print(('ERROR: The target {} cannot be build on host {}.\n' - 'Select another target platform, or change your host system.' - ).format(target_platform, self.distname)) - sys.exit(-1) - self.cross_config = self.platform_info.get_cross_config(self.distname) - - def setup_toolchains(self): - toolchain_names = self.platform_info.toolchains - self.toolchains =[Toolchain.all_toolchains[toolchain_name](self) - for toolchain_name in toolchain_names] - - def finalize_setup(self): - getattr(self, 'setup_{}'.format(self.platform_info.build))() - - def setup_native(self): - self.cmake_crossfile = None - self.meson_crossfile = None - - def _gen_crossfile(self, name): - crossfile = pj(self.build_dir, name) - template_file = pj(SCRIPT_DIR, 'templates', name) - with open(template_file, 'r') as f: - template = f.read() - content = template.format( - toolchain=self.toolchains[0], - **self.cross_config - ) - with open(crossfile, 'w') as outfile: - outfile.write(content) - return crossfile - - def setup_win32(self): - self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') - self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') - - def setup_android(self): - self.cmake_crossfile = self._gen_crossfile('cmake_android_cross_file.txt') - self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') - - def setup_armhf(self): - self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') - self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') - - def setup_iOS(self): - self.cmake_crossfile = self._gen_crossfile('cmake_ios_cross_file.txt') - self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') - - def setup_i586(self): - self.cmake_crossfile = self._gen_crossfile('cmake_i586_cross_file.txt') - self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') - - def __getattr__(self, name): - return getattr(self.options, name) - - def _is_debianlike(self): - return os.path.isfile('/etc/debian_version') - - def _detect_libdir(self): - if self._is_debianlike(): - try: - pc = subprocess.Popen(['dpkg-architecture', '-qDEB_HOST_MULTIARCH'], - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL) - (stdo, _) = pc.communicate() - if pc.returncode == 0: - archpath = stdo.decode().strip() - return 'lib/' + archpath - except Exception: - pass - if os.path.isdir('/usr/lib64') and not os.path.islink('/usr/lib64'): - return 'lib64' - return 'lib' - - def _detect_ninja(self): - for n in ['ninja', 'ninja-build']: - try: - retcode = subprocess.check_call([n, '--version'], - stdout=subprocess.DEVNULL) - except (FileNotFoundError, PermissionError): - # Doesn't exist in PATH or isn't executable - continue - if retcode == 0: - return n - - def _detect_meson(self): - for n in ['meson.py', 'meson']: - try: - retcode = subprocess.check_call([n, '--version'], - stdout=subprocess.DEVNULL) - except (FileNotFoundError, PermissionError): - # Doesn't exist in PATH or isn't executable - continue - if retcode == 0: - return n - - @property - def configure_option(self): - configure_options = [tlc.configure_option for tlc in self.toolchains] - return " ".join(configure_options) - - @property - def cmake_option(self): - cmake_options = [tlc.cmake_option for tlc in self.toolchains] - return " ".join(cmake_options) - - def _set_env(self, env, cross_compile_env, cross_compile_compiler, cross_compile_path): - if env is None: - env = Defaultdict(str, os.environ) - - bin_dirs = [] - if cross_compile_env: - for k, v in self.cross_config.get('env', {}).items(): - if k.startswith('_format_'): - v = v.format(**self.cross_config) - k = k[8:] - env[k] = v - for toolchain in self.toolchains: - toolchain.set_env(env) - if cross_compile_compiler: - for toolchain in self.toolchains: - toolchain.set_compiler(env) - if cross_compile_path: - for tlc in self.toolchains: - bin_dirs += tlc.get_bin_dir() - - pkgconfig_path = pj(self.install_dir, self.libprefix, 'pkgconfig') - env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path]) - - # Add ccache path - for p in ('/usr/lib/ccache', '/usr/lib64/ccache'): - if os.path.isdir(p): - ccache_path = [p] - break - else: - ccache_path = [] - env['PATH'] = ':'.join(bin_dirs + - [pj(self.install_dir, 'bin')] + - ccache_path + - [env['PATH']]) - - env['LD_LIBRARY_PATH'] = ':'.join([env['LD_LIBRARY_PATH'], - pj(self.install_dir, 'lib'), - pj(self.install_dir, self.libprefix) - ]) - - env['CPPFLAGS'] = " ".join(['-I'+pj(self.install_dir, 'include'), env['CPPFLAGS']]) - env['LDFLAGS'] = " ".join(['-L'+pj(self.install_dir, 'lib'), - '-L'+pj(self.install_dir, self.libprefix), - env['LDFLAGS']]) - return env - - def run_command(self, command, cwd, context, env=None, input=None, cross_env_only=False): - os.makedirs(cwd, exist_ok=True) - cross_compile_env = True - cross_compile_compiler = True - cross_compile_path = True - if context.force_native_build: - cross_compile_env = False - cross_compile_compiler = False - cross_compile_path = False - if cross_env_only: - cross_compile_compiler = False - env = self._set_env(env, cross_compile_env, cross_compile_compiler, cross_compile_path) - log = None - try: - if not self.options.verbose: - log = open(context.log_file, 'w') - print("run command '{}'".format(command), file=log) - print("current directory is '{}'".format(cwd), file=log) - print("env is :", file=log) - for k, v in env.items(): - print(" {} : {!r}".format(k, v), file=log) - - kwargs = dict() - if input: - kwargs['stdin'] = subprocess.PIPE - process = subprocess.Popen(command, shell=True, cwd=cwd, env=env, stdout=log or sys.stdout, stderr=subprocess.STDOUT, **kwargs) - if input: - process.communicate(input.encode()) - retcode = process.wait() - if retcode: - raise subprocess.CalledProcessError(retcode, command) - finally: - if log: - log.close() - - def download(self, what, where=None): - where = where or self.archive_dir - download_remote(what, where, not self.options.no_cert_check) - - def install_packages(self): - autoskip_file = pj(self.build_dir, ".install_packages_ok") - if self.distname in ('fedora', 'redhat', 'centos'): - package_installer = 'sudo dnf install {}' - package_checker = 'rpm -q --quiet {}' - elif self.distname in ('debian', 'Ubuntu'): - package_installer = 'sudo apt-get install {}' - package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1' - elif self.distname == 'Darwin': - package_installer = 'brew install {}' - package_checker = 'brew list -1 | grep -q {}' - mapper_name = "{host}_{target}".format( - host=self.distname, - target=self.platform_info) - try: - package_name_mapper = PACKAGE_NAME_MAPPERS[mapper_name] - except KeyError: - print("SKIP : We don't know which packages we must install to compile" - " a {target} {build_type} version on a {host} host.".format( - target=self.platform_info, - host=self.distname)) - return - - packages_list = package_name_mapper.get('COMMON', []) - for dep in self.targetsDict.values(): - packages = package_name_mapper.get(dep.name) - if packages: - packages_list += packages - dep.skip = True - for dep in self.targetsDict.values(): - packages = getattr(dep, 'extra_packages', []) - for package in packages: - packages_list += package_name_mapper.get(package, []) - if not self.options.force_install_packages and os.path.exists(autoskip_file): - print("SKIP") - return - - packages_to_install = [] - for package in packages_list: - print(" - {} : ".format(package), end="") - command = package_checker.format(package) - try: - subprocess.check_call(command, shell=True) - except subprocess.CalledProcessError: - print("NEEDED") - packages_to_install.append(package) - else: - print("SKIP") - - if packages_to_install: - command = package_installer.format(" ".join(packages_to_install)) - print(command) - subprocess.check_call(command, shell=True) - else: - print("SKIP, No package to install.") - - with open(autoskip_file, 'w'): - pass - - -class Builder: - def __init__(self, options): - self.options = options - self.targets = OrderedDict() - self.buildEnv = buildEnv = BuildEnv(options, self.targets) - - _targets = {} - targetDef = options.targets - self.add_targets(targetDef, _targets) - dependencies = self.order_dependencies(_targets, targetDef) - dependencies = list(remove_duplicates(dependencies)) - - if options.build_nodeps: - self.targets[targetDef] = _targets[targetDef] - else: - for dep in dependencies: - if self.options.build_deps_only and dep == targetDef: - continue - self.targets[dep] = _targets[dep] - - def add_targets(self, targetName, targets): - if targetName in targets: - return - targetClass = Dependency.all_deps[targetName] - target = targetClass(self.buildEnv) - targets[targetName] = target - for dep in target.dependencies: - self.add_targets(dep, targets) - - def order_dependencies(self, _targets, targetName): - target = _targets[targetName] - for depName in target.dependencies: - yield from self.order_dependencies(_targets, depName) - yield targetName - - def prepare_sources(self): - if self.options.skip_source_prepare: - print("SKIP") - return - - toolchain_sources = (tlc.source for tlc in self.buildEnv.toolchains if tlc.source) - for toolchain_source in toolchain_sources: - print("prepare sources for toolchain {} :".format(toolchain_source.name)) - toolchain_source.prepare() - - sources = (dep.source for dep in self.targets.values() if not dep.skip) - sources = remove_duplicates(sources, lambda s: s.__class__) - for source in sources: - print("prepare sources {} :".format(source.name)) - source.prepare() - - def build(self): - toolchain_builders = (tlc.builder for tlc in self.buildEnv.toolchains if tlc.builder) - for toolchain_builder in toolchain_builders: - print("build toolchain {} :".format(toolchain_builder.name)) - toolchain_builder.build() - - builders = (dep.builder for dep in self.targets.values() if (dep.builder and not dep.skip)) - for builder in builders: - if self.options.make_dist and builder.name == self.options.targets: - continue - print("build {} :".format(builder.name)) - builder.build() - - if self.options.make_dist: - dep = self.targets[self.options.targets] - builder = dep.builder - print("make dist {}:".format(builder.name)) - builder.make_dist() - - def run(self): - try: - print("[INSTALL PACKAGES]") - self.buildEnv.install_packages() - self.buildEnv.finalize_setup() - print("[PREPARE]") - self.prepare_sources() - print("[BUILD]") - self.build() - # No error, clean intermediate file at end of build if needed. - print("[CLEAN]") - if self.buildEnv.options.clean_at_end: - self.buildEnv.clean_intermediate_directories() - else: - print("SKIP") - except StopBuild: - sys.exit("Stopping build due to errors") - +from .builder import Builder +from .utils import setup_print_progress def parse_args(): parser = argparse.ArgumentParser() diff --git a/kiwixbuild/buildenv.py b/kiwixbuild/buildenv.py new file mode 100644 index 0000000..8be045f --- /dev/null +++ b/kiwixbuild/buildenv.py @@ -0,0 +1,376 @@ + +import os, sys, shutil +import subprocess +import platform + +from .platforms import PlatformInfo +from .toolchains import Toolchain +from .packages import PACKAGE_NAME_MAPPERS +from .utils import ( + pj, + download_remote, + Defaultdict) + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) + + +class PlatformNeutralEnv: + def __init__(self, options): + self.options = options + self.source_dir = pj(options.working_dir, "SOURCE") + self.archive_dir = pj(options.working_dir, "ARCHIVE") + self.toolchain_dir = pj(options.working_dir, "TOOLCHAINS") + self.log_dir = pj(self.working_dir, 'LOGS') + for d in (self.source_dir, + self.archive_dir, + self.toolchain_dir, + self.log_dir): + os.makedirs(d, exist_ok=True) + self.toolchains = {} + self.detect_platform() + self.ninja_command = self._detect_ninja() + if not self.ninja_command: + sys.exit("ERROR: ninja command not found") + self.meson_command = self._detect_meson() + if not self.meson_command: + sys.exit("ERROR: meson command not fount") + self.mesontest_command = "{} test".format(self.meson_command) + + def run_command(self, command, cwd, context, env=None, input=None): + os.makedirs(cwd, exist_ok=True) + if env is None: + env = Defaultdict(str, os.environ) + log = None + try: + if not self.options.verbose: + log = open(context.log_file, 'w') + print("run command '{}'".format(command), file=log) + print("current directory is '{}'".format(cwd), file=log) + print("env is :", file=log) + for k, v in env.items(): + print(" {} : {!r}".format(k, v), file=log) + + kwargs = dict() + if input: + kwargs['stdin'] = subprocess.PIPE + process = subprocess.Popen(command, shell=True, cwd=cwd, env=env, stdout=log or sys.stdout, stderr=subprocess.STDOUT, **kwargs) + if input: + process.communicate(input.encode()) + retcode = process.wait() + if retcode: + raise subprocess.CalledProcessError(retcode, command) + finally: + if log: + log.close() + + def detect_platform(self): + _platform = platform.system() + self.distname = _platform + if _platform == 'Windows': + print('ERROR: kiwix-build is not intented to run on Windows platform.\n' + 'It should probably not work, but well, you still can have a try.') + cont = input('Do you want to continue ? [y/N]') + if cont.lower() != 'y': + sys.exit(0) + if _platform == 'Linux': + self.distname, _, _ = platform.linux_distribution() + self.distname = self.distname.lower() + if self.distname == 'ubuntu': + self.distname = 'debian' + + def download(self, what, where=None): + where = where or self.archive_dir + download_remote(what, where, not self.options.no_cert_check) + + def _detect_ninja(self): + for n in ['ninja', 'ninja-build']: + try: + retcode = subprocess.check_call([n, '--version'], + stdout=subprocess.DEVNULL) + except (FileNotFoundError, PermissionError): + # Doesn't exist in PATH or isn't executable + continue + if retcode == 0: + return n + + def add_toolchain(self, toolchain_name): + if toolchain_name not in self.toolchains: + ToolchainClass = Toolchain.all_toolchains[toolchain_name] + self.toolchains[toolchain_name] = ToolchainClass(self) + return self.toolchains[toolchain_name] + + def _detect_meson(self): + for n in ['meson.py', 'meson']: + try: + retcode = subprocess.check_call([n, '--version'], + stdout=subprocess.DEVNULL) + except (FileNotFoundError, PermissionError): + # Doesn't exist in PATH or isn't executable + continue + if retcode == 0: + return n + + def __getattr__(self, name): + return getattr(self.options, name) + + +class BuildEnv: + def __init__(self, options, neutralEnv, targetsDict): + build_dir = "BUILD_{}".format(options.target_platform) + self.neutralEnv = neutralEnv + self.build_dir = pj(options.working_dir, build_dir) + self.install_dir = pj(self.build_dir, "INSTALL") + for d in (self.build_dir, + self.install_dir): + os.makedirs(d, exist_ok=True) + + self.setup_build(options.target_platform) + self.setup_toolchains() + self.options = options + self.libprefix = options.libprefix or self._detect_libdir() + self.targetsDict = targetsDict + + def clean_intermediate_directories(self): + for subdir in os.listdir(self.build_dir): + subpath = pj(self.build_dir, subdir) + if subpath == self.install_dir: + continue + if os.path.isdir(subpath): + shutil.rmtree(subpath) + else: + os.remove(subpath) + + def setup_build(self, target_platform): + self.platform_info = PlatformInfo.all_platforms[target_platform] + if self.distname not in self.platform_info.compatible_hosts: + print(('ERROR: The target {} cannot be build on host {}.\n' + 'Select another target platform, or change your host system.' + ).format(target_platform, self.distname)) + sys.exit(-1) + self.cross_config = self.platform_info.get_cross_config(self.distname) + + def setup_toolchains(self): + toolchain_names = self.platform_info.toolchains + self.toolchains = [] + for toolchain_name in toolchain_names: + ToolchainClass = Toolchain.all_toolchains[toolchain_name] + if ToolchainClass.neutral: + self.toolchains.append( + self.neutralEnv.add_toolchain(toolchain_name) + ) + else: + self.toolchains.append(ToolchainClass(self)) + + def finalize_setup(self): + getattr(self, 'setup_{}'.format(self.platform_info.build))() + + def setup_native(self): + self.cmake_crossfile = None + self.meson_crossfile = None + + def _gen_crossfile(self, name): + crossfile = pj(self.build_dir, name) + template_file = pj(SCRIPT_DIR, 'templates', name) + with open(template_file, 'r') as f: + template = f.read() + content = template.format( + toolchain=self.toolchains[0], + **self.cross_config + ) + with open(crossfile, 'w') as outfile: + outfile.write(content) + return crossfile + + def setup_win32(self): + self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') + self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') + + def setup_android(self): + self.cmake_crossfile = self._gen_crossfile('cmake_android_cross_file.txt') + self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') + + def setup_armhf(self): + self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') + self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') + + def setup_iOS(self): + self.cmake_crossfile = self._gen_crossfile('cmake_ios_cross_file.txt') + self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') + + def setup_i586(self): + self.cmake_crossfile = self._gen_crossfile('cmake_i586_cross_file.txt') + self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') + + def __getattr__(self, name): + return getattr(self.neutralEnv, name) + + def _is_debianlike(self): + return os.path.isfile('/etc/debian_version') + + def _detect_libdir(self): + if self._is_debianlike(): + try: + pc = subprocess.Popen(['dpkg-architecture', '-qDEB_HOST_MULTIARCH'], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL) + (stdo, _) = pc.communicate() + if pc.returncode == 0: + archpath = stdo.decode().strip() + return 'lib/' + archpath + except Exception: + pass + if os.path.isdir('/usr/lib64') and not os.path.islink('/usr/lib64'): + return 'lib64' + return 'lib' + + @property + def configure_option(self): + configure_options = [tlc.configure_option for tlc in self.toolchains] + return " ".join(configure_options) + + @property + def cmake_option(self): + cmake_options = [tlc.cmake_option for tlc in self.toolchains] + return " ".join(cmake_options) + + def _set_env(self, env, cross_compile_env, cross_compile_compiler, cross_compile_path): + if env is None: + env = Defaultdict(str, os.environ) + + pkgconfig_path = pj(self.install_dir, self.libprefix, 'pkgconfig') + env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path]) + + # Add ccache path + for p in ('/usr/lib/ccache', '/usr/lib64/ccache'): + if os.path.isdir(p): + ccache_path = [p] + break + else: + ccache_path = [] + env['PATH'] = ':'.join([pj(self.install_dir, 'bin')] + + ccache_path + + [env['PATH']]) + + env['LD_LIBRARY_PATH'] = ':'.join([env['LD_LIBRARY_PATH'], + pj(self.install_dir, 'lib'), + pj(self.install_dir, self.libprefix) + ]) + + env['CPPFLAGS'] = " ".join(['-I'+pj(self.install_dir, 'include'), env['CPPFLAGS']]) + env['LDFLAGS'] = " ".join(['-L'+pj(self.install_dir, 'lib'), + '-L'+pj(self.install_dir, self.libprefix), + env['LDFLAGS']]) + + if cross_compile_env: + for k, v in self.cross_config.get('env', {}).items(): + if k.startswith('_format_'): + v = v.format(**self.cross_config) + k = k[8:] + env[k] = v + for toolchain in self.toolchains: + toolchain.set_env(env) + self.platform_info.set_env(env) + if cross_compile_compiler: + for toolchain in self.toolchains: + toolchain.set_compiler(env) + if cross_compile_path: + bin_dirs = [] + for tlc in self.toolchains: + bin_dirs += tlc.get_bin_dir() + bin_dirs += self.platform_info.get_bind_dir() + env['PATH'] = ':'.join(bin_dirs + [env['PATH']]) + return env + + def run_command(self, command, cwd, context, env=None, input=None, cross_env_only=False): + os.makedirs(cwd, exist_ok=True) + cross_compile_env = True + cross_compile_compiler = True + cross_compile_path = True + if context.force_native_build: + cross_compile_env = False + cross_compile_compiler = False + cross_compile_path = False + if cross_env_only: + cross_compile_compiler = False + env = self._set_env(env, cross_compile_env, cross_compile_compiler, cross_compile_path) + log = None + try: + if not self.options.verbose: + log = open(context.log_file, 'w') + print("run command '{}'".format(command), file=log) + print("current directory is '{}'".format(cwd), file=log) + print("env is :", file=log) + for k, v in env.items(): + print(" {} : {!r}".format(k, v), file=log) + + kwargs = dict() + if input: + kwargs['stdin'] = subprocess.PIPE + process = subprocess.Popen(command, shell=True, cwd=cwd, env=env, stdout=log or sys.stdout, stderr=subprocess.STDOUT, **kwargs) + if input: + process.communicate(input.encode()) + retcode = process.wait() + if retcode: + raise subprocess.CalledProcessError(retcode, command) + finally: + if log: + log.close() + + def install_packages(self): + autoskip_file = pj(self.build_dir, ".install_packages_ok") + if self.distname in ('fedora', 'redhat', 'centos'): + package_installer = 'sudo dnf install {}' + package_checker = 'rpm -q --quiet {}' + elif self.distname in ('debian', 'Ubuntu'): + package_installer = 'sudo apt-get install {}' + package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1' + elif self.distname == 'Darwin': + package_installer = 'brew install {}' + package_checker = 'brew list -1 | grep -q {}' + mapper_name = "{host}_{target}".format( + host=self.distname, + target=self.platform_info) + try: + package_name_mapper = PACKAGE_NAME_MAPPERS[mapper_name] + except KeyError: + print("SKIP : We don't know which packages we must install to compile" + " a {target} {build_type} version on a {host} host.".format( + target=self.platform_info, + host=self.distname)) + return + + packages_list = package_name_mapper.get('COMMON', []) + for dep in self.targetsDict.values(): + packages = package_name_mapper.get(dep.name) + if packages: + packages_list += packages + dep.skip = True + for dep in self.targetsDict.values(): + packages = getattr(dep, 'extra_packages', []) + for package in packages: + packages_list += package_name_mapper.get(package, []) + if not self.options.force_install_packages and os.path.exists(autoskip_file): + print("SKIP") + return + + packages_to_install = [] + for package in packages_list: + print(" - {} : ".format(package), end="") + command = package_checker.format(package) + try: + subprocess.check_call(command, shell=True) + except subprocess.CalledProcessError: + print("NEEDED") + packages_to_install.append(package) + else: + print("SKIP") + + if packages_to_install: + command = package_installer.format(" ".join(packages_to_install)) + print(command) + subprocess.check_call(command, shell=True) + else: + print("SKIP, No package to install.") + + with open(autoskip_file, 'w'): + pass diff --git a/kiwixbuild/builder.py b/kiwixbuild/builder.py new file mode 100644 index 0000000..92aab5a --- /dev/null +++ b/kiwixbuild/builder.py @@ -0,0 +1,97 @@ + +import sys +from collections import OrderedDict +from .buildenv import * + +from .utils import remove_duplicates, StopBuild +from .dependencies import Dependency + +class Builder: + def __init__(self, options): + self.options = options + self.targets = OrderedDict() + self.neutralEnv = PlatformNeutralEnv(options) + self.buildEnv = BuildEnv(options, self.neutralEnv, self.targets) + + _targets = {} + targetDef = options.targets + self.add_targets(targetDef, _targets) + dependencies = self.order_dependencies(_targets, targetDef) + dependencies = list(remove_duplicates(dependencies)) + + if options.build_nodeps: + self.targets[targetDef] = _targets[targetDef] + else: + for dep in dependencies: + if self.options.build_deps_only and dep == targetDef: + continue + self.targets[dep] = _targets[dep] + + def add_targets(self, targetName, targets): + if targetName in targets: + return + targetClass = Dependency.all_deps[targetName] + target = targetClass(self.neutralEnv, self.buildEnv) + targets[targetName] = target + for dep in target.dependencies: + self.add_targets(dep, targets) + + def order_dependencies(self, _targets, targetName): + target = _targets[targetName] + for depName in target.dependencies: + yield from self.order_dependencies(_targets, depName) + yield targetName + + def prepare_sources(self): + if self.options.skip_source_prepare: + print("SKIP") + return + + toolchain_sources = (tlc.source for tlc in self.buildEnv.toolchains if tlc.source) + for toolchain_source in toolchain_sources: + print("prepare sources for toolchain {} :".format(toolchain_source.name)) + toolchain_source.prepare() + + sources = (dep.source for dep in self.targets.values() if not dep.skip) + sources = remove_duplicates(sources, lambda s: s.__class__) + for source in sources: + print("prepare sources {} :".format(source.name)) + source.prepare() + + def build(self): + toolchain_builders = (tlc.builder for tlc in self.buildEnv.toolchains if tlc.builder) + for toolchain_builder in toolchain_builders: + print("build toolchain {} :".format(toolchain_builder.name)) + toolchain_builder.build() + + builders = (dep.builder for dep in self.targets.values() if (dep.builder and not dep.skip)) + for builder in builders: + if self.options.make_dist and builder.name == self.options.targets: + continue + print("build {} :".format(builder.name)) + builder.build() + + if self.options.make_dist: + dep = self.targets[self.options.targets] + builder = dep.builder + print("make dist {}:".format(builder.name)) + builder.make_dist() + + def run(self): + try: + print("[INSTALL PACKAGES]") + self.buildEnv.install_packages() + self.buildEnv.finalize_setup() + print("[PREPARE]") + self.prepare_sources() + print("[BUILD]") + self.build() + # No error, clean intermediate file at end of build if needed. + print("[CLEAN]") + if self.buildEnv.options.clean_at_end: + self.buildEnv.clean_intermediate_directories() + else: + print("SKIP") + except StopBuild: + sys.exit("Stopping build due to errors") + diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index c06e451..a1410bf 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -22,7 +22,8 @@ class Dependency(metaclass=_MetaDependency): dependencies = [] force_native_build = False - def __init__(self, buildEnv): + def __init__(self, neutralEnv, buildEnv): + self.neutralEnv = neutralEnv self.buildEnv = buildEnv self.source = self.Source(self) self.builder = self.Builder(self) @@ -40,7 +41,7 @@ class Dependency(metaclass=_MetaDependency): @property def source_path(self): - return pj(self.buildEnv.source_dir, self.source.source_dir) + return pj(self.neutralEnv.source_dir, self.source.source_dir) @property def _log_dir(self): @@ -73,10 +74,10 @@ class Dependency(metaclass=_MetaDependency): class Source: """Base Class to the real preparator A source preparator must install source in the self.source_dir attribute - inside the buildEnv.source_dir.""" + inside the neutralEnv.source_dir.""" def __init__(self, target): self.target = target - self.buildEnv = target.buildEnv + self.neutralEnv = target.neutralEnv @property def name(self): @@ -87,12 +88,12 @@ class Source: return self.target.full_name def _patch(self, context): - source_path = pj(self.buildEnv.source_dir, self.source_dir) + source_path = pj(self.neutralEnv.source_dir, self.source_dir) context.try_skip(source_path) context.force_native_build = True for p in self.patches: with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input: - self.buildEnv.run_command("patch -p1", source_path, context, input=patch_input.read()) + self.neutralEnv.run_command("patch -p1", source_path, context, input=patch_input.read()) def command(self, *args, **kwargs): return self.target.command(*args, **kwargs) @@ -108,18 +109,18 @@ class ReleaseDownload(Source): @property def extract_path(self): - return pj(self.buildEnv.source_dir, self.source_dir) + return pj(self.neutralEnv.source_dir, self.source_dir) def _download(self, context): - context.try_skip(self.buildEnv.archive_dir, self.name) - self.buildEnv.download(self.archive) + context.try_skip(self.neutralEnv.archive_dir, self.name) + self.neutralEnv.download(self.archive) def _extract(self, context): context.try_skip(self.extract_path) if os.path.exists(self.extract_path): shutil.rmtree(self.extract_path) - extract_archive(pj(self.buildEnv.archive_dir, self.archive.name), - self.buildEnv.source_dir, + extract_archive(pj(self.neutralEnv.archive_dir, self.archive.name), + self.neutralEnv.source_dir, topdir=self.archive_top_dir, name=self.source_dir) @@ -141,36 +142,34 @@ class GitClone(Source): @property def source_dir(self): - if self.buildEnv.make_release: + if self.neutralEnv.make_release: return "{}_release".format(self.git_dir) else: return self.git_dir @property def git_path(self): - return pj(self.buildEnv.source_dir, self.source_dir) + return pj(self.neutralEnv.source_dir, self.source_dir) @property def git_ref(self): - if self.buildEnv.make_release: + if self.neutralEnv.make_release: return self.release_git_ref else: return self.base_git_ref def _git_clone(self, context): - context.force_native_build = True if os.path.exists(self.git_path): raise SkipCommand() command = "git clone --depth=1 --branch {} {} {}".format( self.git_ref, self.git_remote, self.source_dir) - self.buildEnv.run_command(command, self.buildEnv.source_dir, context) + self.neutralEnv.run_command(command, self.neutralEnv.source_dir, context) def _git_update(self, context): - context.force_native_build = True command = "git fetch origin {}".format( self.git_ref) - self.buildEnv.run_command(command, self.git_path, context) - self.buildEnv.run_command("git checkout "+self.git_ref, self.git_path, context) + self.neutralEnv.run_command(command, self.git_path, context) + self.neutralEnv.run_command("git checkout "+self.git_ref, self.git_path, context) def prepare(self): self.command('gitclone', self._git_clone) @@ -186,19 +185,17 @@ class SvnClone(Source): @property def svn_path(self): - return pj(self.buildEnv.source_dir, self.svn_dir) + return pj(self.neutralEnv.source_dir, self.svn_dir) def _svn_checkout(self, context): - context.force_native_build = True if os.path.exists(self.svn_path): raise SkipCommand() command = "svn checkout {} {}".format(self.svn_remote, self.svn_dir) - self.buildEnv.run_command(command, self.buildEnv.source_dir, context) + self.neutralEnv.run_command(command, self.neutralEnv.source_dir, context) def _svn_update(self, context): context.try_skip(self.svn_path) - context.force_native_build = True - self.buildEnv.run_command("svn update", self.svn_path, context) + self.neutralEnv.run_command("svn update", self.svn_path, context) def prepare(self): self.command('svncheckout', self._svn_checkout) @@ -358,6 +355,12 @@ class MesonBuilder(Builder): configure_option = "" test_option = "" + def __init__(self, target): + super().__init__(target) + self.meson_command = self.buildEnv.neutralEnv.meson_command + self.mesontest_command = self.buildEnv.neutralEnv.mesontest_command + self.ninja_command = self.buildEnv.neutralEnv.ninja_command + @property def library_type(self): return 'static' if self.buildEnv.platform_info.static else 'shared' @@ -379,7 +382,7 @@ class MesonBuilder(Builder): " --libdir={buildEnv.libprefix}" " {cross_option}") command = command.format( - command=self.buildEnv.meson_command, + command=self.meson_command, library_type=self.library_type, configure_option=configure_option, build_path=self.build_path, @@ -389,7 +392,7 @@ class MesonBuilder(Builder): self.buildEnv.run_command(command, self.source_path, context, cross_env_only=True) def _compile(self, context): - command = "{} -v".format(self.buildEnv.ninja_command) + command = "{} -v".format(self.ninja_command) self.buildEnv.run_command(command, self.build_path, context) def _test(self, context): @@ -398,15 +401,15 @@ class MesonBuilder(Builder): and not self.buildEnv.platform_info.static) ): raise SkipCommand() - command = "{} --verbose {}".format(self.buildEnv.mesontest_command, self.test_option) + command = "{} --verbose {}".format(self.mesontest_command, self.test_option) self.buildEnv.run_command(command, self.build_path, context) def _install(self, context): - command = "{} -v install".format(self.buildEnv.ninja_command) + command = "{} -v install".format(self.ninja_command) self.buildEnv.run_command(command, self.build_path, context) def _make_dist(self, context): - command = "{} -v dist".format(self.buildEnv.ninja_command) + command = "{} -v dist".format(self.ninja_command) self.buildEnv.run_command(command, self.build_path, context) diff --git a/kiwixbuild/dependencies/libaria2.py b/kiwixbuild/dependencies/libaria2.py index 4545fb0..90f1567 100644 --- a/kiwixbuild/dependencies/libaria2.py +++ b/kiwixbuild/dependencies/libaria2.py @@ -20,7 +20,7 @@ class Aria2(Dependency): def _post_prepare_script(self, context): context.try_skip(self.extract_path) command = "autoreconf -i" - self.buildEnv.run_command(command, self.extract_path, context) + self.neutralEnv.run_command(command, self.extract_path, context) class Builder(MakeBuilder): configure_option = "--enable-libaria2 --disable-ssl --disable-bittorent --disable-metalink --without-sqlite3 --without-libxml2 --without-libexpat" diff --git a/kiwixbuild/packages.py b/kiwixbuild/packages.py new file mode 100644 index 0000000..6d33089 --- /dev/null +++ b/kiwixbuild/packages.py @@ -0,0 +1,103 @@ + + +_fedora_common = ['automake', 'libtool', 'cmake', 'git', 'subversion', 'ccache', 'pkgconfig', 'gcc-c++', 'gettext-devel'] +_debian_common = ['automake', 'libtool', 'cmake', 'git', 'subversion', 'ccache', 'pkg-config', 'gcc', 'autopoint'] +PACKAGE_NAME_MAPPERS = { + 'fedora_native_dyn': { + 'COMMON': _fedora_common, + 'uuid': ['libuuid-devel'], + 'xapian-core': None, # Not the right version on fedora 25 + 'ctpp2': None, + 'pugixml': None, # ['pugixml-devel'] but package doesn't provide pkg-config file + 'libmicrohttpd': ['libmicrohttpd-devel'], + 'zlib': ['zlib-devel'], + 'lzma': ['xz-devel'], + 'icu4c': None, + 'zimlib': None, + 'file' : ['file-devel'], + 'gumbo' : ['gumbo-parser-devel'], + }, + 'fedora_native_static': { + 'COMMON': _fedora_common + ['glibc-static', 'libstdc++-static'], + 'zlib': ['zlib-devel', 'zlib-static'], + 'lzma': ['xz-devel', 'xz-static'] + # Either there is no packages, or no static or too old + }, + 'fedora_i586_dyn': { + 'COMMON': _fedora_common + ['glibc-devel.i686', 'libstdc++-devel.i686'], + }, + 'fedora_i586_static': { + 'COMMON': _fedora_common + ['glibc-devel.i686'], + }, + 'fedora_win32_dyn': { + 'COMMON': _fedora_common + ['mingw32-gcc-c++', 'mingw32-bzip2', 'mingw32-win-iconv', 'mingw32-winpthreads', 'wine'], + 'zlib': ['mingw32-zlib'], + 'lzma': ['mingw32-xz-libs'], + 'libmicrohttpd': ['mingw32-libmicrohttpd'], + }, + 'fedora_win32_static': { + 'COMMON': _fedora_common + ['mingw32-gcc-c++', 'mingw32-bzip2-static', 'mingw32-win-iconv-static', 'mingw32-winpthreads-static', 'wine'], + 'zlib': ['mingw32-zlib-static'], + 'lzma': ['mingw32-xz-libs-static'], + 'libmicrohttpd': None, # ['mingw32-libmicrohttpd-static'] packaging dependecy seems buggy, and some static lib are name libfoo.dll.a and + # gcc cannot found them. + }, + 'fedora_armhf_static': { + 'COMMON': _fedora_common + }, + 'fedora_armhf_dyn': { + 'COMMON': _fedora_common + }, + 'fedora_android': { + 'COMMON': _fedora_common + ['java-1.8.0-openjdk-devel'] + }, + 'debian_native_dyn': { + 'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'], + 'zlib': ['zlib1g-dev'], + 'uuid': ['uuid-dev'], + 'ctpp2': ['libctpp2-dev'], + 'ctpp2c': ['ctpp2-utils'], + 'libmicrohttpd': ['libmicrohttpd-dev', 'ccache'] + }, + 'debian_native_static': { + 'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'], + 'zlib': ['zlib1g-dev'], + 'uuid': ['uuid-dev'], + 'ctpp2': ['libctpp2-dev'], + 'ctpp2c': ['ctpp2-utils'], + }, + 'debian_i586_dyn': { + 'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib'], + }, + 'debian_i586_static': { + 'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib'], + }, + 'debian_win32_dyn': { + 'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'], + 'ctpp2c': ['ctpp2-utils'], + }, + 'debian_win32_static': { + 'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'], + 'ctpp2c': ['ctpp2-utils'], + }, + 'debian_armhf_static': { + 'COMMON': _debian_common, + 'ctpp2c': ['ctpp2-utils'], + }, + 'debian_armhf_dyn': { + 'COMMON': _debian_common, + 'ctpp2c': ['ctpp2-utils'], + }, + 'debian_android': { + 'COMMON': _debian_common + ['default-jdk'], + 'ctpp2c': ['ctpp2-utils'], + }, + 'Darwin_native_dyn': { + 'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'], + 'file': ['libmagic'] + }, + 'Darwin_iOS': { + 'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'], + 'file': ['libmagic'] + }, +} diff --git a/kiwixbuild/platforms/base.py b/kiwixbuild/platforms/base.py index b1ea348..6ddd37b 100644 --- a/kiwixbuild/platforms/base.py +++ b/kiwixbuild/platforms/base.py @@ -13,3 +13,9 @@ class PlatformInfo: def __str__(self): return "{}_{}".format(self.build, 'static' if self.static else 'dyn') + + def set_env(self, env): + pass + + def get_bind_dir(self): + return [] diff --git a/kiwixbuild/platforms/ios.py b/kiwixbuild/platforms/ios.py index 8b8e8db..af8b4e4 100644 --- a/kiwixbuild/platforms/ios.py +++ b/kiwixbuild/platforms/ios.py @@ -43,6 +43,15 @@ class iOSPlatformInfo(PlatformInfo): }, } + def set_env(self, env): + env['CFLAGS'] = " -fembed-bitcode -isysroot {SDKROOT} -arch {arch} -miphoneos-version-min=9.0 ".format(SDKROOT=self.root_path, arch=self.arch) + env['CFLAGS'] + env['CXXFLAGS'] = env['CFLAGS'] + " -stdlib=libc++ -std=c++11 "+env['CXXFLAGS'] + env['LDFLAGS'] = " -arch {arch} -isysroot {SDKROOT} ".format(SDKROOT=self.root_path, arch=self.arch) + env['MACOSX_DEPLOYMENT_TARGET'] = "10.7" + + def get_bin_dir(self): + return [pj(self.root_path, 'bin')] + iOSPlatformInfo('iOS_armv7', 'armv7') iOSPlatformInfo('iOS_arm64', 'arm64') iOSPlatformInfo('iOS_i386', 'i386') diff --git a/kiwixbuild/platforms/win32.py b/kiwixbuild/platforms/win32.py index aec524c..c33b68e 100644 --- a/kiwixbuild/platforms/win32.py +++ b/kiwixbuild/platforms/win32.py @@ -4,15 +4,17 @@ from .base import PlatformInfo class Win32PlatformInfo(PlatformInfo): def __init__(self, name, static): super().__init__(name, 'win32', static, ['mingw32_toolchain'], ['fedora', 'debian']) + self.extra_libs = ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4', '-lmsvcr90', '-liphlpapi'] def get_cross_config(self, host): root_paths = { 'fedora': '/usr/i686-w64-mingw32/sys-root/mingw', 'debian': '/usr/i686-w64-mingw32' } + self.root_path = root_paths[host] return { - 'root_path': root_paths[host], - 'extra_libs': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4', '-lmsvcr90', '-liphlpapi'], + 'root_path': self.root_path, + 'extra_libs': self.extra_libs, 'extra_cflags': ['-DWIN32'], 'host_machine': { 'system': 'Windows', @@ -24,6 +26,12 @@ class Win32PlatformInfo(PlatformInfo): } } + def get_bin_dir(self): + return [pj(self.root_path, 'bin')] + + def set_env(self, env): + env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') + env['LIBS'] = " ".join(self.extra_libs) + " " +env['LIBS'] Win32PlatformInfo('win32_dyn', False) Win32PlatformInfo('win32_static', True) diff --git a/kiwixbuild/toolchains/android_ndk.py b/kiwixbuild/toolchains/android_ndk.py index c7cafc3..8a3f3d9 100644 --- a/kiwixbuild/toolchains/android_ndk.py +++ b/kiwixbuild/toolchains/android_ndk.py @@ -7,6 +7,7 @@ from kiwixbuild.utils import Remotefile, add_execution_right pj = os.path.join class android_ndk(Toolchain): + neutral = False name = 'android-ndk' version = 'r13b' gccver = '4.9.x' diff --git a/kiwixbuild/toolchains/android_sdk.py b/kiwixbuild/toolchains/android_sdk.py index db820d2..7eb8934 100644 --- a/kiwixbuild/toolchains/android_sdk.py +++ b/kiwixbuild/toolchains/android_sdk.py @@ -50,8 +50,5 @@ class android_sdk(Toolchain): self.command('build_platform', self._build_platform) self.command('fix_licenses', self._fix_licenses) - def get_bin_dir(self): - return [] - def set_env(self, env): env['ANDROID_HOME'] = self.builder.install_path diff --git a/kiwixbuild/toolchains/armhf.py b/kiwixbuild/toolchains/armhf.py index aaace94..1975665 100644 --- a/kiwixbuild/toolchains/armhf.py +++ b/kiwixbuild/toolchains/armhf.py @@ -3,7 +3,7 @@ import subprocess from .base_toolchain import Toolchain from kiwixbuild.dependencies import GitClone -from kiwixbuild.utils import Remotefile, which +from kiwixbuild.utils import which pj = os.path.join class armhf_toolchain(Toolchain): @@ -52,13 +52,11 @@ class armhf_toolchain(Toolchain): env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') env['CFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS'] env['CXXFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS'] - env['LIBS'] = " ".join(self.buildEnv.cross_config['extra_libs']) + " " +env['LIBS'] env['QEMU_LD_PREFIX'] = pj(self.root_path, "arm-linux-gnueabihf", "libc") env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format( ':'.join([ pj(self.root_path, "arm-linux-gnueabihf", "lib"), - pj(self.buildEnv.install_dir, 'lib'), - pj(self.buildEnv.install_dir, self.buildEnv.libprefix) + env['LD_LIBRARY_PATH'] ])) def set_compiler(self, env): diff --git a/kiwixbuild/toolchains/base_toolchain.py b/kiwixbuild/toolchains/base_toolchain.py index 0396a51..e5040cd 100644 --- a/kiwixbuild/toolchains/base_toolchain.py +++ b/kiwixbuild/toolchains/base_toolchain.py @@ -14,6 +14,7 @@ class _MetaToolchain(type): class Toolchain(metaclass=_MetaToolchain): + neutral = True all_toolchains = {} configure_option = "" cmake_option = "" @@ -21,8 +22,9 @@ class Toolchain(metaclass=_MetaToolchain): Builder = None Source = None - def __init__(self, buildEnv): - self.buildEnv = buildEnv + def __init__(self, neutralEnv): + self.neutralEnv = neutralEnv + self.buildEnv = neutralEnv self.source = self.Source(self) if self.Source else None self.builder = self.Builder(self) if self.Builder else None @@ -34,11 +36,11 @@ class Toolchain(metaclass=_MetaToolchain): @property def source_path(self): - return pj(self.buildEnv.source_dir, self.source.source_dir) + return pj(self.neutralEnv.source_dir, self.source.source_dir) @property def _log_dir(self): - return self.buildEnv.log_dir + return self.neutralEnv.log_dir def set_env(self, env): pass @@ -46,6 +48,9 @@ class Toolchain(metaclass=_MetaToolchain): def set_compiler(self, env): pass + def get_bin_dir(self): + return [] + def command(self, name, function, *args): print(" {} {} : ".format(name, self.name), end="", flush=True) log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name)) diff --git a/kiwixbuild/toolchains/ios.py b/kiwixbuild/toolchains/ios.py index d3f7309..f1dd4be 100644 --- a/kiwixbuild/toolchains/ios.py +++ b/kiwixbuild/toolchains/ios.py @@ -3,10 +3,6 @@ from .base_toolchain import Toolchain from kiwixbuild.utils import pj, xrun_find class iOS_sdk(Toolchain): - @property - def root_path(self): - return self.buildEnv.platform_info.root_path - @property def binaries(self): return { @@ -22,16 +18,6 @@ class iOS_sdk(Toolchain): def configure_option(self): return '--host=arm-apple-darwin' - def get_bin_dir(self): - return [pj(self.root_path, 'bin')] - - def set_env(self, env): - arch = self.buildEnv.platform_info.arch - env['CFLAGS'] = " -fembed-bitcode -isysroot {SDKROOT} -arch {arch} -miphoneos-version-min=9.0 ".format(SDKROOT=self.root_path, arch=arch) + env['CFLAGS'] - env['CXXFLAGS'] = env['CFLAGS'] + " -stdlib=libc++ -std=c++11 "+env['CXXFLAGS'] - env['LDFLAGS'] = " -arch {arch} -isysroot {SDKROOT} ".format(SDKROOT=self.root_path, arch=arch) - env['MACOSX_DEPLOYMENT_TARGET'] = "10.7" - def set_compiler(self, env): env['CC'] = self.binaries['CC'] env['CXX'] = self.binaries['CXX'] diff --git a/kiwixbuild/toolchains/mingw32.py b/kiwixbuild/toolchains/mingw32.py index c27555c..873507e 100644 --- a/kiwixbuild/toolchains/mingw32.py +++ b/kiwixbuild/toolchains/mingw32.py @@ -34,18 +34,10 @@ class mingw32_toolchain(Toolchain): else: return "exec_wrapper = 'wine'" - @property def configure_option(self): return '--host={}'.format(self.arch_full) - def get_bin_dir(self): - return [pj(self.root_path, 'bin')] - - def set_env(self, env): - env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') - env['LIBS'] = " ".join(self.buildEnv.cross_config['extra_libs']) + " " +env['LIBS'] - def set_compiler(self, env): for k, v in self.binaries.items(): env[k] = v