diff --git a/kiwixbuild/buildenv.py b/kiwixbuild/buildenv.py index edae697..afbb6a5 100644 --- a/kiwixbuild/buildenv.py +++ b/kiwixbuild/buildenv.py @@ -3,13 +3,9 @@ import os, sys, shutil import subprocess import platform -from .platforms import PlatformInfo from .toolchains import Toolchain -from .packages import PACKAGE_NAME_MAPPERS from .utils import pj, download_remote, Defaultdict -from . import _global - -SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +from ._global import neutralEnv class PlatformNeutralEnv: @@ -64,12 +60,6 @@ class PlatformNeutralEnv: if retcode == 0: return n - def add_toolchain(self, toolchain_name): - if toolchain_name not in self.toolchains: - ToolchainClass = Toolchain.all_toolchains[toolchain_name] - self.toolchains[toolchain_name] = ToolchainClass(self) - return self.toolchains[toolchain_name] - def _detect_meson(self): for n in ['meson.py', 'meson']: try: @@ -86,18 +76,16 @@ class PlatformNeutralEnv: class BuildEnv: - def __init__(self, options, targetsDict): - build_dir = "BUILD_{}".format(options.target_platform) - self.build_dir = pj(options.working_dir, build_dir) + def __init__(self, platformInfo, targetsDict): + build_dir = "BUILD_{}".format(platformInfo.name) + self.platformInfo = platformInfo + self.build_dir = pj(neutralEnv('working_dir'), build_dir) self.install_dir = pj(self.build_dir, "INSTALL") for d in (self.build_dir, self.install_dir): os.makedirs(d, exist_ok=True) - self.setup_build(options.target_platform) - self.setup_toolchains() - self.options = options - self.libprefix = options.libprefix or self._detect_libdir() + self.libprefix = neutralEnv('libprefix') or self._detect_libdir() self.targetsDict = targetsDict def clean_intermediate_directories(self): @@ -110,67 +98,6 @@ class BuildEnv: else: os.remove(subpath) - def setup_build(self, target_platform): - self.platform_info = PlatformInfo.all_platforms[target_platform] - if self.distname not in self.platform_info.compatible_hosts: - print(('ERROR: The target {} cannot be build on host {}.\n' - 'Select another target platform, or change your host system.' - ).format(target_platform, self.distname)) - sys.exit(-1) - self.cross_config = self.platform_info.get_cross_config() - - def setup_toolchains(self): - toolchain_names = self.platform_info.toolchains - self.toolchains = [] - for toolchain_name in toolchain_names: - ToolchainClass = Toolchain.all_toolchains[toolchain_name] - if ToolchainClass.neutral: - self.toolchains.append( - _global._neutralEnv.add_toolchain(toolchain_name) - ) - else: - self.toolchains.append(ToolchainClass(self)) - - def finalize_setup(self): - getattr(self, 'setup_{}'.format(self.platform_info.build))() - - def setup_native(self): - self.cmake_crossfile = None - self.meson_crossfile = None - - def _gen_crossfile(self, name): - crossfile = pj(self.build_dir, name) - template_file = pj(SCRIPT_DIR, 'templates', name) - with open(template_file, 'r') as f: - template = f.read() - content = template.format( - toolchain=self.toolchains[0], - **self.cross_config - ) - with open(crossfile, 'w') as outfile: - outfile.write(content) - return crossfile - - def setup_win32(self): - self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') - self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') - - def setup_android(self): - self.cmake_crossfile = self._gen_crossfile('cmake_android_cross_file.txt') - self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') - - def setup_armhf(self): - self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') - self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') - - def setup_iOS(self): - self.cmake_crossfile = self._gen_crossfile('cmake_ios_cross_file.txt') - self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') - - def setup_i586(self): - self.cmake_crossfile = self._gen_crossfile('cmake_i586_cross_file.txt') - self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') - def __getattr__(self, name): return _global.neutralEnv(name) @@ -237,75 +164,10 @@ class BuildEnv: v = v.format(**self.cross_config) k = k[8:] env[k] = v - for toolchain in self.toolchains: - toolchain.set_env(env) - self.platform_info.set_env(env) + self.platformInfo.set_env(env) if cross_compile_compiler: - for toolchain in self.toolchains: - toolchain.set_compiler(env) + self.platformInfo.set_compiler(env) if cross_compile_path: - bin_dirs = [] - for tlc in self.toolchains: - bin_dirs += tlc.get_bin_dir() - bin_dirs += self.platform_info.get_bind_dir() + bin_dirs = self.platformInfo.get_bind_dir() env['PATH'] = ':'.join(bin_dirs + [env['PATH']]) return env - - def install_packages(self): - autoskip_file = pj(self.build_dir, ".install_packages_ok") - if self.distname in ('fedora', 'redhat', 'centos'): - package_installer = 'sudo dnf install {}' - package_checker = 'rpm -q --quiet {}' - elif self.distname in ('debian', 'Ubuntu'): - package_installer = 'sudo apt-get install {}' - package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1' - elif self.distname == 'Darwin': - package_installer = 'brew install {}' - package_checker = 'brew list -1 | grep -q {}' - mapper_name = "{host}_{target}".format( - host=self.distname, - target=self.platform_info) - try: - package_name_mapper = PACKAGE_NAME_MAPPERS[mapper_name] - except KeyError: - print("SKIP : We don't know which packages we must install to compile" - " a {target} {build_type} version on a {host} host.".format( - target=self.platform_info, - host=self.distname)) - return - - packages_list = package_name_mapper.get('COMMON', []) - for dep in self.targetsDict.values(): - packages = package_name_mapper.get(dep.name) - if packages: - packages_list += packages - dep.skip = True - for dep in self.targetsDict.values(): - packages = getattr(dep, 'extra_packages', []) - for package in packages: - packages_list += package_name_mapper.get(package, []) - if not self.options.force_install_packages and os.path.exists(autoskip_file): - print("SKIP") - return - - packages_to_install = [] - for package in packages_list: - print(" - {} : ".format(package), end="") - command = package_checker.format(package) - try: - subprocess.check_call(command, shell=True) - except subprocess.CalledProcessError: - print("NEEDED") - packages_to_install.append(package) - else: - print("SKIP") - - if packages_to_install: - command = package_installer.format(" ".join(packages_to_install)) - print(command) - subprocess.check_call(command, shell=True) - else: - print("SKIP, No package to install.") - - with open(autoskip_file, 'w'): - pass diff --git a/kiwixbuild/builder.py b/kiwixbuild/builder.py index a83d3c6..eefa670 100644 --- a/kiwixbuild/builder.py +++ b/kiwixbuild/builder.py @@ -9,8 +9,14 @@ from .dependencies import Dependency class Builder: def __init__(self, options): self.options = options + platformClass = PlatformInfo.all_platforms[options.target_platform] + if neutralEnv('distname') not in platformClass.compatible_hosts: + print(('ERROR: The target platform {} cannot be build on host {}.\n' + 'Select another target platform, or change your host system.' + ).format(options.target_platform, self.distname)) + sys.exit(-1) self.targets = OrderedDict() - self.buildEnv = BuildEnv(options, self.targets) + self.platform = platform = platformClass(self.targets) _targets = {} targetDef = options.targets @@ -30,7 +36,7 @@ class Builder: if targetName in targets: return targetClass = Dependency.all_deps[targetName] - target = targetClass(self.buildEnv) + target = targetClass(self.platform.buildEnv) targets[targetName] = target for dep in target.builder.get_dependencies(self.platform): self.add_targets(dep, targets) @@ -46,7 +52,7 @@ class Builder: print("SKIP") return - toolchain_sources = (tlc.source for tlc in self.buildEnv.toolchains if tlc.source) + toolchain_sources = (tlc.source for tlc in self.platform.buildEnv.toolchains if tlc.source) for toolchain_source in toolchain_sources: print("prepare sources for toolchain {} :".format(toolchain_source.name)) toolchain_source.prepare() @@ -58,7 +64,7 @@ class Builder: source.prepare() def build(self): - toolchain_builders = (tlc.builder for tlc in self.buildEnv.toolchains if tlc.builder) + toolchain_builders = (tlc.builder for tlc in self.platform.buildEnv.toolchains if tlc.builder) for toolchain_builder in toolchain_builders: print("build toolchain {} :".format(toolchain_builder.name)) toolchain_builder.build() @@ -79,16 +85,16 @@ class Builder: def run(self): try: print("[INSTALL PACKAGES]") - self.buildEnv.install_packages() - self.buildEnv.finalize_setup() + self.platform.install_packages() + self.platform.finalize_setup() print("[PREPARE]") self.prepare_sources() print("[BUILD]") self.build() # No error, clean intermediate file at end of build if needed. print("[CLEAN]") - if self.buildEnv.options.clean_at_end: - self.buildEnv.clean_intermediate_directories() + if self.options.clean_at_end: + self.platform.clean_intermediate_directories() else: print("SKIP") except StopBuild: diff --git a/kiwixbuild/dependencies/icu4c.py b/kiwixbuild/dependencies/icu4c.py index 77ade0c..b3029d1 100644 --- a/kiwixbuild/dependencies/icu4c.py +++ b/kiwixbuild/dependencies/icu4c.py @@ -28,7 +28,7 @@ class Icu(Dependency): @property def configure_option(self): options = "--disable-samples --disable-tests --disable-extras --disable-dyload --enable-rpath" - if self.buildEnv.platform_info.build == 'android': + if self.buildEnv.platformInfo.build == 'android': options += " --with-data-packaging=archive" return options diff --git a/kiwixbuild/dependencies/kiwix_lib.py b/kiwixbuild/dependencies/kiwix_lib.py index dee3627..120230d 100644 --- a/kiwixbuild/dependencies/kiwix_lib.py +++ b/kiwixbuild/dependencies/kiwix_lib.py @@ -27,12 +27,12 @@ class Kiwixlib(Dependency): @property def configure_option(self): base_option = "-Dctpp2-install-prefix={buildEnv.install_dir}" - if self.buildEnv.platform_info.build == 'android': + if self.buildEnv.platformInfo.build == 'android': base_option += ' -Dandroid=true' return base_option @property def library_type(self): - if self.buildEnv.platform_info.build == 'android': + if self.buildEnv.platformInfo.build == 'android': return 'shared' return super().library_type diff --git a/kiwixbuild/dependencies/kiwix_tools.py b/kiwixbuild/dependencies/kiwix_tools.py index 1551ce7..cfdce90 100644 --- a/kiwixbuild/dependencies/kiwix_tools.py +++ b/kiwixbuild/dependencies/kiwix_tools.py @@ -15,6 +15,6 @@ class KiwixTools(Dependency): @property def configure_option(self): - if self.buildEnv.platform_info.static: + if self.buildEnv.platformInfo.static: return "-Dstatic-linkage=true" return "" diff --git a/kiwixbuild/dependencies/zim_tools.py b/kiwixbuild/dependencies/zim_tools.py index f3454dd..46332e0 100644 --- a/kiwixbuild/dependencies/zim_tools.py +++ b/kiwixbuild/dependencies/zim_tools.py @@ -15,6 +15,6 @@ class ZimTools(Dependency): @property def configure_option(self): - if self.buildEnv.platform_info.static: + if self.buildEnv.platformInfo.static: return "-Dstatic-linkage=true" return "" diff --git a/kiwixbuild/dependencies/zimwriterfs.py b/kiwixbuild/dependencies/zimwriterfs.py index 3b0808e..72cc92f 100644 --- a/kiwixbuild/dependencies/zimwriterfs.py +++ b/kiwixbuild/dependencies/zimwriterfs.py @@ -23,6 +23,6 @@ class Zimwriterfs(Dependency): @property def configure_option(self): base_option = "-Dmagic-install-prefix={buildEnv.install_dir}" - if self.buildEnv.platform_info.static: + if self.buildEnv.platformInfo.static: base_option += " -Dstatic-linkage=true" return base_option diff --git a/kiwixbuild/dependencies/zlib.py b/kiwixbuild/dependencies/zlib.py index 821eee6..babc294 100644 --- a/kiwixbuild/dependencies/zlib.py +++ b/kiwixbuild/dependencies/zlib.py @@ -28,16 +28,16 @@ class zlib(Dependency): def _configure(self, context): - if self.buildEnv.platform_info.build == 'win32': + if self.buildEnv.platformInfo.build == 'win32': raise SkipCommand() return super()._configure(context) @property def make_option(self): - if self.buildEnv.platform_info.build == 'win32': + if self.buildEnv.platformInfo.build == 'win32': return "--makefile win32/Makefile.gcc PREFIX={host}- SHARED_MODE={static} INCLUDE_PATH={include_path} LIBRARY_PATH={library_path} BINARY_PATH={binary_path}".format( host='i686-w64-mingw32', - static="0" if self.buildEnv.platform_info.static else "1", + static="0" if self.buildEnv.platformInfo.static else "1", include_path=pj(self.buildEnv.install_dir, 'include'), library_path=pj(self.buildEnv.install_dir, self.buildEnv.libprefix), binary_path=pj(self.buildEnv.install_dir, 'bin'), diff --git a/kiwixbuild/platforms/android.py b/kiwixbuild/platforms/android.py index 6caa326..8504d27 100644 --- a/kiwixbuild/platforms/android.py +++ b/kiwixbuild/platforms/android.py @@ -20,8 +20,34 @@ class AndroidPlatformInfo(PlatformInfo): def __str__(self): return "android" + def binaries(self, install_path): + binaries = ((k,'{}-{}'.format(self.arch_full, v)) + for k, v in (('CC', 'gcc'), + ('CXX', 'g++'), + ('AR', 'ar'), + ('STRIP', 'strip'), + ('WINDRES', 'windres'), + ('RANLIB', 'ranlib'), + ('LD', 'ld')) + ) + return {k:pj(install_path, 'bin', v) + for k,v in binaries} + + @property + def ndk_builder(self): + return self.toolchains[0].builder + + @property + def sdk_builder(self): + return self.toolchains[1].builder + def get_cross_config(self): + install_path = self.ndk_builder.install_path return { + 'exec_wrapper_def': '', + 'install_path': install_path, + 'binaries': self.binaries(install_path), + 'root_path': pj(install_path, 'sysroot'), 'extra_libs': [], 'extra_cflags': [], 'host_machine': { @@ -34,7 +60,22 @@ class AndroidPlatformInfo(PlatformInfo): }, } + def get_bin_dir(self): + return [pj(self.ndk_builder.install_path, 'bin')] + def set_env(self, env): + root_path = pj(self.ndk_builder.install_path, 'sysroot') + env['PKG_CONFIG_LIBDIR'] = pj(root_path, 'lib', 'pkgconfig') + env['CFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(root_path) + env['CFLAGS'] + env['CXXFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(root_path) + env['CXXFLAGS'] + env['LDFLAGS'] = '--sysroot={} '.format(root_path) + env['LDFLAGS'] + #env['CFLAGS'] = ' -fPIC -D_FILE_OFFSET_BITS=64 -O3 '+env['CFLAGS'] + #env['CXXFLAGS'] = (' -D__OPTIMIZE__ -fno-strict-aliasing ' + # ' -DU_HAVE_NL_LANGINFO_CODESET=0 ' + # '-DU_STATIC_IMPLEMENTATION -O3 ' + # '-DU_HAVE_STD_STRING -DU_TIMEZONE=0 ')+env['CXXFLAGS'] + env['NDK_DEBUG'] = '0' + env['ANDROID_HOME'] = self.sdk_builder.install_path AndroidPlatformInfo('android_arm', 'arm') AndroidPlatformInfo('android_arm64', 'arm64') diff --git a/kiwixbuild/platforms/armhf.py b/kiwixbuild/platforms/armhf.py index 742d3cd..3c13d0c 100644 --- a/kiwixbuild/platforms/armhf.py +++ b/kiwixbuild/platforms/armhf.py @@ -2,11 +2,15 @@ from .base import PlatformInfo class ArmhfPlatformInfo(PlatformInfo): + arch_full = 'arm-linux-gnueabihf' def __init__(self, name, static): super().__init__(name, 'armhf', static, ['armhf_toolchain'], ['fedora', 'debian']) def get_cross_config(self): return { + 'binaries': self.binaries, + 'exec_wrapper_def': '', + 'root_path': self.root_path, 'extra_libs': [], 'extra_cflags': [], 'host_machine': { @@ -19,6 +23,65 @@ class ArmhfPlatformInfo(PlatformInfo): } } + @property + def tlc_source(self): + return self.toolchains[0].source + + @property + def root_path(self): + return pj(self.tlc_source.source_path, + 'arm-bcm2708', + 'gcc-linaro-{}-raspbian-x64'.format(self.arch_full)) + + @property + def binaries(self): + binaries = ((k,'{}-{}'.format(self.arch_full, v)) + for k, v in (('CC', 'gcc'), + ('CXX', 'g++'), + ('AR', 'ar'), + ('STRIP', 'strip'), + ('WINDRES', 'windres'), + ('RANLIB', 'ranlib'), + ('LD', 'ld')) + ) + return {k:pj(self.root_path, 'bin', v) + for k,v in binaries} + + @property + def exec_wrapper_def(self): + try: + which('qemu-arm') + except subprocess.CalledProcessError: + return "" + else: + return "exec_wrapper = 'qemu-arm'" + + @property + def configure_option(self): + return '--host={}'.format(self.arch_full) + + def get_bin_dir(self): + return [pj(self.root_path, 'bin')] + + def set_env(self, env): + env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') + env['CFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS'] + env['CXXFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS'] + env['QEMU_LD_PREFIX'] = pj(self.root_path, "arm-linux-gnueabihf", "libc") + env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format( + ':'.join([ + pj(self.root_path, self.arch_full, "lib"), + env['LD_LIBRARY_PATH'] + ])) + + def set_compiler(self, env): + env['CC'] = self.binaries['CC'] + env['CXX'] = self.binaries['CXX'] + + def finalize_setup(self): + super().finalize_setup() + self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') + self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') ArmhfPlatformInfo('armhf_dyn', False) ArmhfPlatformInfo('armhf_static', True) diff --git a/kiwixbuild/platforms/base.py b/kiwixbuild/platforms/base.py index 6ddd37b..1d252e2 100644 --- a/kiwixbuild/platforms/base.py +++ b/kiwixbuild/platforms/base.py @@ -1,5 +1,15 @@ +import os +import subprocess +from kiwixbuild.toolchains import Toolchain +from kiwixbuild.packages import PACKAGE_NAME_MAPPERS +from kiwixbuild.utils import pj, remove_duplicates +from kiwixbuild.buildenv import BuildEnv +from kiwixbuild._global import neutralEnv + +_SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +TEMPLATES_DIR = pj(os.path.dirname(_SCRIPT_DIR), 'templates') class PlatformInfo: all_platforms = {} @@ -10,12 +20,127 @@ class PlatformInfo: self.static = static self.toolchains = toolchains self.compatible_hosts = hosts + self.buildEnv = BuildEnv(self) + self.setup_toolchains() def __str__(self): return "{}_{}".format(self.build, 'static' if self.static else 'dyn') + def setup_toolchains(self): + self.toolchains = {} + for tlc_name in self.toolchains: + ToolchainClass = Toolchain.all_toolchains[tlc_name] + self.toolchains[tlc_name] = ToolchainClass() + + def get_cross_config(self): + return {} + def set_env(self, env): pass def get_bind_dir(self): return [] + + def set_compiler(self, env): + pass + + def _gen_crossfile(self, name): + crossfile = pj(self.buildEnv.build_dir, name) + template_file = pj(TEMPLATES_DIR, name) + with open(template_file, 'r') as f: + template = f.read() + content = template.format( + **self.get_cross_config() + ) + with open(crossfile, 'w') as outfile: + outfile.write(content) + return crossfile + + def finalize_setup(self): + self.buildEnv.toolchains = [ + self.toolchains[tlc_name] for tlc_name in self.toolchain_names] + self.buildEnv.cross_config = self.get_cross_config() + self.buildEnv.meson_crossfile = None + self.buildEnv.cmake_crossfile = None + + def get_packages(self): + mapper_name = "{host}_{target}".format( + host=neutralEnv('distname'), + target=self) + try: + package_name_mapper = PACKAGE_NAME_MAPPERS[mapper_name] + except KeyError: + print("SKIP : We don't know which packages we must install to compile" + " a {target} {build_type} version on a {host} host.".format( + target=self.platform_info, + host=neutralEnv('distname'))) + return + + packages_list = package_name_mapper.get('COMMON', []) + to_drop = [] + build_steps = (step for step in target_steps() if step[0] == self.name) + for dep in build_steps: + packages = package_name_mapper.get(dep[1]) + if packages: + packages_list += packages + to_drop.append(dep) + for dep in to_drop: + del target_steps()[dep] + source_dep = [dep[1] for dep in target_steps() if dep[0] == 'source'] + for dep in source_dep: + it_build = (True for dep in target_steps() if dep[0] != 'source' and dep[1] == dep) + if next(it_build, False): + print("removing source ", dep) + del target_steps()[('source', dep)] + build_steps = (step for step in target_steps() if step[0] == self.name) + for dep in build_steps: + packages = getattr(dep, 'extra_packages', []) + for package in packages: + packages_list += package_name_mapper.get(package, []) + return packages_list + + def install_packages(self): + autoskip_file = pj(self.buildEnv.build_dir, ".install_packages_ok") + packages_to_have = self.get_packages() + packages_to_have = remove_duplicates(packages_to_have) + + if not neutralEnv('force_install_packages') and os.path.exists(autoskip_file): + print("SKIP") + return + + distname = neutralEnv('distname') + if distname in ('fedora', 'redhat', 'centos'): + package_installer = 'sudo dnf install {}' + package_checker = 'rpm -q --quiet {}' + elif distname in ('debian', 'Ubuntu'): + package_installer = 'sudo apt-get install {}' + package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1' + elif distname == 'Darwin': + package_installer = 'brew install {}' + package_checker = 'brew list -1 | grep -q {}' + + packages_to_install = [] + for package in packages_to_have: + print(" - {} : ".format(package), end="") + command = package_checker.format(package) + try: + subprocess.check_call(command, shell=True) + except subprocess.CalledProcessError: + print("NEEDED") + packages_to_install.append(package) + else: + print("SKIP") + + if packages_to_install: + command = package_installer.format(" ".join(packages_to_install)) + print(command) + subprocess.check_call(command, shell=True) + else: + print("SKIP, No package to install.") + + with open(autoskip_file, 'w'): + pass + + def clean_intermediate_directories(self): + self.buildEnv.clean_intermediate_directories() + diff --git a/kiwixbuild/platforms/i586.py b/kiwixbuild/platforms/i586.py index 262c97d..e12c953 100644 --- a/kiwixbuild/platforms/i586.py +++ b/kiwixbuild/platforms/i586.py @@ -3,11 +3,14 @@ from .base import PlatformInfo class I586PlatformInfo(PlatformInfo): + arch_full = 'i586-linux-gnu' def __init__(self, name, static): - super().__init__(name, 'i586', static, ['linux_i586_toolchain'], ['fedora', 'debian']) + super().__init__(name, 'i586', static, [], ['fedora', 'debian']) def get_cross_config(self): return { + 'binaries': self.binaries, + 'exec_wrapper_def': '', 'extra_libs': ['-m32', '-march=i586', '-mno-sse'], 'extra_cflags': ['-m32', '-march=i586', '-mno-sse'], 'host_machine': { @@ -20,6 +23,34 @@ class I586PlatformInfo(PlatformInfo): } } + @property + def configure_option(self): + return '--host={}'.format(self.arch_full) + + @property + def binaries(self): + return {k:which(v) + for k, v in (('CC', os.environ.get('CC', 'gcc')), + ('CXX', os.environ.get('CXX', 'g++')), + ('AR', 'ar'), + ('STRIP', 'strip'), + ('RANLIB', 'ranlib'), + ('LD', 'ld')) + } + + def set_env(self, env): + env['CFLAGS'] = "-m32 -march=i586 -mno-sse "+env['CFLAGS'] + env['CXXFLAGS'] = "-m32 -march=i586 -mno-sse "+env['CXXFLAGS'] + env['LDFLAGS'] = "-m32 -march=i586 -mno-sse "+env['LDFLAGS'] + + def get_bin_dir(self): + return [] + + + def finalize_setup(self): + super().finalize_setup() + self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_i586_cross_file.txt') + self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') I586PlatformInfo('i586_dyn', False) I586PlatformInfo('i586_static', True) diff --git a/kiwixbuild/platforms/ios.py b/kiwixbuild/platforms/ios.py index 86a9b55..bbbe804 100644 --- a/kiwixbuild/platforms/ios.py +++ b/kiwixbuild/platforms/ios.py @@ -1,7 +1,8 @@ -from .base import PlatformInfo import subprocess +from .base import PlatformInfo +from kiwixbuild.utils import pj, xrun_find class iOSPlatformInfo(PlatformInfo): @@ -13,7 +14,7 @@ class iOSPlatformInfo(PlatformInfo): } def __init__(self, name, arch): - super().__init__(name, 'iOS', True, ['iOS_sdk'], + super().__init__(name, 'iOS', True, [], hosts=['Darwin']) self.arch = arch self.arch_full, self.cpu, self.sdk_name = self.__arch_infos[arch] @@ -29,9 +30,16 @@ class iOSPlatformInfo(PlatformInfo): def __str__(self): return "iOS" + def finalize_setup(self): + super().finalize_setup() + self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_ios_cross_file.txt') + self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') + def get_cross_config(self): return { 'root_path': self.root_path, + 'binaries': self.binaries, + 'exec_wrapper_def': '', 'extra_libs': ['-fembed-bitcode', '-isysroot', self.root_path, '-arch', self.arch, '-miphoneos-version-min=9.0', '-stdlib=libc++'], 'extra_cflags': ['-fembed-bitcode', '-isysroot', self.root_path, '-arch', self.arch, '-miphoneos-version-min=9.0', '-stdlib=libc++'], 'host_machine': { @@ -53,6 +61,26 @@ class iOSPlatformInfo(PlatformInfo): def get_bin_dir(self): return [pj(self.root_path, 'bin')] + @property + def binaries(self): + return { + 'CC': xrun_find('clang'), + 'CXX': xrun_find('clang++'), + 'AR': '/usr/bin/ar', + 'STRIP': '/usr/bin/strip', + 'RANLIB': '/usr/bin/ranlib', + 'LD': '/usr/bin/ld', + } + + @property + def configure_option(self): + return '--host={}'.format(self.arch_full) + + def set_compiler(self, env): + env['CC'] = self.binaries['CC'] + env['CXX'] = self.binaries['CXX'] + + iOSPlatformInfo('iOS_armv7', 'armv7') iOSPlatformInfo('iOS_arm64', 'arm64') iOSPlatformInfo('iOS_i386', 'i386') diff --git a/kiwixbuild/platforms/native.py b/kiwixbuild/platforms/native.py index 9a23b10..bd324e5 100644 --- a/kiwixbuild/platforms/native.py +++ b/kiwixbuild/platforms/native.py @@ -5,9 +5,6 @@ class NativePlatformInfo(PlatformInfo): def __init__(self, name, static, hosts): super().__init__(name, 'native', static, [], hosts) - def get_cross_config(self): - return {} - NativePlatformInfo('native_dyn', False, ['fedora', 'debian', 'Darwin']) NativePlatformInfo('native_static', True, ['fedora', 'debian']) diff --git a/kiwixbuild/platforms/win32.py b/kiwixbuild/platforms/win32.py index 5d6b643..785acc3 100644 --- a/kiwixbuild/platforms/win32.py +++ b/kiwixbuild/platforms/win32.py @@ -1,13 +1,21 @@ +import subprocess + from .base import PlatformInfo +from kiwixbuild.utils import which, pj +from kiwixbuild._global import neutralEnv class Win32PlatformInfo(PlatformInfo): extra_libs = ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4', '-lmsvcr90', '-liphlpapi'] + arch_full = 'i686-w64-mingw32' def __init__(self, name, static): - super().__init__(name, 'win32', static, ['mingw32_toolchain'], ['fedora', 'debian']) + super().__init__(name, 'win32', static, [], ['fedora', 'debian']) def get_cross_config(self): return { + 'exec_wrapper_def': self.exec_wrapper_def, + 'binaries': self.binaries, + 'root_path': self.root_path, 'extra_libs': self.extra_libs, 'extra_cflags': ['-DWIN32'], 'host_machine': { @@ -20,5 +28,53 @@ class Win32PlatformInfo(PlatformInfo): } } + def finalize_setup(self): + super().finalize_setup() + self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') + self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') + + @property + def root_path(self): + root_paths = { + 'fedora': '/usr/i686-w64-mingw32/sys-root/mingw', + 'debian': '/usr/i686-w64-mingw32' + } + return root_paths[neutralEnv('distname')] + + @property + def binaries(self): + return {k:which('{}-{}'.format(self.arch_full, v)) + for k, v in (('CC', 'gcc'), + ('CXX', 'g++'), + ('AR', 'ar'), + ('STRIP', 'strip'), + ('WINDRES', 'windres'), + ('RANLIB', 'ranlib')) + } + + @property + def exec_wrapper_def(self): + try: + which('wine') + except subprocess.CalledProcessError: + return "" + else: + return "exec_wrapper = 'wine'" + + @property + def configure_option(self): + return '--host={}'.format(self.arch_full) + + def set_compiler(self, env): + for k, v in self.binaries.items(): + env[k] = v + + def get_bin_dir(self): + return [pj(self.root_path, 'bin')] + + def set_env(self, env): + env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') + env['LIBS'] = " ".join(self.extra_libs) + " " +env['LIBS'] + Win32PlatformInfo('win32_dyn', False) Win32PlatformInfo('win32_static', True) diff --git a/kiwixbuild/templates/cmake_android_cross_file.txt b/kiwixbuild/templates/cmake_android_cross_file.txt index 7f80a34..09f121e 100644 --- a/kiwixbuild/templates/cmake_android_cross_file.txt +++ b/kiwixbuild/templates/cmake_android_cross_file.txt @@ -1,7 +1,8 @@ SET(CMAKE_SYSTEM_NAME {host_machine[system]}) -SET(CMAKE_ANDROID_STANDALONE_TOOLCHAIN {toolchain.builder.install_path}) -SET(CMAKE_C_COMPILER "{toolchain.binaries[CC]}") -SET(CMAKE_CXX_COMPILER "{toolchain.binaries[CXX]}") +SET(CMAKE_ANDROID_STANDALONE_TOOLCHAIN {install_path}) -SET(CMAKE_FIND_ROOT_PATH {toolchain.root_path}) +SET(CMAKE_C_COMPILER "{binaries[CC]}") +SET(CMAKE_CXX_COMPILER "{binaries[CXX]}") + +SET(CMAKE_FIND_ROOT_PATH {root_path}) diff --git a/kiwixbuild/templates/cmake_cross_file.txt b/kiwixbuild/templates/cmake_cross_file.txt index 70184cf..f236854 100644 --- a/kiwixbuild/templates/cmake_cross_file.txt +++ b/kiwixbuild/templates/cmake_cross_file.txt @@ -2,11 +2,11 @@ SET(CMAKE_SYSTEM_NAME {host_machine[system]}) SET(CMAKE_SYSTEM_PROCESSOR {host_machine[cpu_family]}) # specify the cross compiler -SET(CMAKE_C_COMPILER "{toolchain.binaries[CC]}") -SET(CMAKE_CXX_COMPILER "{toolchain.binaries[CXX]}") -SET(CMAKE_RC_COMPILER {toolchain.binaries[WINDRES]}) -SET(CMAKE_AR:FILEPATH {toolchain.binaries[AR]}) -SET(CMAKE_RANLIB:FILEPATH {toolchain.binaries[RANLIB]}) +SET(CMAKE_C_COMPILER "{binaries[CC]}") +SET(CMAKE_CXX_COMPILER "{binaries[CXX]}") +SET(CMAKE_RC_COMPILER {binaries[WINDRES]}) +SET(CMAKE_AR:FILEPATH {binaries[AR]}) +SET(CMAKE_RANLIB:FILEPATH {binaries[RANLIB]}) find_program(CCACHE_FOUND ccache) if(CCACHE_FOUND) @@ -15,7 +15,7 @@ if(CCACHE_FOUND) endif(CCACHE_FOUND) # where is the target environment -SET(CMAKE_FIND_ROOT_PATH {toolchain.root_path}) +SET(CMAKE_FIND_ROOT_PATH {root_path}) # search for programs in the build host directories SET(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) diff --git a/kiwixbuild/templates/cmake_i586_cross_file.txt b/kiwixbuild/templates/cmake_i586_cross_file.txt index 03d186a..037d783 100644 --- a/kiwixbuild/templates/cmake_i586_cross_file.txt +++ b/kiwixbuild/templates/cmake_i586_cross_file.txt @@ -2,13 +2,13 @@ SET(CMAKE_SYSTEM_NAME {host_machine[system]}) SET(CMAKE_SYSTEM_PROCESSOR {host_machine[cpu_family]}) # specify the cross compiler -SET(CMAKE_C_COMPILER "{toolchain.binaries[CC]}") -SET(CMAKE_CXX_COMPILER "{toolchain.binaries[CXX]}") +SET(CMAKE_C_COMPILER "{binaries[CC]}") +SET(CMAKE_CXX_COMPILER "{binaries[CXX]}") SET(C_FLAGS "-m32 -march=i586") SET(CXX_FLAGS "-m32 -march=i586") SET(CMAKE_LD_FLAGS "-m32 -march=i586") -SET(CMAKE_AR:FILEPATH {toolchain.binaries[AR]}) -SET(CMAKE_RANLIB:FILEPATH {toolchain.binaries[RANLIB]}) +SET(CMAKE_AR:FILEPATH {binaries[AR]}) +SET(CMAKE_RANLIB:FILEPATH {binaries[RANLIB]}) find_program(CCACHE_FOUND ccache) if(CCACHE_FOUND) diff --git a/kiwixbuild/templates/cmake_ios_cross_file.txt b/kiwixbuild/templates/cmake_ios_cross_file.txt index 7ceaa46..41ccd60 100644 --- a/kiwixbuild/templates/cmake_ios_cross_file.txt +++ b/kiwixbuild/templates/cmake_ios_cross_file.txt @@ -1,7 +1,7 @@ SET(CMAKE_SYSTEM_NAME {host_machine[system]}) -SET(CMAKE_C_COMPILER "{toolchain.binaries[CC]}") -SET(CMAKE_CXX_COMPILER "{toolchain.binaries[CXX]}") +SET(CMAKE_C_COMPILER "{binaries[CC]}") +SET(CMAKE_CXX_COMPILER "{binaries[CXX]}") SET(CMAKE_FIND_ROOT_PATH {root_path}) diff --git a/kiwixbuild/templates/meson_android_cross_file.txt b/kiwixbuild/templates/meson_android_cross_file.txt index d0710b4..2d2f6d6 100644 --- a/kiwixbuild/templates/meson_android_cross_file.txt +++ b/kiwixbuild/templates/meson_android_cross_file.txt @@ -1,9 +1,9 @@ [binaries] pkgconfig = 'pkg-config' -c = '{toolchain.binaries[CC]}' -ar = '{toolchain.binaries[AR]}' -cpp = '{toolchain.binaries[CXX]}' -strip = '{toolchain.binaries[STRIP]}' +c = '{binaries[CC]}' +ar = '{binaries[AR]}' +cpp = '{binaries[CXX]}' +strip = '{binaries[STRIP]}' [properties] c_link_args = {extra_libs!r} diff --git a/kiwixbuild/templates/meson_cross_file.txt b/kiwixbuild/templates/meson_cross_file.txt index 069b2ee..eef32f1 100644 --- a/kiwixbuild/templates/meson_cross_file.txt +++ b/kiwixbuild/templates/meson_cross_file.txt @@ -1,10 +1,10 @@ [binaries] pkgconfig = 'pkg-config' -c = '{toolchain.binaries[CC]}' -ar = '{toolchain.binaries[AR]}' -cpp = '{toolchain.binaries[CXX]}' -strip = '{toolchain.binaries[STRIP]}' -{toolchain.exec_wrapper_def} +c = '{binaries[CC]}' +ar = '{binaries[AR]}' +cpp = '{binaries[CXX]}' +strip = '{binaries[STRIP]}' +{exec_wrapper_def} [properties] c_link_args = {extra_libs!r} diff --git a/kiwixbuild/toolchains/__init__.py b/kiwixbuild/toolchains/__init__.py index d0ca8b1..ebf0cff 100644 --- a/kiwixbuild/toolchains/__init__.py +++ b/kiwixbuild/toolchains/__init__.py @@ -1,4 +1,4 @@ -from . import android_ndk, android_sdk, armhf, ios, mingw32, linux_i586 +from . import android_ndk, android_sdk, armhf from .base_toolchain import Toolchain diff --git a/kiwixbuild/toolchains/android_ndk.py b/kiwixbuild/toolchains/android_ndk.py index 3487f6b..f33e7e8 100644 --- a/kiwixbuild/toolchains/android_ndk.py +++ b/kiwixbuild/toolchains/android_ndk.py @@ -113,22 +113,3 @@ class android_ndk(Toolchain): self.command('build_platform', self._build_platform) self.command('fix_permission_right', self._fix_permission_right) - def get_bin_dir(self): - return [pj(self.builder.install_path, 'bin')] - - def set_env(self, env): - env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') - env['CFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(self.root_path) + env['CFLAGS'] - env['CXXFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(self.root_path) + env['CXXFLAGS'] - env['LDFLAGS'] = '--sysroot={} '.format(self.root_path) + env['LDFLAGS'] - #env['CFLAGS'] = ' -fPIC -D_FILE_OFFSET_BITS=64 -O3 '+env['CFLAGS'] - #env['CXXFLAGS'] = (' -D__OPTIMIZE__ -fno-strict-aliasing ' - # ' -DU_HAVE_NL_LANGINFO_CODESET=0 ' - # '-DU_STATIC_IMPLEMENTATION -O3 ' - # '-DU_HAVE_STD_STRING -DU_TIMEZONE=0 ')+env['CXXFLAGS'] - env['NDK_DEBUG'] = '0' - - def set_compiler(self, env): - env['CC'] = self.binaries['CC'] - env['CXX'] = self.binaries['CXX'] - diff --git a/kiwixbuild/toolchains/android_sdk.py b/kiwixbuild/toolchains/android_sdk.py index 753dfb4..de4b0b7 100644 --- a/kiwixbuild/toolchains/android_sdk.py +++ b/kiwixbuild/toolchains/android_sdk.py @@ -49,6 +49,3 @@ class android_sdk(Toolchain): def build(self): self.command('build_platform', self._build_platform) self.command('fix_licenses', self._fix_licenses) - - def set_env(self, env): - env['ANDROID_HOME'] = self.builder.install_path diff --git a/kiwixbuild/toolchains/armhf.py b/kiwixbuild/toolchains/armhf.py index 1975665..d49aa62 100644 --- a/kiwixbuild/toolchains/armhf.py +++ b/kiwixbuild/toolchains/armhf.py @@ -8,57 +8,7 @@ pj = os.path.join class armhf_toolchain(Toolchain): name = 'armhf' - arch_full = 'arm-linux-gnueabihf' class Source(GitClone): git_remote = "https://github.com/raspberrypi/tools" git_dir = "raspberrypi-tools" - - @property - def root_path(self): - return pj(self.source_path, 'arm-bcm2708', 'gcc-linaro-arm-linux-gnueabihf-raspbian-x64') - - @property - def binaries(self): - binaries = ((k,'{}-{}'.format(self.arch_full, v)) - for k, v in (('CC', 'gcc'), - ('CXX', 'g++'), - ('AR', 'ar'), - ('STRIP', 'strip'), - ('WINDRES', 'windres'), - ('RANLIB', 'ranlib'), - ('LD', 'ld')) - ) - return {k:pj(self.root_path, 'bin', v) - for k,v in binaries} - - @property - def exec_wrapper_def(self): - try: - which('qemu-arm') - except subprocess.CalledProcessError: - return "" - else: - return "exec_wrapper = 'qemu-arm'" - - @property - def configure_option(self): - return '--host={}'.format(self.arch_full) - - def get_bin_dir(self): - return [pj(self.root_path, 'bin')] - - def set_env(self, env): - env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') - env['CFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS'] - env['CXXFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS'] - env['QEMU_LD_PREFIX'] = pj(self.root_path, "arm-linux-gnueabihf", "libc") - env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format( - ':'.join([ - pj(self.root_path, "arm-linux-gnueabihf", "lib"), - env['LD_LIBRARY_PATH'] - ])) - - def set_compiler(self, env): - env['CC'] = self.binaries['CC'] - env['CXX'] = self.binaries['CXX'] diff --git a/kiwixbuild/toolchains/base_toolchain.py b/kiwixbuild/toolchains/base_toolchain.py index 3a459ac..3c626e0 100644 --- a/kiwixbuild/toolchains/base_toolchain.py +++ b/kiwixbuild/toolchains/base_toolchain.py @@ -41,15 +41,6 @@ class Toolchain(metaclass=_MetaToolchain): def _log_dir(self): return neutralEnv('log_dir') - def set_env(self, env): - pass - - def set_compiler(self, env): - pass - - def get_bin_dir(self): - return [] - def command(self, name, function, *args): print(" {} {} : ".format(name, self.name), end="", flush=True) log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name)) diff --git a/kiwixbuild/toolchains/ios.py b/kiwixbuild/toolchains/ios.py deleted file mode 100644 index f1dd4be..0000000 --- a/kiwixbuild/toolchains/ios.py +++ /dev/null @@ -1,23 +0,0 @@ -from .base_toolchain import Toolchain - -from kiwixbuild.utils import pj, xrun_find - -class iOS_sdk(Toolchain): - @property - def binaries(self): - return { - 'CC': xrun_find('clang'), - 'CXX': xrun_find('clang++'), - 'AR': '/usr/bin/ar', - 'STRIP': '/usr/bin/strip', - 'RANLIB': '/usr/bin/ranlib', - 'LD': '/usr/bin/ld', - } - - @property - def configure_option(self): - return '--host=arm-apple-darwin' - - def set_compiler(self, env): - env['CC'] = self.binaries['CC'] - env['CXX'] = self.binaries['CXX'] diff --git a/kiwixbuild/toolchains/linux_i586.py b/kiwixbuild/toolchains/linux_i586.py deleted file mode 100644 index ce89dd8..0000000 --- a/kiwixbuild/toolchains/linux_i586.py +++ /dev/null @@ -1,37 +0,0 @@ -import os - -from .base_toolchain import Toolchain -from kiwixbuild.dependencies import GitClone -from kiwixbuild.utils import which -pj = os.path.join - -class linux_i586_toolchain(Toolchain): - name = 'linux_i586' - arch_full = 'i586-linux-gnu' - - @property - def configure_option(self): - return '--host={}'.format(self.arch_full) - - @property - def binaries(self): - return {k:which(v) - for k, v in (('CC', os.environ.get('CC', 'gcc')), - ('CXX', os.environ.get('CXX', 'g++')), - ('AR', 'ar'), - ('STRIP', 'strip'), - ('RANLIB', 'ranlib'), - ('LD', 'ld')) - } - - @property - def configure_option(self): - return '--host={}'.format(self.arch_full) - - def set_env(self, env): - env['CFLAGS'] = "-m32 -march=i586 -mno-sse "+env['CFLAGS'] - env['CXXFLAGS'] = "-m32 -march=i586 -mno-sse "+env['CXXFLAGS'] - env['LDFLAGS'] = "-m32 -march=i586 -mno-sse "+env['LDFLAGS'] - - def get_bin_dir(self): - return [] diff --git a/kiwixbuild/toolchains/mingw32.py b/kiwixbuild/toolchains/mingw32.py deleted file mode 100644 index 8b29b1a..0000000 --- a/kiwixbuild/toolchains/mingw32.py +++ /dev/null @@ -1,56 +0,0 @@ -import os -import subprocess - -from .base_toolchain import Toolchain -from kiwixbuild.utils import which -from kiwixbuild._global import neutralEnv - -pj = os.path.join - -class mingw32_toolchain(Toolchain): - name = 'mingw32' - arch_full = 'i686-w64-mingw32' - extra_libs = ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4', '-lmsvcr90', '-liphlpapi'] - - @property - def root_path(self): - root_paths = { - 'fedora': '/usr/i686-w64-mingw32/sys-root/mingw', - 'debian': '/usr/i686-w64-mingw32' - } - return root_paths[neutralEnv('distname')] - - @property - def binaries(self): - return {k:which('{}-{}'.format(self.arch_full, v)) - for k, v in (('CC', 'gcc'), - ('CXX', 'g++'), - ('AR', 'ar'), - ('STRIP', 'strip'), - ('WINDRES', 'windres'), - ('RANLIB', 'ranlib')) - } - - @property - def exec_wrapper_def(self): - try: - which('wine') - except subprocess.CalledProcessError: - return "" - else: - return "exec_wrapper = 'wine'" - - @property - def configure_option(self): - return '--host={}'.format(self.arch_full) - - def set_compiler(self, env): - for k, v in self.binaries.items(): - env[k] = v - - def get_bin_dir(self): - return [pj(self.root_path, 'bin')] - - def set_env(self, env): - env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') - env['LIBS'] = " ".join(self.extra_libs) + " " +env['LIBS']