From 00acba9ee933a3cc87c01b840d16883615475681 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 22 Feb 2017 12:42:53 +0100 Subject: [PATCH] Add the notion of toolschain. - For now, only the win32 is here. - There no more nativeBuilEnv as the buildEnv can launch command without using the toolschain. --- dependency_utils.py | 17 +-- kiwix-build.py | 209 +++++++++++++++++++++------------ templates/cmake_cross_file.txt | 12 +- templates/meson_cross_file.txt | 20 ++-- utils.py | 3 +- 5 files changed, 165 insertions(+), 96 deletions(-) diff --git a/dependency_utils.py b/dependency_utils.py index 20bfb89..291f118 100644 --- a/dependency_utils.py +++ b/dependency_utils.py @@ -44,7 +44,7 @@ class Dependency(metaclass=_MetaDependency): def command(self, name, function, *args): print(" {} {} : ".format(name, self.name), end="", flush=True) log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name)) - context = Context(name, log) + context = Context(name, log, self.force_native_build) try: ret = function(*args, context=context) context._finalise() @@ -106,9 +106,10 @@ class ReleaseDownload(Source): def _patch(self, context): context.try_skip(self.extract_path) + context.force_native_build = True for p in self.patches: with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input: - self.buildEnv.run_command("patch -p1", self.extract_path, context, input=patch_input, allow_wrapper=False) + self.buildEnv.run_command("patch -p1", self.extract_path, context, input=patch_input) def prepare(self): self.command('download', self._download) @@ -129,12 +130,14 @@ class GitClone(Source): return pj(self.buildEnv.source_dir, self.git_dir) def _git_clone(self, context): + context.force_native_build = True if os.path.exists(self.git_path): raise SkipCommand() command = "git clone " + self.git_remote self.buildEnv.run_command(command, self.buildEnv.source_dir, context) def _git_update(self, context): + context.force_native_build = True self.buildEnv.run_command("git fetch", self.git_path, context) self.buildEnv.run_command("git checkout "+self.git_ref, self.git_path, context) @@ -188,7 +191,7 @@ class MakeBuilder(Builder): configure_option = "{} {} {}".format( self.configure_option, self.static_configure_option if self.buildEnv.build_static else self.dynamic_configure_option, - self.buildEnv.configure_option) + self.buildEnv.configure_option if not self.target.force_native_build else "") command = "{configure_script} {configure_option} --prefix {install_dir} --libdir {libdir}" command = command.format( configure_script=pj(self.source_path, self.configure_script), @@ -251,7 +254,7 @@ class CMakeBuilder(MakeBuilder): v = v.format(buildEnv=self.buildEnv, env=env) self.configure_env[k[8:]] = v env.update(self.configure_env) - self.buildEnv.run_command(command, self.build_path, context, env=env, allow_wrapper=False) + self.buildEnv.run_command(command, self.build_path, context, env=env, cross_path_only=True) class MesonBuilder(Builder): @@ -281,12 +284,12 @@ class MesonBuilder(Builder): buildEnv=self.buildEnv, cross_option="--cross-file {}".format(self.buildEnv.meson_crossfile) if self.buildEnv.meson_crossfile else "" ) - self.buildEnv.run_command(command, self.source_path, context, allow_wrapper=False) + self.buildEnv.run_command(command, self.source_path, context, cross_path_only=True) def _compile(self, context): command = "{} -v".format(self.buildEnv.ninja_command) - self.buildEnv.run_command(command, self.build_path, context, allow_wrapper=False) + self.buildEnv.run_command(command, self.build_path, context) def _install(self, context): command = "{} -v install".format(self.buildEnv.ninja_command) - self.buildEnv.run_command(command, self.build_path, context, allow_wrapper=False) + self.buildEnv.run_command(command, self.build_path, context) diff --git a/kiwix-build.py b/kiwix-build.py index 371fdba..6d48cce 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -23,16 +23,8 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) CROSS_ENV = { 'fedora_win32': { + 'toolchain_names': ['mingw32_toolchain'], 'root_path': '/usr/i686-w64-mingw32/sys-root/mingw', - 'binaries': { - 'c': 'i686-w64-mingw32-gcc', - 'cpp': 'i686-w64-mingw32-g++', - 'ar': 'i686-w64-mingw32-ar', - 'strip': 'i686-w64-mingw32-strip', - 'windres': 'i686-w64-mingw32-windres', - 'ranlib': 'i686-w64-mingw32-ranlib', - 'exe_wrapper': 'wine' - }, 'properties': { 'c_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4'], 'cpp_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4'] @@ -48,16 +40,8 @@ CROSS_ENV = { } }, 'debian_win32': { + 'toolchain_names': ['mingw32_toolchain'], 'root_path': '/usr/i686-w64-mingw32/', - 'binaries': { - 'c': 'i686-w64-mingw32-gcc', - 'cpp': 'i686-w64-mingw32-g++', - 'ar': 'i686-w64-mingw32-ar', - 'strip': 'i686-w64-mingw32-strip', - 'windres': 'i686-w64-mingw32-windres', - 'ranlib': 'i686-w64-mingw32-ranlib', - 'exe_wrapper': 'wine' - }, 'properties': { 'c_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4'], 'cpp_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4'] @@ -129,14 +113,10 @@ PACKAGE_NAME_MAPPERS = { } -class Which(): - def __getattr__(self, name): - command = "which {}".format(name) - output = subprocess.check_output(command, shell=True) - return output[:-1].decode() - - def __format__(self, format_spec): - return getattr(self, format_spec) +def which(name): + command = "which {}".format(name) + output = subprocess.check_output(command, shell=True) + return output[:-1].decode() class BuildEnv: @@ -165,6 +145,8 @@ class BuildEnv: self.meson_command = self._detect_meson() if not self.meson_command: sys.exit("ERROR: meson command not fount") + self.setup_build(options.build_target) + self.setup_toolchains() self.options = options self.libprefix = options.libprefix or self._detect_libdir() self.targetsDict = targetsDict @@ -187,14 +169,32 @@ class BuildEnv: if self.distname == 'ubuntu': self.distname = 'debian' + def setup_build(self, target): + self.build_target = target + if target == 'native': + self.cross_env = {} + else: + cross_name = "{host}_{target}".format( + host = self.distname, + target = self.build_target) + try: + self.cross_env = CROSS_ENV[cross_name] + except KeyError: + sys.exit("ERROR : We don't know how to set env to compile" + " a {target} version on a {host} host.".format( + target = self.build_target, + host = self.distname + )) + + def setup_toolchains(self): + toolchain_names = self.cross_env.get('toolchain_names', []) + self.toolchains =[Toolchain.all_toolchains[toolchain_name](self) + for toolchain_name in toolchain_names] + def finalize_setup(self): getattr(self, 'setup_{}'.format(self.build_target))() def setup_native(self): - self.cross_env = {} - self.wrapper = None - self.configure_option = "" - self.cmake_option = "" self.cmake_crossfile = None self.meson_crossfile = None @@ -204,7 +204,7 @@ class BuildEnv: with open(template_file, 'r') as f: template = f.read() content = template.format( - which=Which(), + toolchain=self.toolchains[0], **self.cross_env ) with open(crossfile, 'w') as outfile: @@ -212,23 +212,6 @@ class BuildEnv: return crossfile def setup_win32(self): - cross_name = "{host}_{target}".format( - host=self.distname, - target=self.build_target) - try: - self.cross_env = CROSS_ENV[cross_name] - except KeyError: - sys.exit("ERROR : We don't know how to set env to compile" - " a {target} version on a {host} host.".format( - target=self.build_target, - host=self.distname - )) - - self.wrapper = self._gen_crossfile('bash_wrapper.sh') - current_permissions = stat.S_IMODE(os.lstat(self.wrapper).st_mode) - os.chmod(self.wrapper, current_permissions | stat.S_IXUSR) - self.configure_option = "--host=i686-w64-mingw32" - self.cmake_option = "" self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') @@ -276,14 +259,33 @@ class BuildEnv: if retcode == 0: return n - def _set_env(self, env): + @property + def configure_option(self): + configure_options = [tlc.configure_option for tlc in self.toolchains] + return " ".join(configure_options) + + @property + def cmake_option(self): + cmake_options = [tlc.cmake_option for tlc in self.toolchains] + return " ".join(cmake_options) + + def _set_env(self, env, cross_compile_env, cross_compile_path): if env is None: env = Defaultdict(str, os.environ) - for k, v in self.cross_env.get('env', {}).items(): - if k.startswith('_format_'): - v = v.format(**self.cross_env) - k = k[8:] - env[k] = v + + bin_dirs = [] + if cross_compile_env: + for k, v in self.cross_env.get('env', {}).items(): + if k.startswith('_format_'): + v = v.format(**self.cross_env) + k = k[8:] + env[k] = v + for toolchain in self.toolchains: + toolchain.set_env(env) + if cross_compile_path: + for tlc in self.toolchains: + bin_dirs += tlc.get_bin_dir() + pkgconfig_path = pj(self.install_dir, self.libprefix, 'pkgconfig') env['PKG_CONFIG_PATH'] = (env['PKG_CONFIG_PATH'] + ':' + pkgconfig_path if env['PKG_CONFIG_PATH'] @@ -296,7 +298,10 @@ class BuildEnv: break else: ccache_path = [] - env['PATH'] = ':'.join([pj(self.install_dir, 'bin')] + ccache_path + [env['PATH']]) + env['PATH'] = ':'.join(bin_dirs + + [pj(self.install_dir, 'bin')] + + ccache_path + + [env['PATH']]) ld_library_path = ':'.join([ pj(self.install_dir, 'lib'), pj(self.install_dir, 'lib64') @@ -305,15 +310,20 @@ class BuildEnv: if env['LD_LIBRARY_PATH'] else ld_library_path ) - env['CPPFLAGS'] = '-I'+pj(self.install_dir, 'include') - env['LDFLAGS'] = '-L'+pj(self.install_dir, 'lib') + env['CPPFLAGS'] = " ".join(['-I'+pj(self.install_dir, 'include'), env['CPPFLAGS']]) + env['LDFLAGS'] = " ".join(['-L'+pj(self.install_dir, 'lib'), env['LDFLAGS']]) return env - def run_command(self, command, cwd, context, env=None, input=None, allow_wrapper=True): + def run_command(self, command, cwd, context, env=None, input=None, cross_path_only=False): os.makedirs(cwd, exist_ok=True) - if allow_wrapper and self.wrapper: - command = "{} {}".format(self.wrapper, command) - env = self._set_env(env) + cross_compile_env = True + cross_compile_path = True + if context.force_native_build: + cross_compile_env = False + cross_compile_path = False + if cross_path_only: + cross_compile_env = False + env = self._set_env(env, cross_compile_env, cross_compile_path) log = None try: if not self.options.verbose: @@ -414,16 +424,71 @@ class BuildEnv: with open(autoskip_file, 'w'): pass +class _MetaToolchain(type): + def __new__(cls, name, bases, dct): + _class = type.__new__(cls, name, bases, dct) + if name != 'Toolchain': + Toolchain.all_toolchains[name] = _class + return _class + + +class Toolchain(metaclass=_MetaToolchain): + all_toolchains = {} + configure_option = "" + cmake_option = "" + + def __init__(self, buildEnv): + self.buildEnv = buildEnv + + @property + def full_name(self): + return "{name}-{version}".format( + name = self.name, + version = self.version) + + def set_env(self, env): + pass + + +class mingw32_toolchain(Toolchain): + name = 'mingw32' + arch_full = 'i686-w64-mingw32' + + @property + def root_path(self): + return self.buildEnv.cross_env['root_path'] + + @property + def binaries(self): + return {k:which('{}-{}'.format(self.arch_full, v)) + for k, v in (('CC', 'gcc'), + ('CXX', 'g++'), + ('AR', 'ar'), + ('STRIP', 'strip'), + ('WINDRES', 'windres'), + ('RANLIB', 'ranlib')) + } + + @property + def configure_option(self): + return '--host={}'.format(self.arch_full) + + def get_bin_dir(self): + return [pj(self.root_path, 'bin')] + + def set_env(self, env): + for k, v in self.binaries.items(): + env[k] = v + + env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') + env['CFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS'] + env['CXXFLAGS'] =" -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS'] + class Builder: def __init__(self, options, targetDef='KiwixTools'): self.targets = OrderedDict() self.buildEnv = buildEnv = BuildEnv(options, self.targets) - if buildEnv.build_target != 'native': - self.nativeBuildEnv = BuildEnv(options, self.targets) - self.nativeBuildEnv.build_target = 'native' - else: - self.nativeBuildEnv = buildEnv _targets = {} self.add_targets(targetDef, _targets) @@ -437,10 +502,7 @@ class Builder: if targetName in targets: return targetClass = Dependency.all_deps[targetName] - if targetClass.force_native_build: - target = targetClass(self.nativeBuildEnv) - else: - target = targetClass(self.buildEnv) + target = targetClass(self.buildEnv) targets[targetName] = target for dep in target.dependencies: self.add_targets(dep, targets) @@ -452,6 +514,11 @@ class Builder: yield targetName def prepare_sources(self): + toolchain_sources = (tlc.source for tlc in self.buildEnv.toolchains if tlc.source) + for toolchain_source in toolchain_sources: + print("prepare sources for toolchain {} :".format(toolchain_source.name)) + toolchain_source.prepare() + sources = (dep.source for dep in self.targets.values() if not dep.skip) sources = remove_duplicates(sources, lambda s: s.__class__) for source in sources: @@ -469,8 +536,6 @@ class Builder: print("[INSTALL PACKAGES]") self.buildEnv.install_packages() self.buildEnv.finalize_setup() - if self.nativeBuildEnv != self.buildEnv: - self.nativeBuildEnv.finalize_setup() print("[PREPARE]") self.prepare_sources() print("[BUILD]") diff --git a/templates/cmake_cross_file.txt b/templates/cmake_cross_file.txt index 8410a07..75a98a0 100644 --- a/templates/cmake_cross_file.txt +++ b/templates/cmake_cross_file.txt @@ -2,11 +2,11 @@ SET(CMAKE_SYSTEM_NAME Windows) SET(CMAKE_SYSTEM_PROCESSOR x86) # specify the cross compiler -SET(CMAKE_C_COMPILER "{which:{binaries[c]}}") -SET(CMAKE_CXX_COMPILER "{which:{binaries[cpp]}}") -SET(CMAKE_RC_COMPILER {which:{binaries[windres]}}) -SET(CMAKE_AR:FILEPATH {which:{binaries[ar]}}) -SET(CMAKE_RANLIB:FILEPATH {which:{binaries[ranlib]}}) +SET(CMAKE_C_COMPILER "{toolchain.binaries[CC]}") +SET(CMAKE_CXX_COMPILER "{toolchain.binaries[CXX]}") +SET(CMAKE_RC_COMPILER {toolchain.binaries[WINDRES]}) +SET(CMAKE_AR:FILEPATH {toolchain.binaries[AR]}) +SET(CMAKE_RANLIB:FILEPATH {toolchain.binaries[RANLIB]}) find_program(CCACHE_FOUND ccache) if(CCACHE_FOUND) @@ -15,7 +15,7 @@ if(CCACHE_FOUND) endif(CCACHE_FOUND) # where is the target environment -SET(CMAKE_FIND_ROOT_PATH {root_path}) +SET(CMAKE_FIND_ROOT_PATH {toolchain.root_path}) # search for programs in the build host directories SET(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) diff --git a/templates/meson_cross_file.txt b/templates/meson_cross_file.txt index edea054..464cbfe 100644 --- a/templates/meson_cross_file.txt +++ b/templates/meson_cross_file.txt @@ -1,16 +1,16 @@ [binaries] pkgconfig = 'pkg-config' -c = '{which:{binaries[c]}}' -ar = '{which:{binaries[ar]}}' -cpp = '{which:{binaries[cpp]}}' -strip = '{which:{binaries[strip]}}' +c = '{toolchain.binaries[CC]}' +ar = '{toolchain.binaries[AR]}' +cpp = '{toolchain.binaries[CXX]}' +strip = '{toolchain.binaries[STRIP]}' [properties] -c_link_args = ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4'] -cpp_link_args = ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4'] +c_link_args = {properties[c_link_args]!r} +cpp_link_args = {properties[cpp_link_args]!r} [host_machine] -cpu_family = 'x86' -cpu = 'i586' -system = 'windows' -endian = 'little' +cpu_family = '{host_machine[cpu_family]}' +cpu = '{host_machine[cpu]}' +system = '{host_machine[system]}' +endian = '{host_machine[endian]}' diff --git a/utils.py b/utils.py index 9c1464b..af51cc8 100644 --- a/utils.py +++ b/utils.py @@ -45,9 +45,10 @@ class Remotefile(namedtuple('Remotefile', ('name', 'sha256', 'url'))): class Context: - def __init__(self, command_name, log_file): + def __init__(self, command_name, log_file, force_native_build): self.command_name = command_name self.log_file = log_file + self.force_native_build = force_native_build self.autoskip_file = None def try_skip(self, path):