From 816e06a512122788e77a2b6c12920ed0da2bb952 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 28 May 2018 11:16:47 +0200 Subject: [PATCH 01/23] Make the neutralEnv global. No need to pass the neutralEnv from instance to instance. --- kiwixbuild/__init__.py | 3 ++ kiwixbuild/_global.py | 8 +++ kiwixbuild/buildenv.py | 13 ++--- kiwixbuild/builder.py | 5 +- kiwixbuild/dependencies/all_dependencies.py | 4 +- kiwixbuild/dependencies/base.py | 55 +++++++++------------ kiwixbuild/dependencies/kiwix_lib.py | 3 +- kiwixbuild/dependencies/libaria2.py | 3 +- kiwixbuild/dependencies/xapian.py | 3 +- kiwixbuild/toolchains/base_toolchain.py | 9 ++-- kiwixbuild/toolchains/mingw32.py | 3 +- 11 files changed, 57 insertions(+), 52 deletions(-) create mode 100644 kiwixbuild/_global.py diff --git a/kiwixbuild/__init__.py b/kiwixbuild/__init__.py index d84be62..73bb7c2 100644 --- a/kiwixbuild/__init__.py +++ b/kiwixbuild/__init__.py @@ -7,6 +7,7 @@ from .dependencies import Dependency from .platforms import PlatformInfo from .builder import Builder from .utils import setup_print_progress +from . import _global def parse_args(): parser = argparse.ArgumentParser() @@ -65,6 +66,8 @@ def main(): options = parse_args() options.working_dir = os.path.abspath(options.working_dir) setup_print_progress(options.show_progress) + neutralEnv = buildenv.PlatformNeutralEnv(options) + _global.set_neutralEnv(neutralEnv) builder = Builder(options) builder.run() diff --git a/kiwixbuild/_global.py b/kiwixbuild/_global.py new file mode 100644 index 0000000..0b5aa35 --- /dev/null +++ b/kiwixbuild/_global.py @@ -0,0 +1,8 @@ +_neutralEnv = None + +def set_neutralEnv(env): + global _neutralEnv + _neutralEnv = env + +def neutralEnv(what): + return getattr(_neutralEnv, what) diff --git a/kiwixbuild/buildenv.py b/kiwixbuild/buildenv.py index c742519..c103a69 100644 --- a/kiwixbuild/buildenv.py +++ b/kiwixbuild/buildenv.py @@ -6,10 +6,8 @@ import platform from .platforms import PlatformInfo from .toolchains import Toolchain from .packages import PACKAGE_NAME_MAPPERS -from .utils import ( - pj, - download_remote, - Defaultdict) +from .utils import pj, download_remote, Defaultdict +from . import _global SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -115,9 +113,8 @@ class PlatformNeutralEnv: class BuildEnv: - def __init__(self, options, neutralEnv, targetsDict): + def __init__(self, options, targetsDict): build_dir = "BUILD_{}".format(options.target_platform) - self.neutralEnv = neutralEnv self.build_dir = pj(options.working_dir, build_dir) self.install_dir = pj(self.build_dir, "INSTALL") for d in (self.build_dir, @@ -156,7 +153,7 @@ class BuildEnv: ToolchainClass = Toolchain.all_toolchains[toolchain_name] if ToolchainClass.neutral: self.toolchains.append( - self.neutralEnv.add_toolchain(toolchain_name) + _global._neutralEnv.add_toolchain(toolchain_name) ) else: self.toolchains.append(ToolchainClass(self)) @@ -202,7 +199,7 @@ class BuildEnv: self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') def __getattr__(self, name): - return getattr(self.neutralEnv, name) + return _global.neutralEnv(name) def _is_debianlike(self): return os.path.isfile('/etc/debian_version') diff --git a/kiwixbuild/builder.py b/kiwixbuild/builder.py index 92aab5a..52b9525 100644 --- a/kiwixbuild/builder.py +++ b/kiwixbuild/builder.py @@ -10,8 +10,7 @@ class Builder: def __init__(self, options): self.options = options self.targets = OrderedDict() - self.neutralEnv = PlatformNeutralEnv(options) - self.buildEnv = BuildEnv(options, self.neutralEnv, self.targets) + self.buildEnv = BuildEnv(options, self.targets) _targets = {} targetDef = options.targets @@ -31,7 +30,7 @@ class Builder: if targetName in targets: return targetClass = Dependency.all_deps[targetName] - target = targetClass(self.neutralEnv, self.buildEnv) + target = targetClass(self.buildEnv) targets[targetName] = target for dep in target.dependencies: self.add_targets(dep, targets) diff --git a/kiwixbuild/dependencies/all_dependencies.py b/kiwixbuild/dependencies/all_dependencies.py index 34845fd..9eb6837 100644 --- a/kiwixbuild/dependencies/all_dependencies.py +++ b/kiwixbuild/dependencies/all_dependencies.py @@ -3,6 +3,8 @@ from .base import ( NoopSource, NoopBuilder) +from kiwixbuild._global import neutralEnv + class AllBaseDependencies(Dependency): name = "alldependencies" @@ -16,7 +18,7 @@ class AllBaseDependencies(Dependency): else: base_deps += ["icu4c", "libmagic"] if ( self.buildEnv.platform_info.build != 'android' - and self.buildEnv.distname != 'Darwin'): + and neutralEnv('distname') != 'Darwin'): base_deps += ['ctpp2c', 'ctpp2'] if self.buildEnv.platform_info.build == 'android': base_deps += ['Gradle'] diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index a1410bf..ee9e66d 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -4,6 +4,7 @@ import shutil from kiwixbuild.utils import pj, Context, SkipCommand, extract_archive, Defaultdict, StopBuild from kiwixbuild.versions import main_project_versions, base_deps_versions +from kiwixbuild._global import neutralEnv SCRIPT_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) @@ -22,8 +23,7 @@ class Dependency(metaclass=_MetaDependency): dependencies = [] force_native_build = False - def __init__(self, neutralEnv, buildEnv): - self.neutralEnv = neutralEnv + def __init__(self, buildEnv): self.buildEnv = buildEnv self.source = self.Source(self) self.builder = self.Builder(self) @@ -41,7 +41,7 @@ class Dependency(metaclass=_MetaDependency): @property def source_path(self): - return pj(self.neutralEnv.source_dir, self.source.source_dir) + return pj(neutralEnv('source_dir'), self.source.source_dir) @property def _log_dir(self): @@ -77,7 +77,6 @@ class Source: inside the neutralEnv.source_dir.""" def __init__(self, target): self.target = target - self.neutralEnv = target.neutralEnv @property def name(self): @@ -88,12 +87,12 @@ class Source: return self.target.full_name def _patch(self, context): - source_path = pj(self.neutralEnv.source_dir, self.source_dir) + source_path = pj(neutralEnv('source_dir'), self.source_dir) context.try_skip(source_path) context.force_native_build = True for p in self.patches: with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input: - self.neutralEnv.run_command("patch -p1", source_path, context, input=patch_input.read()) + neutralEnv('run_command')("patch -p1", source_path, context, input=patch_input.read()) def command(self, *args, **kwargs): return self.target.command(*args, **kwargs) @@ -109,18 +108,18 @@ class ReleaseDownload(Source): @property def extract_path(self): - return pj(self.neutralEnv.source_dir, self.source_dir) + return pj(neutralEnv('source_dir'), self.source_dir) def _download(self, context): - context.try_skip(self.neutralEnv.archive_dir, self.name) - self.neutralEnv.download(self.archive) + context.try_skip(neutralEnv('archive_dir'), self.name) + neutralEnv('download')(self.archive) def _extract(self, context): context.try_skip(self.extract_path) if os.path.exists(self.extract_path): shutil.rmtree(self.extract_path) - extract_archive(pj(self.neutralEnv.archive_dir, self.archive.name), - self.neutralEnv.source_dir, + extract_archive(pj(neutralEnv('archive_dir'), self.archive.name), + neutralEnv('source_dir'), topdir=self.archive_top_dir, name=self.source_dir) @@ -142,18 +141,18 @@ class GitClone(Source): @property def source_dir(self): - if self.neutralEnv.make_release: + if neutralEnv('make_release'): return "{}_release".format(self.git_dir) else: return self.git_dir @property def git_path(self): - return pj(self.neutralEnv.source_dir, self.source_dir) + return pj(neutralEnv('source_dir'), self.source_dir) @property def git_ref(self): - if self.neutralEnv.make_release: + if neutralEnv('make_release'): return self.release_git_ref else: return self.base_git_ref @@ -163,13 +162,13 @@ class GitClone(Source): raise SkipCommand() command = "git clone --depth=1 --branch {} {} {}".format( self.git_ref, self.git_remote, self.source_dir) - self.neutralEnv.run_command(command, self.neutralEnv.source_dir, context) + neutralEnv('run_command')(command, neutralEnv('source_dir'), context) def _git_update(self, context): command = "git fetch origin {}".format( self.git_ref) - self.neutralEnv.run_command(command, self.git_path, context) - self.neutralEnv.run_command("git checkout "+self.git_ref, self.git_path, context) + neutralEnv('run_command')(command, self.git_path, context) + neutralEnv('run_command')("git checkout "+self.git_ref, self.git_path, context) def prepare(self): self.command('gitclone', self._git_clone) @@ -185,17 +184,17 @@ class SvnClone(Source): @property def svn_path(self): - return pj(self.neutralEnv.source_dir, self.svn_dir) + return pj(neutralEnv('source_dir'), self.svn_dir) def _svn_checkout(self, context): if os.path.exists(self.svn_path): raise SkipCommand() command = "svn checkout {} {}".format(self.svn_remote, self.svn_dir) - self.neutralEnv.run_command(command, self.neutralEnv.source_dir, context) + neutralEnv('run_command')(command, neutralEnv('source_dir'), context) def _svn_update(self, context): context.try_skip(self.svn_path) - self.neutralEnv.run_command("svn update", self.svn_path, context) + neutralEnv('run_command')("svn update", self.svn_path, context) def prepare(self): self.command('svncheckout', self._svn_checkout) @@ -355,12 +354,6 @@ class MesonBuilder(Builder): configure_option = "" test_option = "" - def __init__(self, target): - super().__init__(target) - self.meson_command = self.buildEnv.neutralEnv.meson_command - self.mesontest_command = self.buildEnv.neutralEnv.mesontest_command - self.ninja_command = self.buildEnv.neutralEnv.ninja_command - @property def library_type(self): return 'static' if self.buildEnv.platform_info.static else 'shared' @@ -382,7 +375,7 @@ class MesonBuilder(Builder): " --libdir={buildEnv.libprefix}" " {cross_option}") command = command.format( - command=self.meson_command, + command=neutralEnv('meson_command'), library_type=self.library_type, configure_option=configure_option, build_path=self.build_path, @@ -392,7 +385,7 @@ class MesonBuilder(Builder): self.buildEnv.run_command(command, self.source_path, context, cross_env_only=True) def _compile(self, context): - command = "{} -v".format(self.ninja_command) + command = "{} -v".format(neutralEnv('ninja_command')) self.buildEnv.run_command(command, self.build_path, context) def _test(self, context): @@ -401,15 +394,15 @@ class MesonBuilder(Builder): and not self.buildEnv.platform_info.static) ): raise SkipCommand() - command = "{} --verbose {}".format(self.mesontest_command, self.test_option) + command = "{} --verbose {}".format(neutralEnv('mesontest_command'), self.test_option) self.buildEnv.run_command(command, self.build_path, context) def _install(self, context): - command = "{} -v install".format(self.ninja_command) + command = "{} -v install".format(neutralEnv('ninja_command')) self.buildEnv.run_command(command, self.build_path, context) def _make_dist(self, context): - command = "{} -v dist".format(self.ninja_command) + command = "{} -v dist".format(neutralEnv('ninja_command')) self.buildEnv.run_command(command, self.build_path, context) diff --git a/kiwixbuild/dependencies/kiwix_lib.py b/kiwixbuild/dependencies/kiwix_lib.py index 6d18a34..46e8bc3 100644 --- a/kiwixbuild/dependencies/kiwix_lib.py +++ b/kiwixbuild/dependencies/kiwix_lib.py @@ -2,6 +2,7 @@ from .base import ( Dependency, GitClone, MesonBuilder) +from kiwixbuild._global import neutralEnv class Kiwixlib(Dependency): name = "kiwix-lib" @@ -10,7 +11,7 @@ class Kiwixlib(Dependency): def dependencies(self): base_dependencies = ["pugixml", "libzim", "zlib", "lzma", "libaria2"] if ( self.buildEnv.platform_info.build != 'android' - and self.buildEnv.distname != 'Darwin'): + and neutralEnv('distname') != 'Darwin'): base_dependencies += ['ctpp2c', 'ctpp2'] if self.buildEnv.platform_info.build != 'native': return base_dependencies + ["icu4c_cross-compile"] diff --git a/kiwixbuild/dependencies/libaria2.py b/kiwixbuild/dependencies/libaria2.py index 90f1567..c64686d 100644 --- a/kiwixbuild/dependencies/libaria2.py +++ b/kiwixbuild/dependencies/libaria2.py @@ -5,6 +5,7 @@ from .base import ( ) from kiwixbuild.utils import Remotefile +from kiwixbuild._global import neutralEnv class Aria2(Dependency): name = "libaria2" @@ -20,7 +21,7 @@ class Aria2(Dependency): def _post_prepare_script(self, context): context.try_skip(self.extract_path) command = "autoreconf -i" - self.neutralEnv.run_command(command, self.extract_path, context) + neutralEnv('run_command')(command, self.extract_path, context) class Builder(MakeBuilder): configure_option = "--enable-libaria2 --disable-ssl --disable-bittorent --disable-metalink --without-sqlite3 --without-libxml2 --without-libexpat" diff --git a/kiwixbuild/dependencies/xapian.py b/kiwixbuild/dependencies/xapian.py index 13f9b4b..0236e8f 100644 --- a/kiwixbuild/dependencies/xapian.py +++ b/kiwixbuild/dependencies/xapian.py @@ -5,6 +5,7 @@ from .base import ( ) from kiwixbuild.utils import Remotefile +from kiwixbuild._global import neutralEnv class Xapian(Dependency): @@ -23,6 +24,6 @@ class Xapian(Dependency): def dependencies(self): deps = ['zlib', 'lzma'] if (self.buildEnv.platform_info.build == 'win32' - or self.buildEnv.distname == 'Darwin'): + or neutralEnv('distname') == 'Darwin'): return deps return deps + ['uuid'] diff --git a/kiwixbuild/toolchains/base_toolchain.py b/kiwixbuild/toolchains/base_toolchain.py index e5040cd..3a459ac 100644 --- a/kiwixbuild/toolchains/base_toolchain.py +++ b/kiwixbuild/toolchains/base_toolchain.py @@ -4,6 +4,7 @@ import subprocess pj = os.path.join from kiwixbuild.utils import Context, SkipCommand, StopBuild +from kiwixbuild._global import neutralEnv class _MetaToolchain(type): def __new__(cls, name, bases, dct): @@ -22,9 +23,7 @@ class Toolchain(metaclass=_MetaToolchain): Builder = None Source = None - def __init__(self, neutralEnv): - self.neutralEnv = neutralEnv - self.buildEnv = neutralEnv + def __init__(self): self.source = self.Source(self) if self.Source else None self.builder = self.Builder(self) if self.Builder else None @@ -36,11 +35,11 @@ class Toolchain(metaclass=_MetaToolchain): @property def source_path(self): - return pj(self.neutralEnv.source_dir, self.source.source_dir) + return pj(neutralEnv('source_dir'), self.source.source_dir) @property def _log_dir(self): - return self.neutralEnv.log_dir + return neutralEnv('log_dir') def set_env(self, env): pass diff --git a/kiwixbuild/toolchains/mingw32.py b/kiwixbuild/toolchains/mingw32.py index 7da1283..8b29b1a 100644 --- a/kiwixbuild/toolchains/mingw32.py +++ b/kiwixbuild/toolchains/mingw32.py @@ -3,6 +3,7 @@ import subprocess from .base_toolchain import Toolchain from kiwixbuild.utils import which +from kiwixbuild._global import neutralEnv pj = os.path.join @@ -17,7 +18,7 @@ class mingw32_toolchain(Toolchain): 'fedora': '/usr/i686-w64-mingw32/sys-root/mingw', 'debian': '/usr/i686-w64-mingw32' } - return root_paths[self.neutralEnv.distname] + return root_paths[neutralEnv('distname')] @property def binaries(self): From 7e0b403cccd8dd0f2d23e966946fdcfda8db27f9 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 28 May 2018 11:18:13 +0200 Subject: [PATCH 02/23] Move `run_command` method out of buildEnv class. --- kiwixbuild/buildenv.py | 62 ---------------------------- kiwixbuild/dependencies/base.py | 34 +++++++-------- kiwixbuild/dependencies/ctpp2.py | 4 +- kiwixbuild/dependencies/gumbo.py | 4 +- kiwixbuild/dependencies/libaria2.py | 5 +-- kiwixbuild/toolchains/android_ndk.py | 4 +- kiwixbuild/toolchains/android_sdk.py | 4 +- kiwixbuild/utils.py | 40 ++++++++++++++++++ 8 files changed, 67 insertions(+), 90 deletions(-) diff --git a/kiwixbuild/buildenv.py b/kiwixbuild/buildenv.py index c103a69..edae697 100644 --- a/kiwixbuild/buildenv.py +++ b/kiwixbuild/buildenv.py @@ -34,33 +34,6 @@ class PlatformNeutralEnv: sys.exit("ERROR: meson command not fount") self.mesontest_command = "{} test".format(self.meson_command) - def run_command(self, command, cwd, context, env=None, input=None): - os.makedirs(cwd, exist_ok=True) - if env is None: - env = Defaultdict(str, os.environ) - log = None - try: - if not self.options.verbose: - log = open(context.log_file, 'w') - print("run command '{}'".format(command), file=log) - print("current directory is '{}'".format(cwd), file=log) - print("env is :", file=log) - for k, v in env.items(): - print(" {} : {!r}".format(k, v), file=log) - - kwargs = dict() - if input: - kwargs['stdin'] = subprocess.PIPE - process = subprocess.Popen(command, shell=True, cwd=cwd, env=env, stdout=log or sys.stdout, stderr=subprocess.STDOUT, **kwargs) - if input: - process.communicate(input.encode()) - retcode = process.wait() - if retcode: - raise subprocess.CalledProcessError(retcode, command) - finally: - if log: - log.close() - def detect_platform(self): _platform = platform.system() self.distname = _platform @@ -278,41 +251,6 @@ class BuildEnv: env['PATH'] = ':'.join(bin_dirs + [env['PATH']]) return env - def run_command(self, command, cwd, context, env=None, input=None, cross_env_only=False): - os.makedirs(cwd, exist_ok=True) - cross_compile_env = True - cross_compile_compiler = True - cross_compile_path = True - if context.force_native_build: - cross_compile_env = False - cross_compile_compiler = False - cross_compile_path = False - if cross_env_only: - cross_compile_compiler = False - env = self._set_env(env, cross_compile_env, cross_compile_compiler, cross_compile_path) - log = None - try: - if not self.options.verbose: - log = open(context.log_file, 'w') - print("run command '{}'".format(command), file=log) - print("current directory is '{}'".format(cwd), file=log) - print("env is :", file=log) - for k, v in env.items(): - print(" {} : {!r}".format(k, v), file=log) - - kwargs = dict() - if input: - kwargs['stdin'] = subprocess.PIPE - process = subprocess.Popen(command, shell=True, cwd=cwd, env=env, stdout=log or sys.stdout, stderr=subprocess.STDOUT, **kwargs) - if input: - process.communicate(input.encode()) - retcode = process.wait() - if retcode: - raise subprocess.CalledProcessError(retcode, command) - finally: - if log: - log.close() - def install_packages(self): autoskip_file = pj(self.build_dir, ".install_packages_ok") if self.distname in ('fedora', 'redhat', 'centos'): diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index ee9e66d..b7ee722 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -2,7 +2,7 @@ import subprocess import os import shutil -from kiwixbuild.utils import pj, Context, SkipCommand, extract_archive, Defaultdict, StopBuild +from kiwixbuild.utils import pj, Context, SkipCommand, extract_archive, Defaultdict, StopBuild, run_command from kiwixbuild.versions import main_project_versions, base_deps_versions from kiwixbuild._global import neutralEnv @@ -162,13 +162,13 @@ class GitClone(Source): raise SkipCommand() command = "git clone --depth=1 --branch {} {} {}".format( self.git_ref, self.git_remote, self.source_dir) - neutralEnv('run_command')(command, neutralEnv('source_dir'), context) + run_command(command, neutralEnv('source_dir'), context) def _git_update(self, context): command = "git fetch origin {}".format( self.git_ref) - neutralEnv('run_command')(command, self.git_path, context) - neutralEnv('run_command')("git checkout "+self.git_ref, self.git_path, context) + run_command(command, self.git_path, context) + run_command("git checkout "+self.git_ref, self.git_path, context) def prepare(self): self.command('gitclone', self._git_clone) @@ -190,11 +190,11 @@ class SvnClone(Source): if os.path.exists(self.svn_path): raise SkipCommand() command = "svn checkout {} {}".format(self.svn_remote, self.svn_dir) - neutralEnv('run_command')(command, neutralEnv('source_dir'), context) + run_command(command, neutralEnv('source_dir'), context) def _svn_update(self, context): context.try_skip(self.svn_path) - neutralEnv('run_command')("svn update", self.svn_path, context) + run_command("svn update", self.svn_path, context) def prepare(self): self.command('svncheckout', self._svn_checkout) @@ -294,7 +294,7 @@ class MakeBuilder(Builder): v = v.format(buildEnv=self.buildEnv, env=env) self.configure_env[k[8:]] = v env.update(self.configure_env) - self.buildEnv.run_command(command, self.build_path, context, env=env) + run_command(command, self.build_path, context, buildEnv=self.buildEnv, env=env) def _compile(self, context): context.try_skip(self.build_path) @@ -302,7 +302,7 @@ class MakeBuilder(Builder): make_target=self.make_target, make_option=self.make_option ) - self.buildEnv.run_command(command, self.build_path, context) + run_command(command, self.build_path, context, buildEnv=self.buildEnv) def _install(self, context): context.try_skip(self.build_path) @@ -310,12 +310,12 @@ class MakeBuilder(Builder): make_install_target=self.make_install_target, make_option=self.make_option ) - self.buildEnv.run_command(command, self.build_path, context) + run_command(command, self.build_path, context, buildEnv=self.buildEnv) def _make_dist(self, context): context.try_skip(self.build_path) command = "make dist" - self.buildEnv.run_command(command, self.build_path, context) + run_command(command, self.build_path, context, buildEnv=self.buildEnv) class CMakeBuilder(MakeBuilder): @@ -347,7 +347,7 @@ class CMakeBuilder(MakeBuilder): v = v.format(buildEnv=self.buildEnv, env=env) self.configure_env[k[8:]] = v env.update(self.configure_env) - self.buildEnv.run_command(command, self.build_path, context, env=env, cross_env_only=True) + run_command(command, self.build_path, context, env=env, buildEnv=self.buildEnv, cross_env_only=True) class MesonBuilder(Builder): @@ -382,11 +382,11 @@ class MesonBuilder(Builder): buildEnv=self.buildEnv, cross_option=cross_option ) - self.buildEnv.run_command(command, self.source_path, context, cross_env_only=True) + run_command(command, self.source_path, context, buildEnv=self.buildEnv, cross_env_only=True) def _compile(self, context): command = "{} -v".format(neutralEnv('ninja_command')) - self.buildEnv.run_command(command, self.build_path, context) + run_command(command, self.build_path, context, buildEnv=self.buildEnv) def _test(self, context): if ( self.buildEnv.platform_info.build == 'android' @@ -395,15 +395,15 @@ class MesonBuilder(Builder): ): raise SkipCommand() command = "{} --verbose {}".format(neutralEnv('mesontest_command'), self.test_option) - self.buildEnv.run_command(command, self.build_path, context) + run_command(command, self.build_path, context, buildEnv=self.buildEnv) def _install(self, context): command = "{} -v install".format(neutralEnv('ninja_command')) - self.buildEnv.run_command(command, self.build_path, context) + run_command(command, self.build_path, context, buildEnv=self.buildEnv) def _make_dist(self, context): command = "{} -v dist".format(neutralEnv('ninja_command')) - self.buildEnv.run_command(command, self.build_path, context) + run_command(command, self.build_path, context, buildEnv=self.buildEnv) class GradleBuilder(Builder): @@ -428,4 +428,4 @@ class GradleBuilder(Builder): command = command.format( gradle_target=self.gradle_target, gradle_option=self.gradle_option) - self.buildEnv.run_command(command, self.build_path, context) + run_command(command, self.build_path, context, buildEnv=self.buildEnv) diff --git a/kiwixbuild/dependencies/ctpp2.py b/kiwixbuild/dependencies/ctpp2.py index 4e1410f..4b05b03 100644 --- a/kiwixbuild/dependencies/ctpp2.py +++ b/kiwixbuild/dependencies/ctpp2.py @@ -3,7 +3,7 @@ from .base import ( ReleaseDownload, CMakeBuilder) -from kiwixbuild.utils import Remotefile, pj +from kiwixbuild.utils import Remotefile, pj, run_command class CTPP2(Dependency): name = "ctpp2" @@ -51,4 +51,4 @@ class CTPP2C(CTPP2): ctpp2c=pj(self.build_path, 'ctpp2c'), install_dir=pj(self.buildEnv.install_dir, 'bin') ) - self.buildEnv.run_command(command, self.build_path, context) + run_command(command, self.build_path, context, buildEnv=self.buildEnv) diff --git a/kiwixbuild/dependencies/gumbo.py b/kiwixbuild/dependencies/gumbo.py index 3d9cbe1..078f0fd 100644 --- a/kiwixbuild/dependencies/gumbo.py +++ b/kiwixbuild/dependencies/gumbo.py @@ -4,7 +4,7 @@ from .base import ( MakeBuilder ) -from kiwixbuild.utils import Remotefile +from kiwixbuild.utils import Remotefile, run_command class Gumbo(Dependency): @@ -18,6 +18,6 @@ class Gumbo(Dependency): def _post_prepare_script(self, context): context.try_skip(self.extract_path) command = "./autogen.sh" - self.buildEnv.run_command(command, self.extract_path, context) + run_command(command, self.extract_path, context) Builder = MakeBuilder diff --git a/kiwixbuild/dependencies/libaria2.py b/kiwixbuild/dependencies/libaria2.py index c64686d..d59e443 100644 --- a/kiwixbuild/dependencies/libaria2.py +++ b/kiwixbuild/dependencies/libaria2.py @@ -4,8 +4,7 @@ from .base import ( MakeBuilder ) -from kiwixbuild.utils import Remotefile -from kiwixbuild._global import neutralEnv +from kiwixbuild.utils import Remotefile, run_command class Aria2(Dependency): name = "libaria2" @@ -21,7 +20,7 @@ class Aria2(Dependency): def _post_prepare_script(self, context): context.try_skip(self.extract_path) command = "autoreconf -i" - neutralEnv('run_command')(command, self.extract_path, context) + run_command(command, self.extract_path, context) class Builder(MakeBuilder): configure_option = "--enable-libaria2 --disable-ssl --disable-bittorent --disable-metalink --without-sqlite3 --without-libxml2 --without-libexpat" diff --git a/kiwixbuild/toolchains/android_ndk.py b/kiwixbuild/toolchains/android_ndk.py index 8a3f3d9..3487f6b 100644 --- a/kiwixbuild/toolchains/android_ndk.py +++ b/kiwixbuild/toolchains/android_ndk.py @@ -2,7 +2,7 @@ import os from .base_toolchain import Toolchain from kiwixbuild.dependencies import ReleaseDownload, Builder -from kiwixbuild.utils import Remotefile, add_execution_right +from kiwixbuild.utils import Remotefile, add_execution_right, run_command pj = os.path.join @@ -91,7 +91,7 @@ class android_ndk(Toolchain): api=self.target.api, install_dir=self.install_path ) - self.buildEnv.run_command(command, self.build_path, context) + run_command(command, self.build_path, context, buildEnv=self.buildEnv) def _fix_permission_right(self, context): context.try_skip(self.build_path) diff --git a/kiwixbuild/toolchains/android_sdk.py b/kiwixbuild/toolchains/android_sdk.py index 7eb8934..753dfb4 100644 --- a/kiwixbuild/toolchains/android_sdk.py +++ b/kiwixbuild/toolchains/android_sdk.py @@ -3,7 +3,7 @@ import shutil from .base_toolchain import Toolchain from kiwixbuild.dependencies import ReleaseDownload, Builder -from kiwixbuild.utils import Remotefile +from kiwixbuild.utils import Remotefile, run_command pj = os.path.join @@ -38,7 +38,7 @@ class android_sdk(Toolchain): # - 8 : Android SDK Build-tools, revision 24.0.1 # - 34 : SDK Platform Android 7.0, API 24, revision 2 # - 162 : Android Support Repository, revision 44 - self.buildEnv.run_command(command, self.install_path, context, input="y\n") + run_command(command, self.install_path, context, input="y\n") def _fix_licenses(self, context): context.try_skip(self.install_path) diff --git a/kiwixbuild/utils.py b/kiwixbuild/utils.py index 34d6ae4..7eb0d29 100644 --- a/kiwixbuild/utils.py +++ b/kiwixbuild/utils.py @@ -10,6 +10,8 @@ import ssl import subprocess from collections import namedtuple, defaultdict +from kiwixbuild._global import neutralEnv + pj = os.path.join g_print_progress = True @@ -221,3 +223,41 @@ def extract_archive(archive_path, dest_dir, topdir=None, name=None): if archive is not None: archive.close() + +def run_command(command, cwd, context, buildEnv=None, env=None, input=None, cross_env_only=False): + os.makedirs(cwd, exist_ok=True) + if env is None: + env = Defaultdict(str, os.environ) + if buildEnv is not None: + cross_compile_env = True + cross_compile_compiler = True + cross_compile_path = True + if context.force_native_build: + cross_compile_env = False + cross_compile_compiler = False + cross_compile_path = False + if cross_env_only: + cross_compile_compiler = False + env = buildEnv._set_env(env, cross_compile_env, cross_compile_compiler, cross_compile_path) + log = None + try: + if not neutralEnv('verbose'): + log = open(context.log_file, 'w') + print("run command '{}'".format(command), file=log) + print("current directory is '{}'".format(cwd), file=log) + print("env is :", file=log) + for k, v in env.items(): + print(" {} : {!r}".format(k, v), file=log) + + kwargs = dict() + if input: + kwargs['stdin'] = subprocess.PIPE + process = subprocess.Popen(command, shell=True, cwd=cwd, env=env, stdout=log or sys.stdout, stderr=subprocess.STDOUT, **kwargs) + if input: + process.communicate(input.encode()) + retcode = process.wait() + if retcode: + raise subprocess.CalledProcessError(retcode, command) + finally: + if log: + log.close() From 115fbfa1476c328ae58e9eff7c784bf67e8e5aae Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 28 May 2018 11:44:54 +0200 Subject: [PATCH 03/23] Move dependencies declaration of a target into the builder. This is the builder that depends on other target, not the target itself. --- kiwixbuild/builder.py | 4 +-- kiwixbuild/dependencies/all_dependencies.py | 36 ++++++++++----------- kiwixbuild/dependencies/base.py | 6 +++- kiwixbuild/dependencies/icu4c.py | 3 +- kiwixbuild/dependencies/kiwix_android.py | 3 +- kiwixbuild/dependencies/kiwix_custom_app.py | 3 +- kiwixbuild/dependencies/kiwix_lib.py | 23 ++++++------- kiwixbuild/dependencies/kiwix_tools.py | 3 +- kiwixbuild/dependencies/libaria2.py | 2 +- kiwixbuild/dependencies/libmagic.py | 3 +- kiwixbuild/dependencies/libzim.py | 16 ++++----- kiwixbuild/dependencies/xapian.py | 14 ++++---- kiwixbuild/dependencies/zim_tools.py | 3 +- kiwixbuild/dependencies/zimwriterfs.py | 16 ++++----- 14 files changed, 72 insertions(+), 63 deletions(-) diff --git a/kiwixbuild/builder.py b/kiwixbuild/builder.py index 52b9525..a83d3c6 100644 --- a/kiwixbuild/builder.py +++ b/kiwixbuild/builder.py @@ -32,12 +32,12 @@ class Builder: targetClass = Dependency.all_deps[targetName] target = targetClass(self.buildEnv) targets[targetName] = target - for dep in target.dependencies: + for dep in target.builder.get_dependencies(self.platform): self.add_targets(dep, targets) def order_dependencies(self, _targets, targetName): target = _targets[targetName] - for depName in target.dependencies: + for depName in target.builder.dependencies(self.platform): yield from self.order_dependencies(_targets, depName) yield targetName diff --git a/kiwixbuild/dependencies/all_dependencies.py b/kiwixbuild/dependencies/all_dependencies.py index 9eb6837..c55d727 100644 --- a/kiwixbuild/dependencies/all_dependencies.py +++ b/kiwixbuild/dependencies/all_dependencies.py @@ -8,23 +8,21 @@ from kiwixbuild._global import neutralEnv class AllBaseDependencies(Dependency): name = "alldependencies" - @property - def dependencies(self): - base_deps = ['zlib', 'lzma', 'xapian-core', 'gumbo', 'pugixml', 'libmicrohttpd', 'libaria2'] - if self.buildEnv.platform_info.build != 'native': - base_deps += ["icu4c_cross-compile"] - if self.buildEnv.platform_info.build != 'win32': - base_deps += ["libmagic_cross-compile"] - else: - base_deps += ["icu4c", "libmagic"] - if ( self.buildEnv.platform_info.build != 'android' - and neutralEnv('distname') != 'Darwin'): - base_deps += ['ctpp2c', 'ctpp2'] - if self.buildEnv.platform_info.build == 'android': - base_deps += ['Gradle'] - - return base_deps - - Source = NoopSource - Builder = NoopBuilder + class Builder(NoopBuilder): + @classmethod + def get_dependencies(cls, platformInfo): + base_deps = ['zlib', 'lzma', 'xapian-core', 'gumbo', 'pugixml', 'libmicrohttpd', 'libaria2'] + if platformInfo.build != 'native': + base_deps += ["icu4c_cross-compile"] + if platformInfo.build != 'win32': + base_deps += ["libmagic_cross-compile"] + else: + base_deps += ["icu4c", "libmagic"] + if (platformInfo.build != 'android' and + neutralEnv('distname') != 'Darwin'): + base_deps += ['ctpp2c', 'ctpp2'] + if platformInfo.build == 'android': + base_deps += ['Gradle'] + + return base_deps diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index b7ee722..45fb39f 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -20,7 +20,6 @@ class _MetaDependency(type): class Dependency(metaclass=_MetaDependency): all_deps = {} - dependencies = [] force_native_build = False def __init__(self, buildEnv): @@ -205,11 +204,16 @@ class SvnClone(Source): class Builder: subsource_dir = None + dependencies = [] def __init__(self, target): self.target = target self.buildEnv = target.buildEnv + @classmethod + def get_dependencies(cls, platformInfo): + return cls.dependencies + @property def name(self): return self.target.name diff --git a/kiwixbuild/dependencies/icu4c.py b/kiwixbuild/dependencies/icu4c.py index 84ce484..77ade0c 100644 --- a/kiwixbuild/dependencies/icu4c.py +++ b/kiwixbuild/dependencies/icu4c.py @@ -48,9 +48,10 @@ class Icu_native(Icu): class Icu_cross_compile(Icu): name = "icu4c_cross-compile" - dependencies = ['icu4c_native'] class Builder(Icu.Builder): + dependencies = ['icu4c_native'] + @property def configure_option(self): icu_native_dep = self.buildEnv.targetsDict['icu4c_native'] diff --git a/kiwixbuild/dependencies/kiwix_android.py b/kiwixbuild/dependencies/kiwix_android.py index 9273a01..6e00b33 100644 --- a/kiwixbuild/dependencies/kiwix_android.py +++ b/kiwixbuild/dependencies/kiwix_android.py @@ -9,13 +9,14 @@ from kiwixbuild.utils import pj class KiwixAndroid(Dependency): name = "kiwix-android" - dependencies = ["Gradle", "kiwix-lib"] class Source(GitClone): git_remote = "https://github.com/kiwix/kiwix-android" git_dir = "kiwix-android" class Builder(GradleBuilder): + dependencies = ["Gradle", "kiwix-lib"] + def build(self): if self.buildEnv.options.targets == 'kiwix-android-custom': print("SKIP") diff --git a/kiwixbuild/dependencies/kiwix_custom_app.py b/kiwixbuild/dependencies/kiwix_custom_app.py index 75d4058..5c52efe 100644 --- a/kiwixbuild/dependencies/kiwix_custom_app.py +++ b/kiwixbuild/dependencies/kiwix_custom_app.py @@ -10,7 +10,6 @@ from kiwixbuild.utils import Remotefile, pj, SkipCommand class KiwixCustomApp(Dependency): name = "kiwix-android-custom" - dependencies = ["kiwix-android", "kiwix-lib"] def __init__(self, buildEnv): super().__init__(buildEnv) @@ -21,6 +20,8 @@ class KiwixCustomApp(Dependency): git_dir = "kiwix-android-custom" class Builder(GradleBuilder): + dependencies = ["kiwix-android", "kiwix-lib"] + @property def gradle_target(self): return "assemble{}".format(self.target.custom_name) diff --git a/kiwixbuild/dependencies/kiwix_lib.py b/kiwixbuild/dependencies/kiwix_lib.py index 46e8bc3..dee3627 100644 --- a/kiwixbuild/dependencies/kiwix_lib.py +++ b/kiwixbuild/dependencies/kiwix_lib.py @@ -7,22 +7,23 @@ from kiwixbuild._global import neutralEnv class Kiwixlib(Dependency): name = "kiwix-lib" - @property - def dependencies(self): - base_dependencies = ["pugixml", "libzim", "zlib", "lzma", "libaria2"] - if ( self.buildEnv.platform_info.build != 'android' - and neutralEnv('distname') != 'Darwin'): - base_dependencies += ['ctpp2c', 'ctpp2'] - if self.buildEnv.platform_info.build != 'native': - return base_dependencies + ["icu4c_cross-compile"] - else: - return base_dependencies + ["icu4c"] - class Source(GitClone): git_remote = "https://github.com/kiwix/kiwix-lib.git" git_dir = "kiwix-lib" class Builder(MesonBuilder): + @classmethod + def get_dependencies(cls, platformInfo): + base_dependencies = ["pugixml", "libzim", "zlib", "lzma", "libaria2"] + if (platformInfo.build != 'android' and + neutralEnv('distname') != 'Darwin'): + base_dependencies += ['ctpp2c', 'ctpp2'] + if platformInfo.build != 'native': + return base_dependencies + ["icu4c_cross-compile"] + else: + return base_dependencies + ["icu4c"] + + @property def configure_option(self): base_option = "-Dctpp2-install-prefix={buildEnv.install_dir}" diff --git a/kiwixbuild/dependencies/kiwix_tools.py b/kiwixbuild/dependencies/kiwix_tools.py index 3ff33c0..1551ce7 100644 --- a/kiwixbuild/dependencies/kiwix_tools.py +++ b/kiwixbuild/dependencies/kiwix_tools.py @@ -5,13 +5,14 @@ from .base import ( class KiwixTools(Dependency): name = "kiwix-tools" - dependencies = ["kiwix-lib", "libmicrohttpd", "zlib"] class Source(GitClone): git_remote = "https://github.com/kiwix/kiwix-tools.git" git_dir = "kiwix-tools" class Builder(MesonBuilder): + dependencies = ["kiwix-lib", "libmicrohttpd", "zlib"] + @property def configure_option(self): if self.buildEnv.platform_info.static: diff --git a/kiwixbuild/dependencies/libaria2.py b/kiwixbuild/dependencies/libaria2.py index d59e443..e0a5d38 100644 --- a/kiwixbuild/dependencies/libaria2.py +++ b/kiwixbuild/dependencies/libaria2.py @@ -8,7 +8,6 @@ from kiwixbuild.utils import Remotefile, run_command class Aria2(Dependency): name = "libaria2" - dependencies = ['zlib'] class Source(ReleaseDownload): archive = Remotefile('libaria2-1.33.1.tar.gz', @@ -23,4 +22,5 @@ class Aria2(Dependency): run_command(command, self.extract_path, context) class Builder(MakeBuilder): + dependencies = ['zlib'] configure_option = "--enable-libaria2 --disable-ssl --disable-bittorent --disable-metalink --without-sqlite3 --without-libxml2 --without-libexpat" diff --git a/kiwixbuild/dependencies/libmagic.py b/kiwixbuild/dependencies/libmagic.py index b7691a6..b093191 100644 --- a/kiwixbuild/dependencies/libmagic.py +++ b/kiwixbuild/dependencies/libmagic.py @@ -38,9 +38,10 @@ class LibMagic_native(LibMagicBase): class LibMagic_cross_compile(LibMagicBase): name = "libmagic_cross-compile" - dependencies = ['libmagic_native'] class Builder(LibMagicBase.Builder): + dependencies = ['libmagic_native'] + def _compile(self, context): context.try_skip(self.build_path) command = "make -j4 {make_target} {make_option}".format( diff --git a/kiwixbuild/dependencies/libzim.py b/kiwixbuild/dependencies/libzim.py index eb46df8..3b0c19a 100644 --- a/kiwixbuild/dependencies/libzim.py +++ b/kiwixbuild/dependencies/libzim.py @@ -6,17 +6,17 @@ from .base import ( class Libzim(Dependency): name = "libzim" - @property - def dependencies(self): - base_dependencies = ['zlib', 'lzma', 'xapian-core'] - if self.buildEnv.platform_info.build != 'native': - return base_dependencies + ["icu4c_cross-compile"] - else: - return base_dependencies + ["icu4c"] - class Source(GitClone): git_remote = "https://github.com/openzim/libzim.git" git_dir = "libzim" class Builder(MesonBuilder): test_option = "-t 8" + + @classmethod + def get_dependencies(cls, platformInfo): + base_dependencies = ['zlib', 'lzma', 'xapian-core'] + if platformInfo.build != 'native': + return base_dependencies + ["icu4c_cross-compile"] + else: + return base_dependencies + ["icu4c"] diff --git a/kiwixbuild/dependencies/xapian.py b/kiwixbuild/dependencies/xapian.py index 0236e8f..8fc49a0 100644 --- a/kiwixbuild/dependencies/xapian.py +++ b/kiwixbuild/dependencies/xapian.py @@ -20,10 +20,10 @@ class Xapian(Dependency): configure_env = {'_format_LDFLAGS': "-L{buildEnv.install_dir}/{buildEnv.libprefix}", '_format_CXXFLAGS': "-I{buildEnv.install_dir}/include"} - @property - def dependencies(self): - deps = ['zlib', 'lzma'] - if (self.buildEnv.platform_info.build == 'win32' - or neutralEnv('distname') == 'Darwin'): - return deps - return deps + ['uuid'] + @classmethod + def get_dependencies(cls, platformInfo): + deps = ['zlib', 'lzma'] + if (platformInfo.build == 'win32' + or neutralEnv('distname') == 'Darwin'): + return deps + return deps + ['uuid'] diff --git a/kiwixbuild/dependencies/zim_tools.py b/kiwixbuild/dependencies/zim_tools.py index 0634390..f3454dd 100644 --- a/kiwixbuild/dependencies/zim_tools.py +++ b/kiwixbuild/dependencies/zim_tools.py @@ -5,13 +5,14 @@ from .base import ( class ZimTools(Dependency): name = "zim-tools" - dependencies = ['libzim'] class Source(GitClone): git_remote = "https://github.com/openzim/zim-tools.git" git_dir = "zim-tools" class Builder(MesonBuilder): + dependencies = ['libzim'] + @property def configure_option(self): if self.buildEnv.platform_info.static: diff --git a/kiwixbuild/dependencies/zimwriterfs.py b/kiwixbuild/dependencies/zimwriterfs.py index 075ef8e..3b0808e 100644 --- a/kiwixbuild/dependencies/zimwriterfs.py +++ b/kiwixbuild/dependencies/zimwriterfs.py @@ -6,20 +6,20 @@ from .base import ( class Zimwriterfs(Dependency): name = "zimwriterfs" - @property - def dependencies(self): - base_dependencies = ['libzim', 'zlib', 'xapian-core', 'gumbo'] - if self.buildEnv.platform_info.build != 'native': - return base_dependencies + ["icu4c_cross-compile", "libmagic_cross-compile"] - else: - return base_dependencies + ["icu4c", "libmagic"] - class Source(GitClone): git_remote = "https://github.com/openzim/zimwriterfs.git" git_dir = "zimwriterfs" release_git_ref = "1.1" class Builder(MesonBuilder): + @classmethod + def get_dependencies(cls, platformInfo): + base_dependencies = ['libzim', 'zlib', 'xapian-core', 'gumbo'] + if platformInfo.build != 'native': + return base_dependencies + ["icu4c_cross-compile", "libmagic_cross-compile"] + else: + return base_dependencies + ["icu4c", "libmagic"] + @property def configure_option(self): base_option = "-Dmagic-install-prefix={buildEnv.install_dir}" From 754df8df1de179cda08f891327dce74d84cfd4d6 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 28 May 2018 14:18:51 +0200 Subject: [PATCH 04/23] Make the platform responsible to create the buildenv. It is now the platform who is responsible to: - correctly add the toolchains as needed - setup the cross environment - create the cross files. --- kiwixbuild/buildenv.py | 156 +----------------- kiwixbuild/builder.py | 22 ++- kiwixbuild/dependencies/icu4c.py | 2 +- kiwixbuild/dependencies/kiwix_lib.py | 4 +- kiwixbuild/dependencies/kiwix_tools.py | 2 +- kiwixbuild/dependencies/zim_tools.py | 2 +- kiwixbuild/dependencies/zimwriterfs.py | 2 +- kiwixbuild/dependencies/zlib.py | 6 +- kiwixbuild/platforms/android.py | 41 +++++ kiwixbuild/platforms/armhf.py | 63 +++++++ kiwixbuild/platforms/base.py | 125 ++++++++++++++ kiwixbuild/platforms/i586.py | 33 +++- kiwixbuild/platforms/ios.py | 32 +++- kiwixbuild/platforms/native.py | 3 - kiwixbuild/platforms/win32.py | 58 ++++++- .../templates/cmake_android_cross_file.txt | 9 +- kiwixbuild/templates/cmake_cross_file.txt | 12 +- .../templates/cmake_i586_cross_file.txt | 8 +- kiwixbuild/templates/cmake_ios_cross_file.txt | 4 +- .../templates/meson_android_cross_file.txt | 8 +- kiwixbuild/templates/meson_cross_file.txt | 10 +- kiwixbuild/toolchains/__init__.py | 2 +- kiwixbuild/toolchains/android_ndk.py | 19 --- kiwixbuild/toolchains/android_sdk.py | 3 - kiwixbuild/toolchains/armhf.py | 50 ------ kiwixbuild/toolchains/base_toolchain.py | 9 - kiwixbuild/toolchains/ios.py | 23 --- kiwixbuild/toolchains/linux_i586.py | 37 ----- kiwixbuild/toolchains/mingw32.py | 56 ------- 29 files changed, 407 insertions(+), 394 deletions(-) delete mode 100644 kiwixbuild/toolchains/ios.py delete mode 100644 kiwixbuild/toolchains/linux_i586.py delete mode 100644 kiwixbuild/toolchains/mingw32.py diff --git a/kiwixbuild/buildenv.py b/kiwixbuild/buildenv.py index edae697..afbb6a5 100644 --- a/kiwixbuild/buildenv.py +++ b/kiwixbuild/buildenv.py @@ -3,13 +3,9 @@ import os, sys, shutil import subprocess import platform -from .platforms import PlatformInfo from .toolchains import Toolchain -from .packages import PACKAGE_NAME_MAPPERS from .utils import pj, download_remote, Defaultdict -from . import _global - -SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +from ._global import neutralEnv class PlatformNeutralEnv: @@ -64,12 +60,6 @@ class PlatformNeutralEnv: if retcode == 0: return n - def add_toolchain(self, toolchain_name): - if toolchain_name not in self.toolchains: - ToolchainClass = Toolchain.all_toolchains[toolchain_name] - self.toolchains[toolchain_name] = ToolchainClass(self) - return self.toolchains[toolchain_name] - def _detect_meson(self): for n in ['meson.py', 'meson']: try: @@ -86,18 +76,16 @@ class PlatformNeutralEnv: class BuildEnv: - def __init__(self, options, targetsDict): - build_dir = "BUILD_{}".format(options.target_platform) - self.build_dir = pj(options.working_dir, build_dir) + def __init__(self, platformInfo, targetsDict): + build_dir = "BUILD_{}".format(platformInfo.name) + self.platformInfo = platformInfo + self.build_dir = pj(neutralEnv('working_dir'), build_dir) self.install_dir = pj(self.build_dir, "INSTALL") for d in (self.build_dir, self.install_dir): os.makedirs(d, exist_ok=True) - self.setup_build(options.target_platform) - self.setup_toolchains() - self.options = options - self.libprefix = options.libprefix or self._detect_libdir() + self.libprefix = neutralEnv('libprefix') or self._detect_libdir() self.targetsDict = targetsDict def clean_intermediate_directories(self): @@ -110,67 +98,6 @@ class BuildEnv: else: os.remove(subpath) - def setup_build(self, target_platform): - self.platform_info = PlatformInfo.all_platforms[target_platform] - if self.distname not in self.platform_info.compatible_hosts: - print(('ERROR: The target {} cannot be build on host {}.\n' - 'Select another target platform, or change your host system.' - ).format(target_platform, self.distname)) - sys.exit(-1) - self.cross_config = self.platform_info.get_cross_config() - - def setup_toolchains(self): - toolchain_names = self.platform_info.toolchains - self.toolchains = [] - for toolchain_name in toolchain_names: - ToolchainClass = Toolchain.all_toolchains[toolchain_name] - if ToolchainClass.neutral: - self.toolchains.append( - _global._neutralEnv.add_toolchain(toolchain_name) - ) - else: - self.toolchains.append(ToolchainClass(self)) - - def finalize_setup(self): - getattr(self, 'setup_{}'.format(self.platform_info.build))() - - def setup_native(self): - self.cmake_crossfile = None - self.meson_crossfile = None - - def _gen_crossfile(self, name): - crossfile = pj(self.build_dir, name) - template_file = pj(SCRIPT_DIR, 'templates', name) - with open(template_file, 'r') as f: - template = f.read() - content = template.format( - toolchain=self.toolchains[0], - **self.cross_config - ) - with open(crossfile, 'w') as outfile: - outfile.write(content) - return crossfile - - def setup_win32(self): - self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') - self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') - - def setup_android(self): - self.cmake_crossfile = self._gen_crossfile('cmake_android_cross_file.txt') - self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') - - def setup_armhf(self): - self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') - self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') - - def setup_iOS(self): - self.cmake_crossfile = self._gen_crossfile('cmake_ios_cross_file.txt') - self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') - - def setup_i586(self): - self.cmake_crossfile = self._gen_crossfile('cmake_i586_cross_file.txt') - self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') - def __getattr__(self, name): return _global.neutralEnv(name) @@ -237,75 +164,10 @@ class BuildEnv: v = v.format(**self.cross_config) k = k[8:] env[k] = v - for toolchain in self.toolchains: - toolchain.set_env(env) - self.platform_info.set_env(env) + self.platformInfo.set_env(env) if cross_compile_compiler: - for toolchain in self.toolchains: - toolchain.set_compiler(env) + self.platformInfo.set_compiler(env) if cross_compile_path: - bin_dirs = [] - for tlc in self.toolchains: - bin_dirs += tlc.get_bin_dir() - bin_dirs += self.platform_info.get_bind_dir() + bin_dirs = self.platformInfo.get_bind_dir() env['PATH'] = ':'.join(bin_dirs + [env['PATH']]) return env - - def install_packages(self): - autoskip_file = pj(self.build_dir, ".install_packages_ok") - if self.distname in ('fedora', 'redhat', 'centos'): - package_installer = 'sudo dnf install {}' - package_checker = 'rpm -q --quiet {}' - elif self.distname in ('debian', 'Ubuntu'): - package_installer = 'sudo apt-get install {}' - package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1' - elif self.distname == 'Darwin': - package_installer = 'brew install {}' - package_checker = 'brew list -1 | grep -q {}' - mapper_name = "{host}_{target}".format( - host=self.distname, - target=self.platform_info) - try: - package_name_mapper = PACKAGE_NAME_MAPPERS[mapper_name] - except KeyError: - print("SKIP : We don't know which packages we must install to compile" - " a {target} {build_type} version on a {host} host.".format( - target=self.platform_info, - host=self.distname)) - return - - packages_list = package_name_mapper.get('COMMON', []) - for dep in self.targetsDict.values(): - packages = package_name_mapper.get(dep.name) - if packages: - packages_list += packages - dep.skip = True - for dep in self.targetsDict.values(): - packages = getattr(dep, 'extra_packages', []) - for package in packages: - packages_list += package_name_mapper.get(package, []) - if not self.options.force_install_packages and os.path.exists(autoskip_file): - print("SKIP") - return - - packages_to_install = [] - for package in packages_list: - print(" - {} : ".format(package), end="") - command = package_checker.format(package) - try: - subprocess.check_call(command, shell=True) - except subprocess.CalledProcessError: - print("NEEDED") - packages_to_install.append(package) - else: - print("SKIP") - - if packages_to_install: - command = package_installer.format(" ".join(packages_to_install)) - print(command) - subprocess.check_call(command, shell=True) - else: - print("SKIP, No package to install.") - - with open(autoskip_file, 'w'): - pass diff --git a/kiwixbuild/builder.py b/kiwixbuild/builder.py index a83d3c6..eefa670 100644 --- a/kiwixbuild/builder.py +++ b/kiwixbuild/builder.py @@ -9,8 +9,14 @@ from .dependencies import Dependency class Builder: def __init__(self, options): self.options = options + platformClass = PlatformInfo.all_platforms[options.target_platform] + if neutralEnv('distname') not in platformClass.compatible_hosts: + print(('ERROR: The target platform {} cannot be build on host {}.\n' + 'Select another target platform, or change your host system.' + ).format(options.target_platform, self.distname)) + sys.exit(-1) self.targets = OrderedDict() - self.buildEnv = BuildEnv(options, self.targets) + self.platform = platform = platformClass(self.targets) _targets = {} targetDef = options.targets @@ -30,7 +36,7 @@ class Builder: if targetName in targets: return targetClass = Dependency.all_deps[targetName] - target = targetClass(self.buildEnv) + target = targetClass(self.platform.buildEnv) targets[targetName] = target for dep in target.builder.get_dependencies(self.platform): self.add_targets(dep, targets) @@ -46,7 +52,7 @@ class Builder: print("SKIP") return - toolchain_sources = (tlc.source for tlc in self.buildEnv.toolchains if tlc.source) + toolchain_sources = (tlc.source for tlc in self.platform.buildEnv.toolchains if tlc.source) for toolchain_source in toolchain_sources: print("prepare sources for toolchain {} :".format(toolchain_source.name)) toolchain_source.prepare() @@ -58,7 +64,7 @@ class Builder: source.prepare() def build(self): - toolchain_builders = (tlc.builder for tlc in self.buildEnv.toolchains if tlc.builder) + toolchain_builders = (tlc.builder for tlc in self.platform.buildEnv.toolchains if tlc.builder) for toolchain_builder in toolchain_builders: print("build toolchain {} :".format(toolchain_builder.name)) toolchain_builder.build() @@ -79,16 +85,16 @@ class Builder: def run(self): try: print("[INSTALL PACKAGES]") - self.buildEnv.install_packages() - self.buildEnv.finalize_setup() + self.platform.install_packages() + self.platform.finalize_setup() print("[PREPARE]") self.prepare_sources() print("[BUILD]") self.build() # No error, clean intermediate file at end of build if needed. print("[CLEAN]") - if self.buildEnv.options.clean_at_end: - self.buildEnv.clean_intermediate_directories() + if self.options.clean_at_end: + self.platform.clean_intermediate_directories() else: print("SKIP") except StopBuild: diff --git a/kiwixbuild/dependencies/icu4c.py b/kiwixbuild/dependencies/icu4c.py index 77ade0c..b3029d1 100644 --- a/kiwixbuild/dependencies/icu4c.py +++ b/kiwixbuild/dependencies/icu4c.py @@ -28,7 +28,7 @@ class Icu(Dependency): @property def configure_option(self): options = "--disable-samples --disable-tests --disable-extras --disable-dyload --enable-rpath" - if self.buildEnv.platform_info.build == 'android': + if self.buildEnv.platformInfo.build == 'android': options += " --with-data-packaging=archive" return options diff --git a/kiwixbuild/dependencies/kiwix_lib.py b/kiwixbuild/dependencies/kiwix_lib.py index dee3627..120230d 100644 --- a/kiwixbuild/dependencies/kiwix_lib.py +++ b/kiwixbuild/dependencies/kiwix_lib.py @@ -27,12 +27,12 @@ class Kiwixlib(Dependency): @property def configure_option(self): base_option = "-Dctpp2-install-prefix={buildEnv.install_dir}" - if self.buildEnv.platform_info.build == 'android': + if self.buildEnv.platformInfo.build == 'android': base_option += ' -Dandroid=true' return base_option @property def library_type(self): - if self.buildEnv.platform_info.build == 'android': + if self.buildEnv.platformInfo.build == 'android': return 'shared' return super().library_type diff --git a/kiwixbuild/dependencies/kiwix_tools.py b/kiwixbuild/dependencies/kiwix_tools.py index 1551ce7..cfdce90 100644 --- a/kiwixbuild/dependencies/kiwix_tools.py +++ b/kiwixbuild/dependencies/kiwix_tools.py @@ -15,6 +15,6 @@ class KiwixTools(Dependency): @property def configure_option(self): - if self.buildEnv.platform_info.static: + if self.buildEnv.platformInfo.static: return "-Dstatic-linkage=true" return "" diff --git a/kiwixbuild/dependencies/zim_tools.py b/kiwixbuild/dependencies/zim_tools.py index f3454dd..46332e0 100644 --- a/kiwixbuild/dependencies/zim_tools.py +++ b/kiwixbuild/dependencies/zim_tools.py @@ -15,6 +15,6 @@ class ZimTools(Dependency): @property def configure_option(self): - if self.buildEnv.platform_info.static: + if self.buildEnv.platformInfo.static: return "-Dstatic-linkage=true" return "" diff --git a/kiwixbuild/dependencies/zimwriterfs.py b/kiwixbuild/dependencies/zimwriterfs.py index 3b0808e..72cc92f 100644 --- a/kiwixbuild/dependencies/zimwriterfs.py +++ b/kiwixbuild/dependencies/zimwriterfs.py @@ -23,6 +23,6 @@ class Zimwriterfs(Dependency): @property def configure_option(self): base_option = "-Dmagic-install-prefix={buildEnv.install_dir}" - if self.buildEnv.platform_info.static: + if self.buildEnv.platformInfo.static: base_option += " -Dstatic-linkage=true" return base_option diff --git a/kiwixbuild/dependencies/zlib.py b/kiwixbuild/dependencies/zlib.py index 821eee6..babc294 100644 --- a/kiwixbuild/dependencies/zlib.py +++ b/kiwixbuild/dependencies/zlib.py @@ -28,16 +28,16 @@ class zlib(Dependency): def _configure(self, context): - if self.buildEnv.platform_info.build == 'win32': + if self.buildEnv.platformInfo.build == 'win32': raise SkipCommand() return super()._configure(context) @property def make_option(self): - if self.buildEnv.platform_info.build == 'win32': + if self.buildEnv.platformInfo.build == 'win32': return "--makefile win32/Makefile.gcc PREFIX={host}- SHARED_MODE={static} INCLUDE_PATH={include_path} LIBRARY_PATH={library_path} BINARY_PATH={binary_path}".format( host='i686-w64-mingw32', - static="0" if self.buildEnv.platform_info.static else "1", + static="0" if self.buildEnv.platformInfo.static else "1", include_path=pj(self.buildEnv.install_dir, 'include'), library_path=pj(self.buildEnv.install_dir, self.buildEnv.libprefix), binary_path=pj(self.buildEnv.install_dir, 'bin'), diff --git a/kiwixbuild/platforms/android.py b/kiwixbuild/platforms/android.py index 6caa326..8504d27 100644 --- a/kiwixbuild/platforms/android.py +++ b/kiwixbuild/platforms/android.py @@ -20,8 +20,34 @@ class AndroidPlatformInfo(PlatformInfo): def __str__(self): return "android" + def binaries(self, install_path): + binaries = ((k,'{}-{}'.format(self.arch_full, v)) + for k, v in (('CC', 'gcc'), + ('CXX', 'g++'), + ('AR', 'ar'), + ('STRIP', 'strip'), + ('WINDRES', 'windres'), + ('RANLIB', 'ranlib'), + ('LD', 'ld')) + ) + return {k:pj(install_path, 'bin', v) + for k,v in binaries} + + @property + def ndk_builder(self): + return self.toolchains[0].builder + + @property + def sdk_builder(self): + return self.toolchains[1].builder + def get_cross_config(self): + install_path = self.ndk_builder.install_path return { + 'exec_wrapper_def': '', + 'install_path': install_path, + 'binaries': self.binaries(install_path), + 'root_path': pj(install_path, 'sysroot'), 'extra_libs': [], 'extra_cflags': [], 'host_machine': { @@ -34,7 +60,22 @@ class AndroidPlatformInfo(PlatformInfo): }, } + def get_bin_dir(self): + return [pj(self.ndk_builder.install_path, 'bin')] + def set_env(self, env): + root_path = pj(self.ndk_builder.install_path, 'sysroot') + env['PKG_CONFIG_LIBDIR'] = pj(root_path, 'lib', 'pkgconfig') + env['CFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(root_path) + env['CFLAGS'] + env['CXXFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(root_path) + env['CXXFLAGS'] + env['LDFLAGS'] = '--sysroot={} '.format(root_path) + env['LDFLAGS'] + #env['CFLAGS'] = ' -fPIC -D_FILE_OFFSET_BITS=64 -O3 '+env['CFLAGS'] + #env['CXXFLAGS'] = (' -D__OPTIMIZE__ -fno-strict-aliasing ' + # ' -DU_HAVE_NL_LANGINFO_CODESET=0 ' + # '-DU_STATIC_IMPLEMENTATION -O3 ' + # '-DU_HAVE_STD_STRING -DU_TIMEZONE=0 ')+env['CXXFLAGS'] + env['NDK_DEBUG'] = '0' + env['ANDROID_HOME'] = self.sdk_builder.install_path AndroidPlatformInfo('android_arm', 'arm') AndroidPlatformInfo('android_arm64', 'arm64') diff --git a/kiwixbuild/platforms/armhf.py b/kiwixbuild/platforms/armhf.py index 742d3cd..3c13d0c 100644 --- a/kiwixbuild/platforms/armhf.py +++ b/kiwixbuild/platforms/armhf.py @@ -2,11 +2,15 @@ from .base import PlatformInfo class ArmhfPlatformInfo(PlatformInfo): + arch_full = 'arm-linux-gnueabihf' def __init__(self, name, static): super().__init__(name, 'armhf', static, ['armhf_toolchain'], ['fedora', 'debian']) def get_cross_config(self): return { + 'binaries': self.binaries, + 'exec_wrapper_def': '', + 'root_path': self.root_path, 'extra_libs': [], 'extra_cflags': [], 'host_machine': { @@ -19,6 +23,65 @@ class ArmhfPlatformInfo(PlatformInfo): } } + @property + def tlc_source(self): + return self.toolchains[0].source + + @property + def root_path(self): + return pj(self.tlc_source.source_path, + 'arm-bcm2708', + 'gcc-linaro-{}-raspbian-x64'.format(self.arch_full)) + + @property + def binaries(self): + binaries = ((k,'{}-{}'.format(self.arch_full, v)) + for k, v in (('CC', 'gcc'), + ('CXX', 'g++'), + ('AR', 'ar'), + ('STRIP', 'strip'), + ('WINDRES', 'windres'), + ('RANLIB', 'ranlib'), + ('LD', 'ld')) + ) + return {k:pj(self.root_path, 'bin', v) + for k,v in binaries} + + @property + def exec_wrapper_def(self): + try: + which('qemu-arm') + except subprocess.CalledProcessError: + return "" + else: + return "exec_wrapper = 'qemu-arm'" + + @property + def configure_option(self): + return '--host={}'.format(self.arch_full) + + def get_bin_dir(self): + return [pj(self.root_path, 'bin')] + + def set_env(self, env): + env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') + env['CFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS'] + env['CXXFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS'] + env['QEMU_LD_PREFIX'] = pj(self.root_path, "arm-linux-gnueabihf", "libc") + env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format( + ':'.join([ + pj(self.root_path, self.arch_full, "lib"), + env['LD_LIBRARY_PATH'] + ])) + + def set_compiler(self, env): + env['CC'] = self.binaries['CC'] + env['CXX'] = self.binaries['CXX'] + + def finalize_setup(self): + super().finalize_setup() + self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') + self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') ArmhfPlatformInfo('armhf_dyn', False) ArmhfPlatformInfo('armhf_static', True) diff --git a/kiwixbuild/platforms/base.py b/kiwixbuild/platforms/base.py index 6ddd37b..1d252e2 100644 --- a/kiwixbuild/platforms/base.py +++ b/kiwixbuild/platforms/base.py @@ -1,5 +1,15 @@ +import os +import subprocess +from kiwixbuild.toolchains import Toolchain +from kiwixbuild.packages import PACKAGE_NAME_MAPPERS +from kiwixbuild.utils import pj, remove_duplicates +from kiwixbuild.buildenv import BuildEnv +from kiwixbuild._global import neutralEnv + +_SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +TEMPLATES_DIR = pj(os.path.dirname(_SCRIPT_DIR), 'templates') class PlatformInfo: all_platforms = {} @@ -10,12 +20,127 @@ class PlatformInfo: self.static = static self.toolchains = toolchains self.compatible_hosts = hosts + self.buildEnv = BuildEnv(self) + self.setup_toolchains() def __str__(self): return "{}_{}".format(self.build, 'static' if self.static else 'dyn') + def setup_toolchains(self): + self.toolchains = {} + for tlc_name in self.toolchains: + ToolchainClass = Toolchain.all_toolchains[tlc_name] + self.toolchains[tlc_name] = ToolchainClass() + + def get_cross_config(self): + return {} + def set_env(self, env): pass def get_bind_dir(self): return [] + + def set_compiler(self, env): + pass + + def _gen_crossfile(self, name): + crossfile = pj(self.buildEnv.build_dir, name) + template_file = pj(TEMPLATES_DIR, name) + with open(template_file, 'r') as f: + template = f.read() + content = template.format( + **self.get_cross_config() + ) + with open(crossfile, 'w') as outfile: + outfile.write(content) + return crossfile + + def finalize_setup(self): + self.buildEnv.toolchains = [ + self.toolchains[tlc_name] for tlc_name in self.toolchain_names] + self.buildEnv.cross_config = self.get_cross_config() + self.buildEnv.meson_crossfile = None + self.buildEnv.cmake_crossfile = None + + def get_packages(self): + mapper_name = "{host}_{target}".format( + host=neutralEnv('distname'), + target=self) + try: + package_name_mapper = PACKAGE_NAME_MAPPERS[mapper_name] + except KeyError: + print("SKIP : We don't know which packages we must install to compile" + " a {target} {build_type} version on a {host} host.".format( + target=self.platform_info, + host=neutralEnv('distname'))) + return + + packages_list = package_name_mapper.get('COMMON', []) + to_drop = [] + build_steps = (step for step in target_steps() if step[0] == self.name) + for dep in build_steps: + packages = package_name_mapper.get(dep[1]) + if packages: + packages_list += packages + to_drop.append(dep) + for dep in to_drop: + del target_steps()[dep] + source_dep = [dep[1] for dep in target_steps() if dep[0] == 'source'] + for dep in source_dep: + it_build = (True for dep in target_steps() if dep[0] != 'source' and dep[1] == dep) + if next(it_build, False): + print("removing source ", dep) + del target_steps()[('source', dep)] + build_steps = (step for step in target_steps() if step[0] == self.name) + for dep in build_steps: + packages = getattr(dep, 'extra_packages', []) + for package in packages: + packages_list += package_name_mapper.get(package, []) + return packages_list + + def install_packages(self): + autoskip_file = pj(self.buildEnv.build_dir, ".install_packages_ok") + packages_to_have = self.get_packages() + packages_to_have = remove_duplicates(packages_to_have) + + if not neutralEnv('force_install_packages') and os.path.exists(autoskip_file): + print("SKIP") + return + + distname = neutralEnv('distname') + if distname in ('fedora', 'redhat', 'centos'): + package_installer = 'sudo dnf install {}' + package_checker = 'rpm -q --quiet {}' + elif distname in ('debian', 'Ubuntu'): + package_installer = 'sudo apt-get install {}' + package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1' + elif distname == 'Darwin': + package_installer = 'brew install {}' + package_checker = 'brew list -1 | grep -q {}' + + packages_to_install = [] + for package in packages_to_have: + print(" - {} : ".format(package), end="") + command = package_checker.format(package) + try: + subprocess.check_call(command, shell=True) + except subprocess.CalledProcessError: + print("NEEDED") + packages_to_install.append(package) + else: + print("SKIP") + + if packages_to_install: + command = package_installer.format(" ".join(packages_to_install)) + print(command) + subprocess.check_call(command, shell=True) + else: + print("SKIP, No package to install.") + + with open(autoskip_file, 'w'): + pass + + def clean_intermediate_directories(self): + self.buildEnv.clean_intermediate_directories() + diff --git a/kiwixbuild/platforms/i586.py b/kiwixbuild/platforms/i586.py index 262c97d..e12c953 100644 --- a/kiwixbuild/platforms/i586.py +++ b/kiwixbuild/platforms/i586.py @@ -3,11 +3,14 @@ from .base import PlatformInfo class I586PlatformInfo(PlatformInfo): + arch_full = 'i586-linux-gnu' def __init__(self, name, static): - super().__init__(name, 'i586', static, ['linux_i586_toolchain'], ['fedora', 'debian']) + super().__init__(name, 'i586', static, [], ['fedora', 'debian']) def get_cross_config(self): return { + 'binaries': self.binaries, + 'exec_wrapper_def': '', 'extra_libs': ['-m32', '-march=i586', '-mno-sse'], 'extra_cflags': ['-m32', '-march=i586', '-mno-sse'], 'host_machine': { @@ -20,6 +23,34 @@ class I586PlatformInfo(PlatformInfo): } } + @property + def configure_option(self): + return '--host={}'.format(self.arch_full) + + @property + def binaries(self): + return {k:which(v) + for k, v in (('CC', os.environ.get('CC', 'gcc')), + ('CXX', os.environ.get('CXX', 'g++')), + ('AR', 'ar'), + ('STRIP', 'strip'), + ('RANLIB', 'ranlib'), + ('LD', 'ld')) + } + + def set_env(self, env): + env['CFLAGS'] = "-m32 -march=i586 -mno-sse "+env['CFLAGS'] + env['CXXFLAGS'] = "-m32 -march=i586 -mno-sse "+env['CXXFLAGS'] + env['LDFLAGS'] = "-m32 -march=i586 -mno-sse "+env['LDFLAGS'] + + def get_bin_dir(self): + return [] + + + def finalize_setup(self): + super().finalize_setup() + self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_i586_cross_file.txt') + self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') I586PlatformInfo('i586_dyn', False) I586PlatformInfo('i586_static', True) diff --git a/kiwixbuild/platforms/ios.py b/kiwixbuild/platforms/ios.py index 86a9b55..bbbe804 100644 --- a/kiwixbuild/platforms/ios.py +++ b/kiwixbuild/platforms/ios.py @@ -1,7 +1,8 @@ -from .base import PlatformInfo import subprocess +from .base import PlatformInfo +from kiwixbuild.utils import pj, xrun_find class iOSPlatformInfo(PlatformInfo): @@ -13,7 +14,7 @@ class iOSPlatformInfo(PlatformInfo): } def __init__(self, name, arch): - super().__init__(name, 'iOS', True, ['iOS_sdk'], + super().__init__(name, 'iOS', True, [], hosts=['Darwin']) self.arch = arch self.arch_full, self.cpu, self.sdk_name = self.__arch_infos[arch] @@ -29,9 +30,16 @@ class iOSPlatformInfo(PlatformInfo): def __str__(self): return "iOS" + def finalize_setup(self): + super().finalize_setup() + self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_ios_cross_file.txt') + self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') + def get_cross_config(self): return { 'root_path': self.root_path, + 'binaries': self.binaries, + 'exec_wrapper_def': '', 'extra_libs': ['-fembed-bitcode', '-isysroot', self.root_path, '-arch', self.arch, '-miphoneos-version-min=9.0', '-stdlib=libc++'], 'extra_cflags': ['-fembed-bitcode', '-isysroot', self.root_path, '-arch', self.arch, '-miphoneos-version-min=9.0', '-stdlib=libc++'], 'host_machine': { @@ -53,6 +61,26 @@ class iOSPlatformInfo(PlatformInfo): def get_bin_dir(self): return [pj(self.root_path, 'bin')] + @property + def binaries(self): + return { + 'CC': xrun_find('clang'), + 'CXX': xrun_find('clang++'), + 'AR': '/usr/bin/ar', + 'STRIP': '/usr/bin/strip', + 'RANLIB': '/usr/bin/ranlib', + 'LD': '/usr/bin/ld', + } + + @property + def configure_option(self): + return '--host={}'.format(self.arch_full) + + def set_compiler(self, env): + env['CC'] = self.binaries['CC'] + env['CXX'] = self.binaries['CXX'] + + iOSPlatformInfo('iOS_armv7', 'armv7') iOSPlatformInfo('iOS_arm64', 'arm64') iOSPlatformInfo('iOS_i386', 'i386') diff --git a/kiwixbuild/platforms/native.py b/kiwixbuild/platforms/native.py index 9a23b10..bd324e5 100644 --- a/kiwixbuild/platforms/native.py +++ b/kiwixbuild/platforms/native.py @@ -5,9 +5,6 @@ class NativePlatformInfo(PlatformInfo): def __init__(self, name, static, hosts): super().__init__(name, 'native', static, [], hosts) - def get_cross_config(self): - return {} - NativePlatformInfo('native_dyn', False, ['fedora', 'debian', 'Darwin']) NativePlatformInfo('native_static', True, ['fedora', 'debian']) diff --git a/kiwixbuild/platforms/win32.py b/kiwixbuild/platforms/win32.py index 5d6b643..785acc3 100644 --- a/kiwixbuild/platforms/win32.py +++ b/kiwixbuild/platforms/win32.py @@ -1,13 +1,21 @@ +import subprocess + from .base import PlatformInfo +from kiwixbuild.utils import which, pj +from kiwixbuild._global import neutralEnv class Win32PlatformInfo(PlatformInfo): extra_libs = ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4', '-lmsvcr90', '-liphlpapi'] + arch_full = 'i686-w64-mingw32' def __init__(self, name, static): - super().__init__(name, 'win32', static, ['mingw32_toolchain'], ['fedora', 'debian']) + super().__init__(name, 'win32', static, [], ['fedora', 'debian']) def get_cross_config(self): return { + 'exec_wrapper_def': self.exec_wrapper_def, + 'binaries': self.binaries, + 'root_path': self.root_path, 'extra_libs': self.extra_libs, 'extra_cflags': ['-DWIN32'], 'host_machine': { @@ -20,5 +28,53 @@ class Win32PlatformInfo(PlatformInfo): } } + def finalize_setup(self): + super().finalize_setup() + self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') + self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') + + @property + def root_path(self): + root_paths = { + 'fedora': '/usr/i686-w64-mingw32/sys-root/mingw', + 'debian': '/usr/i686-w64-mingw32' + } + return root_paths[neutralEnv('distname')] + + @property + def binaries(self): + return {k:which('{}-{}'.format(self.arch_full, v)) + for k, v in (('CC', 'gcc'), + ('CXX', 'g++'), + ('AR', 'ar'), + ('STRIP', 'strip'), + ('WINDRES', 'windres'), + ('RANLIB', 'ranlib')) + } + + @property + def exec_wrapper_def(self): + try: + which('wine') + except subprocess.CalledProcessError: + return "" + else: + return "exec_wrapper = 'wine'" + + @property + def configure_option(self): + return '--host={}'.format(self.arch_full) + + def set_compiler(self, env): + for k, v in self.binaries.items(): + env[k] = v + + def get_bin_dir(self): + return [pj(self.root_path, 'bin')] + + def set_env(self, env): + env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') + env['LIBS'] = " ".join(self.extra_libs) + " " +env['LIBS'] + Win32PlatformInfo('win32_dyn', False) Win32PlatformInfo('win32_static', True) diff --git a/kiwixbuild/templates/cmake_android_cross_file.txt b/kiwixbuild/templates/cmake_android_cross_file.txt index 7f80a34..09f121e 100644 --- a/kiwixbuild/templates/cmake_android_cross_file.txt +++ b/kiwixbuild/templates/cmake_android_cross_file.txt @@ -1,7 +1,8 @@ SET(CMAKE_SYSTEM_NAME {host_machine[system]}) -SET(CMAKE_ANDROID_STANDALONE_TOOLCHAIN {toolchain.builder.install_path}) -SET(CMAKE_C_COMPILER "{toolchain.binaries[CC]}") -SET(CMAKE_CXX_COMPILER "{toolchain.binaries[CXX]}") +SET(CMAKE_ANDROID_STANDALONE_TOOLCHAIN {install_path}) -SET(CMAKE_FIND_ROOT_PATH {toolchain.root_path}) +SET(CMAKE_C_COMPILER "{binaries[CC]}") +SET(CMAKE_CXX_COMPILER "{binaries[CXX]}") + +SET(CMAKE_FIND_ROOT_PATH {root_path}) diff --git a/kiwixbuild/templates/cmake_cross_file.txt b/kiwixbuild/templates/cmake_cross_file.txt index 70184cf..f236854 100644 --- a/kiwixbuild/templates/cmake_cross_file.txt +++ b/kiwixbuild/templates/cmake_cross_file.txt @@ -2,11 +2,11 @@ SET(CMAKE_SYSTEM_NAME {host_machine[system]}) SET(CMAKE_SYSTEM_PROCESSOR {host_machine[cpu_family]}) # specify the cross compiler -SET(CMAKE_C_COMPILER "{toolchain.binaries[CC]}") -SET(CMAKE_CXX_COMPILER "{toolchain.binaries[CXX]}") -SET(CMAKE_RC_COMPILER {toolchain.binaries[WINDRES]}) -SET(CMAKE_AR:FILEPATH {toolchain.binaries[AR]}) -SET(CMAKE_RANLIB:FILEPATH {toolchain.binaries[RANLIB]}) +SET(CMAKE_C_COMPILER "{binaries[CC]}") +SET(CMAKE_CXX_COMPILER "{binaries[CXX]}") +SET(CMAKE_RC_COMPILER {binaries[WINDRES]}) +SET(CMAKE_AR:FILEPATH {binaries[AR]}) +SET(CMAKE_RANLIB:FILEPATH {binaries[RANLIB]}) find_program(CCACHE_FOUND ccache) if(CCACHE_FOUND) @@ -15,7 +15,7 @@ if(CCACHE_FOUND) endif(CCACHE_FOUND) # where is the target environment -SET(CMAKE_FIND_ROOT_PATH {toolchain.root_path}) +SET(CMAKE_FIND_ROOT_PATH {root_path}) # search for programs in the build host directories SET(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) diff --git a/kiwixbuild/templates/cmake_i586_cross_file.txt b/kiwixbuild/templates/cmake_i586_cross_file.txt index 03d186a..037d783 100644 --- a/kiwixbuild/templates/cmake_i586_cross_file.txt +++ b/kiwixbuild/templates/cmake_i586_cross_file.txt @@ -2,13 +2,13 @@ SET(CMAKE_SYSTEM_NAME {host_machine[system]}) SET(CMAKE_SYSTEM_PROCESSOR {host_machine[cpu_family]}) # specify the cross compiler -SET(CMAKE_C_COMPILER "{toolchain.binaries[CC]}") -SET(CMAKE_CXX_COMPILER "{toolchain.binaries[CXX]}") +SET(CMAKE_C_COMPILER "{binaries[CC]}") +SET(CMAKE_CXX_COMPILER "{binaries[CXX]}") SET(C_FLAGS "-m32 -march=i586") SET(CXX_FLAGS "-m32 -march=i586") SET(CMAKE_LD_FLAGS "-m32 -march=i586") -SET(CMAKE_AR:FILEPATH {toolchain.binaries[AR]}) -SET(CMAKE_RANLIB:FILEPATH {toolchain.binaries[RANLIB]}) +SET(CMAKE_AR:FILEPATH {binaries[AR]}) +SET(CMAKE_RANLIB:FILEPATH {binaries[RANLIB]}) find_program(CCACHE_FOUND ccache) if(CCACHE_FOUND) diff --git a/kiwixbuild/templates/cmake_ios_cross_file.txt b/kiwixbuild/templates/cmake_ios_cross_file.txt index 7ceaa46..41ccd60 100644 --- a/kiwixbuild/templates/cmake_ios_cross_file.txt +++ b/kiwixbuild/templates/cmake_ios_cross_file.txt @@ -1,7 +1,7 @@ SET(CMAKE_SYSTEM_NAME {host_machine[system]}) -SET(CMAKE_C_COMPILER "{toolchain.binaries[CC]}") -SET(CMAKE_CXX_COMPILER "{toolchain.binaries[CXX]}") +SET(CMAKE_C_COMPILER "{binaries[CC]}") +SET(CMAKE_CXX_COMPILER "{binaries[CXX]}") SET(CMAKE_FIND_ROOT_PATH {root_path}) diff --git a/kiwixbuild/templates/meson_android_cross_file.txt b/kiwixbuild/templates/meson_android_cross_file.txt index d0710b4..2d2f6d6 100644 --- a/kiwixbuild/templates/meson_android_cross_file.txt +++ b/kiwixbuild/templates/meson_android_cross_file.txt @@ -1,9 +1,9 @@ [binaries] pkgconfig = 'pkg-config' -c = '{toolchain.binaries[CC]}' -ar = '{toolchain.binaries[AR]}' -cpp = '{toolchain.binaries[CXX]}' -strip = '{toolchain.binaries[STRIP]}' +c = '{binaries[CC]}' +ar = '{binaries[AR]}' +cpp = '{binaries[CXX]}' +strip = '{binaries[STRIP]}' [properties] c_link_args = {extra_libs!r} diff --git a/kiwixbuild/templates/meson_cross_file.txt b/kiwixbuild/templates/meson_cross_file.txt index 069b2ee..eef32f1 100644 --- a/kiwixbuild/templates/meson_cross_file.txt +++ b/kiwixbuild/templates/meson_cross_file.txt @@ -1,10 +1,10 @@ [binaries] pkgconfig = 'pkg-config' -c = '{toolchain.binaries[CC]}' -ar = '{toolchain.binaries[AR]}' -cpp = '{toolchain.binaries[CXX]}' -strip = '{toolchain.binaries[STRIP]}' -{toolchain.exec_wrapper_def} +c = '{binaries[CC]}' +ar = '{binaries[AR]}' +cpp = '{binaries[CXX]}' +strip = '{binaries[STRIP]}' +{exec_wrapper_def} [properties] c_link_args = {extra_libs!r} diff --git a/kiwixbuild/toolchains/__init__.py b/kiwixbuild/toolchains/__init__.py index d0ca8b1..ebf0cff 100644 --- a/kiwixbuild/toolchains/__init__.py +++ b/kiwixbuild/toolchains/__init__.py @@ -1,4 +1,4 @@ -from . import android_ndk, android_sdk, armhf, ios, mingw32, linux_i586 +from . import android_ndk, android_sdk, armhf from .base_toolchain import Toolchain diff --git a/kiwixbuild/toolchains/android_ndk.py b/kiwixbuild/toolchains/android_ndk.py index 3487f6b..f33e7e8 100644 --- a/kiwixbuild/toolchains/android_ndk.py +++ b/kiwixbuild/toolchains/android_ndk.py @@ -113,22 +113,3 @@ class android_ndk(Toolchain): self.command('build_platform', self._build_platform) self.command('fix_permission_right', self._fix_permission_right) - def get_bin_dir(self): - return [pj(self.builder.install_path, 'bin')] - - def set_env(self, env): - env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') - env['CFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(self.root_path) + env['CFLAGS'] - env['CXXFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(self.root_path) + env['CXXFLAGS'] - env['LDFLAGS'] = '--sysroot={} '.format(self.root_path) + env['LDFLAGS'] - #env['CFLAGS'] = ' -fPIC -D_FILE_OFFSET_BITS=64 -O3 '+env['CFLAGS'] - #env['CXXFLAGS'] = (' -D__OPTIMIZE__ -fno-strict-aliasing ' - # ' -DU_HAVE_NL_LANGINFO_CODESET=0 ' - # '-DU_STATIC_IMPLEMENTATION -O3 ' - # '-DU_HAVE_STD_STRING -DU_TIMEZONE=0 ')+env['CXXFLAGS'] - env['NDK_DEBUG'] = '0' - - def set_compiler(self, env): - env['CC'] = self.binaries['CC'] - env['CXX'] = self.binaries['CXX'] - diff --git a/kiwixbuild/toolchains/android_sdk.py b/kiwixbuild/toolchains/android_sdk.py index 753dfb4..de4b0b7 100644 --- a/kiwixbuild/toolchains/android_sdk.py +++ b/kiwixbuild/toolchains/android_sdk.py @@ -49,6 +49,3 @@ class android_sdk(Toolchain): def build(self): self.command('build_platform', self._build_platform) self.command('fix_licenses', self._fix_licenses) - - def set_env(self, env): - env['ANDROID_HOME'] = self.builder.install_path diff --git a/kiwixbuild/toolchains/armhf.py b/kiwixbuild/toolchains/armhf.py index 1975665..d49aa62 100644 --- a/kiwixbuild/toolchains/armhf.py +++ b/kiwixbuild/toolchains/armhf.py @@ -8,57 +8,7 @@ pj = os.path.join class armhf_toolchain(Toolchain): name = 'armhf' - arch_full = 'arm-linux-gnueabihf' class Source(GitClone): git_remote = "https://github.com/raspberrypi/tools" git_dir = "raspberrypi-tools" - - @property - def root_path(self): - return pj(self.source_path, 'arm-bcm2708', 'gcc-linaro-arm-linux-gnueabihf-raspbian-x64') - - @property - def binaries(self): - binaries = ((k,'{}-{}'.format(self.arch_full, v)) - for k, v in (('CC', 'gcc'), - ('CXX', 'g++'), - ('AR', 'ar'), - ('STRIP', 'strip'), - ('WINDRES', 'windres'), - ('RANLIB', 'ranlib'), - ('LD', 'ld')) - ) - return {k:pj(self.root_path, 'bin', v) - for k,v in binaries} - - @property - def exec_wrapper_def(self): - try: - which('qemu-arm') - except subprocess.CalledProcessError: - return "" - else: - return "exec_wrapper = 'qemu-arm'" - - @property - def configure_option(self): - return '--host={}'.format(self.arch_full) - - def get_bin_dir(self): - return [pj(self.root_path, 'bin')] - - def set_env(self, env): - env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') - env['CFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS'] - env['CXXFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS'] - env['QEMU_LD_PREFIX'] = pj(self.root_path, "arm-linux-gnueabihf", "libc") - env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format( - ':'.join([ - pj(self.root_path, "arm-linux-gnueabihf", "lib"), - env['LD_LIBRARY_PATH'] - ])) - - def set_compiler(self, env): - env['CC'] = self.binaries['CC'] - env['CXX'] = self.binaries['CXX'] diff --git a/kiwixbuild/toolchains/base_toolchain.py b/kiwixbuild/toolchains/base_toolchain.py index 3a459ac..3c626e0 100644 --- a/kiwixbuild/toolchains/base_toolchain.py +++ b/kiwixbuild/toolchains/base_toolchain.py @@ -41,15 +41,6 @@ class Toolchain(metaclass=_MetaToolchain): def _log_dir(self): return neutralEnv('log_dir') - def set_env(self, env): - pass - - def set_compiler(self, env): - pass - - def get_bin_dir(self): - return [] - def command(self, name, function, *args): print(" {} {} : ".format(name, self.name), end="", flush=True) log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name)) diff --git a/kiwixbuild/toolchains/ios.py b/kiwixbuild/toolchains/ios.py deleted file mode 100644 index f1dd4be..0000000 --- a/kiwixbuild/toolchains/ios.py +++ /dev/null @@ -1,23 +0,0 @@ -from .base_toolchain import Toolchain - -from kiwixbuild.utils import pj, xrun_find - -class iOS_sdk(Toolchain): - @property - def binaries(self): - return { - 'CC': xrun_find('clang'), - 'CXX': xrun_find('clang++'), - 'AR': '/usr/bin/ar', - 'STRIP': '/usr/bin/strip', - 'RANLIB': '/usr/bin/ranlib', - 'LD': '/usr/bin/ld', - } - - @property - def configure_option(self): - return '--host=arm-apple-darwin' - - def set_compiler(self, env): - env['CC'] = self.binaries['CC'] - env['CXX'] = self.binaries['CXX'] diff --git a/kiwixbuild/toolchains/linux_i586.py b/kiwixbuild/toolchains/linux_i586.py deleted file mode 100644 index ce89dd8..0000000 --- a/kiwixbuild/toolchains/linux_i586.py +++ /dev/null @@ -1,37 +0,0 @@ -import os - -from .base_toolchain import Toolchain -from kiwixbuild.dependencies import GitClone -from kiwixbuild.utils import which -pj = os.path.join - -class linux_i586_toolchain(Toolchain): - name = 'linux_i586' - arch_full = 'i586-linux-gnu' - - @property - def configure_option(self): - return '--host={}'.format(self.arch_full) - - @property - def binaries(self): - return {k:which(v) - for k, v in (('CC', os.environ.get('CC', 'gcc')), - ('CXX', os.environ.get('CXX', 'g++')), - ('AR', 'ar'), - ('STRIP', 'strip'), - ('RANLIB', 'ranlib'), - ('LD', 'ld')) - } - - @property - def configure_option(self): - return '--host={}'.format(self.arch_full) - - def set_env(self, env): - env['CFLAGS'] = "-m32 -march=i586 -mno-sse "+env['CFLAGS'] - env['CXXFLAGS'] = "-m32 -march=i586 -mno-sse "+env['CXXFLAGS'] - env['LDFLAGS'] = "-m32 -march=i586 -mno-sse "+env['LDFLAGS'] - - def get_bin_dir(self): - return [] diff --git a/kiwixbuild/toolchains/mingw32.py b/kiwixbuild/toolchains/mingw32.py deleted file mode 100644 index 8b29b1a..0000000 --- a/kiwixbuild/toolchains/mingw32.py +++ /dev/null @@ -1,56 +0,0 @@ -import os -import subprocess - -from .base_toolchain import Toolchain -from kiwixbuild.utils import which -from kiwixbuild._global import neutralEnv - -pj = os.path.join - -class mingw32_toolchain(Toolchain): - name = 'mingw32' - arch_full = 'i686-w64-mingw32' - extra_libs = ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4', '-lmsvcr90', '-liphlpapi'] - - @property - def root_path(self): - root_paths = { - 'fedora': '/usr/i686-w64-mingw32/sys-root/mingw', - 'debian': '/usr/i686-w64-mingw32' - } - return root_paths[neutralEnv('distname')] - - @property - def binaries(self): - return {k:which('{}-{}'.format(self.arch_full, v)) - for k, v in (('CC', 'gcc'), - ('CXX', 'g++'), - ('AR', 'ar'), - ('STRIP', 'strip'), - ('WINDRES', 'windres'), - ('RANLIB', 'ranlib')) - } - - @property - def exec_wrapper_def(self): - try: - which('wine') - except subprocess.CalledProcessError: - return "" - else: - return "exec_wrapper = 'wine'" - - @property - def configure_option(self): - return '--host={}'.format(self.arch_full) - - def set_compiler(self, env): - for k, v in self.binaries.items(): - env[k] = v - - def get_bin_dir(self): - return [pj(self.root_path, 'bin')] - - def set_env(self, env): - env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') - env['LIBS'] = " ".join(self.extra_libs) + " " +env['LIBS'] From 041826d0e81d8d65866e3562aa454e53735a8c9f Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 28 May 2018 15:42:30 +0200 Subject: [PATCH 05/23] Move from target logic to steps logic. This is the big change !!!! Instead of handling target as primary object and prepare/build targets, we are handling build steps. A build step may be a source (preparation) or a build (of the source). Actualy, a step is a tuple (context, Builder or Source). The context define the context of the step. It can be : - 'source', for a Source step - 'neutral' or the name of a platform for Build step. Target becomes a "Class only" class. --- kiwixbuild/_global.py | 30 +++++ kiwixbuild/buildenv.py | 7 +- kiwixbuild/builder.py | 125 +++++++++++++------- kiwixbuild/dependencies/base.py | 125 +++++++++++--------- kiwixbuild/dependencies/icu4c.py | 8 +- kiwixbuild/dependencies/kiwix_custom_app.py | 5 +- kiwixbuild/dependencies/libmagic.py | 5 +- kiwixbuild/platforms/android.py | 80 +++++++++---- kiwixbuild/platforms/armhf.py | 19 ++- kiwixbuild/platforms/base.py | 33 ++++-- kiwixbuild/platforms/i586.py | 15 ++- kiwixbuild/platforms/ios.py | 43 ++++--- kiwixbuild/platforms/native.py | 14 ++- kiwixbuild/platforms/win32.py | 13 +- kiwixbuild/toolchains/android_ndk.py | 88 ++++++-------- kiwixbuild/toolchains/android_sdk.py | 2 +- kiwixbuild/toolchains/base_toolchain.py | 14 +-- 17 files changed, 393 insertions(+), 233 deletions(-) diff --git a/kiwixbuild/_global.py b/kiwixbuild/_global.py index 0b5aa35..6e4dcad 100644 --- a/kiwixbuild/_global.py +++ b/kiwixbuild/_global.py @@ -1,4 +1,8 @@ +from collections import OrderedDict as _OrderedDict + _neutralEnv = None +_target_steps = _OrderedDict() +_plt_steps = _OrderedDict() def set_neutralEnv(env): global _neutralEnv @@ -6,3 +10,29 @@ def set_neutralEnv(env): def neutralEnv(what): return getattr(_neutralEnv, what) + +def add_target_step(key, what): + _target_steps[key] = what + +def get_target_step(key, default_context=None): + try: + context, target = key + except ValueError: + context, target = default_context, key + return _target_steps[(context, target)] + +def target_steps(): + return _target_steps + +def add_plt_step(key, what): + _plt_steps[key] = what + +def get_plt_step(key, default_context=None): + try: + context, target = key + except ValueError: + context, target = default_context, key + return _plt_steps[(context, target)] + +def plt_steps(): + return _plt_steps diff --git a/kiwixbuild/buildenv.py b/kiwixbuild/buildenv.py index afbb6a5..672c906 100644 --- a/kiwixbuild/buildenv.py +++ b/kiwixbuild/buildenv.py @@ -76,17 +76,18 @@ class PlatformNeutralEnv: class BuildEnv: - def __init__(self, platformInfo, targetsDict): + def __init__(self, platformInfo): build_dir = "BUILD_{}".format(platformInfo.name) self.platformInfo = platformInfo self.build_dir = pj(neutralEnv('working_dir'), build_dir) self.install_dir = pj(self.build_dir, "INSTALL") + self.log_dir = pj(self.build_dir, 'LOGS') for d in (self.build_dir, - self.install_dir): + self.install_dir, + self.log_dir): os.makedirs(d, exist_ok=True) self.libprefix = neutralEnv('libprefix') or self._detect_libdir() - self.targetsDict = targetsDict def clean_intermediate_directories(self): for subdir in os.listdir(self.build_dir): diff --git a/kiwixbuild/builder.py b/kiwixbuild/builder.py index eefa670..d56724c 100644 --- a/kiwixbuild/builder.py +++ b/kiwixbuild/builder.py @@ -3,8 +3,14 @@ import sys from collections import OrderedDict from .buildenv import * +from .platforms import PlatformInfo from .utils import remove_duplicates, StopBuild from .dependencies import Dependency +from ._global import ( + neutralEnv, + add_target_step, get_target_step, target_steps, + add_plt_step, get_plt_step, plt_steps) +from . import _global class Builder: def __init__(self, options): @@ -15,78 +21,115 @@ class Builder: 'Select another target platform, or change your host system.' ).format(options.target_platform, self.distname)) sys.exit(-1) - self.targets = OrderedDict() - self.platform = platform = platformClass(self.targets) + self.platform = platform = platformClass() _targets = {} - targetDef = options.targets + targetDef = (options.target_platform, options.targets) self.add_targets(targetDef, _targets) dependencies = self.order_dependencies(_targets, targetDef) dependencies = list(remove_duplicates(dependencies)) if options.build_nodeps: - self.targets[targetDef] = _targets[targetDef] + add_target_step(targetDef, _targets[targetDef]) else: for dep in dependencies: if self.options.build_deps_only and dep == targetDef: continue - self.targets[dep] = _targets[dep] + add_target_step(dep, _targets[dep]) - def add_targets(self, targetName, targets): - if targetName in targets: + def add_targets(self, targetDef, targets): + if targetDef in targets: return + targetPlatformName, targetName = targetDef + targetPlatform = PlatformInfo.all_platforms[targetPlatformName] targetClass = Dependency.all_deps[targetName] - target = targetClass(self.platform.buildEnv) - targets[targetName] = target - for dep in target.builder.get_dependencies(self.platform): - self.add_targets(dep, targets) + targets[('source', targetName)] = targetClass.Source + targets[targetDef] = targetClass.Builder + for dep in targetClass.Builder.get_dependencies(targetPlatform): + try: + depPlatform, depName = dep + except ValueError: + depPlatform, depName = targetPlatformName, dep + self.add_targets((depPlatform, depName), targets) - def order_dependencies(self, _targets, targetName): - target = _targets[targetName] - for depName in target.builder.dependencies(self.platform): - yield from self.order_dependencies(_targets, depName) - yield targetName + def order_dependencies(self, _targets, targetDef): + targetPlatformName, targetName = targetDef + if targetPlatformName == 'source': + # Do not try to order sources, they will be added as dep by the + # build step two lines later. + return + target = _targets[targetDef] + targetPlatform = PlatformInfo.all_platforms[targetPlatformName] + yield ('source', targetName) + for dep in target.get_dependencies(targetPlatform): + try: + depPlatform, depName = dep + except ValueError: + depPlatform, depName = targetPlatformName, dep + yield from self.order_dependencies(_targets, (depPlatform, depName)) + yield targetDef + + def prepare_toolchain_sources(self): + tlsourceDefs = (tlDef for tlDef in plt_steps() if tlDef[0]=='source') + for tlsourceDef in tlsourceDefs: + print("prepare sources for toolchain {} :".format(tlsourceDef[1])) + toolchainClass = Toolchain.all_toolchains[tlsourceDef[1]] + source = get_plt_step(tlsourceDef)(toolchainClass) + add_plt_step(tlsourceDef, source) + source.prepare() def prepare_sources(self): if self.options.skip_source_prepare: print("SKIP") return - toolchain_sources = (tlc.source for tlc in self.platform.buildEnv.toolchains if tlc.source) - for toolchain_source in toolchain_sources: - print("prepare sources for toolchain {} :".format(toolchain_source.name)) - toolchain_source.prepare() + sourceDefs = remove_duplicates(tDef for tDef in target_steps() if tDef[0]=='source') + for sourceDef in sourceDefs: - sources = (dep.source for dep in self.targets.values() if not dep.skip) - sources = remove_duplicates(sources, lambda s: s.__class__) - for source in sources: - print("prepare sources {} :".format(source.name)) + print("prepare sources {} :".format(sourceDef[1])) + depClass = Dependency.all_deps[sourceDef[1]] + source = get_target_step(sourceDef)(depClass) + add_target_step(sourceDef, source) source.prepare() - def build(self): - toolchain_builders = (tlc.builder for tlc in self.platform.buildEnv.toolchains if tlc.builder) - for toolchain_builder in toolchain_builders: - print("build toolchain {} :".format(toolchain_builder.name)) - toolchain_builder.build() - - builders = (dep.builder for dep in self.targets.values() if (dep.builder and not dep.skip)) - for builder in builders: - if self.options.make_dist and builder.name == self.options.targets: - continue - print("build {} :".format(builder.name)) + def build_toolchains(self): + tlbuilderDefs = (tlDef for tlDef in plt_steps() if tlDef[0] != 'source') + for tlbuilderDef in tlbuilderDefs: + print("build toolchain {} :".format(tlbuilderDef[1])) + toolchainClass = Toolchain.all_toolchains[tlbuilderDef[1]] + source = get_plt_step(tlbuilderDef[1], 'source') + if tlbuilderDef[0] == 'neutral': + env = _global._neutralEnv + else: + env = PlatformInfo.all_running_platforms[tlbuilderDef[0]].buildEnv + builder = get_plt_step(tlbuilderDef)(toolchainClass, source, env) + add_plt_step(tlbuilderDef, builder) + builder.build() + + def build(self): + builderDefs = (tDef for tDef in target_steps() if tDef[0] != 'source') + for builderDef in builderDefs: + depClass = Dependency.all_deps[builderDef[1]] + source = get_target_step(builderDef[1], 'source') + env = PlatformInfo.all_running_platforms[builderDef[0]].buildEnv + builder = get_target_step(builderDef)(depClass, source, env) + if self.options.make_dist and builderDef[1] == self.options.targets: + print("make dist {}:".format(builder.name)) + builder.make_dist() + continue + print("build {} ({}):".format(builder.name, builderDef[0])) + add_target_step(builderDef, builder) builder.build() - if self.options.make_dist: - dep = self.targets[self.options.targets] - builder = dep.builder - print("make dist {}:".format(builder.name)) - builder.make_dist() def run(self): try: + print("[SETUP PLATFORMS]") + self.prepare_toolchain_sources() + self.build_toolchains() + self.platform.finalize_setup() print("[INSTALL PACKAGES]") self.platform.install_packages() - self.platform.finalize_setup() print("[PREPARE]") self.prepare_sources() print("[BUILD]") diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index 45fb39f..9d93795 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -22,34 +22,51 @@ class Dependency(metaclass=_MetaDependency): all_deps = {} force_native_build = False - def __init__(self, buildEnv): - self.buildEnv = buildEnv - self.source = self.Source(self) - self.builder = self.Builder(self) - self.skip = False + @classmethod + def version(cls): + return base_deps_versions.get(cls.name, None) + + @classmethod + def full_name(cls): + if cls.version(): + return "{}-{}".format(cls.name, cls.version()) + return cls.name + + +class Source: + """Base Class to the real preparator + A source preparator must install source in the self.source_dir attribute + inside the neutralEnv.source_dir.""" + def __init__(self, target): + self.target = target @property - def version(self): - return base_deps_versions.get(self.name, None) + def name(self): + return self.target.name @property - def full_name(self): - if self.version: - return "{}-{}".format(self.name, self.version) - return self.name + def source_dir(self): + return self.target.full_name() @property def source_path(self): - return pj(neutralEnv('source_dir'), self.source.source_dir) + return pj(neutralEnv('source_dir'), self.source_dir) @property def _log_dir(self): - return self.buildEnv.log_dir + return neutralEnv('log_dir') + + def _patch(self, context): + context.try_skip(self.source_path) + context.force_native_build = True + for p in self.patches: + with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input: + run_command("patch -p1", self.source_path, context, input=patch_input.read()) def command(self, name, function, *args): print(" {} {} : ".format(name, self.name), end="", flush=True) log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name)) - context = Context(name, log, self.force_native_build) + context = Context(name, log, True) try: ret = function(*args, context=context) context._finalise() @@ -70,33 +87,6 @@ class Dependency(metaclass=_MetaDependency): raise -class Source: - """Base Class to the real preparator - A source preparator must install source in the self.source_dir attribute - inside the neutralEnv.source_dir.""" - def __init__(self, target): - self.target = target - - @property - def name(self): - return self.target.name - - @property - def source_dir(self): - return self.target.full_name - - def _patch(self, context): - source_path = pj(neutralEnv('source_dir'), self.source_dir) - context.try_skip(source_path) - context.force_native_build = True - for p in self.patches: - with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input: - neutralEnv('run_command')("patch -p1", source_path, context, input=patch_input.read()) - - def command(self, *args, **kwargs): - return self.target.command(*args, **kwargs) - - class NoopSource(Source): def prepare(self): pass @@ -206,9 +196,10 @@ class Builder: subsource_dir = None dependencies = [] - def __init__(self, target): + def __init__(self, target, source, buildEnv): self.target = target - self.buildEnv = target.buildEnv + self.source = source + self.buildEnv = buildEnv @classmethod def get_dependencies(cls, platformInfo): @@ -220,17 +211,41 @@ class Builder: @property def source_path(self): - base_source_path = self.target.source_path + base_source_path = self.source.source_path if self.subsource_dir: return pj(base_source_path, self.subsource_dir) return base_source_path @property def build_path(self): - return pj(self.buildEnv.build_dir, self.target.full_name) + return pj(self.buildEnv.build_dir, self.target.full_name()) - def command(self, *args, **kwargs): - return self.target.command(*args, **kwargs) + @property + def _log_dir(self): + return self.buildEnv.log_dir + + def command(self, name, function, *args): + print(" {} {} : ".format(name, self.name), end="", flush=True) + log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name)) + context = Context(name, log, self.target.force_native_build) + try: + ret = function(*args, context=context) + context._finalise() + print("OK") + return ret + except SkipCommand: + print("SKIP") + except subprocess.CalledProcessError: + print("ERROR") + try: + with open(log, 'r') as f: + print(f.read()) + except: + pass + raise StopBuild() + except: + print("ERROR") + raise def build(self): if hasattr(self, '_pre_build_script'): @@ -274,8 +289,8 @@ class MakeBuilder(Builder): def all_configure_option(self): option = self.configure_option_template.format( dep_options=self.configure_option, - static_option=self.static_configure_option if self.buildEnv.platform_info.static else self.dynamic_configure_option, - env_option=self.buildEnv.configure_option if not self.target.force_native_build else "", + static_option=self.static_configure_option if self.buildEnv.platformInfo.static else self.dynamic_configure_option, + env_option=self.buildEnv.platformInfo.configure_option if not self.target.force_native_build else "", install_dir=self.buildEnv.install_dir, libdir=pj(self.buildEnv.install_dir, self.buildEnv.libprefix) ) @@ -289,7 +304,7 @@ class MakeBuilder(Builder): configure_option=self.all_configure_option ) env = Defaultdict(str, os.environ) - if self.buildEnv.platform_info.static: + if self.buildEnv.platformInfo.static: env['CFLAGS'] = env['CFLAGS'] + ' -fPIC' if self.configure_env: for k in self.configure_env: @@ -342,7 +357,7 @@ class CMakeBuilder(MakeBuilder): cross_option=cross_option ) env = Defaultdict(str, os.environ) - if self.buildEnv.platform_info.static: + if self.buildEnv.platformInfo.static: env['CFLAGS'] = env['CFLAGS'] + ' -fPIC' if self.configure_env: for k in self.configure_env: @@ -360,7 +375,7 @@ class MesonBuilder(Builder): @property def library_type(self): - return 'static' if self.buildEnv.platform_info.static else 'shared' + return 'static' if self.buildEnv.platformInfo.static else 'shared' def _configure(self, context): context.try_skip(self.build_path) @@ -393,9 +408,9 @@ class MesonBuilder(Builder): run_command(command, self.build_path, context, buildEnv=self.buildEnv) def _test(self, context): - if ( self.buildEnv.platform_info.build == 'android' - or (self.buildEnv.platform_info.build != 'native' - and not self.buildEnv.platform_info.static) + if ( self.buildEnv.platformInfo.build == 'android' + or (self.buildEnv.platformInfo.build != 'native' + and not self.buildEnv.platformInfo.static) ): raise SkipCommand() command = "{} --verbose {}".format(neutralEnv('mesontest_command'), self.test_option) diff --git a/kiwixbuild/dependencies/icu4c.py b/kiwixbuild/dependencies/icu4c.py index b3029d1..d9c8a0b 100644 --- a/kiwixbuild/dependencies/icu4c.py +++ b/kiwixbuild/dependencies/icu4c.py @@ -5,7 +5,7 @@ from .base import ( ) from kiwixbuild.utils import SkipCommand - +from kiwixbuild._global import get_target_step class Icu(Dependency): name = "icu4c" @@ -54,5 +54,7 @@ class Icu_cross_compile(Icu): @property def configure_option(self): - icu_native_dep = self.buildEnv.targetsDict['icu4c_native'] - return super().configure_option + " --with-cross-build={} --disable-tools".format(icu_native_dep.builder.build_path) + icu_native_builder = get_target_step('icu4c_native', self.buildEnv.platformInfo.name) + return (super().configure_option + + " --with-cross-build={} --disable-tools" + ).format(icu_native_builder.build_path) diff --git a/kiwixbuild/dependencies/kiwix_custom_app.py b/kiwixbuild/dependencies/kiwix_custom_app.py index 5c52efe..43b90a9 100644 --- a/kiwixbuild/dependencies/kiwix_custom_app.py +++ b/kiwixbuild/dependencies/kiwix_custom_app.py @@ -7,6 +7,7 @@ from .base import ( GradleBuilder) from kiwixbuild.utils import Remotefile, pj, SkipCommand +from kiwixbuild._global import get_target_step class KiwixCustomApp(Dependency): name = "kiwix-android-custom" @@ -78,9 +79,9 @@ class KiwixCustomApp(Dependency): def _configure(self, context): # Copy kiwix-android in build dir. - kiwix_android_dep = self.buildEnv.targetsDict['kiwix-android'] + kiwix_android_source = get_target_step('kiwix-android', 'source') if not os.path.exists(self.build_path): - shutil.copytree(kiwix_android_dep.source_path, self.build_path) + shutil.copytree(kiwix_android_source.source_path, self.build_path) # Copy kiwix-lib application in build dir try: diff --git a/kiwixbuild/dependencies/libmagic.py b/kiwixbuild/dependencies/libmagic.py index b093191..ab0f88e 100644 --- a/kiwixbuild/dependencies/libmagic.py +++ b/kiwixbuild/dependencies/libmagic.py @@ -7,6 +7,7 @@ from .base import ( ) from kiwixbuild.utils import Remotefile, pj, Defaultdict, SkipCommand +from kiwixbuild._global import get_target_step class LibMagicBase(Dependency): name = "libmagic" @@ -48,7 +49,7 @@ class LibMagic_cross_compile(LibMagicBase): make_target=self.make_target, make_option=self.make_option ) - libmagic_native_dep = self.buildEnv.targetsDict['libmagic_native'] + libmagic_native_builder = get_target_step('libmagic_native', self.buildEnv.platformInfo.name) env = Defaultdict(str, os.environ) - env['PATH'] = ':'.join([pj(libmagic_native_dep.builder.build_path, 'src'), env['PATH']]) + env['PATH'] = ':'.join([pj(libmagic_native_builder.build_path, 'src'), env['PATH']]) self.buildEnv.run_command(command, self.build_path, context, env=env) diff --git a/kiwixbuild/platforms/android.py b/kiwixbuild/platforms/android.py index 8504d27..4187a19 100644 --- a/kiwixbuild/platforms/android.py +++ b/kiwixbuild/platforms/android.py @@ -1,21 +1,13 @@ from .base import PlatformInfo +from kiwixbuild.utils import pj +from kiwixbuild._global import get_plt_step class AndroidPlatformInfo(PlatformInfo): - __arch_infos = { - 'arm' : ('arm-linux-androideabi', 'arm', 'armeabi'), - 'arm64': ('aarch64-linux-android', 'aarch64', 'arm64-v8a'), - 'mips': ('mipsel-linux-android', 'mipsel', 'mips'), - 'mips64': ('mips64el-linux-android', 'mips64el', 'mips64'), - 'x86': ('i686-linux-android', 'i686', 'x86'), - 'x86_64': ('x86_64-linux-android', 'x86_64', 'x86_64'), - } - - def __init__(self, name, arch): - super().__init__(name, 'android', True, ['android_ndk', 'android_sdk'], - hosts=['fedora', 'debian']) - self.arch = arch - self.arch_full, self.cpu, self.abi = self.__arch_infos[arch] + build = 'android' + static = True + toolchain_names = ['android-ndk', 'android-sdk'] + compatible_hosts = ['fedora', 'debian'] def __str__(self): return "android" @@ -35,11 +27,11 @@ class AndroidPlatformInfo(PlatformInfo): @property def ndk_builder(self): - return self.toolchains[0].builder + return get_plt_step('android_ndk', self.name) @property def sdk_builder(self): - return self.toolchains[1].builder + return get_plt_step('android_sdk', 'neutral') def get_cross_config(self): install_path = self.ndk_builder.install_path @@ -77,9 +69,53 @@ class AndroidPlatformInfo(PlatformInfo): env['NDK_DEBUG'] = '0' env['ANDROID_HOME'] = self.sdk_builder.install_path -AndroidPlatformInfo('android_arm', 'arm') -AndroidPlatformInfo('android_arm64', 'arm64') -AndroidPlatformInfo('android_mips', 'mips') -AndroidPlatformInfo('android_mips64', 'mips64') -AndroidPlatformInfo('android_x86', 'x86') -AndroidPlatformInfo('android_x86_64', 'x86_64') + def set_compiler(self, env): + binaries = self.binaries(self.ndk_builder.install_path) + env['CC'] = binaries['CC'] + env['CXX'] = binaries['CXX'] + + @property + def configure_option(self): + return '--host={}'.format(self.arch_full) + + def finalize_setup(self): + super().finalize_setup() + self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_android_cross_file.txt') + self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') + + +class AndroidArm(AndroidPlatformInfo): + name = 'android_arm' + arch = cpu = 'arm' + arch_full = 'arm-linux-androideabi' + abi = 'armeabi' + +class AndroidArm(AndroidPlatformInfo): + name = 'android_arm64' + arch = 'arm64' + arch_full = 'aarch64-linux-android' + cpu = 'aarch64' + abi = 'arm64-v8a' + +class AndroidArm(AndroidPlatformInfo): + name = 'android_mips' + arch = abi = 'mips' + arch_full = 'mipsel-linux-android' + cpu = 'mipsel' + +class AndroidArm(AndroidPlatformInfo): + name = 'android_mips64' + arch = abi = 'mips64' + arch_full = 'mips64el-linux-android' + cpu = 'mips64el' + +class AndroidArm(AndroidPlatformInfo): + name = 'android_x86' + arch = abi = 'x86' + arch_full = 'i686-linux-android' + cpu = 'i686' + +class AndroidArm(AndroidPlatformInfo): + name = 'android_x86_64' + arch = cpu = abi = 'x86_64' + arch_full = 'x86_64-linux-android' diff --git a/kiwixbuild/platforms/armhf.py b/kiwixbuild/platforms/armhf.py index 3c13d0c..e3077d9 100644 --- a/kiwixbuild/platforms/armhf.py +++ b/kiwixbuild/platforms/armhf.py @@ -1,10 +1,14 @@ from .base import PlatformInfo +from kiwixbuild.utils import pj +from kiwixbuild._global import get_plt_step + class ArmhfPlatformInfo(PlatformInfo): + build = 'armhf' arch_full = 'arm-linux-gnueabihf' - def __init__(self, name, static): - super().__init__(name, 'armhf', static, ['armhf_toolchain'], ['fedora', 'debian']) + toolchain_names = ['armhf'] + compatible_hosts = ['fedora', 'debian'] def get_cross_config(self): return { @@ -25,7 +29,7 @@ class ArmhfPlatformInfo(PlatformInfo): @property def tlc_source(self): - return self.toolchains[0].source + return get_plt_step('armhf', 'source') @property def root_path(self): @@ -83,5 +87,10 @@ class ArmhfPlatformInfo(PlatformInfo): self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') -ArmhfPlatformInfo('armhf_dyn', False) -ArmhfPlatformInfo('armhf_static', True) +class ArmhfDyn(ArmhfPlatformInfo): + name = 'armhf_dyn' + static = False + +class ArmhfStatic(ArmhfPlatformInfo): + name = 'armhf_static' + static = True diff --git a/kiwixbuild/platforms/base.py b/kiwixbuild/platforms/base.py index 1d252e2..2d1babe 100644 --- a/kiwixbuild/platforms/base.py +++ b/kiwixbuild/platforms/base.py @@ -6,20 +6,28 @@ from kiwixbuild.toolchains import Toolchain from kiwixbuild.packages import PACKAGE_NAME_MAPPERS from kiwixbuild.utils import pj, remove_duplicates from kiwixbuild.buildenv import BuildEnv -from kiwixbuild._global import neutralEnv +from kiwixbuild._global import neutralEnv, add_plt_step, target_steps _SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) TEMPLATES_DIR = pj(os.path.dirname(_SCRIPT_DIR), 'templates') -class PlatformInfo: - all_platforms = {} +class _MetaPlatform(type): + def __new__(cls, name, bases, dct): + _class = type.__new__(cls, name, bases, dct) + if name not in ('PlatformInfo', 'MetaPlatformInfo') and 'name' in dct: + dep_name = dct['name'] + PlatformInfo.all_platforms[dep_name] = _class + return _class - def __init__(self, name, build, static, toolchains, hosts=None): - self.all_platforms[name] = self - self.build = build - self.static = static - self.toolchains = toolchains - self.compatible_hosts = hosts + +class PlatformInfo(metaclass=_MetaPlatform): + all_platforms = {} + all_running_platforms = {} + toolchain_names = [] + configure_option = "" + + def __init__(self): + self.all_running_platforms[self.name] = self self.buildEnv = BuildEnv(self) self.setup_toolchains() @@ -28,9 +36,14 @@ class PlatformInfo: def setup_toolchains(self): self.toolchains = {} - for tlc_name in self.toolchains: + for tlc_name in self.toolchain_names: ToolchainClass = Toolchain.all_toolchains[tlc_name] self.toolchains[tlc_name] = ToolchainClass() + if ToolchainClass.Source is not None: + add_plt_step(('source', tlc_name), ToolchainClass.Source) + if ToolchainClass.Builder is not None: + plt_name = 'neutral' if ToolchainClass.neutral else self.name + add_plt_step((plt_name, tlc_name), ToolchainClass.Builder) def get_cross_config(self): return {} diff --git a/kiwixbuild/platforms/i586.py b/kiwixbuild/platforms/i586.py index e12c953..b52dd9e 100644 --- a/kiwixbuild/platforms/i586.py +++ b/kiwixbuild/platforms/i586.py @@ -1,11 +1,13 @@ +import os from .base import PlatformInfo +from kiwixbuild.utils import which class I586PlatformInfo(PlatformInfo): + build = 'i586' arch_full = 'i586-linux-gnu' - def __init__(self, name, static): - super().__init__(name, 'i586', static, [], ['fedora', 'debian']) + compatible_hosts = ['fedora', 'debian'] def get_cross_config(self): return { @@ -52,5 +54,10 @@ class I586PlatformInfo(PlatformInfo): self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_i586_cross_file.txt') self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') -I586PlatformInfo('i586_dyn', False) -I586PlatformInfo('i586_static', True) +class I586Dyn(I586PlatformInfo): + name = 'i586_dyn' + static = False + +class I586Static(I586PlatformInfo): + name = 'i586_static' + static = True diff --git a/kiwixbuild/platforms/ios.py b/kiwixbuild/platforms/ios.py index bbbe804..499a8b3 100644 --- a/kiwixbuild/platforms/ios.py +++ b/kiwixbuild/platforms/ios.py @@ -6,18 +6,12 @@ from kiwixbuild.utils import pj, xrun_find class iOSPlatformInfo(PlatformInfo): - __arch_infos = { - 'armv7': ('arm-apple-darwin', 'armv7', 'iphoneos'), - 'arm64': ('arm-apple-darwin', 'arm64', 'iphoneos'), - 'i386': ('', 'i386', 'iphonesimulator'), - 'x86_64': ('', 'x86_64', 'iphonesimulator'), - } + build = 'iOS' + static = True + compatible_hosts = ['Darwin'] - def __init__(self, name, arch): - super().__init__(name, 'iOS', True, [], - hosts=['Darwin']) - self.arch = arch - self.arch_full, self.cpu, self.sdk_name = self.__arch_infos[arch] + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self._root_path = None @property @@ -81,7 +75,26 @@ class iOSPlatformInfo(PlatformInfo): env['CXX'] = self.binaries['CXX'] -iOSPlatformInfo('iOS_armv7', 'armv7') -iOSPlatformInfo('iOS_arm64', 'arm64') -iOSPlatformInfo('iOS_i386', 'i386') -iOSPlatformInfo('iOS_x86_64', 'x86_64') +class iOSArmv7(iOSPlatformInfo): + name = 'iOS_armv7' + arch = cpu = 'armv7' + arch_full = 'arm-apple-darwin' + sdk_name = 'iphoneos' + +class iOSArm64(iOSPlatformInfo): + name = 'iOS_arm64' + arch = cpu = 'arm64' + arch_full = 'arm-apple-darwin' + sdk_name = 'iphoneos' + +class iOSi386(iOSPlatformInfo): + name = 'iOS_i386' + arch = cpu = 'i386' + arch_full = '' + sdk_name = 'iphonesimulator' + +class iOSx64(iOSPlatformInfo): + name = 'iOS_x86_64' + arch = cpu = 'x86_64' + arch_full = '' + sdk_name = 'iphonesimulator' diff --git a/kiwixbuild/platforms/native.py b/kiwixbuild/platforms/native.py index bd324e5..069d9f2 100644 --- a/kiwixbuild/platforms/native.py +++ b/kiwixbuild/platforms/native.py @@ -2,9 +2,15 @@ from .base import PlatformInfo class NativePlatformInfo(PlatformInfo): - def __init__(self, name, static, hosts): - super().__init__(name, 'native', static, [], hosts) + build = 'native' -NativePlatformInfo('native_dyn', False, ['fedora', 'debian', 'Darwin']) -NativePlatformInfo('native_static', True, ['fedora', 'debian']) +class NativeDyn(NativePlatformInfo): + name = 'native_dyn' + static = False + compatible_hosts = ['fedora', 'debian', 'Darwin'] + +class NativeStatic(NativePlatformInfo): + name = 'native_static' + static = True + compatible_hosts = ['fedora', 'debian'] diff --git a/kiwixbuild/platforms/win32.py b/kiwixbuild/platforms/win32.py index 785acc3..81ec903 100644 --- a/kiwixbuild/platforms/win32.py +++ b/kiwixbuild/platforms/win32.py @@ -7,9 +7,9 @@ from kiwixbuild._global import neutralEnv class Win32PlatformInfo(PlatformInfo): extra_libs = ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4', '-lmsvcr90', '-liphlpapi'] + build = 'win32' + compatible_hosts = ['fedora', 'debian'] arch_full = 'i686-w64-mingw32' - def __init__(self, name, static): - super().__init__(name, 'win32', static, [], ['fedora', 'debian']) def get_cross_config(self): return { @@ -76,5 +76,10 @@ class Win32PlatformInfo(PlatformInfo): env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') env['LIBS'] = " ".join(self.extra_libs) + " " +env['LIBS'] -Win32PlatformInfo('win32_dyn', False) -Win32PlatformInfo('win32_static', True) +class Win32Dyn(Win32PlatformInfo): + name = 'win32_dyn' + static = False + +class Win32Static(Win32PlatformInfo): + name = 'win32_static' + static = True diff --git a/kiwixbuild/toolchains/android_ndk.py b/kiwixbuild/toolchains/android_ndk.py index f33e7e8..597f82f 100644 --- a/kiwixbuild/toolchains/android_ndk.py +++ b/kiwixbuild/toolchains/android_ndk.py @@ -12,55 +12,11 @@ class android_ndk(Toolchain): version = 'r13b' gccver = '4.9.x' - @property - def api(self): - return '21' if self.arch in ('arm64', 'mips64', 'x86_64') else '14' - - @property - def platform(self): - return 'android-'+self.api - - @property - def arch(self): - return self.buildEnv.platform_info.arch - - @property - def arch_full(self): - return self.buildEnv.platform_info.arch_full - - @property - def toolchain(self): - return self.arch_full+"-4.9" - - @property - def root_path(self): - return pj(self.builder.install_path, 'sysroot') - - @property - def binaries(self): - binaries = ((k,'{}-{}'.format(self.arch_full, v)) - for k, v in (('CC', 'gcc'), - ('CXX', 'g++'), - ('AR', 'ar'), - ('STRIP', 'strip'), - ('WINDRES', 'windres'), - ('RANLIB', 'ranlib'), - ('LD', 'ld')) - ) - return {k:pj(self.builder.install_path, 'bin', v) - for k,v in binaries} - - @property - def configure_option(self): - return '--host={}'.format(self.arch_full) - - @property - def full_name(self): - return "{name}-{version}-{arch}-{api}".format( - name = self.name, - version = self.version, - arch = self.arch, - api = self.api) + @classmethod + def full_name(cls): + return "{name}-{version}".format( + name = cls.name, + version = cls.version) class Source(ReleaseDownload): archive = Remotefile('android-ndk-r13b-linux-x86_64.zip', @@ -75,11 +31,34 @@ class android_ndk(Toolchain): class Builder(Builder): - @property def install_path(self): return self.build_path + @property + def api(self): + return '21' if self.arch in ('arm64', 'mips64', 'x86_64') else '14' + + @property + def platform(self): + return 'android-'+self.api + + @property + def arch(self): + return self.buildEnv.platformInfo.arch + + @property + def arch_full(self): + return self.buildEnv.platformInfo.arch_full + + @property + def toolchain(self): + return self.arch_full+"-4.9" + + @property + def configure_option(self): + return '--host={}'.format(self.arch_full) + def _build_platform(self, context): context.try_skip(self.build_path) script = pj(self.source_path, 'build/tools/make_standalone_toolchain.py') @@ -87,17 +66,18 @@ class android_ndk(Toolchain): command = '{script} --arch={arch} --api={api} --install-dir={install_dir} --force' command = command.format( script=script, - arch=self.target.arch, - api=self.target.api, + arch=self.arch, + api=self.api, install_dir=self.install_path ) + context.force_native_build = True run_command(command, self.build_path, context, buildEnv=self.buildEnv) def _fix_permission_right(self, context): context.try_skip(self.build_path) bin_dirs = [pj(self.install_path, 'bin'), - pj(self.install_path, self.target.arch_full, 'bin'), - pj(self.install_path, 'libexec', 'gcc', self.target.arch_full, self.target.gccver) + pj(self.install_path, self.arch_full, 'bin'), + pj(self.install_path, 'libexec', 'gcc', self.arch_full, self.target.gccver) ] for root, dirs, files in os.walk(self.install_path): if not root in bin_dirs: diff --git a/kiwixbuild/toolchains/android_sdk.py b/kiwixbuild/toolchains/android_sdk.py index de4b0b7..c58e05f 100644 --- a/kiwixbuild/toolchains/android_sdk.py +++ b/kiwixbuild/toolchains/android_sdk.py @@ -20,7 +20,7 @@ class android_sdk(Toolchain): @property def install_path(self): - return pj(self.buildEnv.toolchain_dir, self.target.full_name) + return pj(self.buildEnv.toolchain_dir, self.target.full_name()) def _build_platform(self, context): context.try_skip(self.install_path) diff --git a/kiwixbuild/toolchains/base_toolchain.py b/kiwixbuild/toolchains/base_toolchain.py index 3c626e0..c51de7a 100644 --- a/kiwixbuild/toolchains/base_toolchain.py +++ b/kiwixbuild/toolchains/base_toolchain.py @@ -10,11 +10,12 @@ class _MetaToolchain(type): def __new__(cls, name, bases, dct): _class = type.__new__(cls, name, bases, dct) if name != 'Toolchain': - Toolchain.all_toolchains[name] = _class + Toolchain.all_toolchains[dct['name']] = _class return _class class Toolchain(metaclass=_MetaToolchain): + force_native_build = False neutral = True all_toolchains = {} configure_option = "" @@ -23,15 +24,12 @@ class Toolchain(metaclass=_MetaToolchain): Builder = None Source = None - def __init__(self): - self.source = self.Source(self) if self.Source else None - self.builder = self.Builder(self) if self.Builder else None - @property - def full_name(self): + @classmethod + def full_name(cls): return "{name}-{version}".format( - name = self.name, - version = self.version) + name = cls.name, + version = cls.version) @property def source_path(self): From daad1c98c675d354be33a14022ea2b2b8c0cc004 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 28 May 2018 16:13:33 +0200 Subject: [PATCH 06/23] Make the options global. --- kiwixbuild/__init__.py | 7 +++--- kiwixbuild/_global.py | 8 +++++++ kiwixbuild/buildenv.py | 24 ++++++++------------- kiwixbuild/builder.py | 21 +++++++++--------- kiwixbuild/dependencies/base.py | 6 +++--- kiwixbuild/dependencies/kiwix_android.py | 3 ++- kiwixbuild/dependencies/kiwix_custom_app.py | 8 +++---- kiwixbuild/platforms/base.py | 5 +++-- kiwixbuild/utils.py | 16 +++++--------- 9 files changed, 47 insertions(+), 51 deletions(-) diff --git a/kiwixbuild/__init__.py b/kiwixbuild/__init__.py index 73bb7c2..ff70f55 100644 --- a/kiwixbuild/__init__.py +++ b/kiwixbuild/__init__.py @@ -6,7 +6,6 @@ import argparse from .dependencies import Dependency from .platforms import PlatformInfo from .builder import Builder -from .utils import setup_print_progress from . import _global def parse_args(): @@ -65,9 +64,9 @@ def parse_args(): def main(): options = parse_args() options.working_dir = os.path.abspath(options.working_dir) - setup_print_progress(options.show_progress) - neutralEnv = buildenv.PlatformNeutralEnv(options) + _global.set_options(options) + neutralEnv = buildenv.PlatformNeutralEnv() _global.set_neutralEnv(neutralEnv) - builder = Builder(options) + builder = Builder() builder.run() diff --git a/kiwixbuild/_global.py b/kiwixbuild/_global.py index 6e4dcad..42e5285 100644 --- a/kiwixbuild/_global.py +++ b/kiwixbuild/_global.py @@ -1,6 +1,7 @@ from collections import OrderedDict as _OrderedDict _neutralEnv = None +_options = None _target_steps = _OrderedDict() _plt_steps = _OrderedDict() @@ -11,6 +12,13 @@ def set_neutralEnv(env): def neutralEnv(what): return getattr(_neutralEnv, what) +def set_options(options): + global _options + _options = options + +def option(what): + return getattr(_options, what) + def add_target_step(key, what): _target_steps[key] = what diff --git a/kiwixbuild/buildenv.py b/kiwixbuild/buildenv.py index 672c906..816ca04 100644 --- a/kiwixbuild/buildenv.py +++ b/kiwixbuild/buildenv.py @@ -5,15 +5,15 @@ import platform from .toolchains import Toolchain from .utils import pj, download_remote, Defaultdict -from ._global import neutralEnv +from ._global import neutralEnv, option class PlatformNeutralEnv: - def __init__(self, options): - self.options = options - self.source_dir = pj(options.working_dir, "SOURCE") - self.archive_dir = pj(options.working_dir, "ARCHIVE") - self.toolchain_dir = pj(options.working_dir, "TOOLCHAINS") + def __init__(self): + self.working_dir = option('working_dir') + self.source_dir = pj(self.working_dir, "SOURCE") + self.archive_dir = pj(self.working_dir, "ARCHIVE") + self.toolchain_dir = pj(self.working_dir, "TOOLCHAINS") self.log_dir = pj(self.working_dir, 'LOGS') for d in (self.source_dir, self.archive_dir, @@ -47,7 +47,7 @@ class PlatformNeutralEnv: def download(self, what, where=None): where = where or self.archive_dir - download_remote(what, where, not self.options.no_cert_check) + download_remote(what, where) def _detect_ninja(self): for n in ['ninja', 'ninja-build']: @@ -71,15 +71,12 @@ class PlatformNeutralEnv: if retcode == 0: return n - def __getattr__(self, name): - return getattr(self.options, name) - class BuildEnv: def __init__(self, platformInfo): build_dir = "BUILD_{}".format(platformInfo.name) self.platformInfo = platformInfo - self.build_dir = pj(neutralEnv('working_dir'), build_dir) + self.build_dir = pj(option('working_dir'), build_dir) self.install_dir = pj(self.build_dir, "INSTALL") self.log_dir = pj(self.build_dir, 'LOGS') for d in (self.build_dir, @@ -87,7 +84,7 @@ class BuildEnv: self.log_dir): os.makedirs(d, exist_ok=True) - self.libprefix = neutralEnv('libprefix') or self._detect_libdir() + self.libprefix = option('libprefix') or self._detect_libdir() def clean_intermediate_directories(self): for subdir in os.listdir(self.build_dir): @@ -99,9 +96,6 @@ class BuildEnv: else: os.remove(subpath) - def __getattr__(self, name): - return _global.neutralEnv(name) - def _is_debianlike(self): return os.path.isfile('/etc/debian_version') diff --git a/kiwixbuild/builder.py b/kiwixbuild/builder.py index d56724c..587811a 100644 --- a/kiwixbuild/builder.py +++ b/kiwixbuild/builder.py @@ -7,33 +7,32 @@ from .platforms import PlatformInfo from .utils import remove_duplicates, StopBuild from .dependencies import Dependency from ._global import ( - neutralEnv, + neutralEnv, option, add_target_step, get_target_step, target_steps, add_plt_step, get_plt_step, plt_steps) from . import _global class Builder: - def __init__(self, options): - self.options = options - platformClass = PlatformInfo.all_platforms[options.target_platform] + def __init__(self): + platformClass = PlatformInfo.all_platforms[option('target_platform')] if neutralEnv('distname') not in platformClass.compatible_hosts: print(('ERROR: The target platform {} cannot be build on host {}.\n' 'Select another target platform, or change your host system.' - ).format(options.target_platform, self.distname)) + ).format(option('target_platform'), self.distname)) sys.exit(-1) self.platform = platform = platformClass() _targets = {} - targetDef = (options.target_platform, options.targets) + targetDef = (option('target_platform'), option('targets')) self.add_targets(targetDef, _targets) dependencies = self.order_dependencies(_targets, targetDef) dependencies = list(remove_duplicates(dependencies)) - if options.build_nodeps: + if option('build_nodeps'): add_target_step(targetDef, _targets[targetDef]) else: for dep in dependencies: - if self.options.build_deps_only and dep == targetDef: + if option('build_deps_only') and dep == targetDef: continue add_target_step(dep, _targets[dep]) @@ -79,7 +78,7 @@ class Builder: source.prepare() def prepare_sources(self): - if self.options.skip_source_prepare: + if option('skip_source_prepare'): print("SKIP") return @@ -113,7 +112,7 @@ class Builder: source = get_target_step(builderDef[1], 'source') env = PlatformInfo.all_running_platforms[builderDef[0]].buildEnv builder = get_target_step(builderDef)(depClass, source, env) - if self.options.make_dist and builderDef[1] == self.options.targets: + if option('make_dist') and builderDef[1] == option('targets'): print("make dist {}:".format(builder.name)) builder.make_dist() continue @@ -136,7 +135,7 @@ class Builder: self.build() # No error, clean intermediate file at end of build if needed. print("[CLEAN]") - if self.options.clean_at_end: + if option('clean_at_end'): self.platform.clean_intermediate_directories() else: print("SKIP") diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index 9d93795..60b16c8 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -4,7 +4,7 @@ import shutil from kiwixbuild.utils import pj, Context, SkipCommand, extract_archive, Defaultdict, StopBuild, run_command from kiwixbuild.versions import main_project_versions, base_deps_versions -from kiwixbuild._global import neutralEnv +from kiwixbuild._global import neutralEnv, option SCRIPT_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) @@ -130,7 +130,7 @@ class GitClone(Source): @property def source_dir(self): - if neutralEnv('make_release'): + if option('make_release'): return "{}_release".format(self.git_dir) else: return self.git_dir @@ -141,7 +141,7 @@ class GitClone(Source): @property def git_ref(self): - if neutralEnv('make_release'): + if option('make_release'): return self.release_git_ref else: return self.base_git_ref diff --git a/kiwixbuild/dependencies/kiwix_android.py b/kiwixbuild/dependencies/kiwix_android.py index 6e00b33..fa19b09 100644 --- a/kiwixbuild/dependencies/kiwix_android.py +++ b/kiwixbuild/dependencies/kiwix_android.py @@ -6,6 +6,7 @@ from .base import ( GradleBuilder) from kiwixbuild.utils import pj +from kiwixbuild._global import option class KiwixAndroid(Dependency): name = "kiwix-android" @@ -18,7 +19,7 @@ class KiwixAndroid(Dependency): dependencies = ["Gradle", "kiwix-lib"] def build(self): - if self.buildEnv.options.targets == 'kiwix-android-custom': + if option('targets') == 'kiwix-android-custom': print("SKIP") else: super().build() diff --git a/kiwixbuild/dependencies/kiwix_custom_app.py b/kiwixbuild/dependencies/kiwix_custom_app.py index 43b90a9..9422301 100644 --- a/kiwixbuild/dependencies/kiwix_custom_app.py +++ b/kiwixbuild/dependencies/kiwix_custom_app.py @@ -7,14 +7,14 @@ from .base import ( GradleBuilder) from kiwixbuild.utils import Remotefile, pj, SkipCommand -from kiwixbuild._global import get_target_step +from kiwixbuild._global import option, get_target_step class KiwixCustomApp(Dependency): name = "kiwix-android-custom" def __init__(self, buildEnv): super().__init__(buildEnv) - self.custom_name = buildEnv.options.android_custom_app + self.custom_name = option('android_custom_app') class Source(GitClone): git_remote = "https://github.com/kiwix/kiwix-android-custom" @@ -51,7 +51,7 @@ class KiwixCustomApp(Dependency): def _get_zim_size(self): try: - zim_size = self.buildEnv.options.zim_file_size + zim_size = option('zim_file_size') except AttributeError: with open(pj(self.source_path, self.target.custom_name, 'info.json')) as f: app_info = json.load(f) @@ -64,7 +64,7 @@ class KiwixCustomApp(Dependency): self.command('compile', self._compile) def _download_zim(self, context): - zim_url = self.buildEnv.options.zim_file_url + zim_url = option('zim_file_url') if zim_url is None: raise SkipCommand() with open(pj(self.source_path, self.target.custom_name, 'info.json')) as f: diff --git a/kiwixbuild/platforms/base.py b/kiwixbuild/platforms/base.py index 2d1babe..dc21532 100644 --- a/kiwixbuild/platforms/base.py +++ b/kiwixbuild/platforms/base.py @@ -2,11 +2,12 @@ import os import subprocess +from kiwixbuild.dependencies import Dependency from kiwixbuild.toolchains import Toolchain from kiwixbuild.packages import PACKAGE_NAME_MAPPERS from kiwixbuild.utils import pj, remove_duplicates from kiwixbuild.buildenv import BuildEnv -from kiwixbuild._global import neutralEnv, add_plt_step, target_steps +from kiwixbuild._global import neutralEnv, option, add_plt_step, target_steps _SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) TEMPLATES_DIR = pj(os.path.dirname(_SCRIPT_DIR), 'templates') @@ -117,7 +118,7 @@ class PlatformInfo(metaclass=_MetaPlatform): packages_to_have = self.get_packages() packages_to_have = remove_duplicates(packages_to_have) - if not neutralEnv('force_install_packages') and os.path.exists(autoskip_file): + if not option('force_install_packages') and os.path.exists(autoskip_file): print("SKIP") return diff --git a/kiwixbuild/utils.py b/kiwixbuild/utils.py index 7eb0d29..b76dc5a 100644 --- a/kiwixbuild/utils.py +++ b/kiwixbuild/utils.py @@ -10,11 +10,10 @@ import ssl import subprocess from collections import namedtuple, defaultdict -from kiwixbuild._global import neutralEnv +from kiwixbuild._global import neutralEnv, option pj = os.path.join -g_print_progress = True REMOTE_PREFIX = 'http://download.kiwix.org/dev/' @@ -30,11 +29,6 @@ def xrun_find(name): return output[:-1].decode() -def setup_print_progress(print_progress): - global g_print_progress - g_print_progress = print_progress - - class Defaultdict(defaultdict): def __getattr__(self, name): return self[name] @@ -69,7 +63,7 @@ def get_sha256(path): def print_progress(progress): - if g_print_progress: + if option('show_progress'): text = "{}\033[{}D".format(progress, len(progress)) print(text, end="") @@ -92,7 +86,7 @@ def copy_tree(src, dst, post_copy_function=None): post_copy_function(dstfile) -def download_remote(what, where, check_certificate=True): +def download_remote(what, where): file_path = pj(where, what.name) file_url = what.url or (REMOTE_PREFIX + what.name) if os.path.exists(file_path): @@ -100,7 +94,7 @@ def download_remote(what, where, check_certificate=True): raise SkipCommand() os.remove(file_path) - if not check_certificate: + if option('no_cert_check'): context = ssl.create_default_context() context.check_hostname = False context.verify_mode = ssl.CERT_NONE @@ -241,7 +235,7 @@ def run_command(command, cwd, context, buildEnv=None, env=None, input=None, cros env = buildEnv._set_env(env, cross_compile_env, cross_compile_compiler, cross_compile_path) log = None try: - if not neutralEnv('verbose'): + if not option('verbose'): log = open(context.log_file, 'w') print("run command '{}'".format(command), file=log) print("current directory is '{}'".format(cwd), file=log) From 41d23b62492b97f64f89e0e20019c3a3d3d69845 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 28 May 2018 17:46:26 +0200 Subject: [PATCH 07/23] Create the associated platform at demand. --- kiwixbuild/builder.py | 25 ++++++++++--------------- kiwixbuild/platforms/base.py | 11 +++++++++++ 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/kiwixbuild/builder.py b/kiwixbuild/builder.py index 587811a..dccf8d4 100644 --- a/kiwixbuild/builder.py +++ b/kiwixbuild/builder.py @@ -14,14 +14,6 @@ from . import _global class Builder: def __init__(self): - platformClass = PlatformInfo.all_platforms[option('target_platform')] - if neutralEnv('distname') not in platformClass.compatible_hosts: - print(('ERROR: The target platform {} cannot be build on host {}.\n' - 'Select another target platform, or change your host system.' - ).format(option('target_platform'), self.distname)) - sys.exit(-1) - self.platform = platform = platformClass() - _targets = {} targetDef = (option('target_platform'), option('targets')) self.add_targets(targetDef, _targets) @@ -40,7 +32,7 @@ class Builder: if targetDef in targets: return targetPlatformName, targetName = targetDef - targetPlatform = PlatformInfo.all_platforms[targetPlatformName] + targetPlatform = PlatformInfo.get_platform(targetPlatformName) targetClass = Dependency.all_deps[targetName] targets[('source', targetName)] = targetClass.Source targets[targetDef] = targetClass.Builder @@ -58,7 +50,7 @@ class Builder: # build step two lines later. return target = _targets[targetDef] - targetPlatform = PlatformInfo.all_platforms[targetPlatformName] + targetPlatform = PlatformInfo.get_platform(targetPlatformName) yield ('source', targetName) for dep in target.get_dependencies(targetPlatform): try: @@ -100,7 +92,7 @@ class Builder: if tlbuilderDef[0] == 'neutral': env = _global._neutralEnv else: - env = PlatformInfo.all_running_platforms[tlbuilderDef[0]].buildEnv + env = PlatformInfo.get_platform(tlbuilderDef[0]).buildEnv builder = get_plt_step(tlbuilderDef)(toolchainClass, source, env) add_plt_step(tlbuilderDef, builder) builder.build() @@ -110,7 +102,7 @@ class Builder: for builderDef in builderDefs: depClass = Dependency.all_deps[builderDef[1]] source = get_target_step(builderDef[1], 'source') - env = PlatformInfo.all_running_platforms[builderDef[0]].buildEnv + env = PlatformInfo.get_platform(builderDef[0]).buildEnv builder = get_target_step(builderDef)(depClass, source, env) if option('make_dist') and builderDef[1] == option('targets'): print("make dist {}:".format(builder.name)) @@ -126,9 +118,11 @@ class Builder: print("[SETUP PLATFORMS]") self.prepare_toolchain_sources() self.build_toolchains() - self.platform.finalize_setup() + for platform in PlatformInfo.all_running_platforms.values(): + platform.finalize_setup() print("[INSTALL PACKAGES]") - self.platform.install_packages() + for platform in PlatformInfo.all_running_platforms.values(): + platform.install_packages() print("[PREPARE]") self.prepare_sources() print("[BUILD]") @@ -136,7 +130,8 @@ class Builder: # No error, clean intermediate file at end of build if needed. print("[CLEAN]") if option('clean_at_end'): - self.platform.clean_intermediate_directories() + for platform in PlatformInfo.all_running_platforms.values(): + platform.clean_intermediate_directories() else: print("SKIP") except StopBuild: diff --git a/kiwixbuild/platforms/base.py b/kiwixbuild/platforms/base.py index dc21532..4c2b012 100644 --- a/kiwixbuild/platforms/base.py +++ b/kiwixbuild/platforms/base.py @@ -27,7 +27,18 @@ class PlatformInfo(metaclass=_MetaPlatform): toolchain_names = [] configure_option = "" + @classmethod + def get_platform(cls, name): + if name not in cls.all_running_platforms: + cls.all_running_platforms[name] = cls.all_platforms[name]() + return cls.all_running_platforms[name] + def __init__(self): + if neutralEnv('distname') not in self.compatible_hosts: + print(('ERROR: The target platform {} cannot be build on host {}.\n' + 'Select another target platform, or change your host system.' + ).format(self.name, neutralEnv('distname'))) + sys.exit(-1) self.all_running_platforms[self.name] = self self.buildEnv = BuildEnv(self) self.setup_toolchains() From 211c0f9651bceb887fb50aa761efd4251ef22f3a Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 28 May 2018 19:50:36 +0200 Subject: [PATCH 08/23] buildEnv doesn't need to know about toolchains. Having access to the platform is enough. --- kiwixbuild/buildenv.py | 11 ----------- kiwixbuild/platforms/base.py | 4 ---- 2 files changed, 15 deletions(-) diff --git a/kiwixbuild/buildenv.py b/kiwixbuild/buildenv.py index 816ca04..186affd 100644 --- a/kiwixbuild/buildenv.py +++ b/kiwixbuild/buildenv.py @@ -20,7 +20,6 @@ class PlatformNeutralEnv: self.toolchain_dir, self.log_dir): os.makedirs(d, exist_ok=True) - self.toolchains = {} self.detect_platform() self.ninja_command = self._detect_ninja() if not self.ninja_command: @@ -115,16 +114,6 @@ class BuildEnv: return 'lib64' return 'lib' - @property - def configure_option(self): - configure_options = [tlc.configure_option for tlc in self.toolchains] - return " ".join(configure_options) - - @property - def cmake_option(self): - cmake_options = [tlc.cmake_option for tlc in self.toolchains] - return " ".join(cmake_options) - def _set_env(self, env, cross_compile_env, cross_compile_compiler, cross_compile_path): if env is None: env = Defaultdict(str, os.environ) diff --git a/kiwixbuild/platforms/base.py b/kiwixbuild/platforms/base.py index 4c2b012..242ebf1 100644 --- a/kiwixbuild/platforms/base.py +++ b/kiwixbuild/platforms/base.py @@ -47,10 +47,8 @@ class PlatformInfo(metaclass=_MetaPlatform): return "{}_{}".format(self.build, 'static' if self.static else 'dyn') def setup_toolchains(self): - self.toolchains = {} for tlc_name in self.toolchain_names: ToolchainClass = Toolchain.all_toolchains[tlc_name] - self.toolchains[tlc_name] = ToolchainClass() if ToolchainClass.Source is not None: add_plt_step(('source', tlc_name), ToolchainClass.Source) if ToolchainClass.Builder is not None: @@ -82,8 +80,6 @@ class PlatformInfo(metaclass=_MetaPlatform): return crossfile def finalize_setup(self): - self.buildEnv.toolchains = [ - self.toolchains[tlc_name] for tlc_name in self.toolchain_names] self.buildEnv.cross_config = self.get_cross_config() self.buildEnv.meson_crossfile = None self.buildEnv.cmake_crossfile = None From 9496ffb86c58621b9929a1a7504e6ccc61cf8666 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 28 May 2018 19:50:53 +0200 Subject: [PATCH 09/23] Clean android_ndk --- kiwixbuild/toolchains/android_ndk.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/kiwixbuild/toolchains/android_ndk.py b/kiwixbuild/toolchains/android_ndk.py index 597f82f..9be7d5f 100644 --- a/kiwixbuild/toolchains/android_ndk.py +++ b/kiwixbuild/toolchains/android_ndk.py @@ -51,14 +51,6 @@ class android_ndk(Toolchain): def arch_full(self): return self.buildEnv.platformInfo.arch_full - @property - def toolchain(self): - return self.arch_full+"-4.9" - - @property - def configure_option(self): - return '--host={}'.format(self.arch_full) - def _build_platform(self, context): context.try_skip(self.build_path) script = pj(self.source_path, 'build/tools/make_standalone_toolchain.py') From 4d6b6994bc4943bd263bc9f74e955cc20815ff15 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 29 May 2018 11:40:42 +0200 Subject: [PATCH 10/23] Merge toolchain and dependencies. Toolchain now became "simple" dependencies. They are referenced by Platform. Platform are now created at demande and a neutralPlatform now exists. --- kiwixbuild/_global.py | 14 ----- kiwixbuild/buildenv.py | 3 +- kiwixbuild/builder.py | 63 ++++++++----------- kiwixbuild/dependencies/__init__.py | 3 + .../android_ndk.py | 16 +---- .../android_sdk.py | 7 +-- kiwixbuild/dependencies/armhf.py | 11 ++++ kiwixbuild/dependencies/base.py | 2 +- kiwixbuild/platforms/__init__.py | 1 + kiwixbuild/platforms/android.py | 6 +- kiwixbuild/platforms/armhf.py | 4 +- kiwixbuild/platforms/base.py | 35 ++++++----- kiwixbuild/platforms/neutral.py | 9 +++ kiwixbuild/toolchains/__init__.py | 4 -- kiwixbuild/toolchains/armhf.py | 14 ----- kiwixbuild/toolchains/base_toolchain.py | 63 ------------------- kiwixbuild/versions.py | 6 +- 17 files changed, 87 insertions(+), 174 deletions(-) rename kiwixbuild/{toolchains => dependencies}/android_ndk.py (86%) rename kiwixbuild/{toolchains => dependencies}/android_sdk.py (92%) create mode 100644 kiwixbuild/dependencies/armhf.py create mode 100644 kiwixbuild/platforms/neutral.py delete mode 100644 kiwixbuild/toolchains/__init__.py delete mode 100644 kiwixbuild/toolchains/armhf.py delete mode 100644 kiwixbuild/toolchains/base_toolchain.py diff --git a/kiwixbuild/_global.py b/kiwixbuild/_global.py index 42e5285..f513eda 100644 --- a/kiwixbuild/_global.py +++ b/kiwixbuild/_global.py @@ -3,7 +3,6 @@ from collections import OrderedDict as _OrderedDict _neutralEnv = None _options = None _target_steps = _OrderedDict() -_plt_steps = _OrderedDict() def set_neutralEnv(env): global _neutralEnv @@ -31,16 +30,3 @@ def get_target_step(key, default_context=None): def target_steps(): return _target_steps - -def add_plt_step(key, what): - _plt_steps[key] = what - -def get_plt_step(key, default_context=None): - try: - context, target = key - except ValueError: - context, target = default_context, key - return _plt_steps[(context, target)] - -def plt_steps(): - return _plt_steps diff --git a/kiwixbuild/buildenv.py b/kiwixbuild/buildenv.py index 186affd..51fc163 100644 --- a/kiwixbuild/buildenv.py +++ b/kiwixbuild/buildenv.py @@ -3,7 +3,6 @@ import os, sys, shutil import subprocess import platform -from .toolchains import Toolchain from .utils import pj, download_remote, Defaultdict from ._global import neutralEnv, option @@ -77,9 +76,11 @@ class BuildEnv: self.platformInfo = platformInfo self.build_dir = pj(option('working_dir'), build_dir) self.install_dir = pj(self.build_dir, "INSTALL") + self.toolchain_dir = pj(self.build_dir, "TOOLCHAINS") self.log_dir = pj(self.build_dir, 'LOGS') for d in (self.build_dir, self.install_dir, + self.toolchain_dir, self.log_dir): os.makedirs(d, exist_ok=True) diff --git a/kiwixbuild/builder.py b/kiwixbuild/builder.py index dccf8d4..4c279e8 100644 --- a/kiwixbuild/builder.py +++ b/kiwixbuild/builder.py @@ -8,8 +8,7 @@ from .utils import remove_duplicates, StopBuild from .dependencies import Dependency from ._global import ( neutralEnv, option, - add_target_step, get_target_step, target_steps, - add_plt_step, get_plt_step, plt_steps) + add_target_step, get_target_step, target_steps) from . import _global class Builder: @@ -20,6 +19,7 @@ class Builder: dependencies = self.order_dependencies(_targets, targetDef) dependencies = list(remove_duplicates(dependencies)) + PlatformInfo.get_platform('neutral', _targets) if option('build_nodeps'): add_target_step(targetDef, _targets[targetDef]) else: @@ -27,12 +27,13 @@ class Builder: if option('build_deps_only') and dep == targetDef: continue add_target_step(dep, _targets[dep]) + self.instanciate_steps() def add_targets(self, targetDef, targets): if targetDef in targets: return targetPlatformName, targetName = targetDef - targetPlatform = PlatformInfo.get_platform(targetPlatformName) + targetPlatform = PlatformInfo.get_platform(targetPlatformName, targets) targetClass = Dependency.all_deps[targetName] targets[('source', targetName)] = targetClass.Source targets[targetDef] = targetClass.Builder @@ -44,6 +45,12 @@ class Builder: self.add_targets((depPlatform, depName), targets) def order_dependencies(self, _targets, targetDef): + for pltName in PlatformInfo.all_running_platforms: + plt = PlatformInfo.all_platforms[pltName] + for tlcName in plt.toolchain_names: + tlc = Dependency.all_deps[tlcName] + yield('source', tlcName) + yield('neutral' if tlc.neutral else pltName, tlcName) targetPlatformName, targetName = targetDef if targetPlatformName == 'source': # Do not try to order sources, they will be added as dep by the @@ -60,14 +67,18 @@ class Builder: yield from self.order_dependencies(_targets, (depPlatform, depName)) yield targetDef - def prepare_toolchain_sources(self): - tlsourceDefs = (tlDef for tlDef in plt_steps() if tlDef[0]=='source') - for tlsourceDef in tlsourceDefs: - print("prepare sources for toolchain {} :".format(tlsourceDef[1])) - toolchainClass = Toolchain.all_toolchains[tlsourceDef[1]] - source = get_plt_step(tlsourceDef)(toolchainClass) - add_plt_step(tlsourceDef, source) - source.prepare() + def instanciate_steps(self): + for stepDef in list(target_steps()): + stepPlatform, stepName = stepDef + stepClass = Dependency.all_deps[stepName] + if stepPlatform == 'source': + source = get_target_step(stepDef)(stepClass) + add_target_step(stepDef, source) + else: + source = get_target_step(stepName, 'source') + env = PlatformInfo.get_platform(stepPlatform).buildEnv + builder = get_target_step(stepDef)(stepClass, source, env) + add_target_step(stepDef, builder) def prepare_sources(self): if option('skip_source_prepare'): @@ -76,36 +87,16 @@ class Builder: sourceDefs = remove_duplicates(tDef for tDef in target_steps() if tDef[0]=='source') for sourceDef in sourceDefs: - print("prepare sources {} :".format(sourceDef[1])) - depClass = Dependency.all_deps[sourceDef[1]] - source = get_target_step(sourceDef)(depClass) - add_target_step(sourceDef, source) + source = get_target_step(sourceDef) source.prepare() - def build_toolchains(self): - tlbuilderDefs = (tlDef for tlDef in plt_steps() if tlDef[0] != 'source') - for tlbuilderDef in tlbuilderDefs: - print("build toolchain {} :".format(tlbuilderDef[1])) - toolchainClass = Toolchain.all_toolchains[tlbuilderDef[1]] - source = get_plt_step(tlbuilderDef[1], 'source') - if tlbuilderDef[0] == 'neutral': - env = _global._neutralEnv - else: - env = PlatformInfo.get_platform(tlbuilderDef[0]).buildEnv - builder = get_plt_step(tlbuilderDef)(toolchainClass, source, env) - add_plt_step(tlbuilderDef, builder) - builder.build() - def build(self): builderDefs = (tDef for tDef in target_steps() if tDef[0] != 'source') for builderDef in builderDefs: - depClass = Dependency.all_deps[builderDef[1]] - source = get_target_step(builderDef[1], 'source') - env = PlatformInfo.get_platform(builderDef[0]).buildEnv - builder = get_target_step(builderDef)(depClass, source, env) - if option('make_dist') and builderDef[1] == option('targets'): - print("make dist {}:".format(builder.name)) + builder = get_target_step(builderDef) + if option('make_dist') and builderName == option('targets'): + print("make dist {} ({}):".format(builder.name, builderDef[0])) builder.make_dist() continue print("build {} ({}):".format(builder.name, builderDef[0])) @@ -116,8 +107,6 @@ class Builder: def run(self): try: print("[SETUP PLATFORMS]") - self.prepare_toolchain_sources() - self.build_toolchains() for platform in PlatformInfo.all_running_platforms.values(): platform.finalize_setup() print("[INSTALL PACKAGES]") diff --git a/kiwixbuild/dependencies/__init__.py b/kiwixbuild/dependencies/__init__.py index 8007287..b457188 100644 --- a/kiwixbuild/dependencies/__init__.py +++ b/kiwixbuild/dependencies/__init__.py @@ -2,6 +2,9 @@ from .base import * from . import ( all_dependencies, + android_ndk, + android_sdk, + armhf, ctpp2, gradle, gumbo, diff --git a/kiwixbuild/toolchains/android_ndk.py b/kiwixbuild/dependencies/android_ndk.py similarity index 86% rename from kiwixbuild/toolchains/android_ndk.py rename to kiwixbuild/dependencies/android_ndk.py index 9be7d5f..fa2b00e 100644 --- a/kiwixbuild/toolchains/android_ndk.py +++ b/kiwixbuild/dependencies/android_ndk.py @@ -1,23 +1,15 @@ import os -from .base_toolchain import Toolchain -from kiwixbuild.dependencies import ReleaseDownload, Builder +from .base import Dependency, ReleaseDownload, Builder from kiwixbuild.utils import Remotefile, add_execution_right, run_command pj = os.path.join -class android_ndk(Toolchain): +class android_ndk(Dependency): neutral = False name = 'android-ndk' - version = 'r13b' gccver = '4.9.x' - @classmethod - def full_name(cls): - return "{name}-{version}".format( - name = cls.name, - version = cls.version) - class Source(ReleaseDownload): archive = Remotefile('android-ndk-r13b-linux-x86_64.zip', '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c', @@ -25,9 +17,7 @@ class android_ndk(Toolchain): @property def source_dir(self): - return "{}-{}".format( - self.target.name, - self.target.version) + return self.target.full_name() class Builder(Builder): diff --git a/kiwixbuild/toolchains/android_sdk.py b/kiwixbuild/dependencies/android_sdk.py similarity index 92% rename from kiwixbuild/toolchains/android_sdk.py rename to kiwixbuild/dependencies/android_sdk.py index c58e05f..d09f433 100644 --- a/kiwixbuild/toolchains/android_sdk.py +++ b/kiwixbuild/dependencies/android_sdk.py @@ -1,15 +1,14 @@ import os import shutil -from .base_toolchain import Toolchain -from kiwixbuild.dependencies import ReleaseDownload, Builder +from .base import Dependency, ReleaseDownload, Builder from kiwixbuild.utils import Remotefile, run_command pj = os.path.join -class android_sdk(Toolchain): +class android_sdk(Dependency): + neutral = True name = 'android-sdk' - version = 'r25.2.3' class Source(ReleaseDownload): archive = Remotefile('tools_r25.2.3-linux.zip', diff --git a/kiwixbuild/dependencies/armhf.py b/kiwixbuild/dependencies/armhf.py new file mode 100644 index 0000000..4ba5200 --- /dev/null +++ b/kiwixbuild/dependencies/armhf.py @@ -0,0 +1,11 @@ +from .base import Dependency, GitClone, NoopBuilder + +class armhf_toolchain(Dependency): + neutral = True + name = 'armhf' + + class Source(GitClone): + git_remote = "https://github.com/raspberrypi/tools" + git_dir = "raspberrypi-tools" + + Builder = NoopBuilder diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index 60b16c8..940a669 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -350,7 +350,7 @@ class CMakeBuilder(MakeBuilder): " {source_path}" " {cross_option}") command = command.format( - configure_option="{} {}".format(self.buildEnv.cmake_option, self.configure_option), + configure_option=self.configure_option, install_dir=self.buildEnv.install_dir, libdir=self.buildEnv.libprefix, source_path=self.source_path, diff --git a/kiwixbuild/platforms/__init__.py b/kiwixbuild/platforms/__init__.py index 5975eca..4b63719 100644 --- a/kiwixbuild/platforms/__init__.py +++ b/kiwixbuild/platforms/__init__.py @@ -7,5 +7,6 @@ from . import ( i586, ios, native, + neutral, win32 ) diff --git a/kiwixbuild/platforms/android.py b/kiwixbuild/platforms/android.py index 4187a19..1fdc46e 100644 --- a/kiwixbuild/platforms/android.py +++ b/kiwixbuild/platforms/android.py @@ -1,6 +1,6 @@ from .base import PlatformInfo from kiwixbuild.utils import pj -from kiwixbuild._global import get_plt_step +from kiwixbuild._global import get_target_step class AndroidPlatformInfo(PlatformInfo): @@ -27,11 +27,11 @@ class AndroidPlatformInfo(PlatformInfo): @property def ndk_builder(self): - return get_plt_step('android_ndk', self.name) + return get_target_step('android-ndk', self.name) @property def sdk_builder(self): - return get_plt_step('android_sdk', 'neutral') + return get_target_step('android-sdk', 'neutral') def get_cross_config(self): install_path = self.ndk_builder.install_path diff --git a/kiwixbuild/platforms/armhf.py b/kiwixbuild/platforms/armhf.py index e3077d9..354af4b 100644 --- a/kiwixbuild/platforms/armhf.py +++ b/kiwixbuild/platforms/armhf.py @@ -1,7 +1,7 @@ from .base import PlatformInfo from kiwixbuild.utils import pj -from kiwixbuild._global import get_plt_step +from kiwixbuild._global import get_target_step class ArmhfPlatformInfo(PlatformInfo): @@ -29,7 +29,7 @@ class ArmhfPlatformInfo(PlatformInfo): @property def tlc_source(self): - return get_plt_step('armhf', 'source') + return get_target_step('armhf', 'source') @property def root_path(self): diff --git a/kiwixbuild/platforms/base.py b/kiwixbuild/platforms/base.py index 242ebf1..cfa7779 100644 --- a/kiwixbuild/platforms/base.py +++ b/kiwixbuild/platforms/base.py @@ -1,13 +1,12 @@ -import os +import os, sys import subprocess from kiwixbuild.dependencies import Dependency -from kiwixbuild.toolchains import Toolchain from kiwixbuild.packages import PACKAGE_NAME_MAPPERS from kiwixbuild.utils import pj, remove_duplicates from kiwixbuild.buildenv import BuildEnv -from kiwixbuild._global import neutralEnv, option, add_plt_step, target_steps +from kiwixbuild._global import neutralEnv, option, target_steps _SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) TEMPLATES_DIR = pj(os.path.dirname(_SCRIPT_DIR), 'templates') @@ -28,12 +27,17 @@ class PlatformInfo(metaclass=_MetaPlatform): configure_option = "" @classmethod - def get_platform(cls, name): + def get_platform(cls, name, targets=None): if name not in cls.all_running_platforms: - cls.all_running_platforms[name] = cls.all_platforms[name]() + if targets is None: + print("Should not got there.") + print(cls.all_running_platforms) + raise KeyError(name) + sys.exit(-1) + cls.all_running_platforms[name] = cls.all_platforms[name](targets) return cls.all_running_platforms[name] - def __init__(self): + def __init__(self, targets): if neutralEnv('distname') not in self.compatible_hosts: print(('ERROR: The target platform {} cannot be build on host {}.\n' 'Select another target platform, or change your host system.' @@ -41,19 +45,17 @@ class PlatformInfo(metaclass=_MetaPlatform): sys.exit(-1) self.all_running_platforms[self.name] = self self.buildEnv = BuildEnv(self) - self.setup_toolchains() + self.setup_toolchains(targets) def __str__(self): return "{}_{}".format(self.build, 'static' if self.static else 'dyn') - def setup_toolchains(self): + def setup_toolchains(self, targets): for tlc_name in self.toolchain_names: - ToolchainClass = Toolchain.all_toolchains[tlc_name] - if ToolchainClass.Source is not None: - add_plt_step(('source', tlc_name), ToolchainClass.Source) - if ToolchainClass.Builder is not None: - plt_name = 'neutral' if ToolchainClass.neutral else self.name - add_plt_step((plt_name, tlc_name), ToolchainClass.Builder) + ToolchainClass = Dependency.all_deps[tlc_name] + targets[('source', tlc_name)] = ToolchainClass.Source + plt_name = 'neutral' if ToolchainClass.neutral else self.name + targets[(plt_name, tlc_name)] = ToolchainClass.Builder def get_cross_config(self): return {} @@ -93,9 +95,10 @@ class PlatformInfo(metaclass=_MetaPlatform): except KeyError: print("SKIP : We don't know which packages we must install to compile" " a {target} {build_type} version on a {host} host.".format( - target=self.platform_info, + target=self, + build_type=self.static, host=neutralEnv('distname'))) - return + return [] packages_list = package_name_mapper.get('COMMON', []) to_drop = [] diff --git a/kiwixbuild/platforms/neutral.py b/kiwixbuild/platforms/neutral.py new file mode 100644 index 0000000..bbe339c --- /dev/null +++ b/kiwixbuild/platforms/neutral.py @@ -0,0 +1,9 @@ +from .base import PlatformInfo + +class NeutralPlatformInfo(PlatformInfo): + name = 'neutral' + static = '' + compatible_hosts = ['fedora', 'debian', 'Darwin'] + + def __str__(self): + return "neutral" diff --git a/kiwixbuild/toolchains/__init__.py b/kiwixbuild/toolchains/__init__.py deleted file mode 100644 index ebf0cff..0000000 --- a/kiwixbuild/toolchains/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ - -from . import android_ndk, android_sdk, armhf - -from .base_toolchain import Toolchain diff --git a/kiwixbuild/toolchains/armhf.py b/kiwixbuild/toolchains/armhf.py deleted file mode 100644 index d49aa62..0000000 --- a/kiwixbuild/toolchains/armhf.py +++ /dev/null @@ -1,14 +0,0 @@ -import os -import subprocess - -from .base_toolchain import Toolchain -from kiwixbuild.dependencies import GitClone -from kiwixbuild.utils import which -pj = os.path.join - -class armhf_toolchain(Toolchain): - name = 'armhf' - - class Source(GitClone): - git_remote = "https://github.com/raspberrypi/tools" - git_dir = "raspberrypi-tools" diff --git a/kiwixbuild/toolchains/base_toolchain.py b/kiwixbuild/toolchains/base_toolchain.py deleted file mode 100644 index c51de7a..0000000 --- a/kiwixbuild/toolchains/base_toolchain.py +++ /dev/null @@ -1,63 +0,0 @@ -import os -import subprocess - -pj = os.path.join - -from kiwixbuild.utils import Context, SkipCommand, StopBuild -from kiwixbuild._global import neutralEnv - -class _MetaToolchain(type): - def __new__(cls, name, bases, dct): - _class = type.__new__(cls, name, bases, dct) - if name != 'Toolchain': - Toolchain.all_toolchains[dct['name']] = _class - return _class - - -class Toolchain(metaclass=_MetaToolchain): - force_native_build = False - neutral = True - all_toolchains = {} - configure_option = "" - cmake_option = "" - exec_wrapper_def = "" - Builder = None - Source = None - - - @classmethod - def full_name(cls): - return "{name}-{version}".format( - name = cls.name, - version = cls.version) - - @property - def source_path(self): - return pj(neutralEnv('source_dir'), self.source.source_dir) - - @property - def _log_dir(self): - return neutralEnv('log_dir') - - def command(self, name, function, *args): - print(" {} {} : ".format(name, self.name), end="", flush=True) - log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name)) - context = Context(name, log, True) - try: - ret = function(*args, context=context) - context._finalise() - print("OK") - return ret - except SkipCommand: - print("SKIP") - except subprocess.CalledProcessError: - print("ERROR") - try: - with open(log, 'r') as f: - print(f.read()) - except: - pass - raise StopBuild() - except: - print("ERROR") - raise diff --git a/kiwixbuild/versions.py b/kiwixbuild/versions.py index e2253ec..f822d96 100644 --- a/kiwixbuild/versions.py +++ b/kiwixbuild/versions.py @@ -10,7 +10,7 @@ main_project_versions = { # This is the "version" of the whole base_deps_versions dict. # Change this when you change base_deps_versions. -base_deps_meta_version = '3' +base_deps_meta_version = '4' base_deps_versions = { @@ -25,5 +25,7 @@ base_deps_versions = { 'icu4c' : '58.2', 'Gradle' : '4.6', 'libaria2' : '1.33.1', - 'libmagic' : '5.33' + 'libmagic' : '5.33', + 'android-sdk' : 'r25.2.3', + 'android-ndk' : 'r13b' } From 7e8ed1a623c65df7fd564d5e6321e556e5a6adca Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 29 May 2018 12:32:55 +0200 Subject: [PATCH 11/23] Correctly install packages. --- kiwixbuild/__init__.py | 4 +- kiwixbuild/builder.py | 95 ++++++++++++++++++++++++++++++------ kiwixbuild/platforms/base.py | 80 ------------------------------ 3 files changed, 82 insertions(+), 97 deletions(-) diff --git a/kiwixbuild/__init__.py b/kiwixbuild/__init__.py index ff70f55..1a194a5 100644 --- a/kiwixbuild/__init__.py +++ b/kiwixbuild/__init__.py @@ -35,8 +35,8 @@ def parse_args(): help="Skip SSL certificate verification during download") subgroup.add_argument('--clean-at-end', action='store_true', help="Clean all intermediate files after the (successfull) build") - subgroup.add_argument('--force-install-packages', action='store_true', - help="Allways check for needed packages before compiling") + subgroup.add_argument('--dont-install-packages', action='store_true', + help="Do not try to install packages before compiling") subgroup = parser.add_argument_group('custom app', description="Android custom app specific options") subgroup.add_argument('--android-custom-app', diff --git a/kiwixbuild/builder.py b/kiwixbuild/builder.py index 4c279e8..5765c9f 100644 --- a/kiwixbuild/builder.py +++ b/kiwixbuild/builder.py @@ -6,6 +6,7 @@ from .buildenv import * from .platforms import PlatformInfo from .utils import remove_duplicates, StopBuild from .dependencies import Dependency +from .packages import PACKAGE_NAME_MAPPERS from ._global import ( neutralEnv, option, add_target_step, get_target_step, target_steps) @@ -13,20 +14,23 @@ from . import _global class Builder: def __init__(self): - _targets = {} - targetDef = (option('target_platform'), option('targets')) - self.add_targets(targetDef, _targets) - dependencies = self.order_dependencies(_targets, targetDef) + self._targets = {} + PlatformInfo.get_platform('neutral', self._targets) + + self.targetDef = (option('target_platform'), option('targets')) + self.add_targets(self.targetDef, self._targets) + + def finalize_target_steps(self): + dependencies = self.order_dependencies(self.targetDef) dependencies = list(remove_duplicates(dependencies)) - PlatformInfo.get_platform('neutral', _targets) if option('build_nodeps'): - add_target_step(targetDef, _targets[targetDef]) + add_target_step(self.targetDef, self._targets[self.targetDef]) else: for dep in dependencies: - if option('build_deps_only') and dep == targetDef: + if option('build_deps_only') and dep == self.targetDef: continue - add_target_step(dep, _targets[dep]) + add_target_step(dep, self._targets[dep]) self.instanciate_steps() def add_targets(self, targetDef, targets): @@ -44,7 +48,7 @@ class Builder: depPlatform, depName = targetPlatformName, dep self.add_targets((depPlatform, depName), targets) - def order_dependencies(self, _targets, targetDef): + def order_dependencies(self, targetDef): for pltName in PlatformInfo.all_running_platforms: plt = PlatformInfo.all_platforms[pltName] for tlcName in plt.toolchain_names: @@ -56,15 +60,16 @@ class Builder: # Do not try to order sources, they will be added as dep by the # build step two lines later. return - target = _targets[targetDef] + target = self._targets[targetDef] targetPlatform = PlatformInfo.get_platform(targetPlatformName) - yield ('source', targetName) for dep in target.get_dependencies(targetPlatform): try: depPlatform, depName = dep except ValueError: depPlatform, depName = targetPlatformName, dep - yield from self.order_dependencies(_targets, (depPlatform, depName)) + if (depPlatform, depName) in self._targets: + yield from self.order_dependencies((depPlatform, depName)) + yield ('source', targetName) yield targetDef def instanciate_steps(self): @@ -103,15 +108,75 @@ class Builder: add_target_step(builderDef, builder) builder.build() + def _get_packages(self): + packages_list = [] + for platform in PlatformInfo.all_running_platforms.values(): + mapper_name = "{host}_{target}".format( + host=neutralEnv('distname'), + target=platform) + package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {}) + packages_list += package_name_mapper.get('COMMON', []) + + to_drop = [] + for builderDef in self._targets: + platformName, builderName = builderDef + mapper_name = "{host}_{target}".format( + host=neutralEnv('distname'), + target=platformName) + package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {}) + packages = package_name_mapper.get(builderName) + if packages: + packages_list += packages + to_drop.append(builderDef) + for dep in to_drop: + del self._targets[dep] + return packages_list + + def install_packages(self): + packages_to_have = self._get_packages() + packages_to_have = remove_duplicates(packages_to_have) + + distname = neutralEnv('distname') + if distname in ('fedora', 'redhat', 'centos'): + package_installer = 'sudo dnf install {}' + package_checker = 'rpm -q --quiet {}' + elif distname in ('debian', 'Ubuntu'): + package_installer = 'sudo apt-get install {}' + package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1' + elif distname == 'Darwin': + package_installer = 'brew install {}' + package_checker = 'brew list -1 | grep -q {}' + + packages_to_install = [] + for package in packages_to_have: + print(" - {} : ".format(package), end="") + command = package_checker.format(package) + try: + subprocess.check_call(command, shell=True) + except subprocess.CalledProcessError: + print("NEEDED") + packages_to_install.append(package) + else: + print("SKIP") + + if packages_to_install: + command = package_installer.format(" ".join(packages_to_install)) + print(command) + subprocess.check_call(command, shell=True) + else: + print("SKIP, No package to install.") def run(self): try: + print("[INSTALL PACKAGES]") + if option('dont_install_packages'): + print("SKIP") + else: + self.install_packages() + self.finalize_target_steps() print("[SETUP PLATFORMS]") for platform in PlatformInfo.all_running_platforms.values(): platform.finalize_setup() - print("[INSTALL PACKAGES]") - for platform in PlatformInfo.all_running_platforms.values(): - platform.install_packages() print("[PREPARE]") self.prepare_sources() print("[BUILD]") diff --git a/kiwixbuild/platforms/base.py b/kiwixbuild/platforms/base.py index cfa7779..2d92c36 100644 --- a/kiwixbuild/platforms/base.py +++ b/kiwixbuild/platforms/base.py @@ -3,7 +3,6 @@ import os, sys import subprocess from kiwixbuild.dependencies import Dependency -from kiwixbuild.packages import PACKAGE_NAME_MAPPERS from kiwixbuild.utils import pj, remove_duplicates from kiwixbuild.buildenv import BuildEnv from kiwixbuild._global import neutralEnv, option, target_steps @@ -86,85 +85,6 @@ class PlatformInfo(metaclass=_MetaPlatform): self.buildEnv.meson_crossfile = None self.buildEnv.cmake_crossfile = None - def get_packages(self): - mapper_name = "{host}_{target}".format( - host=neutralEnv('distname'), - target=self) - try: - package_name_mapper = PACKAGE_NAME_MAPPERS[mapper_name] - except KeyError: - print("SKIP : We don't know which packages we must install to compile" - " a {target} {build_type} version on a {host} host.".format( - target=self, - build_type=self.static, - host=neutralEnv('distname'))) - return [] - - packages_list = package_name_mapper.get('COMMON', []) - to_drop = [] - build_steps = (step for step in target_steps() if step[0] == self.name) - for dep in build_steps: - packages = package_name_mapper.get(dep[1]) - if packages: - packages_list += packages - to_drop.append(dep) - for dep in to_drop: - del target_steps()[dep] - source_dep = [dep[1] for dep in target_steps() if dep[0] == 'source'] - for dep in source_dep: - it_build = (True for dep in target_steps() if dep[0] != 'source' and dep[1] == dep) - if next(it_build, False): - print("removing source ", dep) - del target_steps()[('source', dep)] - build_steps = (step for step in target_steps() if step[0] == self.name) - for dep in build_steps: - packages = getattr(dep, 'extra_packages', []) - for package in packages: - packages_list += package_name_mapper.get(package, []) - return packages_list - - def install_packages(self): - autoskip_file = pj(self.buildEnv.build_dir, ".install_packages_ok") - packages_to_have = self.get_packages() - packages_to_have = remove_duplicates(packages_to_have) - - if not option('force_install_packages') and os.path.exists(autoskip_file): - print("SKIP") - return - - distname = neutralEnv('distname') - if distname in ('fedora', 'redhat', 'centos'): - package_installer = 'sudo dnf install {}' - package_checker = 'rpm -q --quiet {}' - elif distname in ('debian', 'Ubuntu'): - package_installer = 'sudo apt-get install {}' - package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1' - elif distname == 'Darwin': - package_installer = 'brew install {}' - package_checker = 'brew list -1 | grep -q {}' - - packages_to_install = [] - for package in packages_to_have: - print(" - {} : ".format(package), end="") - command = package_checker.format(package) - try: - subprocess.check_call(command, shell=True) - except subprocess.CalledProcessError: - print("NEEDED") - packages_to_install.append(package) - else: - print("SKIP") - - if packages_to_install: - command = package_installer.format(" ".join(packages_to_install)) - print(command) - subprocess.check_call(command, shell=True) - else: - print("SKIP, No package to install.") - - with open(autoskip_file, 'w'): - pass - def clean_intermediate_directories(self): self.buildEnv.clean_intermediate_directories() From a87ba06bf39671b5db3d20fcb4251c86fca3aede Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 29 May 2018 17:41:10 +0200 Subject: [PATCH 12/23] Do not infinit loop when ordering steps --- kiwixbuild/builder.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/kiwixbuild/builder.py b/kiwixbuild/builder.py index 5765c9f..bd12604 100644 --- a/kiwixbuild/builder.py +++ b/kiwixbuild/builder.py @@ -21,7 +21,7 @@ class Builder: self.add_targets(self.targetDef, self._targets) def finalize_target_steps(self): - dependencies = self.order_dependencies(self.targetDef) + dependencies = self.order_steps(self.targetDef) dependencies = list(remove_duplicates(dependencies)) if option('build_nodeps'): @@ -48,27 +48,35 @@ class Builder: depPlatform, depName = targetPlatformName, dep self.add_targets((depPlatform, depName), targets) - def order_dependencies(self, targetDef): + def order_steps(self, targetDef): for pltName in PlatformInfo.all_running_platforms: plt = PlatformInfo.all_platforms[pltName] for tlcName in plt.toolchain_names: tlc = Dependency.all_deps[tlcName] yield('source', tlcName) yield('neutral' if tlc.neutral else pltName, tlcName) + _targets =dict(self._targets) + yield from self.order_dependencies(targetDef, _targets) + + def order_dependencies(self, targetDef, targets): targetPlatformName, targetName = targetDef if targetPlatformName == 'source': # Do not try to order sources, they will be added as dep by the # build step two lines later. return - target = self._targets[targetDef] + try: + target = targets.pop(targetDef) + except KeyError: + return + targetPlatform = PlatformInfo.get_platform(targetPlatformName) for dep in target.get_dependencies(targetPlatform): try: depPlatform, depName = dep except ValueError: depPlatform, depName = targetPlatformName, dep - if (depPlatform, depName) in self._targets: - yield from self.order_dependencies((depPlatform, depName)) + if (depPlatform, depName) in targets: + yield from self.order_dependencies((depPlatform, depName), targets) yield ('source', targetName) yield targetDef From c8c0192675b4adc285199db647988ec8ab0d3b28 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 29 May 2018 17:42:42 +0200 Subject: [PATCH 13/23] Simplify cross-compilation Now we can make reference to build step in another target platform, we can simplify cross-compilation fo libmagic and icu4c. --- kiwixbuild/dependencies/all_dependencies.py | 10 ++--- kiwixbuild/dependencies/icu4c.py | 41 +++++++-------------- kiwixbuild/dependencies/kiwix_lib.py | 7 +--- kiwixbuild/dependencies/libmagic.py | 39 +++++++------------- kiwixbuild/dependencies/libzim.py | 9 +---- kiwixbuild/dependencies/zimwriterfs.py | 8 +--- 6 files changed, 33 insertions(+), 81 deletions(-) diff --git a/kiwixbuild/dependencies/all_dependencies.py b/kiwixbuild/dependencies/all_dependencies.py index c55d727..3d55d29 100644 --- a/kiwixbuild/dependencies/all_dependencies.py +++ b/kiwixbuild/dependencies/all_dependencies.py @@ -12,13 +12,9 @@ class AllBaseDependencies(Dependency): class Builder(NoopBuilder): @classmethod def get_dependencies(cls, platformInfo): - base_deps = ['zlib', 'lzma', 'xapian-core', 'gumbo', 'pugixml', 'libmicrohttpd', 'libaria2'] - if platformInfo.build != 'native': - base_deps += ["icu4c_cross-compile"] - if platformInfo.build != 'win32': - base_deps += ["libmagic_cross-compile"] - else: - base_deps += ["icu4c", "libmagic"] + base_deps = ['zlib', 'lzma', 'xapian-core', 'gumbo', 'pugixml', 'libmicrohttpd', 'libaria2', 'icu4c'] + if platformInfo.build != 'win32': + base_deps += ["libmagic"] if (platformInfo.build != 'android' and neutralEnv('distname') != 'Darwin'): base_deps += ['ctpp2c', 'ctpp2'] diff --git a/kiwixbuild/dependencies/icu4c.py b/kiwixbuild/dependencies/icu4c.py index d9c8a0b..1ac58e1 100644 --- a/kiwixbuild/dependencies/icu4c.py +++ b/kiwixbuild/dependencies/icu4c.py @@ -25,36 +25,21 @@ class Icu(Dependency): class Builder(MakeBuilder): subsource_dir = "source" + @classmethod + def get_dependencies(cls, platformInfo): + plt = 'native_static' if platformInfo.static else 'native_dyn' + return [(plt, 'icu4c')] + @property def configure_option(self): options = "--disable-samples --disable-tests --disable-extras --disable-dyload --enable-rpath" - if self.buildEnv.platformInfo.build == 'android': + platformInfo = self.buildEnv.platformInfo + if platformInfo.build != 'native': + icu_native_builder = get_target_step( + 'icu4c', + 'native_static' if platformInfo.static else 'native_dyn') + options += " --with-cross-build={} --disable-tools".format( + icu_native_builder.build_path) + if platformInfo.build == 'android': options += " --with-data-packaging=archive" return options - - -class Icu_native(Icu): - name = "icu4c_native" - force_native_build = True - - class Builder(Icu.Builder): - @property - def build_path(self): - return super().build_path+"_native" - - def _install(self, context): - raise SkipCommand() - - -class Icu_cross_compile(Icu): - name = "icu4c_cross-compile" - - class Builder(Icu.Builder): - dependencies = ['icu4c_native'] - - @property - def configure_option(self): - icu_native_builder = get_target_step('icu4c_native', self.buildEnv.platformInfo.name) - return (super().configure_option - + " --with-cross-build={} --disable-tools" - ).format(icu_native_builder.build_path) diff --git a/kiwixbuild/dependencies/kiwix_lib.py b/kiwixbuild/dependencies/kiwix_lib.py index 120230d..8647264 100644 --- a/kiwixbuild/dependencies/kiwix_lib.py +++ b/kiwixbuild/dependencies/kiwix_lib.py @@ -14,14 +14,11 @@ class Kiwixlib(Dependency): class Builder(MesonBuilder): @classmethod def get_dependencies(cls, platformInfo): - base_dependencies = ["pugixml", "libzim", "zlib", "lzma", "libaria2"] + base_dependencies = ["pugixml", "libzim", "zlib", "lzma", "libaria2", "icu4c"] if (platformInfo.build != 'android' and neutralEnv('distname') != 'Darwin'): base_dependencies += ['ctpp2c', 'ctpp2'] - if platformInfo.build != 'native': - return base_dependencies + ["icu4c_cross-compile"] - else: - return base_dependencies + ["icu4c"] + return base_dependencies @property diff --git a/kiwixbuild/dependencies/libmagic.py b/kiwixbuild/dependencies/libmagic.py index ab0f88e..8c4d53a 100644 --- a/kiwixbuild/dependencies/libmagic.py +++ b/kiwixbuild/dependencies/libmagic.py @@ -6,10 +6,10 @@ from .base import ( MakeBuilder, ) -from kiwixbuild.utils import Remotefile, pj, Defaultdict, SkipCommand +from kiwixbuild.utils import Remotefile, pj, Defaultdict, SkipCommand, run_command from kiwixbuild._global import get_target_step -class LibMagicBase(Dependency): +class LibMagic(Dependency): name = "libmagic" class Source(ReleaseDownload): @@ -19,37 +19,24 @@ class LibMagicBase(Dependency): '1c52c8c3d271cd898d5511c36a68059cda94036111ab293f01f83c3525b737c6', 'https://fossies.org/linux/misc/file-5.33.tar.gz') - Builder = MakeBuilder + class Builder(MakeBuilder): - -class LibMagic_native(LibMagicBase): - name = "libmagic_native" - force_native_build = True - - class Builder(LibMagicBase.Builder): - static_configure_option = dynamic_configure_option = "--disable-shared --enable-static" - - @property - def build_path(self): - return super().build_path+"_native" - - def _install(self, context): - raise SkipCommand() - - -class LibMagic_cross_compile(LibMagicBase): - name = "libmagic_cross-compile" - - class Builder(LibMagicBase.Builder): - dependencies = ['libmagic_native'] + @classmethod + def get_dependencies(cls, platformInfo): + if platformInfo.build != 'native': + return [('native_static', 'libmagic')] + return [] def _compile(self, context): + platformInfo = self.buildEnv.platformInfo + if platformInfo.build == 'native': + return super()._compile(context) context.try_skip(self.build_path) command = "make -j4 {make_target} {make_option}".format( make_target=self.make_target, make_option=self.make_option ) - libmagic_native_builder = get_target_step('libmagic_native', self.buildEnv.platformInfo.name) + libmagic_native_builder = get_target_step('libmagic', 'native_static') env = Defaultdict(str, os.environ) env['PATH'] = ':'.join([pj(libmagic_native_builder.build_path, 'src'), env['PATH']]) - self.buildEnv.run_command(command, self.build_path, context, env=env) + run_command(command, self.build_path, context, buildEnv=self.buildEnv, env=env) diff --git a/kiwixbuild/dependencies/libzim.py b/kiwixbuild/dependencies/libzim.py index 3b0c19a..aa01108 100644 --- a/kiwixbuild/dependencies/libzim.py +++ b/kiwixbuild/dependencies/libzim.py @@ -12,11 +12,4 @@ class Libzim(Dependency): class Builder(MesonBuilder): test_option = "-t 8" - - @classmethod - def get_dependencies(cls, platformInfo): - base_dependencies = ['zlib', 'lzma', 'xapian-core'] - if platformInfo.build != 'native': - return base_dependencies + ["icu4c_cross-compile"] - else: - return base_dependencies + ["icu4c"] + dependencies = ['zlib', 'lzma', 'xapian-core', 'icu4c'] diff --git a/kiwixbuild/dependencies/zimwriterfs.py b/kiwixbuild/dependencies/zimwriterfs.py index 72cc92f..3ac977c 100644 --- a/kiwixbuild/dependencies/zimwriterfs.py +++ b/kiwixbuild/dependencies/zimwriterfs.py @@ -12,13 +12,7 @@ class Zimwriterfs(Dependency): release_git_ref = "1.1" class Builder(MesonBuilder): - @classmethod - def get_dependencies(cls, platformInfo): - base_dependencies = ['libzim', 'zlib', 'xapian-core', 'gumbo'] - if platformInfo.build != 'native': - return base_dependencies + ["icu4c_cross-compile", "libmagic_cross-compile"] - else: - return base_dependencies + ["icu4c", "libmagic"] + dependencies = ['libzim', 'zlib', 'xapian-core', 'gumbo', 'icu4c', 'libmagic'] @property def configure_option(self): From 3eae47240b97a53e236e8d9062024aeaecbfb3c7 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 30 May 2018 11:33:36 +0200 Subject: [PATCH 14/23] Make the platform responsible to add the targets. --- kiwixbuild/builder.py | 26 ++++++-------------------- kiwixbuild/platforms/base.py | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/kiwixbuild/builder.py b/kiwixbuild/builder.py index bd12604..8d784e6 100644 --- a/kiwixbuild/builder.py +++ b/kiwixbuild/builder.py @@ -17,37 +17,23 @@ class Builder: self._targets = {} PlatformInfo.get_platform('neutral', self._targets) - self.targetDef = (option('target_platform'), option('targets')) - self.add_targets(self.targetDef, self._targets) + platform = PlatformInfo.get_platform(option('target_platform'), self._targets) + platform.add_targets(option('targets'), self._targets) def finalize_target_steps(self): - dependencies = self.order_steps(self.targetDef) + targetDef = (option('target_platform'), option('targets')) + dependencies = self.order_steps(targetDef) dependencies = list(remove_duplicates(dependencies)) if option('build_nodeps'): - add_target_step(self.targetDef, self._targets[self.targetDef]) + add_target_step(targetDef, self._targets[targetDef]) else: for dep in dependencies: - if option('build_deps_only') and dep == self.targetDef: + if option('build_deps_only') and dep == targetDef: continue add_target_step(dep, self._targets[dep]) self.instanciate_steps() - def add_targets(self, targetDef, targets): - if targetDef in targets: - return - targetPlatformName, targetName = targetDef - targetPlatform = PlatformInfo.get_platform(targetPlatformName, targets) - targetClass = Dependency.all_deps[targetName] - targets[('source', targetName)] = targetClass.Source - targets[targetDef] = targetClass.Builder - for dep in targetClass.Builder.get_dependencies(targetPlatform): - try: - depPlatform, depName = dep - except ValueError: - depPlatform, depName = targetPlatformName, dep - self.add_targets((depPlatform, depName), targets) - def order_steps(self, targetDef): for pltName in PlatformInfo.all_running_platforms: plt = PlatformInfo.all_platforms[pltName] diff --git a/kiwixbuild/platforms/base.py b/kiwixbuild/platforms/base.py index 2d92c36..7f1e135 100644 --- a/kiwixbuild/platforms/base.py +++ b/kiwixbuild/platforms/base.py @@ -56,6 +56,20 @@ class PlatformInfo(metaclass=_MetaPlatform): plt_name = 'neutral' if ToolchainClass.neutral else self.name targets[(plt_name, tlc_name)] = ToolchainClass.Builder + def add_targets(self, targetName, targets): + if (self.name, targetName) in targets: + return + targetClass = Dependency.all_deps[targetName] + targets[('source', targetName)] = targetClass.Source + targets[(self.name, targetName)] = targetClass.Builder + for dep in targetClass.Builder.get_dependencies(self): + try: + depPlatformName, depName = dep + except ValueError: + depPlatformName, depName = self.name, dep + depPlatform = self.get_platform(depPlatformName, targets) + depPlatform.add_targets(depName, targets) + def get_cross_config(self): return {} From 74ba5885a06de4dcea0b34f6fecd23b7e3d074f3 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 30 May 2018 17:09:47 +0200 Subject: [PATCH 15/23] Add a MetaPlatformInfo. A metaplatform allow to regroup sereval platform together. When a target is added to the platform, the target is dispatched/add to all sub platforms. As the step (metaplatformName, target) is not really added, we have to track which steps are really added, so `add_targets` need to return the list of added targets. --- kiwixbuild/builder.py | 11 ++++++----- kiwixbuild/platforms/base.py | 15 ++++++++++++++- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/kiwixbuild/builder.py b/kiwixbuild/builder.py index 8d784e6..0734270 100644 --- a/kiwixbuild/builder.py +++ b/kiwixbuild/builder.py @@ -18,17 +18,18 @@ class Builder: PlatformInfo.get_platform('neutral', self._targets) platform = PlatformInfo.get_platform(option('target_platform'), self._targets) - platform.add_targets(option('targets'), self._targets) + self.targetDefs = platform.add_targets(option('targets'), self._targets) def finalize_target_steps(self): - targetDef = (option('target_platform'), option('targets')) - dependencies = self.order_steps(targetDef) - dependencies = list(remove_duplicates(dependencies)) + steps = [] + for targetDef in self.targetDefs: + steps += self.order_steps(targetDef) + steps = list(remove_duplicates(steps)) if option('build_nodeps'): add_target_step(targetDef, self._targets[targetDef]) else: - for dep in dependencies: + for dep in steps: if option('build_deps_only') and dep == targetDef: continue add_target_step(dep, self._targets[dep]) diff --git a/kiwixbuild/platforms/base.py b/kiwixbuild/platforms/base.py index 7f1e135..94316ec 100644 --- a/kiwixbuild/platforms/base.py +++ b/kiwixbuild/platforms/base.py @@ -58,7 +58,7 @@ class PlatformInfo(metaclass=_MetaPlatform): def add_targets(self, targetName, targets): if (self.name, targetName) in targets: - return + return [] targetClass = Dependency.all_deps[targetName] targets[('source', targetName)] = targetClass.Source targets[(self.name, targetName)] = targetClass.Builder @@ -69,6 +69,7 @@ class PlatformInfo(metaclass=_MetaPlatform): depPlatformName, depName = self.name, dep depPlatform = self.get_platform(depPlatformName, targets) depPlatform.add_targets(depName, targets) + return [(self.name, targetName)] def get_cross_config(self): return {} @@ -102,3 +103,15 @@ class PlatformInfo(metaclass=_MetaPlatform): def clean_intermediate_directories(self): self.buildEnv.clean_intermediate_directories() + + +class MetaPlatformInfo(PlatformInfo): + subPlatformNames = [] + + def add_targets(self, targetName, targets): + targetDefs = [] + for platformName in self.subPlatformNames: + print("radd {}".format(platformName)) + platform = self.get_platform(platformName, targets) + targetDefs += platform.add_targets(targetName, targets) + return targetDefs From 6de2cc80b622e41ece8fdbf1137ddf9f6750b116 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 30 May 2018 17:32:23 +0200 Subject: [PATCH 16/23] Add an android metaPlatformInfo to build all android architecture. --- kiwixbuild/platforms/android.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/kiwixbuild/platforms/android.py b/kiwixbuild/platforms/android.py index 1fdc46e..4c7be67 100644 --- a/kiwixbuild/platforms/android.py +++ b/kiwixbuild/platforms/android.py @@ -1,4 +1,4 @@ -from .base import PlatformInfo +from .base import PlatformInfo, MetaPlatformInfo from kiwixbuild.utils import pj from kiwixbuild._global import get_target_step @@ -6,7 +6,7 @@ from kiwixbuild._global import get_target_step class AndroidPlatformInfo(PlatformInfo): build = 'android' static = True - toolchain_names = ['android-ndk', 'android-sdk'] + toolchain_names = ['android-ndk'] compatible_hosts = ['fedora', 'debian'] def __str__(self): @@ -29,10 +29,6 @@ class AndroidPlatformInfo(PlatformInfo): def ndk_builder(self): return get_target_step('android-ndk', self.name) - @property - def sdk_builder(self): - return get_target_step('android-sdk', 'neutral') - def get_cross_config(self): install_path = self.ndk_builder.install_path return { @@ -67,7 +63,6 @@ class AndroidPlatformInfo(PlatformInfo): # '-DU_STATIC_IMPLEMENTATION -O3 ' # '-DU_HAVE_STD_STRING -DU_TIMEZONE=0 ')+env['CXXFLAGS'] env['NDK_DEBUG'] = '0' - env['ANDROID_HOME'] = self.sdk_builder.install_path def set_compiler(self, env): binaries = self.binaries(self.ndk_builder.install_path) @@ -119,3 +114,19 @@ class AndroidArm(AndroidPlatformInfo): name = 'android_x86_64' arch = cpu = abi = 'x86_64' arch_full = 'x86_64-linux-android' + +class Android(MetaPlatformInfo): + name = "android" + toolchain_names = ['android-sdk'] + subPlatformNames = ['android_arm', 'android_arm64', 'android_mips', 'android_mips64', 'android_x86', 'android_x86_64'] + compatible_hosts = ['fedora', 'debian'] + + def __str__(self): + return self.name + + @property + def sdk_builder(self): + return get_target_step('android-sdk', 'neutral') + + def set_env(self, env): + env['ANDROID_HOME'] = self.sdk_builder.install_path From 9b85791705b77f9579b514c8973bcb3c04b52ba5 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 30 May 2018 17:56:06 +0200 Subject: [PATCH 17/23] Gradle is a dependency of the android platform, not of other dependencies. --- kiwixbuild/dependencies/all_dependencies.py | 2 -- kiwixbuild/dependencies/gradle.py | 11 ++++++++--- kiwixbuild/dependencies/kiwix_android.py | 2 +- kiwixbuild/platforms/android.py | 7 ++++++- kiwixbuild/versions.py | 2 +- 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/kiwixbuild/dependencies/all_dependencies.py b/kiwixbuild/dependencies/all_dependencies.py index 3d55d29..096504b 100644 --- a/kiwixbuild/dependencies/all_dependencies.py +++ b/kiwixbuild/dependencies/all_dependencies.py @@ -18,7 +18,5 @@ class AllBaseDependencies(Dependency): if (platformInfo.build != 'android' and neutralEnv('distname') != 'Darwin'): base_deps += ['ctpp2c', 'ctpp2'] - if platformInfo.build == 'android': - base_deps += ['Gradle'] return base_deps diff --git a/kiwixbuild/dependencies/gradle.py b/kiwixbuild/dependencies/gradle.py index 42ed104..e62c464 100644 --- a/kiwixbuild/dependencies/gradle.py +++ b/kiwixbuild/dependencies/gradle.py @@ -6,7 +6,8 @@ from .base import ( from kiwixbuild.utils import Remotefile, pj, copy_tree, add_execution_right class Gradle(Dependency): - name = "Gradle" + neutral = True + name = "gradle" class Source(ReleaseDownload): archive = Remotefile('gradle-4.6-bin.zip', @@ -14,14 +15,18 @@ class Gradle(Dependency): 'https://services.gradle.org/distributions/gradle-4.6-bin.zip') class Builder(BaseBuilder): + @property + def install_path(self): + return self.buildEnv.install_dir + def build(self): self.command('install', self._install) def _install(self, context): copy_tree( pj(self.source_path, "bin"), - pj(self.buildEnv.install_dir, "bin"), + pj(self.install_path, "bin"), post_copy_function = add_execution_right) copy_tree( pj(self.source_path, "lib"), - pj(self.buildEnv.install_dir, "lib")) + pj(self.install_path, "lib")) diff --git a/kiwixbuild/dependencies/kiwix_android.py b/kiwixbuild/dependencies/kiwix_android.py index fa19b09..54a03a1 100644 --- a/kiwixbuild/dependencies/kiwix_android.py +++ b/kiwixbuild/dependencies/kiwix_android.py @@ -16,7 +16,7 @@ class KiwixAndroid(Dependency): git_dir = "kiwix-android" class Builder(GradleBuilder): - dependencies = ["Gradle", "kiwix-lib"] + dependencies = ["kiwix-lib"] def build(self): if option('targets') == 'kiwix-android-custom': diff --git a/kiwixbuild/platforms/android.py b/kiwixbuild/platforms/android.py index 4c7be67..b0d650b 100644 --- a/kiwixbuild/platforms/android.py +++ b/kiwixbuild/platforms/android.py @@ -117,7 +117,7 @@ class AndroidArm(AndroidPlatformInfo): class Android(MetaPlatformInfo): name = "android" - toolchain_names = ['android-sdk'] + toolchain_names = ['android-sdk', 'gradle'] subPlatformNames = ['android_arm', 'android_arm64', 'android_mips', 'android_mips64', 'android_x86', 'android_x86_64'] compatible_hosts = ['fedora', 'debian'] @@ -128,5 +128,10 @@ class Android(MetaPlatformInfo): def sdk_builder(self): return get_target_step('android-sdk', 'neutral') + @property + def gradle_builder(self): + return get_target_step('gradle', 'neutral') + def set_env(self, env): env['ANDROID_HOME'] = self.sdk_builder.install_path + env['PATH'] = ':'.join([pj(self.gradle_builder.install_path, 'bin'), env['PATH']]) diff --git a/kiwixbuild/versions.py b/kiwixbuild/versions.py index f822d96..4f9fddd 100644 --- a/kiwixbuild/versions.py +++ b/kiwixbuild/versions.py @@ -23,7 +23,7 @@ base_deps_versions = { 'libmicrohttpd' : '0.9.46', 'gumbo' : '0.10.1', 'icu4c' : '58.2', - 'Gradle' : '4.6', + 'gradle' : '4.6', 'libaria2' : '1.33.1', 'libmagic' : '5.33', 'android-sdk' : 'r25.2.3', From db82455f03209a59d89e8082e329f0c04899b321 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 30 May 2018 19:25:35 +0200 Subject: [PATCH 18/23] Be able to build multi-arch android apk. By building kiwix-android on the `android` platform, we can now build `kiwix-lib` on all `android_` platforms, and so have all architectures in the same apk. Fix #108 --- README.md | 31 +++++++++++++++-- kiwixbuild/__init__.py | 6 ++++ kiwixbuild/builder.py | 2 +- kiwixbuild/dependencies/all_dependencies.py | 2 +- kiwixbuild/dependencies/base.py | 2 +- kiwixbuild/dependencies/icu4c.py | 2 +- kiwixbuild/dependencies/kiwix_android.py | 38 ++++++++++++++++----- kiwixbuild/dependencies/kiwix_lib.py | 2 +- kiwixbuild/dependencies/libmagic.py | 2 +- kiwixbuild/dependencies/xapian.py | 2 +- kiwixbuild/platforms/android.py | 13 +++++-- kiwixbuild/platforms/base.py | 2 +- 12 files changed, 84 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index b7011ad..60c93b0 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,7 @@ platforms : - native_static - win32_dyn - win32_static +- android - android_arm - android_arm64 - android_mips @@ -95,9 +96,35 @@ So, if you want to compile `kiwix-tools` for win32 using static linkage: $ kiwix-build --target-platform win32_dyn ``` -Or, android apk for android_arm : +## Android + +Android apk (kiwix-android) is a bit a special case. +`kiwix-android` itself is architecture independent (it is written in +java) but it use `kiwix-lib` who is architecture dependent. + +When building `kiwix-lib`, you should directly use the +target-platform `android_`: + ``` -$ kiwix-build --target-platform android_arm kiwix-android +$ kiwix-build kiwix-android --target-platform android_arm +``` + +But, `kiwix-android` apk can also be multi arch (ie, it includes +`kiwix-lib` for several architectures). To do so, you must ask to build +`kiwix-android` using the `android` platform: + +``` +$ kiwix-build --target-platform android kiwix-android +$ kiwix-build kiwix-android # because `android` platform is the default `kiwix-android` +``` + +By default, when using platform `android`, `kiwix-lib` will be build for +all architectures. This can be change by using the option `--android-arch` : + +``` +$ kiwix-build kiwix-android # apk for all architectures +$ kiwix-build kiwix-android --android-arch arm # apk for arm architectures (equivalent to `kiwix-android --target-platform android_arm`) +$ kiwix-build kiwix-anrdoid --android-arch arm --android-arch arm64 # apk for arm and arm64 architectures ``` # Outputs diff --git a/kiwixbuild/__init__.py b/kiwixbuild/__init__.py index 1a194a5..0f7301c 100644 --- a/kiwixbuild/__init__.py +++ b/kiwixbuild/__init__.py @@ -37,6 +37,10 @@ def parse_args(): help="Clean all intermediate files after the (successfull) build") subgroup.add_argument('--dont-install-packages', action='store_true', help="Do not try to install packages before compiling") + subgroup.add_argument('--android-arch', action='append', + help=("Specify the architecture to build for android application/libraries.\n" + "Can be specified several times to build for several architectures.\n" + "If not specified, all architectures will be build.")) subgroup = parser.add_argument_group('custom app', description="Android custom app specific options") subgroup.add_argument('--android-custom-app', @@ -59,6 +63,8 @@ def parse_args(): err = True if err: sys.exit(1) + if not options.android_arch: + options.android_arch = ['arm', 'arm64', 'mips', 'mips64', 'x86', 'x86_64'] return options def main(): diff --git a/kiwixbuild/builder.py b/kiwixbuild/builder.py index 0734270..7b45126 100644 --- a/kiwixbuild/builder.py +++ b/kiwixbuild/builder.py @@ -57,7 +57,7 @@ class Builder: return targetPlatform = PlatformInfo.get_platform(targetPlatformName) - for dep in target.get_dependencies(targetPlatform): + for dep in target.get_dependencies(targetPlatform, True): try: depPlatform, depName = dep except ValueError: diff --git a/kiwixbuild/dependencies/all_dependencies.py b/kiwixbuild/dependencies/all_dependencies.py index 096504b..eb91cab 100644 --- a/kiwixbuild/dependencies/all_dependencies.py +++ b/kiwixbuild/dependencies/all_dependencies.py @@ -11,7 +11,7 @@ class AllBaseDependencies(Dependency): Source = NoopSource class Builder(NoopBuilder): @classmethod - def get_dependencies(cls, platformInfo): + def get_dependencies(cls, platformInfo, allDeps): base_deps = ['zlib', 'lzma', 'xapian-core', 'gumbo', 'pugixml', 'libmicrohttpd', 'libaria2', 'icu4c'] if platformInfo.build != 'win32': base_deps += ["libmagic"] diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index 940a669..645aab9 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -202,7 +202,7 @@ class Builder: self.buildEnv = buildEnv @classmethod - def get_dependencies(cls, platformInfo): + def get_dependencies(cls, platformInfo, allDeps): return cls.dependencies @property diff --git a/kiwixbuild/dependencies/icu4c.py b/kiwixbuild/dependencies/icu4c.py index 1ac58e1..14dbf8e 100644 --- a/kiwixbuild/dependencies/icu4c.py +++ b/kiwixbuild/dependencies/icu4c.py @@ -26,7 +26,7 @@ class Icu(Dependency): subsource_dir = "source" @classmethod - def get_dependencies(cls, platformInfo): + def get_dependencies(cls, platformInfo, allDeps): plt = 'native_static' if platformInfo.static else 'native_dyn' return [(plt, 'icu4c')] diff --git a/kiwixbuild/dependencies/kiwix_android.py b/kiwixbuild/dependencies/kiwix_android.py index 54a03a1..7c15a2c 100644 --- a/kiwixbuild/dependencies/kiwix_android.py +++ b/kiwixbuild/dependencies/kiwix_android.py @@ -5,8 +5,8 @@ from .base import ( GitClone, GradleBuilder) -from kiwixbuild.utils import pj -from kiwixbuild._global import option +from kiwixbuild.utils import pj, copy_tree +from kiwixbuild._global import option, get_target_step class KiwixAndroid(Dependency): name = "kiwix-android" @@ -18,6 +18,15 @@ class KiwixAndroid(Dependency): class Builder(GradleBuilder): dependencies = ["kiwix-lib"] + @classmethod + def get_dependencies(cls, platformInfo, allDeps): + if not allDeps: + return super().get_dependencies(platformInfo, allDeps) + else: + deps = [('android_{}'.format(arch), 'kiwix-lib') + for arch in option('android_arch')] + return deps + def build(self): if option('targets') == 'kiwix-android-custom': print("SKIP") @@ -31,12 +40,25 @@ class KiwixAndroid(Dependency): shutil.rmtree(pj(self.build_path, 'kiwixlib', 'src', 'main')) except FileNotFoundError: pass - shutil.copytree(pj(self.buildEnv.install_dir, 'kiwix-lib'), - pj(self.build_path, 'kiwixlib', 'src', 'main')) + for arch in option('android_arch'): + try: + kiwix_builder = get_target_step('kiwix-lib', 'android_{}'.format(arch)) + except KeyError: + pass + else: + copy_tree(pj(kiwix_builder.buildEnv.install_dir, 'kiwix-lib'), + pj(self.build_path, 'kiwixlib', 'src', 'main')) os.makedirs( pj(self.build_path, 'app', 'src', 'main', 'assets', 'icu'), exist_ok=True) - shutil.copy2(pj(self.buildEnv.install_dir, 'share', 'icu', '58.2', - 'icudt58l.dat'), - pj(self.build_path, 'app', 'src', 'main', 'assets', - 'icu', 'icudt58l.dat')) + for arch in option('android_arch'): + try: + kiwix_builder = get_target_step('kiwix-lib', 'android_{}'.format(arch)) + except KeyError: + pass + else: + shutil.copy2(pj(kiwix_builder.buildEnv.install_dir, 'share', 'icu', '58.2', + 'icudt58l.dat'), + pj(self.build_path, 'app', 'src', 'main', 'assets', + 'icu', 'icudt58l.dat')) + break diff --git a/kiwixbuild/dependencies/kiwix_lib.py b/kiwixbuild/dependencies/kiwix_lib.py index 8647264..a72e5fe 100644 --- a/kiwixbuild/dependencies/kiwix_lib.py +++ b/kiwixbuild/dependencies/kiwix_lib.py @@ -13,7 +13,7 @@ class Kiwixlib(Dependency): class Builder(MesonBuilder): @classmethod - def get_dependencies(cls, platformInfo): + def get_dependencies(cls, platformInfo, allDeps): base_dependencies = ["pugixml", "libzim", "zlib", "lzma", "libaria2", "icu4c"] if (platformInfo.build != 'android' and neutralEnv('distname') != 'Darwin'): diff --git a/kiwixbuild/dependencies/libmagic.py b/kiwixbuild/dependencies/libmagic.py index 8c4d53a..f25e9da 100644 --- a/kiwixbuild/dependencies/libmagic.py +++ b/kiwixbuild/dependencies/libmagic.py @@ -22,7 +22,7 @@ class LibMagic(Dependency): class Builder(MakeBuilder): @classmethod - def get_dependencies(cls, platformInfo): + def get_dependencies(cls, platformInfo, allDeps): if platformInfo.build != 'native': return [('native_static', 'libmagic')] return [] diff --git a/kiwixbuild/dependencies/xapian.py b/kiwixbuild/dependencies/xapian.py index 8fc49a0..3d76cbc 100644 --- a/kiwixbuild/dependencies/xapian.py +++ b/kiwixbuild/dependencies/xapian.py @@ -21,7 +21,7 @@ class Xapian(Dependency): '_format_CXXFLAGS': "-I{buildEnv.install_dir}/include"} @classmethod - def get_dependencies(cls, platformInfo): + def get_dependencies(cls, platformInfo, allDeps): deps = ['zlib', 'lzma'] if (platformInfo.build == 'win32' or neutralEnv('distname') == 'Darwin'): diff --git a/kiwixbuild/platforms/android.py b/kiwixbuild/platforms/android.py index b0d650b..ce06276 100644 --- a/kiwixbuild/platforms/android.py +++ b/kiwixbuild/platforms/android.py @@ -1,6 +1,6 @@ from .base import PlatformInfo, MetaPlatformInfo from kiwixbuild.utils import pj -from kiwixbuild._global import get_target_step +from kiwixbuild._global import get_target_step, option class AndroidPlatformInfo(PlatformInfo): @@ -118,9 +118,18 @@ class AndroidArm(AndroidPlatformInfo): class Android(MetaPlatformInfo): name = "android" toolchain_names = ['android-sdk', 'gradle'] - subPlatformNames = ['android_arm', 'android_arm64', 'android_mips', 'android_mips64', 'android_x86', 'android_x86_64'] compatible_hosts = ['fedora', 'debian'] + @property + def subPlatformNames(self): + return ['android_{}'.format(arch) for arch in option('android_arch')] + + def add_targets(self, targetName, targets): + if targetName != 'kiwix-android': + return super().add_targets(targetName, targets) + else: + return AndroidPlatformInfo.add_targets(self, targetName, targets) + def __str__(self): return self.name diff --git a/kiwixbuild/platforms/base.py b/kiwixbuild/platforms/base.py index 94316ec..44af29b 100644 --- a/kiwixbuild/platforms/base.py +++ b/kiwixbuild/platforms/base.py @@ -62,7 +62,7 @@ class PlatformInfo(metaclass=_MetaPlatform): targetClass = Dependency.all_deps[targetName] targets[('source', targetName)] = targetClass.Source targets[(self.name, targetName)] = targetClass.Builder - for dep in targetClass.Builder.get_dependencies(self): + for dep in targetClass.Builder.get_dependencies(self, False): try: depPlatformName, depName = dep except ValueError: From 5f5fbfe08219430adebe326527d17420b567523b Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 30 May 2018 19:27:14 +0200 Subject: [PATCH 19/23] Rename `targets` option to `target`. --- kiwixbuild/__init__.py | 6 +++--- kiwixbuild/builder.py | 4 ++-- kiwixbuild/dependencies/kiwix_android.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/kiwixbuild/__init__.py b/kiwixbuild/__init__.py index 0f7301c..0c421a6 100644 --- a/kiwixbuild/__init__.py +++ b/kiwixbuild/__init__.py @@ -10,7 +10,7 @@ from . import _global def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument('targets', default='kiwix-tools', nargs='?', metavar='TARGET', + parser.add_argument('target', default='kiwix-tools', nargs='?', metavar='TARGET', choices=Dependency.all_deps.keys()) parser.add_argument('--working-dir', default=".") parser.add_argument('--libprefix', default=None) @@ -23,7 +23,7 @@ def parse_args(): parser.add_argument('--skip-source-prepare', action='store_true', help="Skip the source download part") parser.add_argument('--build-deps-only', action='store_true', - help="Build only the dependencies of the specified targets.") + help="Build only the dependencies of the specified target.") parser.add_argument('--build-nodeps', action='store_true', help="Build only the target, not its dependencies.") parser.add_argument('--make-dist', action='store_true', @@ -51,7 +51,7 @@ def parse_args(): help="The size of the zim file.") options = parser.parse_args() - if options.targets == 'kiwix-android-custom': + if options.target == 'kiwix-android-custom': err = False if not options.android_custom_app: print("You need to specify ANDROID_CUSTOM_APP if you " diff --git a/kiwixbuild/builder.py b/kiwixbuild/builder.py index 7b45126..758e027 100644 --- a/kiwixbuild/builder.py +++ b/kiwixbuild/builder.py @@ -18,7 +18,7 @@ class Builder: PlatformInfo.get_platform('neutral', self._targets) platform = PlatformInfo.get_platform(option('target_platform'), self._targets) - self.targetDefs = platform.add_targets(option('targets'), self._targets) + self.targetDefs = platform.add_targets(option('target'), self._targets) def finalize_target_steps(self): steps = [] @@ -95,7 +95,7 @@ class Builder: builderDefs = (tDef for tDef in target_steps() if tDef[0] != 'source') for builderDef in builderDefs: builder = get_target_step(builderDef) - if option('make_dist') and builderName == option('targets'): + if option('make_dist') and builderName == option('target'): print("make dist {} ({}):".format(builder.name, builderDef[0])) builder.make_dist() continue diff --git a/kiwixbuild/dependencies/kiwix_android.py b/kiwixbuild/dependencies/kiwix_android.py index 7c15a2c..4a2de5b 100644 --- a/kiwixbuild/dependencies/kiwix_android.py +++ b/kiwixbuild/dependencies/kiwix_android.py @@ -28,7 +28,7 @@ class KiwixAndroid(Dependency): return deps def build(self): - if option('targets') == 'kiwix-android-custom': + if option('target') == 'kiwix-android-custom': print("SKIP") else: super().build() From cfdf03c85491e33ac0bf41c70bdcc1686a705bef Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 30 May 2018 19:28:29 +0200 Subject: [PATCH 20/23] Select the default platform depending of the target. Default platform `android` is better for `kiwix-android` (because `kiwix-android` can build only on `android` platform). --- README.md | 6 ++++-- kiwixbuild/__init__.py | 2 +- kiwixbuild/builder.py | 8 +++++++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 60c93b0..b901f4a 100644 --- a/README.md +++ b/README.md @@ -71,8 +71,10 @@ By default, `kiwix-build` will build `kiwix-tools` . ## Target platform -By default, `kiwix-build` will build everything for the current (native) -platform using dynamic linkage (`native_dyn`). +If no target platform is specified, a default one will be infered from +the specified target : +- `kiwix-android` will be build using the platform `android` +- Other targets will be build using the platform `native_dyn` But you can select another target platform using the option `--target-platform`. For now, there is ten different supported diff --git a/kiwixbuild/__init__.py b/kiwixbuild/__init__.py index 0c421a6..47b3695 100644 --- a/kiwixbuild/__init__.py +++ b/kiwixbuild/__init__.py @@ -14,7 +14,7 @@ def parse_args(): choices=Dependency.all_deps.keys()) parser.add_argument('--working-dir', default=".") parser.add_argument('--libprefix', default=None) - parser.add_argument('--target-platform', default="native_dyn", choices=PlatformInfo.all_platforms) + parser.add_argument('--target-platform', choices=PlatformInfo.all_platforms) parser.add_argument('--verbose', '-v', action="store_true", help=("Print all logs on stdout instead of in specific" " log files per commands")) diff --git a/kiwixbuild/builder.py b/kiwixbuild/builder.py index 758e027..8b4ee44 100644 --- a/kiwixbuild/builder.py +++ b/kiwixbuild/builder.py @@ -17,7 +17,13 @@ class Builder: self._targets = {} PlatformInfo.get_platform('neutral', self._targets) - platform = PlatformInfo.get_platform(option('target_platform'), self._targets) + target_platform = option('target_platform') + if not target_platform: + if option('target') == 'kiwix-android': + target_platform = 'android' + else: + target_platform = 'native_dyn' + platform = PlatformInfo.get_platform(target_platform, self._targets) self.targetDefs = platform.add_targets(option('target'), self._targets) def finalize_target_steps(self): From 369b805e59e0956e0f929bb9e35d54e04df38a0f Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 31 May 2018 15:17:24 +0200 Subject: [PATCH 21/23] Add ios_multi_arch fatlib. Fix #113 --- README.md | 21 +++++++++++ kiwixbuild/__init__.py | 7 ++++ kiwixbuild/builder.py | 4 ++ kiwixbuild/dependencies/__init__.py | 1 + kiwixbuild/dependencies/ios_fat_lib.py | 52 ++++++++++++++++++++++++++ kiwixbuild/platforms/base.py | 6 --- kiwixbuild/platforms/ios.py | 25 ++++++++++--- 7 files changed, 105 insertions(+), 11 deletions(-) create mode 100644 kiwixbuild/dependencies/ios_fat_lib.py diff --git a/README.md b/README.md index b901f4a..4361e28 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,27 @@ $ kiwix-build kiwix-android --android-arch arm # apk for arm architectures (equi $ kiwix-build kiwix-anrdoid --android-arch arm --android-arch arm64 # apk for arm and arm64 architectures ``` +## IOS + +When building for ios, we may want to compile a "fat library", a library +for several architectures. + +To do so, you should directly use the target-platfrom `ios_multi`. +As for `android`, `kiwix-build` will build the library several times +(once for each platform) and then create the fat library. + +``` +$ kiwix-build --target-platform iOS_multi kiwix-lib +``` + +You can specify the supported architectures with the option `--ios-arch`: + +``` +$ kiwix-build --target-platform iOS_multi kiwix-lib # all architetures +$ kiwix-build --target-platform iOS_multi --ios-arch arm --ios-arch arm64 # arm and arm64 arch only +``` + + # Outputs Kiwix-build.py will create several directories: diff --git a/kiwixbuild/__init__.py b/kiwixbuild/__init__.py index 47b3695..41855f9 100644 --- a/kiwixbuild/__init__.py +++ b/kiwixbuild/__init__.py @@ -41,6 +41,10 @@ def parse_args(): help=("Specify the architecture to build for android application/libraries.\n" "Can be specified several times to build for several architectures.\n" "If not specified, all architectures will be build.")) + subgroup.add_argument('--ios-arch', action='append', + help=("Specify the architecture to build for ios application/libraries.\n" + "Can be specified several times to build for several architectures.\n" + "If not specified, all architectures will be build.")) subgroup = parser.add_argument_group('custom app', description="Android custom app specific options") subgroup.add_argument('--android-custom-app', @@ -65,6 +69,9 @@ def parse_args(): sys.exit(1) if not options.android_arch: options.android_arch = ['arm', 'arm64', 'mips', 'mips64', 'x86', 'x86_64'] + if not options.ios_arch: + options.ios_arch = ['armv7', 'arm64', 'i386', 'x86_64'] + return options def main(): diff --git a/kiwixbuild/builder.py b/kiwixbuild/builder.py index 8b4ee44..4ee7dd5 100644 --- a/kiwixbuild/builder.py +++ b/kiwixbuild/builder.py @@ -24,6 +24,10 @@ class Builder: else: target_platform = 'native_dyn' platform = PlatformInfo.get_platform(target_platform, self._targets) + if neutralEnv('distname') not in platform.compatible_hosts: + print(('ERROR: The target platform {} cannot be build on host {}.\n' + 'Select another target platform or change your host system.' + ).format(platform.name, neutralEnv('distname'))) self.targetDefs = platform.add_targets(option('target'), self._targets) def finalize_target_steps(self): diff --git a/kiwixbuild/dependencies/__init__.py b/kiwixbuild/dependencies/__init__.py index b457188..5850450 100644 --- a/kiwixbuild/dependencies/__init__.py +++ b/kiwixbuild/dependencies/__init__.py @@ -9,6 +9,7 @@ from . import ( gradle, gumbo, icu4c, + ios_fat_lib, kiwix_android, kiwix_custom_app, kiwix_lib, diff --git a/kiwixbuild/dependencies/ios_fat_lib.py b/kiwixbuild/dependencies/ios_fat_lib.py new file mode 100644 index 0000000..fddf107 --- /dev/null +++ b/kiwixbuild/dependencies/ios_fat_lib.py @@ -0,0 +1,52 @@ +import os + +from kiwixbuild.platforms import PlatformInfo +from kiwixbuild.utils import pj, copy_tree, run_command +from kiwixbuild._global import option +from .base import ( + Dependency, + NoopSource, + Builder as BaseBuilder) + + + +class IOSFatLib(Dependency): + name = "_ios_fat_lib" + + Source = NoopSource + + class Builder(BaseBuilder): + + @classmethod + def get_dependencies(self, platfomInfo, alldeps): + base_target = option('target') + return [('iOS_{}'.format(arch), base_target) for arch in option('ios_arch')] + + def _copy_headers(self, context): + plt = PlatformInfo.get_platform('iOS_{}'.format(option('ios_arch')[0])) + include_src = pj(plt.buildEnv.install_dir, 'include') + include_dst = pj(self.buildEnv.install_dir, 'include') + copy_tree(include_src, include_dst) + + def _merge_libs(self, context): + lib_dirs = [] + for arch in option('ios_arch'): + plt = PlatformInfo.get_platform('iOS_{}'.format(arch)) + lib_dirs.append(pj(plt.buildEnv.install_dir, 'lib')) + libs = [] + for f in os.listdir(lib_dirs[0]): + if os.path.islink(pj(lib_dirs[0], f)): + continue + if f.endswith('.a') or f.endswith('.dylib'): + libs.append(f) + os.makedirs(pj(self.buildEnv.install_dir, 'lib'), exist_ok=True) + command_tmp = "lipo -create {input} -output {output}" + for l in libs: + command = command_tmp.format( + input=" ".join(pj(d, l) for d in lib_dirs), + output=pj(self.buildEnv.install_dir, 'lib', l)) + run_command(command, self.buildEnv.install_dir, context) + + def build(self): + self.command('copy_headers', self._copy_headers) + self.command('merge_libs', self._merge_libs) diff --git a/kiwixbuild/platforms/base.py b/kiwixbuild/platforms/base.py index 44af29b..71edc13 100644 --- a/kiwixbuild/platforms/base.py +++ b/kiwixbuild/platforms/base.py @@ -32,16 +32,10 @@ class PlatformInfo(metaclass=_MetaPlatform): print("Should not got there.") print(cls.all_running_platforms) raise KeyError(name) - sys.exit(-1) cls.all_running_platforms[name] = cls.all_platforms[name](targets) return cls.all_running_platforms[name] def __init__(self, targets): - if neutralEnv('distname') not in self.compatible_hosts: - print(('ERROR: The target platform {} cannot be build on host {}.\n' - 'Select another target platform, or change your host system.' - ).format(self.name, neutralEnv('distname'))) - sys.exit(-1) self.all_running_platforms[self.name] = self self.buildEnv = BuildEnv(self) self.setup_toolchains(targets) diff --git a/kiwixbuild/platforms/ios.py b/kiwixbuild/platforms/ios.py index 499a8b3..1459741 100644 --- a/kiwixbuild/platforms/ios.py +++ b/kiwixbuild/platforms/ios.py @@ -1,9 +1,9 @@ import subprocess -from .base import PlatformInfo +from .base import PlatformInfo, MetaPlatformInfo from kiwixbuild.utils import pj, xrun_find - +from kiwixbuild._global import option class iOSPlatformInfo(PlatformInfo): build = 'iOS' @@ -84,17 +84,32 @@ class iOSArmv7(iOSPlatformInfo): class iOSArm64(iOSPlatformInfo): name = 'iOS_arm64' arch = cpu = 'arm64' - arch_full = 'arm-apple-darwin' + arch_full = 'aarch64-apple-darwin' sdk_name = 'iphoneos' class iOSi386(iOSPlatformInfo): name = 'iOS_i386' arch = cpu = 'i386' - arch_full = '' + arch_full = 'i386-apple-darwin' sdk_name = 'iphonesimulator' class iOSx64(iOSPlatformInfo): name = 'iOS_x86_64' arch = cpu = 'x86_64' - arch_full = '' + arch_full = 'x86_64-apple-darwin' sdk_name = 'iphonesimulator' + +class IOS(MetaPlatformInfo): + name = "iOS_multi" + compatible_hosts = ['Darwin'] + + @property + def subPlatformNames(self): + return ['iOS_{}'.format(arch) for arch in option('ios_arch')] + + def add_targets(self, targetName, targets): + super().add_targets(targetName, targets) + return PlatformInfo.add_targets(self, '_ios_fat_lib', targets) + + def __str__(self): + return self.name From daefadecc948eb5591dbadd8e661f8d216e7214b Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 6 Jun 2018 18:08:35 +0200 Subject: [PATCH 22/23] Better gradle option. Enable `--no-daemon` : Using the daemon make gradle keep "configuration" (like plugin path). If gradle installation path change, it will break kiwix-android build. Enable `--build-cache` : This is not the case by default :/ This should greatly improve compilation speed. --- kiwixbuild/dependencies/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index 645aab9..13d6de5 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -427,7 +427,7 @@ class MesonBuilder(Builder): class GradleBuilder(Builder): gradle_target = "build" - gradle_option = "-i" + gradle_option = "-i --no-daemon --build-cache" def build(self): self.command('configure', self._configure) From 4dea44d3055662d9c71219c289dec9ce23e1ab6d Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 31 May 2018 19:01:33 +0200 Subject: [PATCH 23/23] Fix travis CI --- .travis.yml | 2 ++ kiwixbuild/dependencies/all_dependencies.py | 10 ++++--- kiwixbuild/versions.py | 2 +- travis/compile_all.py | 33 +++++++++++++-------- 4 files changed, 29 insertions(+), 18 deletions(-) diff --git a/.travis.yml b/.travis.yml index 51c3c86..c3f6c76 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,6 +54,7 @@ env: - PLATFORM="armhf_static" - PLATFORM="i586_dyn" - PLATFORM="i586_static" + - PLATFORM="android" - PLATFORM="android_arm" - PLATFORM="android_arm64" - PLATFORM="android_x86" @@ -66,6 +67,7 @@ matrix: - env: PLATFORM="android_mips64" - env: PLATFORM="android_x86" - env: PLATFORM="android_x86_64" + - env: PLATFORM="android" include: - env: PLATFORM="android_mips" if: type!=cron AND type!=pull_request diff --git a/kiwixbuild/dependencies/all_dependencies.py b/kiwixbuild/dependencies/all_dependencies.py index eb91cab..22883eb 100644 --- a/kiwixbuild/dependencies/all_dependencies.py +++ b/kiwixbuild/dependencies/all_dependencies.py @@ -12,11 +12,13 @@ class AllBaseDependencies(Dependency): class Builder(NoopBuilder): @classmethod def get_dependencies(cls, platformInfo, allDeps): - base_deps = ['zlib', 'lzma', 'xapian-core', 'gumbo', 'pugixml', 'libmicrohttpd', 'libaria2', 'icu4c'] - if platformInfo.build != 'win32': - base_deps += ["libmagic"] + base_deps = ['zlib', 'lzma', 'xapian-core', 'pugixml', 'libaria2', 'icu4c'] + # zimwriterfs + if platformInfo.build not in ('android', 'win32'): + base_deps += ['libmagic', 'gumbo'] + # kiwix-tools if (platformInfo.build != 'android' and neutralEnv('distname') != 'Darwin'): - base_deps += ['ctpp2c', 'ctpp2'] + base_deps += ['libmicrohttpd', 'ctpp2c', 'ctpp2'] return base_deps diff --git a/kiwixbuild/versions.py b/kiwixbuild/versions.py index 4f9fddd..d018329 100644 --- a/kiwixbuild/versions.py +++ b/kiwixbuild/versions.py @@ -10,7 +10,7 @@ main_project_versions = { # This is the "version" of the whole base_deps_versions dict. # Change this when you change base_deps_versions. -base_deps_meta_version = '4' +base_deps_meta_version = '5' base_deps_versions = { diff --git a/travis/compile_all.py b/travis/compile_all.py index 265d446..d4716ad 100755 --- a/travis/compile_all.py +++ b/travis/compile_all.py @@ -66,8 +66,14 @@ def run_kiwix_build(target, platform, build_deps_only=False, make_release=False, command = ['kiwix-build'] command.append(target) command.append('--hide-progress') - command.append('--force-install-packages') - command.extend(['--target-platform', platform]) + if target == 'kiwix-android' and platform.startswith('android_'): + command.extend(['--target-platform', 'android', '--android-arch', platform[8:]]) + elif platform == 'android': + command.extend(['--target-platform', 'android']) + for arch in ('arm', 'arm64', 'x86', 'x86_64'): + command.extend(['--android-arch', arch]) + else: + command.extend(['--target-platform', platform]) if build_deps_only: command.append('--build-deps-only') if make_release: @@ -121,12 +127,11 @@ def make_archive(project, platform): def make_deps_archive(target, full=False): - (BASE_DIR/'.install_packages_ok').unlink() - archive_name = "deps_{}_{}_{}.tar.gz".format( TRAVIS_OS_NAME, PLATFORM, target) files_to_archive = [BASE_DIR/'INSTALL'] - files_to_archive += BASE_DIR.glob('**/android-ndk*') + files_to_archive += HOME.glob('BUILD_*/android-ndk*') + files_to_archive += HOME.glob('BUILD_*/android-sdk*') if (BASE_DIR/'meson_cross_file.txt').exists(): files_to_archive.append(BASE_DIR/'meson_cross_file.txt') @@ -138,17 +143,19 @@ def make_deps_archive(target, full=False): if full: files_to_archive += ARCHIVE_DIR.glob(".*_ok") files_to_archive += BASE_DIR.glob('*/.*_ok') + files_to_archive += (HOME/"BUILD_native_dyn").glob('*/.*_ok') + files_to_archive += (HOME/"BUILD_native_static").glob('*/.*_ok') + files_to_archive += HOME.glob('BUILD_android*/.*_ok') files_to_archive += SOURCE_DIR.glob('*/.*_ok') files_to_archive += [SOURCE_DIR/'pugixml-{}'.format( base_deps_versions['pugixml'])] - files_to_archive += [BASE_DIR/'pugixml-{}'.format( - base_deps_versions['pugixml'])] - if (TOOLCHAINS_DIR).exists(): - files_to_archive.append(TOOLCHAINS_DIR) + files_to_archive += HOME.glob('BUILD_*/pugixml-{}'.format( + base_deps_versions['pugixml'])) + files_to_archive += HOME.glob('**/TOOLCHAINS') relative_path = HOME with tarfile.open(str(relative_path/archive_name), 'w:gz') as tar: - for name in files_to_archive: + for name in set(files_to_archive): tar.add(str(name), arcname=str(name.relative_to(relative_path))) return relative_path/archive_name @@ -281,10 +288,10 @@ elif PLATFORM == 'armhf_static': make_archive('kiwix-tools', 'linux-armhf') elif PLATFORM == 'i586_static': make_archive('kiwix-tools', 'linux-i586') -elif PLATFORM.startswith('android_') and 'kiwix-android' in TARGETS: +elif PLATFORM.startswith('android') and 'kiwix-android' in TARGETS: APK_NAME = "kiwix-{}".format(PLATFORM) - source_debug_dir = BASE_DIR/'kiwix-android'/'app'/'build'/'outputs'/'apk'/'kiwix'/'debug' - source_release_dir = BASE_DIR/'kiwix-android'/'app'/'build'/'outputs'/'apk'/'kiwix'/'release' + source_debug_dir = HOME/'BUILD_android'/'kiwix-android'/'app'/'build'/'outputs'/'apk'/'kiwix'/'debug' + source_release_dir = HOME/'BUILD_android'/'kiwix-android'/'app'/'build'/'outputs'/'apk'/'kiwix'/'release' shutil.copy(str(source_debug_dir/'app-kiwix-debug.apk'), str(NIGHTLY_KIWIX_ARCHIVES_DIR/"{}-debug.apk".format(APK_NAME))) shutil.copy(str(source_release_dir/'app-kiwix-release-unsigned.apk'),