From 4d6b6994bc4943bd263bc9f74e955cc20815ff15 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 29 May 2018 11:40:42 +0200 Subject: [PATCH] Merge toolchain and dependencies. Toolchain now became "simple" dependencies. They are referenced by Platform. Platform are now created at demande and a neutralPlatform now exists. --- kiwixbuild/_global.py | 14 ----- kiwixbuild/buildenv.py | 3 +- kiwixbuild/builder.py | 63 ++++++++----------- kiwixbuild/dependencies/__init__.py | 3 + .../android_ndk.py | 16 +---- .../android_sdk.py | 7 +-- kiwixbuild/dependencies/armhf.py | 11 ++++ kiwixbuild/dependencies/base.py | 2 +- kiwixbuild/platforms/__init__.py | 1 + kiwixbuild/platforms/android.py | 6 +- kiwixbuild/platforms/armhf.py | 4 +- kiwixbuild/platforms/base.py | 35 ++++++----- kiwixbuild/platforms/neutral.py | 9 +++ kiwixbuild/toolchains/__init__.py | 4 -- kiwixbuild/toolchains/armhf.py | 14 ----- kiwixbuild/toolchains/base_toolchain.py | 63 ------------------- kiwixbuild/versions.py | 6 +- 17 files changed, 87 insertions(+), 174 deletions(-) rename kiwixbuild/{toolchains => dependencies}/android_ndk.py (86%) rename kiwixbuild/{toolchains => dependencies}/android_sdk.py (92%) create mode 100644 kiwixbuild/dependencies/armhf.py create mode 100644 kiwixbuild/platforms/neutral.py delete mode 100644 kiwixbuild/toolchains/__init__.py delete mode 100644 kiwixbuild/toolchains/armhf.py delete mode 100644 kiwixbuild/toolchains/base_toolchain.py diff --git a/kiwixbuild/_global.py b/kiwixbuild/_global.py index 42e5285..f513eda 100644 --- a/kiwixbuild/_global.py +++ b/kiwixbuild/_global.py @@ -3,7 +3,6 @@ from collections import OrderedDict as _OrderedDict _neutralEnv = None _options = None _target_steps = _OrderedDict() -_plt_steps = _OrderedDict() def set_neutralEnv(env): global _neutralEnv @@ -31,16 +30,3 @@ def get_target_step(key, default_context=None): def target_steps(): return _target_steps - -def add_plt_step(key, what): - _plt_steps[key] = what - -def get_plt_step(key, default_context=None): - try: - context, target = key - except ValueError: - context, target = default_context, key - return _plt_steps[(context, target)] - -def plt_steps(): - return _plt_steps diff --git a/kiwixbuild/buildenv.py b/kiwixbuild/buildenv.py index 186affd..51fc163 100644 --- a/kiwixbuild/buildenv.py +++ b/kiwixbuild/buildenv.py @@ -3,7 +3,6 @@ import os, sys, shutil import subprocess import platform -from .toolchains import Toolchain from .utils import pj, download_remote, Defaultdict from ._global import neutralEnv, option @@ -77,9 +76,11 @@ class BuildEnv: self.platformInfo = platformInfo self.build_dir = pj(option('working_dir'), build_dir) self.install_dir = pj(self.build_dir, "INSTALL") + self.toolchain_dir = pj(self.build_dir, "TOOLCHAINS") self.log_dir = pj(self.build_dir, 'LOGS') for d in (self.build_dir, self.install_dir, + self.toolchain_dir, self.log_dir): os.makedirs(d, exist_ok=True) diff --git a/kiwixbuild/builder.py b/kiwixbuild/builder.py index dccf8d4..4c279e8 100644 --- a/kiwixbuild/builder.py +++ b/kiwixbuild/builder.py @@ -8,8 +8,7 @@ from .utils import remove_duplicates, StopBuild from .dependencies import Dependency from ._global import ( neutralEnv, option, - add_target_step, get_target_step, target_steps, - add_plt_step, get_plt_step, plt_steps) + add_target_step, get_target_step, target_steps) from . import _global class Builder: @@ -20,6 +19,7 @@ class Builder: dependencies = self.order_dependencies(_targets, targetDef) dependencies = list(remove_duplicates(dependencies)) + PlatformInfo.get_platform('neutral', _targets) if option('build_nodeps'): add_target_step(targetDef, _targets[targetDef]) else: @@ -27,12 +27,13 @@ class Builder: if option('build_deps_only') and dep == targetDef: continue add_target_step(dep, _targets[dep]) + self.instanciate_steps() def add_targets(self, targetDef, targets): if targetDef in targets: return targetPlatformName, targetName = targetDef - targetPlatform = PlatformInfo.get_platform(targetPlatformName) + targetPlatform = PlatformInfo.get_platform(targetPlatformName, targets) targetClass = Dependency.all_deps[targetName] targets[('source', targetName)] = targetClass.Source targets[targetDef] = targetClass.Builder @@ -44,6 +45,12 @@ class Builder: self.add_targets((depPlatform, depName), targets) def order_dependencies(self, _targets, targetDef): + for pltName in PlatformInfo.all_running_platforms: + plt = PlatformInfo.all_platforms[pltName] + for tlcName in plt.toolchain_names: + tlc = Dependency.all_deps[tlcName] + yield('source', tlcName) + yield('neutral' if tlc.neutral else pltName, tlcName) targetPlatformName, targetName = targetDef if targetPlatformName == 'source': # Do not try to order sources, they will be added as dep by the @@ -60,14 +67,18 @@ class Builder: yield from self.order_dependencies(_targets, (depPlatform, depName)) yield targetDef - def prepare_toolchain_sources(self): - tlsourceDefs = (tlDef for tlDef in plt_steps() if tlDef[0]=='source') - for tlsourceDef in tlsourceDefs: - print("prepare sources for toolchain {} :".format(tlsourceDef[1])) - toolchainClass = Toolchain.all_toolchains[tlsourceDef[1]] - source = get_plt_step(tlsourceDef)(toolchainClass) - add_plt_step(tlsourceDef, source) - source.prepare() + def instanciate_steps(self): + for stepDef in list(target_steps()): + stepPlatform, stepName = stepDef + stepClass = Dependency.all_deps[stepName] + if stepPlatform == 'source': + source = get_target_step(stepDef)(stepClass) + add_target_step(stepDef, source) + else: + source = get_target_step(stepName, 'source') + env = PlatformInfo.get_platform(stepPlatform).buildEnv + builder = get_target_step(stepDef)(stepClass, source, env) + add_target_step(stepDef, builder) def prepare_sources(self): if option('skip_source_prepare'): @@ -76,36 +87,16 @@ class Builder: sourceDefs = remove_duplicates(tDef for tDef in target_steps() if tDef[0]=='source') for sourceDef in sourceDefs: - print("prepare sources {} :".format(sourceDef[1])) - depClass = Dependency.all_deps[sourceDef[1]] - source = get_target_step(sourceDef)(depClass) - add_target_step(sourceDef, source) + source = get_target_step(sourceDef) source.prepare() - def build_toolchains(self): - tlbuilderDefs = (tlDef for tlDef in plt_steps() if tlDef[0] != 'source') - for tlbuilderDef in tlbuilderDefs: - print("build toolchain {} :".format(tlbuilderDef[1])) - toolchainClass = Toolchain.all_toolchains[tlbuilderDef[1]] - source = get_plt_step(tlbuilderDef[1], 'source') - if tlbuilderDef[0] == 'neutral': - env = _global._neutralEnv - else: - env = PlatformInfo.get_platform(tlbuilderDef[0]).buildEnv - builder = get_plt_step(tlbuilderDef)(toolchainClass, source, env) - add_plt_step(tlbuilderDef, builder) - builder.build() - def build(self): builderDefs = (tDef for tDef in target_steps() if tDef[0] != 'source') for builderDef in builderDefs: - depClass = Dependency.all_deps[builderDef[1]] - source = get_target_step(builderDef[1], 'source') - env = PlatformInfo.get_platform(builderDef[0]).buildEnv - builder = get_target_step(builderDef)(depClass, source, env) - if option('make_dist') and builderDef[1] == option('targets'): - print("make dist {}:".format(builder.name)) + builder = get_target_step(builderDef) + if option('make_dist') and builderName == option('targets'): + print("make dist {} ({}):".format(builder.name, builderDef[0])) builder.make_dist() continue print("build {} ({}):".format(builder.name, builderDef[0])) @@ -116,8 +107,6 @@ class Builder: def run(self): try: print("[SETUP PLATFORMS]") - self.prepare_toolchain_sources() - self.build_toolchains() for platform in PlatformInfo.all_running_platforms.values(): platform.finalize_setup() print("[INSTALL PACKAGES]") diff --git a/kiwixbuild/dependencies/__init__.py b/kiwixbuild/dependencies/__init__.py index 8007287..b457188 100644 --- a/kiwixbuild/dependencies/__init__.py +++ b/kiwixbuild/dependencies/__init__.py @@ -2,6 +2,9 @@ from .base import * from . import ( all_dependencies, + android_ndk, + android_sdk, + armhf, ctpp2, gradle, gumbo, diff --git a/kiwixbuild/toolchains/android_ndk.py b/kiwixbuild/dependencies/android_ndk.py similarity index 86% rename from kiwixbuild/toolchains/android_ndk.py rename to kiwixbuild/dependencies/android_ndk.py index 9be7d5f..fa2b00e 100644 --- a/kiwixbuild/toolchains/android_ndk.py +++ b/kiwixbuild/dependencies/android_ndk.py @@ -1,23 +1,15 @@ import os -from .base_toolchain import Toolchain -from kiwixbuild.dependencies import ReleaseDownload, Builder +from .base import Dependency, ReleaseDownload, Builder from kiwixbuild.utils import Remotefile, add_execution_right, run_command pj = os.path.join -class android_ndk(Toolchain): +class android_ndk(Dependency): neutral = False name = 'android-ndk' - version = 'r13b' gccver = '4.9.x' - @classmethod - def full_name(cls): - return "{name}-{version}".format( - name = cls.name, - version = cls.version) - class Source(ReleaseDownload): archive = Remotefile('android-ndk-r13b-linux-x86_64.zip', '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c', @@ -25,9 +17,7 @@ class android_ndk(Toolchain): @property def source_dir(self): - return "{}-{}".format( - self.target.name, - self.target.version) + return self.target.full_name() class Builder(Builder): diff --git a/kiwixbuild/toolchains/android_sdk.py b/kiwixbuild/dependencies/android_sdk.py similarity index 92% rename from kiwixbuild/toolchains/android_sdk.py rename to kiwixbuild/dependencies/android_sdk.py index c58e05f..d09f433 100644 --- a/kiwixbuild/toolchains/android_sdk.py +++ b/kiwixbuild/dependencies/android_sdk.py @@ -1,15 +1,14 @@ import os import shutil -from .base_toolchain import Toolchain -from kiwixbuild.dependencies import ReleaseDownload, Builder +from .base import Dependency, ReleaseDownload, Builder from kiwixbuild.utils import Remotefile, run_command pj = os.path.join -class android_sdk(Toolchain): +class android_sdk(Dependency): + neutral = True name = 'android-sdk' - version = 'r25.2.3' class Source(ReleaseDownload): archive = Remotefile('tools_r25.2.3-linux.zip', diff --git a/kiwixbuild/dependencies/armhf.py b/kiwixbuild/dependencies/armhf.py new file mode 100644 index 0000000..4ba5200 --- /dev/null +++ b/kiwixbuild/dependencies/armhf.py @@ -0,0 +1,11 @@ +from .base import Dependency, GitClone, NoopBuilder + +class armhf_toolchain(Dependency): + neutral = True + name = 'armhf' + + class Source(GitClone): + git_remote = "https://github.com/raspberrypi/tools" + git_dir = "raspberrypi-tools" + + Builder = NoopBuilder diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index 60b16c8..940a669 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -350,7 +350,7 @@ class CMakeBuilder(MakeBuilder): " {source_path}" " {cross_option}") command = command.format( - configure_option="{} {}".format(self.buildEnv.cmake_option, self.configure_option), + configure_option=self.configure_option, install_dir=self.buildEnv.install_dir, libdir=self.buildEnv.libprefix, source_path=self.source_path, diff --git a/kiwixbuild/platforms/__init__.py b/kiwixbuild/platforms/__init__.py index 5975eca..4b63719 100644 --- a/kiwixbuild/platforms/__init__.py +++ b/kiwixbuild/platforms/__init__.py @@ -7,5 +7,6 @@ from . import ( i586, ios, native, + neutral, win32 ) diff --git a/kiwixbuild/platforms/android.py b/kiwixbuild/platforms/android.py index 4187a19..1fdc46e 100644 --- a/kiwixbuild/platforms/android.py +++ b/kiwixbuild/platforms/android.py @@ -1,6 +1,6 @@ from .base import PlatformInfo from kiwixbuild.utils import pj -from kiwixbuild._global import get_plt_step +from kiwixbuild._global import get_target_step class AndroidPlatformInfo(PlatformInfo): @@ -27,11 +27,11 @@ class AndroidPlatformInfo(PlatformInfo): @property def ndk_builder(self): - return get_plt_step('android_ndk', self.name) + return get_target_step('android-ndk', self.name) @property def sdk_builder(self): - return get_plt_step('android_sdk', 'neutral') + return get_target_step('android-sdk', 'neutral') def get_cross_config(self): install_path = self.ndk_builder.install_path diff --git a/kiwixbuild/platforms/armhf.py b/kiwixbuild/platforms/armhf.py index e3077d9..354af4b 100644 --- a/kiwixbuild/platforms/armhf.py +++ b/kiwixbuild/platforms/armhf.py @@ -1,7 +1,7 @@ from .base import PlatformInfo from kiwixbuild.utils import pj -from kiwixbuild._global import get_plt_step +from kiwixbuild._global import get_target_step class ArmhfPlatformInfo(PlatformInfo): @@ -29,7 +29,7 @@ class ArmhfPlatformInfo(PlatformInfo): @property def tlc_source(self): - return get_plt_step('armhf', 'source') + return get_target_step('armhf', 'source') @property def root_path(self): diff --git a/kiwixbuild/platforms/base.py b/kiwixbuild/platforms/base.py index 242ebf1..cfa7779 100644 --- a/kiwixbuild/platforms/base.py +++ b/kiwixbuild/platforms/base.py @@ -1,13 +1,12 @@ -import os +import os, sys import subprocess from kiwixbuild.dependencies import Dependency -from kiwixbuild.toolchains import Toolchain from kiwixbuild.packages import PACKAGE_NAME_MAPPERS from kiwixbuild.utils import pj, remove_duplicates from kiwixbuild.buildenv import BuildEnv -from kiwixbuild._global import neutralEnv, option, add_plt_step, target_steps +from kiwixbuild._global import neutralEnv, option, target_steps _SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) TEMPLATES_DIR = pj(os.path.dirname(_SCRIPT_DIR), 'templates') @@ -28,12 +27,17 @@ class PlatformInfo(metaclass=_MetaPlatform): configure_option = "" @classmethod - def get_platform(cls, name): + def get_platform(cls, name, targets=None): if name not in cls.all_running_platforms: - cls.all_running_platforms[name] = cls.all_platforms[name]() + if targets is None: + print("Should not got there.") + print(cls.all_running_platforms) + raise KeyError(name) + sys.exit(-1) + cls.all_running_platforms[name] = cls.all_platforms[name](targets) return cls.all_running_platforms[name] - def __init__(self): + def __init__(self, targets): if neutralEnv('distname') not in self.compatible_hosts: print(('ERROR: The target platform {} cannot be build on host {}.\n' 'Select another target platform, or change your host system.' @@ -41,19 +45,17 @@ class PlatformInfo(metaclass=_MetaPlatform): sys.exit(-1) self.all_running_platforms[self.name] = self self.buildEnv = BuildEnv(self) - self.setup_toolchains() + self.setup_toolchains(targets) def __str__(self): return "{}_{}".format(self.build, 'static' if self.static else 'dyn') - def setup_toolchains(self): + def setup_toolchains(self, targets): for tlc_name in self.toolchain_names: - ToolchainClass = Toolchain.all_toolchains[tlc_name] - if ToolchainClass.Source is not None: - add_plt_step(('source', tlc_name), ToolchainClass.Source) - if ToolchainClass.Builder is not None: - plt_name = 'neutral' if ToolchainClass.neutral else self.name - add_plt_step((plt_name, tlc_name), ToolchainClass.Builder) + ToolchainClass = Dependency.all_deps[tlc_name] + targets[('source', tlc_name)] = ToolchainClass.Source + plt_name = 'neutral' if ToolchainClass.neutral else self.name + targets[(plt_name, tlc_name)] = ToolchainClass.Builder def get_cross_config(self): return {} @@ -93,9 +95,10 @@ class PlatformInfo(metaclass=_MetaPlatform): except KeyError: print("SKIP : We don't know which packages we must install to compile" " a {target} {build_type} version on a {host} host.".format( - target=self.platform_info, + target=self, + build_type=self.static, host=neutralEnv('distname'))) - return + return [] packages_list = package_name_mapper.get('COMMON', []) to_drop = [] diff --git a/kiwixbuild/platforms/neutral.py b/kiwixbuild/platforms/neutral.py new file mode 100644 index 0000000..bbe339c --- /dev/null +++ b/kiwixbuild/platforms/neutral.py @@ -0,0 +1,9 @@ +from .base import PlatformInfo + +class NeutralPlatformInfo(PlatformInfo): + name = 'neutral' + static = '' + compatible_hosts = ['fedora', 'debian', 'Darwin'] + + def __str__(self): + return "neutral" diff --git a/kiwixbuild/toolchains/__init__.py b/kiwixbuild/toolchains/__init__.py deleted file mode 100644 index ebf0cff..0000000 --- a/kiwixbuild/toolchains/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ - -from . import android_ndk, android_sdk, armhf - -from .base_toolchain import Toolchain diff --git a/kiwixbuild/toolchains/armhf.py b/kiwixbuild/toolchains/armhf.py deleted file mode 100644 index d49aa62..0000000 --- a/kiwixbuild/toolchains/armhf.py +++ /dev/null @@ -1,14 +0,0 @@ -import os -import subprocess - -from .base_toolchain import Toolchain -from kiwixbuild.dependencies import GitClone -from kiwixbuild.utils import which -pj = os.path.join - -class armhf_toolchain(Toolchain): - name = 'armhf' - - class Source(GitClone): - git_remote = "https://github.com/raspberrypi/tools" - git_dir = "raspberrypi-tools" diff --git a/kiwixbuild/toolchains/base_toolchain.py b/kiwixbuild/toolchains/base_toolchain.py deleted file mode 100644 index c51de7a..0000000 --- a/kiwixbuild/toolchains/base_toolchain.py +++ /dev/null @@ -1,63 +0,0 @@ -import os -import subprocess - -pj = os.path.join - -from kiwixbuild.utils import Context, SkipCommand, StopBuild -from kiwixbuild._global import neutralEnv - -class _MetaToolchain(type): - def __new__(cls, name, bases, dct): - _class = type.__new__(cls, name, bases, dct) - if name != 'Toolchain': - Toolchain.all_toolchains[dct['name']] = _class - return _class - - -class Toolchain(metaclass=_MetaToolchain): - force_native_build = False - neutral = True - all_toolchains = {} - configure_option = "" - cmake_option = "" - exec_wrapper_def = "" - Builder = None - Source = None - - - @classmethod - def full_name(cls): - return "{name}-{version}".format( - name = cls.name, - version = cls.version) - - @property - def source_path(self): - return pj(neutralEnv('source_dir'), self.source.source_dir) - - @property - def _log_dir(self): - return neutralEnv('log_dir') - - def command(self, name, function, *args): - print(" {} {} : ".format(name, self.name), end="", flush=True) - log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name)) - context = Context(name, log, True) - try: - ret = function(*args, context=context) - context._finalise() - print("OK") - return ret - except SkipCommand: - print("SKIP") - except subprocess.CalledProcessError: - print("ERROR") - try: - with open(log, 'r') as f: - print(f.read()) - except: - pass - raise StopBuild() - except: - print("ERROR") - raise diff --git a/kiwixbuild/versions.py b/kiwixbuild/versions.py index e2253ec..f822d96 100644 --- a/kiwixbuild/versions.py +++ b/kiwixbuild/versions.py @@ -10,7 +10,7 @@ main_project_versions = { # This is the "version" of the whole base_deps_versions dict. # Change this when you change base_deps_versions. -base_deps_meta_version = '3' +base_deps_meta_version = '4' base_deps_versions = { @@ -25,5 +25,7 @@ base_deps_versions = { 'icu4c' : '58.2', 'Gradle' : '4.6', 'libaria2' : '1.33.1', - 'libmagic' : '5.33' + 'libmagic' : '5.33', + 'android-sdk' : 'r25.2.3', + 'android-ndk' : 'r13b' }