Move from target logic to steps logic.

This is the big change !!!!

Instead of handling target as primary object and prepare/build targets,
we are handling build steps.

A build step may be a source (preparation) or a build (of the source).
Actualy, a step is a tuple (context, Builder or Source).

The context define the context of the step. It can be :
- 'source', for a Source step
- 'neutral' or the name of a platform for Build step.

Target becomes a "Class only" class.

kiwixbuild/_global.py

@@ -1,4 +1,8 @@
+from collections import OrderedDict as _OrderedDict
+
 _neutralEnv = None
+_target_steps = _OrderedDict()
+_plt_steps = _OrderedDict()
 
 def set_neutralEnv(env):
     global _neutralEnv
@@ -6,3 +10,29 @@ def set_neutralEnv(env):
 
 def neutralEnv(what):
     return getattr(_neutralEnv, what)
+
+def add_target_step(key, what):
+    _target_steps[key] = what
+
+def get_target_step(key, default_context=None):
+    try:
+        context, target = key
+    except ValueError:
+        context, target = default_context, key
+    return _target_steps[(context, target)]
+
+def target_steps():
+    return _target_steps
+
+def add_plt_step(key, what):
+    _plt_steps[key] = what
+
+def get_plt_step(key, default_context=None):
+    try:
+        context, target = key
+    except ValueError:
+        context, target = default_context, key
+    return _plt_steps[(context, target)]
+
+def plt_steps():
+    return _plt_steps

kiwixbuild/buildenv.py

@@ -76,17 +76,18 @@ class PlatformNeutralEnv:
 
 
 class BuildEnv:
-    def __init__(self, platformInfo, targetsDict):
+    def __init__(self, platformInfo):
         build_dir = "BUILD_{}".format(platformInfo.name)
         self.platformInfo = platformInfo
         self.build_dir = pj(neutralEnv('working_dir'), build_dir)
         self.install_dir = pj(self.build_dir, "INSTALL")
+        self.log_dir = pj(self.build_dir, 'LOGS')
         for d in (self.build_dir,
-                  self.install_dir):
+                  self.install_dir,
+                  self.log_dir):
             os.makedirs(d, exist_ok=True)
 
         self.libprefix = neutralEnv('libprefix') or self._detect_libdir()
-        self.targetsDict = targetsDict
 
     def clean_intermediate_directories(self):
         for subdir in os.listdir(self.build_dir):

kiwixbuild/builder.py

@@ -3,8 +3,14 @@ import sys
 from collections import OrderedDict
 from .buildenv import *
 
+from .platforms import PlatformInfo
 from .utils import remove_duplicates, StopBuild
 from .dependencies import Dependency
+from ._global import (
+    neutralEnv,
+    add_target_step, get_target_step, target_steps,
+    add_plt_step, get_plt_step, plt_steps)
+from . import _global
 
 class Builder:
     def __init__(self, options):
@@ -15,78 +21,115 @@ class Builder:
                    'Select another target platform, or change your host system.'
                   ).format(options.target_platform, self.distname))
             sys.exit(-1)
-        self.targets = OrderedDict()
+        self.platform = platform = platformClass()
-        self.platform = platform = platformClass(self.targets)
 
         _targets = {}
-        targetDef = options.targets
+        targetDef = (options.target_platform, options.targets)
         self.add_targets(targetDef, _targets)
         dependencies = self.order_dependencies(_targets, targetDef)
         dependencies = list(remove_duplicates(dependencies))
 
         if options.build_nodeps:
-            self.targets[targetDef] = _targets[targetDef]
+            add_target_step(targetDef, _targets[targetDef])
         else:
             for dep in dependencies:
                 if self.options.build_deps_only and dep == targetDef:
                     continue
-                self.targets[dep] = _targets[dep]
+                add_target_step(dep, _targets[dep])
 
-    def add_targets(self, targetName, targets):
+    def add_targets(self, targetDef, targets):
-        if targetName in targets:
+        if targetDef in targets:
             return
+        targetPlatformName, targetName = targetDef
+        targetPlatform = PlatformInfo.all_platforms[targetPlatformName]
         targetClass = Dependency.all_deps[targetName]
-        target = targetClass(self.platform.buildEnv)
+        targets[('source', targetName)] = targetClass.Source
-        targets[targetName] = target
+        targets[targetDef] = targetClass.Builder
-        for dep in target.builder.get_dependencies(self.platform):
+        for dep in targetClass.Builder.get_dependencies(targetPlatform):
-            self.add_targets(dep, targets)
+            try:
+                depPlatform, depName = dep
+            except ValueError:
+                depPlatform, depName = targetPlatformName, dep
+            self.add_targets((depPlatform, depName), targets)
 
-    def order_dependencies(self, _targets, targetName):
+    def order_dependencies(self, _targets, targetDef):
-        target = _targets[targetName]
+        targetPlatformName, targetName = targetDef
-        for depName in target.builder.dependencies(self.platform):
+        if targetPlatformName == 'source':
-            yield from self.order_dependencies(_targets, depName)
+            # Do not try to order sources, they will be added as dep by the
-        yield targetName
+            # build step two lines later.
+            return
+        target = _targets[targetDef]
+        targetPlatform = PlatformInfo.all_platforms[targetPlatformName]
+        yield ('source', targetName)
+        for dep in target.get_dependencies(targetPlatform):
+            try:
+                depPlatform, depName = dep
+            except ValueError:
+                depPlatform, depName = targetPlatformName, dep
+            yield from self.order_dependencies(_targets, (depPlatform, depName))
+        yield targetDef
+
+    def prepare_toolchain_sources(self):
+        tlsourceDefs = (tlDef for tlDef in plt_steps() if tlDef[0]=='source')
+        for tlsourceDef in tlsourceDefs:
+            print("prepare sources for toolchain {} :".format(tlsourceDef[1]))
+            toolchainClass = Toolchain.all_toolchains[tlsourceDef[1]]
+            source = get_plt_step(tlsourceDef)(toolchainClass)
+            add_plt_step(tlsourceDef, source)
+            source.prepare()
 
     def prepare_sources(self):
         if self.options.skip_source_prepare:
             print("SKIP")
             return
 
-        toolchain_sources = (tlc.source for tlc in self.platform.buildEnv.toolchains if tlc.source)
+        sourceDefs = remove_duplicates(tDef for tDef in target_steps() if tDef[0]=='source')
-        for toolchain_source in toolchain_sources:
+        for sourceDef in sourceDefs:
-            print("prepare sources for toolchain {} :".format(toolchain_source.name))
-            toolchain_source.prepare()
 
-        sources = (dep.source for dep in self.targets.values() if not dep.skip)
+            print("prepare sources {} :".format(sourceDef[1]))
-        sources = remove_duplicates(sources, lambda s: s.__class__)
+            depClass = Dependency.all_deps[sourceDef[1]]
-        for source in sources:
+            source = get_target_step(sourceDef)(depClass)
-            print("prepare sources {} :".format(source.name))
+            add_target_step(sourceDef, source)
             source.prepare()
 
-    def build(self):
+    def build_toolchains(self):
-        toolchain_builders = (tlc.builder for tlc in self.platform.buildEnv.toolchains if tlc.builder)
+        tlbuilderDefs = (tlDef for tlDef in plt_steps() if tlDef[0] != 'source')
-        for toolchain_builder in toolchain_builders:
+        for tlbuilderDef in tlbuilderDefs:
-            print("build toolchain {} :".format(toolchain_builder.name))
+            print("build toolchain {} :".format(tlbuilderDef[1]))
-            toolchain_builder.build()
+            toolchainClass = Toolchain.all_toolchains[tlbuilderDef[1]]
-
+            source = get_plt_step(tlbuilderDef[1], 'source')
-        builders = (dep.builder for dep in self.targets.values() if (dep.builder and not dep.skip))
+            if tlbuilderDef[0] == 'neutral':
-        for builder in builders:
+                env = _global._neutralEnv
-            if self.options.make_dist and builder.name == self.options.targets:
+            else:
-                continue
+                env = PlatformInfo.all_running_platforms[tlbuilderDef[0]].buildEnv
-            print("build {} :".format(builder.name))
+            builder = get_plt_step(tlbuilderDef)(toolchainClass, source, env)
+            add_plt_step(tlbuilderDef, builder)
+            builder.build()
+
+    def build(self):
+        builderDefs = (tDef for tDef in target_steps() if tDef[0] != 'source')
+        for builderDef in builderDefs:
+            depClass = Dependency.all_deps[builderDef[1]]
+            source = get_target_step(builderDef[1], 'source')
+            env = PlatformInfo.all_running_platforms[builderDef[0]].buildEnv
+            builder = get_target_step(builderDef)(depClass, source, env)
+            if self.options.make_dist and builderDef[1] == self.options.targets:
+                print("make dist {}:".format(builder.name))
+                builder.make_dist()
+                continue
+            print("build {} ({}):".format(builder.name, builderDef[0]))
+            add_target_step(builderDef, builder)
             builder.build()
 
-        if self.options.make_dist:
-            dep = self.targets[self.options.targets]
-            builder = dep.builder
-            print("make dist {}:".format(builder.name))
-            builder.make_dist()
 
     def run(self):
         try:
+            print("[SETUP PLATFORMS]")
+            self.prepare_toolchain_sources()
+            self.build_toolchains()
+            self.platform.finalize_setup()
             print("[INSTALL PACKAGES]")
             self.platform.install_packages()
-            self.platform.finalize_setup()
             print("[PREPARE]")
             self.prepare_sources()
             print("[BUILD]")

kiwixbuild/dependencies/base.py

@@ -22,34 +22,51 @@ class Dependency(metaclass=_MetaDependency):
     all_deps = {}
     force_native_build = False
 
-    def __init__(self, buildEnv):
+    @classmethod
-        self.buildEnv = buildEnv
+    def version(cls):
-        self.source = self.Source(self)
+        return base_deps_versions.get(cls.name, None)
-        self.builder = self.Builder(self)
+
-        self.skip = False
+    @classmethod
+    def full_name(cls):
+        if cls.version():
+            return "{}-{}".format(cls.name, cls.version())
+        return cls.name
+
+
+class Source:
+    """Base Class to the real preparator
+       A source preparator must install source in the self.source_dir attribute
+       inside the neutralEnv.source_dir."""
+    def __init__(self, target):
+        self.target = target
 
     @property
-    def version(self):
+    def name(self):
-        return base_deps_versions.get(self.name, None)
+        return self.target.name
 
     @property
-    def full_name(self):
+    def source_dir(self):
-        if self.version:
+        return self.target.full_name()
-            return "{}-{}".format(self.name, self.version)
-        return self.name
 
     @property
     def source_path(self):
-        return pj(neutralEnv('source_dir'), self.source.source_dir)
+        return pj(neutralEnv('source_dir'), self.source_dir)
 
     @property
     def _log_dir(self):
-        return self.buildEnv.log_dir
+        return neutralEnv('log_dir')
+
+    def _patch(self, context):
+        context.try_skip(self.source_path)
+        context.force_native_build = True
+        for p in self.patches:
+            with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input:
+                run_command("patch -p1", self.source_path, context, input=patch_input.read())
 
     def command(self, name, function, *args):
         print("  {} {} : ".format(name, self.name), end="", flush=True)
         log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
-        context = Context(name, log, self.force_native_build)
+        context = Context(name, log, True)
         try:
             ret = function(*args, context=context)
             context._finalise()
@@ -70,33 +87,6 @@ class Dependency(metaclass=_MetaDependency):
             raise
 
 
-class Source:
-    """Base Class to the real preparator
-       A source preparator must install source in the self.source_dir attribute
-       inside the neutralEnv.source_dir."""
-    def __init__(self, target):
-        self.target = target
-
-    @property
-    def name(self):
-        return self.target.name
-
-    @property
-    def source_dir(self):
-        return self.target.full_name
-
-    def _patch(self, context):
-        source_path = pj(neutralEnv('source_dir'), self.source_dir)
-        context.try_skip(source_path)
-        context.force_native_build = True
-        for p in self.patches:
-            with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input:
-                neutralEnv('run_command')("patch -p1", source_path, context, input=patch_input.read())
-
-    def command(self, *args, **kwargs):
-        return self.target.command(*args, **kwargs)
-
-
 class NoopSource(Source):
     def prepare(self):
         pass
@@ -206,9 +196,10 @@ class Builder:
     subsource_dir = None
     dependencies = []
 
-    def __init__(self, target):
+    def __init__(self, target, source, buildEnv):
         self.target = target
-        self.buildEnv = target.buildEnv
+        self.source = source
+        self.buildEnv = buildEnv
 
     @classmethod
     def get_dependencies(cls, platformInfo):
@@ -220,17 +211,41 @@ class Builder:
 
     @property
     def source_path(self):
-        base_source_path = self.target.source_path
+        base_source_path = self.source.source_path
         if self.subsource_dir:
             return pj(base_source_path, self.subsource_dir)
         return base_source_path
 
     @property
     def build_path(self):
-        return pj(self.buildEnv.build_dir, self.target.full_name)
+        return pj(self.buildEnv.build_dir, self.target.full_name())
 
-    def command(self, *args, **kwargs):
+    @property
-        return self.target.command(*args, **kwargs)
+    def _log_dir(self):
+        return self.buildEnv.log_dir
+
+    def command(self, name, function, *args):
+        print("  {} {} : ".format(name, self.name), end="", flush=True)
+        log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
+        context = Context(name, log, self.target.force_native_build)
+        try:
+            ret = function(*args, context=context)
+            context._finalise()
+            print("OK")
+            return ret
+        except SkipCommand:
+            print("SKIP")
+        except subprocess.CalledProcessError:
+            print("ERROR")
+            try:
+                with open(log, 'r') as f:
+                    print(f.read())
+            except:
+                pass
+            raise StopBuild()
+        except:
+            print("ERROR")
+            raise
 
     def build(self):
         if hasattr(self, '_pre_build_script'):
@@ -274,8 +289,8 @@ class MakeBuilder(Builder):
     def all_configure_option(self):
         option = self.configure_option_template.format(
             dep_options=self.configure_option,
-            static_option=self.static_configure_option if self.buildEnv.platform_info.static else self.dynamic_configure_option,
+            static_option=self.static_configure_option if self.buildEnv.platformInfo.static else self.dynamic_configure_option,
-            env_option=self.buildEnv.configure_option if not self.target.force_native_build else "",
+            env_option=self.buildEnv.platformInfo.configure_option if not self.target.force_native_build else "",
             install_dir=self.buildEnv.install_dir,
             libdir=pj(self.buildEnv.install_dir, self.buildEnv.libprefix)
             )
@@ -289,7 +304,7 @@ class MakeBuilder(Builder):
             configure_option=self.all_configure_option
         )
         env = Defaultdict(str, os.environ)
-        if self.buildEnv.platform_info.static:
+        if self.buildEnv.platformInfo.static:
             env['CFLAGS'] = env['CFLAGS'] + ' -fPIC'
         if self.configure_env:
             for k in self.configure_env:
@@ -342,7 +357,7 @@ class CMakeBuilder(MakeBuilder):
             cross_option=cross_option
         )
         env = Defaultdict(str, os.environ)
-        if self.buildEnv.platform_info.static:
+        if self.buildEnv.platformInfo.static:
             env['CFLAGS'] = env['CFLAGS'] + ' -fPIC'
         if self.configure_env:
             for k in self.configure_env:
@@ -360,7 +375,7 @@ class MesonBuilder(Builder):
 
     @property
     def library_type(self):
-        return 'static' if self.buildEnv.platform_info.static else 'shared'
+        return 'static' if self.buildEnv.platformInfo.static else 'shared'
 
     def _configure(self, context):
         context.try_skip(self.build_path)
@@ -393,9 +408,9 @@ class MesonBuilder(Builder):
         run_command(command, self.build_path, context, buildEnv=self.buildEnv)
 
     def _test(self, context):
-        if ( self.buildEnv.platform_info.build == 'android'
+        if ( self.buildEnv.platformInfo.build == 'android'
-             or (self.buildEnv.platform_info.build != 'native'
+             or (self.buildEnv.platformInfo.build != 'native'
-                 and not self.buildEnv.platform_info.static)
+                 and not self.buildEnv.platformInfo.static)
            ):
             raise SkipCommand()
         command = "{} --verbose {}".format(neutralEnv('mesontest_command'), self.test_option)

kiwixbuild/dependencies/icu4c.py

@@ -5,7 +5,7 @@ from .base import (
 )
 
 from kiwixbuild.utils import SkipCommand
-
+from kiwixbuild._global import get_target_step
 
 class Icu(Dependency):
     name = "icu4c"
@@ -54,5 +54,7 @@ class Icu_cross_compile(Icu):
 
         @property
         def configure_option(self):
-            icu_native_dep = self.buildEnv.targetsDict['icu4c_native']
+            icu_native_builder = get_target_step('icu4c_native', self.buildEnv.platformInfo.name)
-            return super().configure_option + " --with-cross-build={} --disable-tools".format(icu_native_dep.builder.build_path)
+            return (super().configure_option
+                  + " --with-cross-build={} --disable-tools"
+                  ).format(icu_native_builder.build_path)

kiwixbuild/dependencies/kiwix_custom_app.py

@@ -7,6 +7,7 @@ from .base import (
     GradleBuilder)
 
 from kiwixbuild.utils import Remotefile, pj, SkipCommand
+from kiwixbuild._global import get_target_step
 
 class KiwixCustomApp(Dependency):
     name = "kiwix-android-custom"
@@ -78,9 +79,9 @@ class KiwixCustomApp(Dependency):
 
         def _configure(self, context):
             # Copy kiwix-android in build dir.
-            kiwix_android_dep = self.buildEnv.targetsDict['kiwix-android']
+            kiwix_android_source = get_target_step('kiwix-android', 'source')
             if not os.path.exists(self.build_path):
-                shutil.copytree(kiwix_android_dep.source_path, self.build_path)
+                shutil.copytree(kiwix_android_source.source_path, self.build_path)
 
             # Copy kiwix-lib application in build dir
             try:

kiwixbuild/dependencies/libmagic.py

@@ -7,6 +7,7 @@ from .base import (
 )
 
 from kiwixbuild.utils import Remotefile, pj, Defaultdict, SkipCommand
+from kiwixbuild._global import get_target_step
 
 class LibMagicBase(Dependency):
     name = "libmagic"
@@ -48,7 +49,7 @@ class LibMagic_cross_compile(LibMagicBase):
                 make_target=self.make_target,
                 make_option=self.make_option
             )
-            libmagic_native_dep = self.buildEnv.targetsDict['libmagic_native']
+            libmagic_native_builder = get_target_step('libmagic_native', self.buildEnv.platformInfo.name)
             env = Defaultdict(str, os.environ)
-            env['PATH'] = ':'.join([pj(libmagic_native_dep.builder.build_path, 'src'), env['PATH']])
+            env['PATH'] = ':'.join([pj(libmagic_native_builder.build_path, 'src'), env['PATH']])
             self.buildEnv.run_command(command, self.build_path, context, env=env)

kiwixbuild/platforms/android.py

@@ -1,21 +1,13 @@
 from .base import PlatformInfo
+from kiwixbuild.utils import pj
+from kiwixbuild._global import get_plt_step
 
 
 class AndroidPlatformInfo(PlatformInfo):
-    __arch_infos = {
+    build = 'android'
-        'arm' : ('arm-linux-androideabi', 'arm', 'armeabi'),
+    static = True
-        'arm64': ('aarch64-linux-android', 'aarch64', 'arm64-v8a'),
+    toolchain_names = ['android-ndk', 'android-sdk']
-        'mips': ('mipsel-linux-android', 'mipsel', 'mips'),
+    compatible_hosts = ['fedora', 'debian']
-        'mips64': ('mips64el-linux-android', 'mips64el', 'mips64'),
-        'x86': ('i686-linux-android', 'i686', 'x86'),
-        'x86_64': ('x86_64-linux-android', 'x86_64', 'x86_64'),
-    }
-
-    def __init__(self, name, arch):
-        super().__init__(name, 'android', True, ['android_ndk', 'android_sdk'],
-                         hosts=['fedora', 'debian'])
-        self.arch = arch
-        self.arch_full, self.cpu, self.abi = self.__arch_infos[arch]
 
     def __str__(self):
         return "android"
@@ -35,11 +27,11 @@ class AndroidPlatformInfo(PlatformInfo):
 
     @property
     def ndk_builder(self):
-        return self.toolchains[0].builder
+        return get_plt_step('android_ndk', self.name)
 
     @property
     def sdk_builder(self):
-        return self.toolchains[1].builder
+        return get_plt_step('android_sdk', 'neutral')
 
     def get_cross_config(self):
         install_path = self.ndk_builder.install_path
@@ -77,9 +69,53 @@ class AndroidPlatformInfo(PlatformInfo):
         env['NDK_DEBUG'] = '0'
         env['ANDROID_HOME'] = self.sdk_builder.install_path
 
-AndroidPlatformInfo('android_arm', 'arm')
+    def set_compiler(self, env):
-AndroidPlatformInfo('android_arm64', 'arm64')
+        binaries = self.binaries(self.ndk_builder.install_path)
-AndroidPlatformInfo('android_mips', 'mips')
+        env['CC'] = binaries['CC']
-AndroidPlatformInfo('android_mips64', 'mips64')
+        env['CXX'] = binaries['CXX']
-AndroidPlatformInfo('android_x86', 'x86')
+
-AndroidPlatformInfo('android_x86_64', 'x86_64')
+    @property
+    def configure_option(self):
+        return '--host={}'.format(self.arch_full)
+
+    def finalize_setup(self):
+        super().finalize_setup()
+        self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_android_cross_file.txt')
+        self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
+
+
+class AndroidArm(AndroidPlatformInfo):
+    name = 'android_arm'
+    arch = cpu = 'arm'
+    arch_full = 'arm-linux-androideabi'
+    abi = 'armeabi'
+
+class AndroidArm(AndroidPlatformInfo):
+    name = 'android_arm64'
+    arch = 'arm64'
+    arch_full = 'aarch64-linux-android'
+    cpu = 'aarch64'
+    abi = 'arm64-v8a'
+
+class AndroidArm(AndroidPlatformInfo):
+    name = 'android_mips'
+    arch = abi = 'mips'
+    arch_full = 'mipsel-linux-android'
+    cpu = 'mipsel'
+
+class AndroidArm(AndroidPlatformInfo):
+    name = 'android_mips64'
+    arch = abi = 'mips64'
+    arch_full = 'mips64el-linux-android'
+    cpu = 'mips64el'
+
+class AndroidArm(AndroidPlatformInfo):
+    name = 'android_x86'
+    arch = abi = 'x86'
+    arch_full = 'i686-linux-android'
+    cpu = 'i686'
+
+class AndroidArm(AndroidPlatformInfo):
+    name = 'android_x86_64'
+    arch = cpu = abi = 'x86_64'
+    arch_full = 'x86_64-linux-android'

kiwixbuild/platforms/armhf.py

@@ -1,10 +1,14 @@
 from .base import PlatformInfo
 
+from kiwixbuild.utils import pj
+from kiwixbuild._global import get_plt_step
+
 
 class ArmhfPlatformInfo(PlatformInfo):
+    build = 'armhf'
     arch_full = 'arm-linux-gnueabihf'
-    def __init__(self, name, static):
+    toolchain_names = ['armhf']
-        super().__init__(name, 'armhf', static, ['armhf_toolchain'], ['fedora', 'debian'])
+    compatible_hosts = ['fedora', 'debian']
 
     def get_cross_config(self):
         return {
@@ -25,7 +29,7 @@ class ArmhfPlatformInfo(PlatformInfo):
 
     @property
     def tlc_source(self):
-        return self.toolchains[0].source
+        return get_plt_step('armhf', 'source')
 
     @property
     def root_path(self):
@@ -83,5 +87,10 @@ class ArmhfPlatformInfo(PlatformInfo):
         self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
         self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
 
-ArmhfPlatformInfo('armhf_dyn', False)
+class ArmhfDyn(ArmhfPlatformInfo):
-ArmhfPlatformInfo('armhf_static', True)
+    name = 'armhf_dyn'
+    static = False
+
+class ArmhfStatic(ArmhfPlatformInfo):
+    name = 'armhf_static'
+    static = True

kiwixbuild/platforms/base.py

@@ -6,20 +6,28 @@ from kiwixbuild.toolchains import Toolchain
 from kiwixbuild.packages import PACKAGE_NAME_MAPPERS
 from kiwixbuild.utils import pj, remove_duplicates
 from kiwixbuild.buildenv import BuildEnv
-from kiwixbuild._global import neutralEnv
+from kiwixbuild._global import neutralEnv, add_plt_step, target_steps
 
 _SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
 TEMPLATES_DIR = pj(os.path.dirname(_SCRIPT_DIR), 'templates')
 
-class PlatformInfo:
+class _MetaPlatform(type):
-    all_platforms = {}
+    def __new__(cls, name, bases, dct):
+        _class = type.__new__(cls, name, bases, dct)
+        if name not in ('PlatformInfo', 'MetaPlatformInfo') and 'name' in dct:
+            dep_name = dct['name']
+            PlatformInfo.all_platforms[dep_name] = _class
+        return _class
 
-    def __init__(self, name, build, static, toolchains, hosts=None):
+
-        self.all_platforms[name] = self
+class PlatformInfo(metaclass=_MetaPlatform):
-        self.build = build
+    all_platforms = {}
-        self.static = static
+    all_running_platforms = {}
-        self.toolchains = toolchains
+    toolchain_names = []
-        self.compatible_hosts = hosts
+    configure_option = ""
+
+    def __init__(self):
+        self.all_running_platforms[self.name] = self
         self.buildEnv = BuildEnv(self)
         self.setup_toolchains()
 
@@ -28,9 +36,14 @@ class PlatformInfo:
 
     def setup_toolchains(self):
         self.toolchains = {}
-        for tlc_name in self.toolchains:
+        for tlc_name in self.toolchain_names:
             ToolchainClass = Toolchain.all_toolchains[tlc_name]
             self.toolchains[tlc_name] = ToolchainClass()
+            if ToolchainClass.Source is not None:
+                add_plt_step(('source', tlc_name), ToolchainClass.Source)
+            if ToolchainClass.Builder is not None:
+                plt_name = 'neutral' if ToolchainClass.neutral else self.name
+                add_plt_step((plt_name, tlc_name), ToolchainClass.Builder)
 
     def get_cross_config(self):
         return {}

kiwixbuild/platforms/i586.py

@@ -1,11 +1,13 @@
+import os
 
 from .base import PlatformInfo
+from kiwixbuild.utils import which
 
 
 class I586PlatformInfo(PlatformInfo):
+    build = 'i586'
     arch_full = 'i586-linux-gnu'
-    def __init__(self, name, static):
+    compatible_hosts = ['fedora', 'debian']
-        super().__init__(name, 'i586', static, [], ['fedora', 'debian'])
 
     def get_cross_config(self):
         return {
@@ -52,5 +54,10 @@ class I586PlatformInfo(PlatformInfo):
         self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_i586_cross_file.txt')
         self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
 
-I586PlatformInfo('i586_dyn', False)
+class I586Dyn(I586PlatformInfo):
-I586PlatformInfo('i586_static', True)
+    name = 'i586_dyn'
+    static = False
+
+class I586Static(I586PlatformInfo):
+    name = 'i586_static'
+    static = True

kiwixbuild/platforms/ios.py

@@ -6,18 +6,12 @@ from kiwixbuild.utils import pj, xrun_find
 
 
 class iOSPlatformInfo(PlatformInfo):
-    __arch_infos = {
+    build = 'iOS'
-        'armv7': ('arm-apple-darwin', 'armv7', 'iphoneos'),
+    static = True
-        'arm64': ('arm-apple-darwin', 'arm64', 'iphoneos'),
+    compatible_hosts = ['Darwin']
-        'i386': ('', 'i386', 'iphonesimulator'),
-        'x86_64': ('', 'x86_64', 'iphonesimulator'),
-    }
 
-    def __init__(self, name, arch):
+    def __init__(self, *args, **kwargs):
-        super().__init__(name, 'iOS', True, [],
+        super().__init__(*args, **kwargs)
-                         hosts=['Darwin'])
-        self.arch = arch
-        self.arch_full, self.cpu, self.sdk_name = self.__arch_infos[arch]
         self._root_path = None
 
     @property
@@ -81,7 +75,26 @@ class iOSPlatformInfo(PlatformInfo):
         env['CXX'] = self.binaries['CXX']
 
 
-iOSPlatformInfo('iOS_armv7', 'armv7')
+class iOSArmv7(iOSPlatformInfo):
-iOSPlatformInfo('iOS_arm64', 'arm64')
+    name = 'iOS_armv7'
-iOSPlatformInfo('iOS_i386', 'i386')
+    arch = cpu = 'armv7'
-iOSPlatformInfo('iOS_x86_64', 'x86_64')
+    arch_full =  'arm-apple-darwin'
+    sdk_name = 'iphoneos'
+
+class iOSArm64(iOSPlatformInfo):
+    name = 'iOS_arm64'
+    arch = cpu = 'arm64'
+    arch_full =  'arm-apple-darwin'
+    sdk_name = 'iphoneos'
+
+class iOSi386(iOSPlatformInfo):
+    name = 'iOS_i386'
+    arch = cpu = 'i386'
+    arch_full =  ''
+    sdk_name = 'iphonesimulator'
+
+class iOSx64(iOSPlatformInfo):
+    name = 'iOS_x86_64'
+    arch = cpu = 'x86_64'
+    arch_full =  ''
+    sdk_name = 'iphonesimulator'

kiwixbuild/platforms/native.py

@@ -2,9 +2,15 @@ from .base import PlatformInfo
 
 
 class NativePlatformInfo(PlatformInfo):
-    def __init__(self, name, static, hosts):
+    build = 'native'
-        super().__init__(name, 'native', static, [], hosts)
 
 
-NativePlatformInfo('native_dyn', False, ['fedora', 'debian', 'Darwin'])
+class NativeDyn(NativePlatformInfo):
-NativePlatformInfo('native_static', True, ['fedora', 'debian'])
+    name = 'native_dyn'
+    static = False
+    compatible_hosts = ['fedora', 'debian', 'Darwin']
+
+class NativeStatic(NativePlatformInfo):
+    name = 'native_static'
+    static = True
+    compatible_hosts = ['fedora', 'debian']

kiwixbuild/platforms/win32.py

@@ -7,9 +7,9 @@ from kiwixbuild._global import neutralEnv
 
 class Win32PlatformInfo(PlatformInfo):
     extra_libs = ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4', '-lmsvcr90', '-liphlpapi']
+    build = 'win32'
+    compatible_hosts = ['fedora', 'debian']
     arch_full = 'i686-w64-mingw32'
-    def __init__(self, name, static):
-        super().__init__(name, 'win32', static, [], ['fedora', 'debian'])
 
     def get_cross_config(self):
         return {
@@ -76,5 +76,10 @@ class Win32PlatformInfo(PlatformInfo):
         env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
         env['LIBS'] = " ".join(self.extra_libs) + " " +env['LIBS']
 
-Win32PlatformInfo('win32_dyn', False)
+class Win32Dyn(Win32PlatformInfo):
-Win32PlatformInfo('win32_static', True)
+    name = 'win32_dyn'
+    static = False
+
+class Win32Static(Win32PlatformInfo):
+    name = 'win32_static'
+    static = True

kiwixbuild/toolchains/android_ndk.py

@@ -12,55 +12,11 @@ class android_ndk(Toolchain):
     version = 'r13b'
     gccver = '4.9.x'
 
-    @property
+    @classmethod
-    def api(self):
+    def full_name(cls):
-        return '21' if self.arch in ('arm64', 'mips64', 'x86_64') else '14'
+        return "{name}-{version}".format(
-
+            name = cls.name,
-    @property
+            version = cls.version)
-    def platform(self):
-        return 'android-'+self.api
-
-    @property
-    def arch(self):
-        return self.buildEnv.platform_info.arch
-
-    @property
-    def arch_full(self):
-        return self.buildEnv.platform_info.arch_full
-
-    @property
-    def toolchain(self):
-        return self.arch_full+"-4.9"
-
-    @property
-    def root_path(self):
-        return pj(self.builder.install_path, 'sysroot')
-
-    @property
-    def binaries(self):
-        binaries = ((k,'{}-{}'.format(self.arch_full, v))
-                for k, v in (('CC', 'gcc'),
-                             ('CXX', 'g++'),
-                             ('AR', 'ar'),
-                             ('STRIP', 'strip'),
-                             ('WINDRES', 'windres'),
-                             ('RANLIB', 'ranlib'),
-                             ('LD', 'ld'))
-               )
-        return {k:pj(self.builder.install_path, 'bin', v)
-                for k,v in binaries}
-
-    @property
-    def configure_option(self):
-        return '--host={}'.format(self.arch_full)
-
-    @property
-    def full_name(self):
-        return "{name}-{version}-{arch}-{api}".format(
-            name = self.name,
-            version = self.version,
-            arch = self.arch,
-            api = self.api)
 
     class Source(ReleaseDownload):
         archive = Remotefile('android-ndk-r13b-linux-x86_64.zip',
@@ -75,11 +31,34 @@ class android_ndk(Toolchain):
 
 
     class Builder(Builder):
-
         @property
         def install_path(self):
             return self.build_path
 
+        @property
+        def api(self):
+            return '21' if self.arch in ('arm64', 'mips64', 'x86_64') else '14'
+
+        @property
+        def platform(self):
+            return 'android-'+self.api
+
+        @property
+        def arch(self):
+            return self.buildEnv.platformInfo.arch
+
+        @property
+        def arch_full(self):
+            return self.buildEnv.platformInfo.arch_full
+
+        @property
+        def toolchain(self):
+            return self.arch_full+"-4.9"
+
+        @property
+        def configure_option(self):
+            return '--host={}'.format(self.arch_full)
+
         def _build_platform(self, context):
             context.try_skip(self.build_path)
             script = pj(self.source_path, 'build/tools/make_standalone_toolchain.py')
@@ -87,17 +66,18 @@ class android_ndk(Toolchain):
             command = '{script} --arch={arch} --api={api} --install-dir={install_dir} --force'
             command = command.format(
                 script=script,
-                arch=self.target.arch,
+                arch=self.arch,
-                api=self.target.api,
+                api=self.api,
                 install_dir=self.install_path
             )
+            context.force_native_build = True
             run_command(command, self.build_path, context, buildEnv=self.buildEnv)
 
         def _fix_permission_right(self, context):
             context.try_skip(self.build_path)
             bin_dirs = [pj(self.install_path, 'bin'),
-                        pj(self.install_path, self.target.arch_full, 'bin'),
+                        pj(self.install_path, self.arch_full, 'bin'),
-                        pj(self.install_path, 'libexec', 'gcc', self.target.arch_full, self.target.gccver)
+                        pj(self.install_path, 'libexec', 'gcc', self.arch_full, self.target.gccver)
                        ]
             for root, dirs, files in os.walk(self.install_path):
                 if not root in bin_dirs:

kiwixbuild/toolchains/android_sdk.py

@@ -20,7 +20,7 @@ class android_sdk(Toolchain):
 
         @property
         def install_path(self):
-            return pj(self.buildEnv.toolchain_dir, self.target.full_name)
+            return pj(self.buildEnv.toolchain_dir, self.target.full_name())
 
         def _build_platform(self, context):
             context.try_skip(self.install_path)

kiwixbuild/toolchains/base_toolchain.py

@@ -10,11 +10,12 @@ class _MetaToolchain(type):
     def __new__(cls, name, bases, dct):
         _class = type.__new__(cls, name, bases, dct)
         if name != 'Toolchain':
-            Toolchain.all_toolchains[name] = _class
+            Toolchain.all_toolchains[dct['name']] = _class
         return _class
 
 
 class Toolchain(metaclass=_MetaToolchain):
+    force_native_build = False
     neutral = True
     all_toolchains = {}
     configure_option = ""
@@ -23,15 +24,12 @@ class Toolchain(metaclass=_MetaToolchain):
     Builder = None
     Source = None
 
-    def __init__(self):
-        self.source = self.Source(self) if self.Source else None
-        self.builder = self.Builder(self) if self.Builder else None
 
-    @property
+    @classmethod
-    def full_name(self):
+    def full_name(cls):
         return "{name}-{version}".format(
-            name = self.name,
+            name = cls.name,
-            version = self.version)
+            version = cls.version)
 
     @property
     def source_path(self):