Merge toolchain and dependencies.

Toolchain now became "simple" dependencies.
They are referenced by Platform.

Platform are now created at demande and a neutralPlatform now exists.

kiwixbuild/_global.py

@@ -3,7 +3,6 @@ from collections import OrderedDict as _OrderedDict
 _neutralEnv = None
 _options = None
 _target_steps = _OrderedDict()
-_plt_steps = _OrderedDict()
 
 def set_neutralEnv(env):
     global _neutralEnv
@@ -31,16 +30,3 @@ def get_target_step(key, default_context=None):
 
 def target_steps():
     return _target_steps
-
-def add_plt_step(key, what):
-    _plt_steps[key] = what
-
-def get_plt_step(key, default_context=None):
-    try:
-        context, target = key
-    except ValueError:
-        context, target = default_context, key
-    return _plt_steps[(context, target)]
-
-def plt_steps():
-    return _plt_steps

kiwixbuild/buildenv.py

@@ -3,7 +3,6 @@ import os, sys, shutil
 import subprocess
 import platform
 
-from .toolchains import Toolchain
 from .utils import pj, download_remote, Defaultdict
 from ._global import neutralEnv, option
 
@@ -77,9 +76,11 @@ class BuildEnv:
         self.platformInfo = platformInfo
         self.build_dir = pj(option('working_dir'), build_dir)
         self.install_dir = pj(self.build_dir, "INSTALL")
+        self.toolchain_dir = pj(self.build_dir, "TOOLCHAINS")
         self.log_dir = pj(self.build_dir, 'LOGS')
         for d in (self.build_dir,
                   self.install_dir,
+                  self.toolchain_dir,
                   self.log_dir):
             os.makedirs(d, exist_ok=True)
 

kiwixbuild/builder.py

@@ -8,8 +8,7 @@ from .utils import remove_duplicates, StopBuild
 from .dependencies import Dependency
 from ._global import (
     neutralEnv, option,
-    add_target_step, get_target_step, target_steps,
+    add_target_step, get_target_step, target_steps)
-    add_plt_step, get_plt_step, plt_steps)
 from . import _global
 
 class Builder:
@@ -20,6 +19,7 @@ class Builder:
         dependencies = self.order_dependencies(_targets, targetDef)
         dependencies = list(remove_duplicates(dependencies))
 
+        PlatformInfo.get_platform('neutral', _targets)
         if option('build_nodeps'):
             add_target_step(targetDef, _targets[targetDef])
         else:
@@ -27,12 +27,13 @@ class Builder:
                 if option('build_deps_only') and dep == targetDef:
                     continue
                 add_target_step(dep, _targets[dep])
+        self.instanciate_steps()
 
     def add_targets(self, targetDef, targets):
         if targetDef in targets:
             return
         targetPlatformName, targetName = targetDef
-        targetPlatform = PlatformInfo.get_platform(targetPlatformName)
+        targetPlatform = PlatformInfo.get_platform(targetPlatformName, targets)
         targetClass = Dependency.all_deps[targetName]
         targets[('source', targetName)] = targetClass.Source
         targets[targetDef] = targetClass.Builder
@@ -44,6 +45,12 @@ class Builder:
             self.add_targets((depPlatform, depName), targets)
 
     def order_dependencies(self, _targets, targetDef):
+        for pltName in PlatformInfo.all_running_platforms:
+            plt = PlatformInfo.all_platforms[pltName]
+            for tlcName in plt.toolchain_names:
+                tlc = Dependency.all_deps[tlcName]
+                yield('source', tlcName)
+                yield('neutral' if tlc.neutral else pltName, tlcName)
         targetPlatformName, targetName = targetDef
         if targetPlatformName == 'source':
             # Do not try to order sources, they will be added as dep by the
@@ -60,14 +67,18 @@ class Builder:
             yield from self.order_dependencies(_targets, (depPlatform, depName))
         yield targetDef
 
-    def prepare_toolchain_sources(self):
+    def instanciate_steps(self):
-        tlsourceDefs = (tlDef for tlDef in plt_steps() if tlDef[0]=='source')
+        for stepDef in list(target_steps()):
-        for tlsourceDef in tlsourceDefs:
+            stepPlatform, stepName = stepDef
-            print("prepare sources for toolchain {} :".format(tlsourceDef[1]))
+            stepClass = Dependency.all_deps[stepName]
-            toolchainClass = Toolchain.all_toolchains[tlsourceDef[1]]
+            if stepPlatform == 'source':
-            source = get_plt_step(tlsourceDef)(toolchainClass)
+                source = get_target_step(stepDef)(stepClass)
-            add_plt_step(tlsourceDef, source)
+                add_target_step(stepDef, source)
-            source.prepare()
+            else:
+                source = get_target_step(stepName, 'source')
+                env = PlatformInfo.get_platform(stepPlatform).buildEnv
+                builder = get_target_step(stepDef)(stepClass, source, env)
+                add_target_step(stepDef, builder)
 
     def prepare_sources(self):
         if option('skip_source_prepare'):
@@ -76,36 +87,16 @@ class Builder:
 
         sourceDefs = remove_duplicates(tDef for tDef in target_steps() if tDef[0]=='source')
         for sourceDef in sourceDefs:
-
             print("prepare sources {} :".format(sourceDef[1]))
-            depClass = Dependency.all_deps[sourceDef[1]]
+            source = get_target_step(sourceDef)
-            source = get_target_step(sourceDef)(depClass)
-            add_target_step(sourceDef, source)
             source.prepare()
 
-    def build_toolchains(self):
-        tlbuilderDefs = (tlDef for tlDef in plt_steps() if tlDef[0] != 'source')
-        for tlbuilderDef in tlbuilderDefs:
-            print("build toolchain {} :".format(tlbuilderDef[1]))
-            toolchainClass = Toolchain.all_toolchains[tlbuilderDef[1]]
-            source = get_plt_step(tlbuilderDef[1], 'source')
-            if tlbuilderDef[0] == 'neutral':
-                env = _global._neutralEnv
-            else:
-                env = PlatformInfo.get_platform(tlbuilderDef[0]).buildEnv
-            builder = get_plt_step(tlbuilderDef)(toolchainClass, source, env)
-            add_plt_step(tlbuilderDef, builder)
-            builder.build()
-
     def build(self):
         builderDefs = (tDef for tDef in target_steps() if tDef[0] != 'source')
         for builderDef in builderDefs:
-            depClass = Dependency.all_deps[builderDef[1]]
+            builder = get_target_step(builderDef)
-            source = get_target_step(builderDef[1], 'source')
+            if option('make_dist') and builderName == option('targets'):
-            env = PlatformInfo.get_platform(builderDef[0]).buildEnv
+                print("make dist {} ({}):".format(builder.name, builderDef[0]))
-            builder = get_target_step(builderDef)(depClass, source, env)
-            if option('make_dist') and builderDef[1] == option('targets'):
-                print("make dist {}:".format(builder.name))
                 builder.make_dist()
                 continue
             print("build {} ({}):".format(builder.name, builderDef[0]))
@@ -116,8 +107,6 @@ class Builder:
     def run(self):
         try:
             print("[SETUP PLATFORMS]")
-            self.prepare_toolchain_sources()
-            self.build_toolchains()
             for platform in PlatformInfo.all_running_platforms.values():
                 platform.finalize_setup()
             print("[INSTALL PACKAGES]")

kiwixbuild/dependencies/__init__.py

@@ -2,6 +2,9 @@
 from .base import *
 from . import (
     all_dependencies,
+    android_ndk,
+    android_sdk,
+    armhf,
     ctpp2,
     gradle,
     gumbo,

kiwixbuild/dependencies/android_ndk.py

@@ -1,23 +1,15 @@
 import os
 
-from .base_toolchain import Toolchain
+from .base import Dependency, ReleaseDownload, Builder
-from kiwixbuild.dependencies import ReleaseDownload, Builder
 from kiwixbuild.utils import Remotefile, add_execution_right, run_command
 
 pj = os.path.join
 
-class android_ndk(Toolchain):
+class android_ndk(Dependency):
     neutral = False
     name = 'android-ndk'
-    version = 'r13b'
     gccver = '4.9.x'
 
-    @classmethod
-    def full_name(cls):
-        return "{name}-{version}".format(
-            name = cls.name,
-            version = cls.version)
-
     class Source(ReleaseDownload):
         archive = Remotefile('android-ndk-r13b-linux-x86_64.zip',
                              '3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c',
@@ -25,9 +17,7 @@ class android_ndk(Toolchain):
 
         @property
         def source_dir(self):
-            return "{}-{}".format(
+            return self.target.full_name()
-                self.target.name,
-                self.target.version)
 
 
     class Builder(Builder):

kiwixbuild/dependencies/android_sdk.py

@@ -1,15 +1,14 @@
 import os
 import shutil
 
-from .base_toolchain import Toolchain
+from .base import Dependency, ReleaseDownload, Builder
-from kiwixbuild.dependencies import ReleaseDownload, Builder
 from kiwixbuild.utils import Remotefile, run_command
 
 pj = os.path.join
 
-class android_sdk(Toolchain):
+class android_sdk(Dependency):
+    neutral = True
     name = 'android-sdk'
-    version = 'r25.2.3'
 
     class Source(ReleaseDownload):
         archive = Remotefile('tools_r25.2.3-linux.zip',

kiwixbuild/dependencies/armhf.py

@@ -0,0 +1,11 @@
+from .base import Dependency, GitClone, NoopBuilder
+
+class armhf_toolchain(Dependency):
+    neutral = True
+    name = 'armhf'
+
+    class Source(GitClone):
+        git_remote = "https://github.com/raspberrypi/tools"
+        git_dir = "raspberrypi-tools"
+
+    Builder = NoopBuilder

kiwixbuild/dependencies/base.py

@@ -350,7 +350,7 @@ class CMakeBuilder(MakeBuilder):
                    " {source_path}"
                    " {cross_option}")
         command = command.format(
-            configure_option="{} {}".format(self.buildEnv.cmake_option, self.configure_option),
+            configure_option=self.configure_option,
             install_dir=self.buildEnv.install_dir,
             libdir=self.buildEnv.libprefix,
             source_path=self.source_path,

kiwixbuild/platforms/__init__.py

@@ -7,5 +7,6 @@ from . import (
     i586,
     ios,
     native,
+    neutral,
     win32
 )

kiwixbuild/platforms/android.py

@@ -1,6 +1,6 @@
 from .base import PlatformInfo
 from kiwixbuild.utils import pj
-from kiwixbuild._global import get_plt_step
+from kiwixbuild._global import get_target_step
 
 
 class AndroidPlatformInfo(PlatformInfo):
@@ -27,11 +27,11 @@ class AndroidPlatformInfo(PlatformInfo):
 
     @property
     def ndk_builder(self):
-        return get_plt_step('android_ndk', self.name)
+        return get_target_step('android-ndk', self.name)
 
     @property
     def sdk_builder(self):
-        return get_plt_step('android_sdk', 'neutral')
+        return get_target_step('android-sdk', 'neutral')
 
     def get_cross_config(self):
         install_path = self.ndk_builder.install_path

kiwixbuild/platforms/armhf.py

@@ -1,7 +1,7 @@
 from .base import PlatformInfo
 
 from kiwixbuild.utils import pj
-from kiwixbuild._global import get_plt_step
+from kiwixbuild._global import get_target_step
 
 
 class ArmhfPlatformInfo(PlatformInfo):
@@ -29,7 +29,7 @@ class ArmhfPlatformInfo(PlatformInfo):
 
     @property
     def tlc_source(self):
-        return get_plt_step('armhf', 'source')
+        return get_target_step('armhf', 'source')
 
     @property
     def root_path(self):

kiwixbuild/platforms/base.py

@@ -1,13 +1,12 @@
 
-import os
+import os, sys
 import subprocess
 
 from kiwixbuild.dependencies import Dependency
-from kiwixbuild.toolchains import Toolchain
 from kiwixbuild.packages import PACKAGE_NAME_MAPPERS
 from kiwixbuild.utils import pj, remove_duplicates
 from kiwixbuild.buildenv import BuildEnv
-from kiwixbuild._global import neutralEnv, option, add_plt_step, target_steps
+from kiwixbuild._global import neutralEnv, option, target_steps
 
 _SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
 TEMPLATES_DIR = pj(os.path.dirname(_SCRIPT_DIR), 'templates')
@@ -28,12 +27,17 @@ class PlatformInfo(metaclass=_MetaPlatform):
     configure_option = ""
 
     @classmethod
-    def get_platform(cls, name):
+    def get_platform(cls, name, targets=None):
         if name not in cls.all_running_platforms:
-            cls.all_running_platforms[name] = cls.all_platforms[name]()
+            if targets is None:
+                print("Should not got there.")
+                print(cls.all_running_platforms)
+                raise KeyError(name)
+                sys.exit(-1)
+            cls.all_running_platforms[name] = cls.all_platforms[name](targets)
         return cls.all_running_platforms[name]
 
-    def __init__(self):
+    def __init__(self, targets):
         if neutralEnv('distname') not in self.compatible_hosts:
             print(('ERROR: The target platform {} cannot be build on host {}.\n'
                    'Select another target platform, or change your host system.'
@@ -41,19 +45,17 @@ class PlatformInfo(metaclass=_MetaPlatform):
             sys.exit(-1)
         self.all_running_platforms[self.name] = self
         self.buildEnv = BuildEnv(self)
-        self.setup_toolchains()
+        self.setup_toolchains(targets)
 
     def __str__(self):
         return "{}_{}".format(self.build, 'static' if self.static else 'dyn')
 
-    def setup_toolchains(self):
+    def setup_toolchains(self, targets):
         for tlc_name in self.toolchain_names:
-            ToolchainClass = Toolchain.all_toolchains[tlc_name]
+            ToolchainClass = Dependency.all_deps[tlc_name]
-            if ToolchainClass.Source is not None:
+            targets[('source', tlc_name)] = ToolchainClass.Source
-                add_plt_step(('source', tlc_name), ToolchainClass.Source)
-            if ToolchainClass.Builder is not None:
             plt_name = 'neutral' if ToolchainClass.neutral else self.name
-                add_plt_step((plt_name, tlc_name), ToolchainClass.Builder)
+            targets[(plt_name, tlc_name)] = ToolchainClass.Builder
 
     def get_cross_config(self):
         return {}
@@ -93,9 +95,10 @@ class PlatformInfo(metaclass=_MetaPlatform):
         except KeyError:
             print("SKIP : We don't know which packages we must install to compile"
                   " a {target} {build_type} version on a {host} host.".format(
-                      target=self.platform_info,
+                      target=self,
+                      build_type=self.static,
                       host=neutralEnv('distname')))
-            return
+            return []
 
         packages_list = package_name_mapper.get('COMMON', [])
         to_drop = []

kiwixbuild/platforms/neutral.py

@@ -0,0 +1,9 @@
+from .base import PlatformInfo
+
+class NeutralPlatformInfo(PlatformInfo):
+    name = 'neutral'
+    static = ''
+    compatible_hosts = ['fedora', 'debian', 'Darwin']
+
+    def __str__(self):
+        return "neutral"

kiwixbuild/toolchains/__init__.py

@@ -1,4 +0,0 @@
-
-from . import android_ndk, android_sdk, armhf
-
-from .base_toolchain import Toolchain

kiwixbuild/toolchains/armhf.py

@@ -1,14 +0,0 @@
-import os
-import subprocess
-
-from .base_toolchain import Toolchain
-from kiwixbuild.dependencies import GitClone
-from kiwixbuild.utils import which
-pj = os.path.join
-
-class armhf_toolchain(Toolchain):
-    name = 'armhf'
-
-    class Source(GitClone):
-        git_remote = "https://github.com/raspberrypi/tools"
-        git_dir = "raspberrypi-tools"

kiwixbuild/toolchains/base_toolchain.py

@@ -1,63 +0,0 @@
-import os
-import subprocess
-
-pj = os.path.join
-
-from kiwixbuild.utils import Context, SkipCommand, StopBuild
-from kiwixbuild._global import neutralEnv
-
-class _MetaToolchain(type):
-    def __new__(cls, name, bases, dct):
-        _class = type.__new__(cls, name, bases, dct)
-        if name != 'Toolchain':
-            Toolchain.all_toolchains[dct['name']] = _class
-        return _class
-
-
-class Toolchain(metaclass=_MetaToolchain):
-    force_native_build = False
-    neutral = True
-    all_toolchains = {}
-    configure_option = ""
-    cmake_option = ""
-    exec_wrapper_def = ""
-    Builder = None
-    Source = None
-
-
-    @classmethod
-    def full_name(cls):
-        return "{name}-{version}".format(
-            name = cls.name,
-            version = cls.version)
-
-    @property
-    def source_path(self):
-        return pj(neutralEnv('source_dir'), self.source.source_dir)
-
-    @property
-    def _log_dir(self):
-        return neutralEnv('log_dir')
-
-    def command(self, name, function, *args):
-        print("  {} {} : ".format(name, self.name), end="", flush=True)
-        log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
-        context = Context(name, log, True)
-        try:
-            ret = function(*args, context=context)
-            context._finalise()
-            print("OK")
-            return ret
-        except SkipCommand:
-            print("SKIP")
-        except subprocess.CalledProcessError:
-            print("ERROR")
-            try:
-                with open(log, 'r') as f:
-                    print(f.read())
-            except:
-                pass
-            raise StopBuild()
-        except:
-            print("ERROR")
-            raise

kiwixbuild/versions.py

@@ -10,7 +10,7 @@ main_project_versions = {
 
 # This is the "version" of the whole base_deps_versions dict.
 # Change this when you change base_deps_versions.
-base_deps_meta_version = '3'
+base_deps_meta_version = '4'
 
 
 base_deps_versions = {
@@ -25,5 +25,7 @@ base_deps_versions = {
   'icu4c' : '58.2',
   'Gradle' : '4.6',
   'libaria2' : '1.33.1',
-  'libmagic' : '5.33'
+  'libmagic' : '5.33',
+  'android-sdk' : 'r25.2.3',
+  'android-ndk' : 'r13b'
 }