Add the notion of toolschain.

- For now, only the win32 is here.
- There no more nativeBuilEnv as the buildEnv can launch command without
  using the toolschain.

dependency_utils.py

@@ -44,7 +44,7 @@ class Dependency(metaclass=_MetaDependency):
     def command(self, name, function, *args):
         print("  {} {} : ".format(name, self.name), end="", flush=True)
         log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
-        context = Context(name, log)
+        context = Context(name, log, self.force_native_build)
         try:
             ret = function(*args, context=context)
             context._finalise()
@@ -106,9 +106,10 @@ class ReleaseDownload(Source):
 
     def _patch(self, context):
         context.try_skip(self.extract_path)
+        context.force_native_build = True
         for p in self.patches:
             with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input:
-                self.buildEnv.run_command("patch -p1", self.extract_path, context, input=patch_input, allow_wrapper=False)
+                self.buildEnv.run_command("patch -p1", self.extract_path, context, input=patch_input)
 
     def prepare(self):
         self.command('download', self._download)
@@ -129,12 +130,14 @@ class GitClone(Source):
         return pj(self.buildEnv.source_dir, self.git_dir)
 
     def _git_clone(self, context):
+        context.force_native_build = True
         if os.path.exists(self.git_path):
             raise SkipCommand()
         command = "git clone " + self.git_remote
         self.buildEnv.run_command(command, self.buildEnv.source_dir, context)
 
     def _git_update(self, context):
+        context.force_native_build = True
         self.buildEnv.run_command("git fetch", self.git_path, context)
         self.buildEnv.run_command("git checkout "+self.git_ref, self.git_path, context)
 
@@ -188,7 +191,7 @@ class MakeBuilder(Builder):
         configure_option = "{} {} {}".format(
             self.configure_option,
             self.static_configure_option if self.buildEnv.build_static else self.dynamic_configure_option,
-            self.buildEnv.configure_option)
+            self.buildEnv.configure_option if not self.target.force_native_build else "")
         command = "{configure_script} {configure_option} --prefix {install_dir} --libdir {libdir}"
         command = command.format(
             configure_script=pj(self.source_path, self.configure_script),
@@ -251,7 +254,7 @@ class CMakeBuilder(MakeBuilder):
                     v = v.format(buildEnv=self.buildEnv, env=env)
                     self.configure_env[k[8:]] = v
             env.update(self.configure_env)
-        self.buildEnv.run_command(command, self.build_path, context, env=env, allow_wrapper=False)
+        self.buildEnv.run_command(command, self.build_path, context, env=env, cross_path_only=True)
 
 
 class MesonBuilder(Builder):
@@ -281,12 +284,12 @@ class MesonBuilder(Builder):
             buildEnv=self.buildEnv,
             cross_option="--cross-file {}".format(self.buildEnv.meson_crossfile) if self.buildEnv.meson_crossfile else ""
         )
-        self.buildEnv.run_command(command, self.source_path, context, allow_wrapper=False)
+        self.buildEnv.run_command(command, self.source_path, context, cross_path_only=True)
 
     def _compile(self, context):
         command = "{} -v".format(self.buildEnv.ninja_command)
-        self.buildEnv.run_command(command, self.build_path, context, allow_wrapper=False)
+        self.buildEnv.run_command(command, self.build_path, context)
 
     def _install(self, context):
         command = "{} -v install".format(self.buildEnv.ninja_command)
-        self.buildEnv.run_command(command, self.build_path, context, allow_wrapper=False)
+        self.buildEnv.run_command(command, self.build_path, context)

kiwix-build.py

@@ -23,16 +23,8 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
 
 CROSS_ENV = {
     'fedora_win32': {
+        'toolchain_names': ['mingw32_toolchain'],
         'root_path': '/usr/i686-w64-mingw32/sys-root/mingw',
-        'binaries': {
-            'c': 'i686-w64-mingw32-gcc',
-            'cpp': 'i686-w64-mingw32-g++',
-            'ar': 'i686-w64-mingw32-ar',
-            'strip': 'i686-w64-mingw32-strip',
-            'windres': 'i686-w64-mingw32-windres',
-            'ranlib': 'i686-w64-mingw32-ranlib',
-            'exe_wrapper': 'wine'
-        },
         'properties': {
             'c_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4'],
             'cpp_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4']
@@ -48,16 +40,8 @@ CROSS_ENV = {
         }
     },
     'debian_win32': {
+        'toolchain_names': ['mingw32_toolchain'],
         'root_path': '/usr/i686-w64-mingw32/',
-        'binaries': {
-            'c': 'i686-w64-mingw32-gcc',
-            'cpp': 'i686-w64-mingw32-g++',
-            'ar': 'i686-w64-mingw32-ar',
-            'strip': 'i686-w64-mingw32-strip',
-            'windres': 'i686-w64-mingw32-windres',
-            'ranlib': 'i686-w64-mingw32-ranlib',
-            'exe_wrapper': 'wine'
-        },
         'properties': {
             'c_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4'],
             'cpp_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4']
@@ -129,15 +113,11 @@ PACKAGE_NAME_MAPPERS = {
 }
 
 
-class Which():
+def which(name):
-    def __getattr__(self, name):
     command = "which {}".format(name)
     output = subprocess.check_output(command, shell=True)
     return output[:-1].decode()
 
-    def __format__(self, format_spec):
-        return getattr(self, format_spec)
-
 
 class BuildEnv:
     build_targets = ['native', 'win32']
@@ -165,6 +145,8 @@ class BuildEnv:
         self.meson_command = self._detect_meson()
         if not self.meson_command:
             sys.exit("ERROR: meson command not fount")
+        self.setup_build(options.build_target)
+        self.setup_toolchains()
         self.options = options
         self.libprefix = options.libprefix or self._detect_libdir()
         self.targetsDict = targetsDict
@@ -187,31 +169,11 @@ class BuildEnv:
             if self.distname == 'ubuntu':
                 self.distname = 'debian'
 
-    def finalize_setup(self):
+    def setup_build(self, target):
-        getattr(self, 'setup_{}'.format(self.build_target))()
+        self.build_target = target
-
+        if target == 'native':
-    def setup_native(self):
             self.cross_env = {}
-        self.wrapper = None
+        else:
-        self.configure_option = ""
-        self.cmake_option = ""
-        self.cmake_crossfile = None
-        self.meson_crossfile = None
-
-    def _gen_crossfile(self, name):
-        crossfile = pj(self.build_dir, name)
-        template_file = pj(SCRIPT_DIR, 'templates', name)
-        with open(template_file, 'r') as f:
-            template = f.read()
-        content = template.format(
-            which=Which(),
-            **self.cross_env
-        )
-        with open(crossfile, 'w') as outfile:
-            outfile.write(content)
-        return crossfile
-
-    def setup_win32(self):
             cross_name = "{host}_{target}".format(
                 host = self.distname,
                 target = self.build_target)
@@ -224,11 +186,32 @@ class BuildEnv:
                             host = self.distname
                         ))
 
-        self.wrapper = self._gen_crossfile('bash_wrapper.sh')
+    def setup_toolchains(self):
-        current_permissions = stat.S_IMODE(os.lstat(self.wrapper).st_mode)
+        toolchain_names = self.cross_env.get('toolchain_names', [])
-        os.chmod(self.wrapper, current_permissions | stat.S_IXUSR)
+        self.toolchains =[Toolchain.all_toolchains[toolchain_name](self)
-        self.configure_option = "--host=i686-w64-mingw32"
+                              for toolchain_name in toolchain_names]
-        self.cmake_option = ""
+
+    def finalize_setup(self):
+        getattr(self, 'setup_{}'.format(self.build_target))()
+
+    def setup_native(self):
+        self.cmake_crossfile = None
+        self.meson_crossfile = None
+
+    def _gen_crossfile(self, name):
+        crossfile = pj(self.build_dir, name)
+        template_file = pj(SCRIPT_DIR, 'templates', name)
+        with open(template_file, 'r') as f:
+            template = f.read()
+        content = template.format(
+            toolchain=self.toolchains[0],
+            **self.cross_env
+        )
+        with open(crossfile, 'w') as outfile:
+            outfile.write(content)
+        return crossfile
+
+    def setup_win32(self):
         self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
         self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
 
@@ -276,14 +259,33 @@ class BuildEnv:
             if retcode == 0:
                 return n
 
-    def _set_env(self, env):
+    @property
+    def configure_option(self):
+        configure_options = [tlc.configure_option for tlc in self.toolchains]
+        return " ".join(configure_options)
+
+    @property
+    def cmake_option(self):
+        cmake_options = [tlc.cmake_option for tlc in self.toolchains]
+        return " ".join(cmake_options)
+
+    def _set_env(self, env, cross_compile_env, cross_compile_path):
         if env is None:
             env = Defaultdict(str, os.environ)
+
+        bin_dirs = []
+        if cross_compile_env:
             for k, v in self.cross_env.get('env', {}).items():
                 if k.startswith('_format_'):
                     v = v.format(**self.cross_env)
                     k = k[8:]
                 env[k] = v
+            for toolchain in self.toolchains:
+                toolchain.set_env(env)
+        if cross_compile_path:
+            for tlc in self.toolchains:
+                bin_dirs += tlc.get_bin_dir()
+
         pkgconfig_path = pj(self.install_dir, self.libprefix, 'pkgconfig')
         env['PKG_CONFIG_PATH'] = (env['PKG_CONFIG_PATH'] + ':' + pkgconfig_path
                                   if env['PKG_CONFIG_PATH']
@@ -296,7 +298,10 @@ class BuildEnv:
                 break
         else:
             ccache_path = []
-        env['PATH'] = ':'.join([pj(self.install_dir, 'bin')] + ccache_path + [env['PATH']])
+        env['PATH'] = ':'.join(bin_dirs +
+                               [pj(self.install_dir, 'bin')] +
+                               ccache_path +
+                               [env['PATH']])
         ld_library_path = ':'.join([
             pj(self.install_dir, 'lib'),
             pj(self.install_dir, 'lib64')
@@ -305,15 +310,20 @@ class BuildEnv:
                                   if env['LD_LIBRARY_PATH']
                                   else ld_library_path
                                   )
-        env['CPPFLAGS'] = '-I'+pj(self.install_dir, 'include')
+        env['CPPFLAGS'] = " ".join(['-I'+pj(self.install_dir, 'include'), env['CPPFLAGS']])
-        env['LDFLAGS'] = '-L'+pj(self.install_dir, 'lib')
+        env['LDFLAGS'] = " ".join(['-L'+pj(self.install_dir, 'lib'), env['LDFLAGS']])
         return env
 
-    def run_command(self, command, cwd, context, env=None, input=None, allow_wrapper=True):
+    def run_command(self, command, cwd, context, env=None, input=None, cross_path_only=False):
         os.makedirs(cwd, exist_ok=True)
-        if allow_wrapper and self.wrapper:
+        cross_compile_env = True
-            command = "{} {}".format(self.wrapper, command)
+        cross_compile_path = True
-        env = self._set_env(env)
+        if context.force_native_build:
+            cross_compile_env = False
+            cross_compile_path = False
+        if cross_path_only:
+            cross_compile_env = False
+        env = self._set_env(env, cross_compile_env, cross_compile_path)
         log = None
         try:
             if not self.options.verbose:
@@ -414,16 +424,71 @@ class BuildEnv:
         with open(autoskip_file, 'w'):
             pass
 
+class _MetaToolchain(type):
+    def __new__(cls, name, bases, dct):
+        _class = type.__new__(cls, name, bases, dct)
+        if name != 'Toolchain':
+            Toolchain.all_toolchains[name] = _class
+        return _class
+
+
+class Toolchain(metaclass=_MetaToolchain):
+    all_toolchains = {}
+    configure_option = ""
+    cmake_option = ""
+
+    def __init__(self, buildEnv):
+        self.buildEnv = buildEnv
+
+    @property
+    def full_name(self):
+        return "{name}-{version}".format(
+            name = self.name,
+            version = self.version)
+
+    def set_env(self, env):
+        pass
+
+
+class mingw32_toolchain(Toolchain):
+    name = 'mingw32'
+    arch_full = 'i686-w64-mingw32'
+
+    @property
+    def root_path(self):
+        return self.buildEnv.cross_env['root_path']
+
+    @property
+    def binaries(self):
+        return {k:which('{}-{}'.format(self.arch_full, v))
+                for k, v in (('CC', 'gcc'),
+                             ('CXX', 'g++'),
+                             ('AR', 'ar'),
+                             ('STRIP', 'strip'),
+                             ('WINDRES', 'windres'),
+                             ('RANLIB', 'ranlib'))
+               }
+
+    @property
+    def configure_option(self):
+        return '--host={}'.format(self.arch_full)
+
+    def get_bin_dir(self):
+        return [pj(self.root_path, 'bin')]
+
+    def set_env(self, env):
+        for k, v in self.binaries.items():
+            env[k] = v
+
+        env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
+        env['CFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS']
+        env['CXXFLAGS'] =" -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
+
 
 class Builder:
     def __init__(self, options, targetDef='KiwixTools'):
         self.targets = OrderedDict()
         self.buildEnv = buildEnv = BuildEnv(options, self.targets)
-        if buildEnv.build_target != 'native':
-            self.nativeBuildEnv = BuildEnv(options, self.targets)
-            self.nativeBuildEnv.build_target = 'native'
-        else:
-            self.nativeBuildEnv = buildEnv
 
         _targets = {}
         self.add_targets(targetDef, _targets)
@@ -437,9 +502,6 @@ class Builder:
         if targetName in targets:
             return
         targetClass = Dependency.all_deps[targetName]
-        if targetClass.force_native_build:
-            target = targetClass(self.nativeBuildEnv)
-        else:
         target = targetClass(self.buildEnv)
         targets[targetName] = target
         for dep in target.dependencies:
@@ -452,6 +514,11 @@ class Builder:
         yield targetName
 
     def prepare_sources(self):
+        toolchain_sources = (tlc.source for tlc in self.buildEnv.toolchains if tlc.source)
+        for toolchain_source in toolchain_sources:
+            print("prepare sources for toolchain {} :".format(toolchain_source.name))
+            toolchain_source.prepare()
+
         sources = (dep.source for dep in self.targets.values() if not dep.skip)
         sources = remove_duplicates(sources, lambda s: s.__class__)
         for source in sources:
@@ -469,8 +536,6 @@ class Builder:
             print("[INSTALL PACKAGES]")
             self.buildEnv.install_packages()
             self.buildEnv.finalize_setup()
-            if self.nativeBuildEnv != self.buildEnv:
-                self.nativeBuildEnv.finalize_setup()
             print("[PREPARE]")
             self.prepare_sources()
             print("[BUILD]")

templates/cmake_cross_file.txt

@@ -2,11 +2,11 @@ SET(CMAKE_SYSTEM_NAME Windows)
 SET(CMAKE_SYSTEM_PROCESSOR x86)
 
 # specify the cross compiler
-SET(CMAKE_C_COMPILER "{which:{binaries[c]}}")
+SET(CMAKE_C_COMPILER "{toolchain.binaries[CC]}")
-SET(CMAKE_CXX_COMPILER "{which:{binaries[cpp]}}")
+SET(CMAKE_CXX_COMPILER "{toolchain.binaries[CXX]}")
-SET(CMAKE_RC_COMPILER {which:{binaries[windres]}})
+SET(CMAKE_RC_COMPILER {toolchain.binaries[WINDRES]})
-SET(CMAKE_AR:FILEPATH {which:{binaries[ar]}})
+SET(CMAKE_AR:FILEPATH {toolchain.binaries[AR]})
-SET(CMAKE_RANLIB:FILEPATH {which:{binaries[ranlib]}})
+SET(CMAKE_RANLIB:FILEPATH {toolchain.binaries[RANLIB]})
 
 find_program(CCACHE_FOUND ccache)
 if(CCACHE_FOUND)
@@ -15,7 +15,7 @@ if(CCACHE_FOUND)
 endif(CCACHE_FOUND)
 
 # where is the target environment
-SET(CMAKE_FIND_ROOT_PATH {root_path})
+SET(CMAKE_FIND_ROOT_PATH {toolchain.root_path})
 
 # search for programs in the build host directories
 SET(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)

templates/meson_cross_file.txt

@@ -1,16 +1,16 @@
 [binaries]
 pkgconfig = 'pkg-config'
-c = '{which:{binaries[c]}}'
+c = '{toolchain.binaries[CC]}'
-ar = '{which:{binaries[ar]}}'
+ar = '{toolchain.binaries[AR]}'
-cpp = '{which:{binaries[cpp]}}'
+cpp = '{toolchain.binaries[CXX]}'
-strip = '{which:{binaries[strip]}}'
+strip = '{toolchain.binaries[STRIP]}'
 
 [properties]
-c_link_args = ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4']
+c_link_args = {properties[c_link_args]!r}
-cpp_link_args = ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4']
+cpp_link_args = {properties[cpp_link_args]!r}
 
 [host_machine]
-cpu_family = 'x86'
+cpu_family = '{host_machine[cpu_family]}'
-cpu = 'i586'
+cpu = '{host_machine[cpu]}'
-system = 'windows'
+system = '{host_machine[system]}'
-endian = 'little'
+endian = '{host_machine[endian]}'

utils.py

@@ -45,9 +45,10 @@ class Remotefile(namedtuple('Remotefile', ('name', 'sha256', 'url'))):
 
 
 class Context:
-    def __init__(self, command_name, log_file):
+    def __init__(self, command_name, log_file, force_native_build):
         self.command_name = command_name
         self.log_file = log_file
+        self.force_native_build = force_native_build
         self.autoskip_file = None
 
     def try_skip(self, path):