Merge pull request #483 from kiwix/libzim_testing_suite

kiwixbuild/builder.py

@@ -34,14 +34,10 @@ class Builder:
 
         if option('build_nodeps'):
             # add all platform steps
-            for pltName in PlatformInfo.all_running_platforms:
+            for dep in steps:
-                plt = PlatformInfo.all_platforms[pltName]
+                stepClass = Dependency.all_deps[dep[1]]
-                for tlcName in plt.toolchain_names:
+                if stepClass.dont_skip:
-                    tlc = Dependency.all_deps[tlcName]
+                    add_target_step(dep, self._targets[dep])
-                    src_plt_step = ('source', tlcName)
-                    add_target_step(src_plt_step, self._targets[src_plt_step])
-                    blt_plt_step = ('neutral' if tlc.neutral else pltName, tlcName)
-                    add_target_step(blt_plt_step, self._targets[blt_plt_step])
 
             src_targetDef = ('source', targetDef[1])
             add_target_step(src_targetDef, self._targets[src_targetDef])

kiwixbuild/dependencies/__init__.py

@@ -25,6 +25,7 @@ from . import (
     uuid,
     xapian,
     zim_tools,
+    zim_testing_suite,
     zlib,
     zstd
 )

kiwixbuild/dependencies/all_dependencies.py

@@ -12,7 +12,7 @@ class AllBaseDependencies(Dependency):
     class Builder(NoopBuilder):
         @classmethod
         def get_dependencies(cls, platformInfo, allDeps):
-            base_deps = ['zlib', 'lzma', 'zstd', 'xapian-core', 'pugixml', 'libcurl', 'icu4c', 'mustache', 'libmicrohttpd']
+            base_deps = ['zlib', 'lzma', 'zstd', 'xapian-core', 'pugixml', 'libcurl', 'icu4c', 'mustache', 'libmicrohttpd', 'zim-testing-suite']
             # zimtools
             # We do not build zimtools at all on "android" and "iOS"
             if platformInfo.build not in ('android', 'iOS'):

kiwixbuild/dependencies/android_ndk.py

@@ -6,6 +6,7 @@ from kiwixbuild.utils import Remotefile, add_execution_right, run_command
 pj = os.path.join
 
 class android_ndk(Dependency):
+    dont_skip = True
     neutral = False
     name = 'android-ndk'
     gccver = '4.9.x'

kiwixbuild/dependencies/android_sdk.py

@@ -7,6 +7,7 @@ from kiwixbuild.utils import Remotefile, run_command
 pj = os.path.join
 
 class android_sdk(Dependency):
+    dont_skip = True
     neutral = True
     name = 'android-sdk'
 

kiwixbuild/dependencies/armhf.py

@@ -2,6 +2,7 @@ from .base import Dependency, ReleaseDownload, NoopBuilder
 from kiwixbuild.utils import Remotefile
 
 class armhf_toolchain(Dependency):
+    dont_skip = True
     neutral = True
     name = 'armhf'
 

kiwixbuild/dependencies/base.py

@@ -5,7 +5,7 @@ import time
 
 from kiwixbuild.utils import pj, Context, SkipCommand, WarningMessage, extract_archive, Defaultdict, StopBuild, run_command, colorize
 from kiwixbuild.versions import main_project_versions, base_deps_versions
-from kiwixbuild._global import neutralEnv, option
+from kiwixbuild._global import neutralEnv, option, get_target_step
 
 SCRIPT_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 
@@ -23,6 +23,7 @@ class Dependency(metaclass=_MetaDependency):
     all_deps = {}
     force_build = False
     force_native_build = False
+    dont_skip = False
 
     @classmethod
     def version(cls):
@@ -50,6 +51,10 @@ class Source:
     def name(self):
         return self.target.name
 
+    @property
+    def full_name(self):
+        return self.target.full_name()
+
     @property
     def source_dir(self):
         return self.target.full_name()
@@ -109,7 +114,7 @@ class ReleaseDownload(Source):
         return pj(neutralEnv('source_dir'), self.source_dir)
 
     def _download(self, context):
-        context.try_skip(neutralEnv('archive_dir'), self.name)
+        context.try_skip(neutralEnv('archive_dir'), self.full_name)
         neutralEnv('download')(self.archive)
 
     def _extract(self, context):
@@ -292,6 +297,20 @@ class Builder:
         if getattr(self, 'configure_option', ''):
             module['config-opts'] = self.configure_option.split(' ')
 
+    def get_env(self, *, cross_comp_flags, cross_compilers, cross_path):
+        env = self.buildEnv.get_env(cross_comp_flags=cross_comp_flags, cross_compilers=cross_compilers, cross_path=cross_path)
+        for dep in self.get_dependencies(self.buildEnv.platformInfo, False):
+            try:
+                builder = get_target_step(dep, self.buildEnv.platformInfo.name)
+                builder.set_env(env)
+            except KeyError:
+                # Some target may be missing (installed by a package, ...)
+                pass
+        return env
+
+    def set_env(self, env):
+        pass
+
 
 class NoopBuilder(Builder):
     def build(self):
@@ -352,7 +371,7 @@ class MakeBuilder(Builder):
             configure_script=pj(self.source_path, self.configure_script),
             configure_option=self.all_configure_option
         )
-        env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
+        env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
         self.set_configure_env(env)
         run_command(command, self.build_path, context, env=env)
 
@@ -362,7 +381,7 @@ class MakeBuilder(Builder):
             make_target=self.make_target,
             make_option=self.make_option
         )
-        env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
+        env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
         run_command(command, self.build_path, context, env=env)
 
     def _install(self, context):
@@ -371,13 +390,13 @@ class MakeBuilder(Builder):
             make_install_target=self.make_install_target,
             make_option=self.make_option
         )
-        env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
+        env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
         run_command(command, self.build_path, context, env=env)
 
     def _make_dist(self, context):
         context.try_skip(self.build_path)
         command = "make dist"
-        env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
+        env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
         run_command(command, self.build_path, context, env=env)
 
 
@@ -402,7 +421,7 @@ class CMakeBuilder(MakeBuilder):
             source_path=self.source_path,
             cross_option=cross_option
         )
-        env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=False, cross_path=True)
+        env = self.get_env(cross_comp_flags=True, cross_compilers=False, cross_path=True)
         self.set_configure_env(env)
         run_command(command, self.build_path, context, env=env)
 
@@ -437,7 +456,7 @@ class QMakeBuilder(MakeBuilder):
             source_path=self.source_path,
             cross_option=cross_option
         )
-        env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=False, cross_path=True)
+        env = self.get_env(cross_comp_flags=True, cross_compilers=False, cross_path=True)
         self.set_configure_env(env)
         run_command(command, self.build_path, context, env=env)
 
@@ -496,13 +515,13 @@ class MesonBuilder(Builder):
             buildEnv=self.buildEnv,
             cross_option=cross_option
         )
-        env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True)
+        env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True)
         run_command(command, self.source_path, context, env=env)
 
     def _compile(self, context):
         context.try_skip(self.build_path)
         command = "{} -v".format(neutralEnv('ninja_command'))
-        env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True)
+        env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True)
         run_command(command, self.build_path, context, env=env)
 
     def _test(self, context):
@@ -513,18 +532,18 @@ class MesonBuilder(Builder):
            ):
             raise SkipCommand()
         command = "{} --verbose {}".format(neutralEnv('mesontest_command'), self.test_option)
-        env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True)
+        env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True)
         run_command(command, self.build_path, context, env=env)
 
     def _install(self, context):
         context.try_skip(self.build_path)
         command = "{} -v install".format(neutralEnv('ninja_command'))
-        env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True)
+        env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True)
         run_command(command, self.build_path, context, env=env)
 
     def _make_dist(self, context):
         command = "{} -v dist".format(neutralEnv('ninja_command'))
-        env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True)
+        env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True)
         run_command(command, self.build_path, context, env=env)
 
 
@@ -551,5 +570,5 @@ class GradleBuilder(Builder):
         command = command.format(
             gradle_target=self.gradle_target,
             gradle_option=self.gradle_option)
-        env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=True, cross_path=True)
+        env = self.get_env(cross_comp_flags=False, cross_compilers=True, cross_path=True)
         run_command(command, self.build_path, context, env=env)

kiwixbuild/dependencies/libzim.py

@@ -2,7 +2,7 @@ from .base import (
     Dependency,
     GitClone,
     MesonBuilder)
-from kiwixbuild._global import option
+from kiwixbuild._global import option, get_target_step
 
 class Libzim(Dependency):
     name = "libzim"
@@ -14,18 +14,20 @@ class Libzim(Dependency):
 
     class Builder(MesonBuilder):
         test_option = "-t 8"
-        dependencies = ['lzma', 'zstd', 'xapian-core', 'icu4c']
+        dependencies = ['lzma', 'zstd', 'xapian-core', 'icu4c', 'zim-testing-suite']
         strip_option = ''
 
         @property
         def configure_option(self):
             platformInfo = self.buildEnv.platformInfo
+            zim_testing_suite = get_target_step('zim-testing-suite', platformInfo.name)
+            config_options = ['-Dtest_data_dir={}'.format(zim_testing_suite.source_path)]
             if platformInfo.build == 'android':
-                return "-DUSE_BUFFER_HEADER=false"
+                config_options.append("-DUSE_BUFFER_HEADER=false")
             if platformInfo.build == 'iOS':
-                return "-Db_bitcode=true"
+                config_options.append("-Db_bitcode=true")
             if platformInfo.name == 'native_mixed' and option('target') == 'libzim':
-                return "-Dstatic-linkage=true"
+                config_options.append("-Dstatic-linkage=true")
             if platformInfo.name == "flatpak":
-                return "--wrap-mode=nodownload"
+                config_options.append("--wrap-mode=nodownload")
-            return ""
+            return " ".join(config_options)

kiwixbuild/dependencies/zim_testing_suite.py

@@ -0,0 +1,19 @@
+from .base import (
+    Dependency,
+    ReleaseDownload,
+    NoopBuilder
+)
+
+from kiwixbuild.utils import Remotefile
+
+
+class ZimTestingSuite(Dependency):
+    name = "zim-testing-suite"
+    dont_skip = True
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('zim-testing-suite-0.2.tar.gz',
+                             '04a6db258a48a09ebf25fdf5856029f11269190467b46d54e02c26f2236e2b32',
+                             'https://github.com/openzim/zim-testing-suite/releases/download/v0.2/zim-testing-suite-0.2.tar.gz')
+
+    Builder = NoopBuilder

kiwixbuild/versions.py

@@ -39,7 +39,7 @@ release_versions = {
 
 # This is the "version" of the whole base_deps_versions dict.
 # Change this when you change base_deps_versions.
-base_deps_meta_version = '71'
+base_deps_meta_version = '72'
 
 base_deps_versions = {
   'zlib' : '1.2.8',
@@ -60,4 +60,5 @@ base_deps_versions = {
   'qt' : '5.10.1',
   'qtwebengine' : '5.10.1',
   'org.kde' : '5.12',
+  'zim-testing-suite': '0.2',
 }