diff --git a/kiwixbuild/builder.py b/kiwixbuild/builder.py index 304d698..78f8ca3 100644 --- a/kiwixbuild/builder.py +++ b/kiwixbuild/builder.py @@ -34,14 +34,10 @@ class Builder: if option('build_nodeps'): # add all platform steps - for pltName in PlatformInfo.all_running_platforms: - plt = PlatformInfo.all_platforms[pltName] - for tlcName in plt.toolchain_names: - tlc = Dependency.all_deps[tlcName] - src_plt_step = ('source', tlcName) - add_target_step(src_plt_step, self._targets[src_plt_step]) - blt_plt_step = ('neutral' if tlc.neutral else pltName, tlcName) - add_target_step(blt_plt_step, self._targets[blt_plt_step]) + for dep in steps: + stepClass = Dependency.all_deps[dep[1]] + if stepClass.dont_skip: + add_target_step(dep, self._targets[dep]) src_targetDef = ('source', targetDef[1]) add_target_step(src_targetDef, self._targets[src_targetDef]) diff --git a/kiwixbuild/dependencies/__init__.py b/kiwixbuild/dependencies/__init__.py index 3558c53..f6ea52d 100644 --- a/kiwixbuild/dependencies/__init__.py +++ b/kiwixbuild/dependencies/__init__.py @@ -25,6 +25,7 @@ from . import ( uuid, xapian, zim_tools, + zim_testing_suite, zlib, zstd ) diff --git a/kiwixbuild/dependencies/all_dependencies.py b/kiwixbuild/dependencies/all_dependencies.py index 45dacef..feabe5a 100644 --- a/kiwixbuild/dependencies/all_dependencies.py +++ b/kiwixbuild/dependencies/all_dependencies.py @@ -12,7 +12,7 @@ class AllBaseDependencies(Dependency): class Builder(NoopBuilder): @classmethod def get_dependencies(cls, platformInfo, allDeps): - base_deps = ['zlib', 'lzma', 'zstd', 'xapian-core', 'pugixml', 'libcurl', 'icu4c', 'mustache', 'libmicrohttpd'] + base_deps = ['zlib', 'lzma', 'zstd', 'xapian-core', 'pugixml', 'libcurl', 'icu4c', 'mustache', 'libmicrohttpd', 'zim-testing-suite'] # zimtools # We do not build zimtools at all on "android" and "iOS" if platformInfo.build not in ('android', 'iOS'): diff --git a/kiwixbuild/dependencies/android_ndk.py b/kiwixbuild/dependencies/android_ndk.py index 57e0792..fb38a03 100644 --- a/kiwixbuild/dependencies/android_ndk.py +++ b/kiwixbuild/dependencies/android_ndk.py @@ -6,6 +6,7 @@ from kiwixbuild.utils import Remotefile, add_execution_right, run_command pj = os.path.join class android_ndk(Dependency): + dont_skip = True neutral = False name = 'android-ndk' gccver = '4.9.x' diff --git a/kiwixbuild/dependencies/android_sdk.py b/kiwixbuild/dependencies/android_sdk.py index 39c1a9e..7319a59 100644 --- a/kiwixbuild/dependencies/android_sdk.py +++ b/kiwixbuild/dependencies/android_sdk.py @@ -7,6 +7,7 @@ from kiwixbuild.utils import Remotefile, run_command pj = os.path.join class android_sdk(Dependency): + dont_skip = True neutral = True name = 'android-sdk' diff --git a/kiwixbuild/dependencies/armhf.py b/kiwixbuild/dependencies/armhf.py index b79735e..35adca3 100644 --- a/kiwixbuild/dependencies/armhf.py +++ b/kiwixbuild/dependencies/armhf.py @@ -2,6 +2,7 @@ from .base import Dependency, ReleaseDownload, NoopBuilder from kiwixbuild.utils import Remotefile class armhf_toolchain(Dependency): + dont_skip = True neutral = True name = 'armhf' diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index 3ed677d..a4702ca 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -5,7 +5,7 @@ import time from kiwixbuild.utils import pj, Context, SkipCommand, WarningMessage, extract_archive, Defaultdict, StopBuild, run_command, colorize from kiwixbuild.versions import main_project_versions, base_deps_versions -from kiwixbuild._global import neutralEnv, option +from kiwixbuild._global import neutralEnv, option, get_target_step SCRIPT_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) @@ -23,6 +23,7 @@ class Dependency(metaclass=_MetaDependency): all_deps = {} force_build = False force_native_build = False + dont_skip = False @classmethod def version(cls): @@ -50,6 +51,10 @@ class Source: def name(self): return self.target.name + @property + def full_name(self): + return self.target.full_name() + @property def source_dir(self): return self.target.full_name() @@ -109,7 +114,7 @@ class ReleaseDownload(Source): return pj(neutralEnv('source_dir'), self.source_dir) def _download(self, context): - context.try_skip(neutralEnv('archive_dir'), self.name) + context.try_skip(neutralEnv('archive_dir'), self.full_name) neutralEnv('download')(self.archive) def _extract(self, context): @@ -292,6 +297,20 @@ class Builder: if getattr(self, 'configure_option', ''): module['config-opts'] = self.configure_option.split(' ') + def get_env(self, *, cross_comp_flags, cross_compilers, cross_path): + env = self.buildEnv.get_env(cross_comp_flags=cross_comp_flags, cross_compilers=cross_compilers, cross_path=cross_path) + for dep in self.get_dependencies(self.buildEnv.platformInfo, False): + try: + builder = get_target_step(dep, self.buildEnv.platformInfo.name) + builder.set_env(env) + except KeyError: + # Some target may be missing (installed by a package, ...) + pass + return env + + def set_env(self, env): + pass + class NoopBuilder(Builder): def build(self): @@ -352,7 +371,7 @@ class MakeBuilder(Builder): configure_script=pj(self.source_path, self.configure_script), configure_option=self.all_configure_option ) - env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) + env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) self.set_configure_env(env) run_command(command, self.build_path, context, env=env) @@ -362,7 +381,7 @@ class MakeBuilder(Builder): make_target=self.make_target, make_option=self.make_option ) - env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) + env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) run_command(command, self.build_path, context, env=env) def _install(self, context): @@ -371,13 +390,13 @@ class MakeBuilder(Builder): make_install_target=self.make_install_target, make_option=self.make_option ) - env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) + env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) run_command(command, self.build_path, context, env=env) def _make_dist(self, context): context.try_skip(self.build_path) command = "make dist" - env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) + env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) run_command(command, self.build_path, context, env=env) @@ -402,7 +421,7 @@ class CMakeBuilder(MakeBuilder): source_path=self.source_path, cross_option=cross_option ) - env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=False, cross_path=True) + env = self.get_env(cross_comp_flags=True, cross_compilers=False, cross_path=True) self.set_configure_env(env) run_command(command, self.build_path, context, env=env) @@ -437,7 +456,7 @@ class QMakeBuilder(MakeBuilder): source_path=self.source_path, cross_option=cross_option ) - env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=False, cross_path=True) + env = self.get_env(cross_comp_flags=True, cross_compilers=False, cross_path=True) self.set_configure_env(env) run_command(command, self.build_path, context, env=env) @@ -496,13 +515,13 @@ class MesonBuilder(Builder): buildEnv=self.buildEnv, cross_option=cross_option ) - env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) + env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) run_command(command, self.source_path, context, env=env) def _compile(self, context): context.try_skip(self.build_path) command = "{} -v".format(neutralEnv('ninja_command')) - env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) + env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) run_command(command, self.build_path, context, env=env) def _test(self, context): @@ -513,18 +532,18 @@ class MesonBuilder(Builder): ): raise SkipCommand() command = "{} --verbose {}".format(neutralEnv('mesontest_command'), self.test_option) - env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) + env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) run_command(command, self.build_path, context, env=env) def _install(self, context): context.try_skip(self.build_path) command = "{} -v install".format(neutralEnv('ninja_command')) - env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) + env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) run_command(command, self.build_path, context, env=env) def _make_dist(self, context): command = "{} -v dist".format(neutralEnv('ninja_command')) - env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) + env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) run_command(command, self.build_path, context, env=env) @@ -551,5 +570,5 @@ class GradleBuilder(Builder): command = command.format( gradle_target=self.gradle_target, gradle_option=self.gradle_option) - env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=True, cross_path=True) + env = self.get_env(cross_comp_flags=False, cross_compilers=True, cross_path=True) run_command(command, self.build_path, context, env=env) diff --git a/kiwixbuild/dependencies/libzim.py b/kiwixbuild/dependencies/libzim.py index 8640d2f..58b166c 100644 --- a/kiwixbuild/dependencies/libzim.py +++ b/kiwixbuild/dependencies/libzim.py @@ -2,7 +2,7 @@ from .base import ( Dependency, GitClone, MesonBuilder) -from kiwixbuild._global import option +from kiwixbuild._global import option, get_target_step class Libzim(Dependency): name = "libzim" @@ -14,18 +14,20 @@ class Libzim(Dependency): class Builder(MesonBuilder): test_option = "-t 8" - dependencies = ['lzma', 'zstd', 'xapian-core', 'icu4c'] + dependencies = ['lzma', 'zstd', 'xapian-core', 'icu4c', 'zim-testing-suite'] strip_option = '' @property def configure_option(self): platformInfo = self.buildEnv.platformInfo + zim_testing_suite = get_target_step('zim-testing-suite', platformInfo.name) + config_options = ['-Dtest_data_dir={}'.format(zim_testing_suite.source_path)] if platformInfo.build == 'android': - return "-DUSE_BUFFER_HEADER=false" + config_options.append("-DUSE_BUFFER_HEADER=false") if platformInfo.build == 'iOS': - return "-Db_bitcode=true" + config_options.append("-Db_bitcode=true") if platformInfo.name == 'native_mixed' and option('target') == 'libzim': - return "-Dstatic-linkage=true" + config_options.append("-Dstatic-linkage=true") if platformInfo.name == "flatpak": - return "--wrap-mode=nodownload" - return "" + config_options.append("--wrap-mode=nodownload") + return " ".join(config_options) diff --git a/kiwixbuild/dependencies/zim_testing_suite.py b/kiwixbuild/dependencies/zim_testing_suite.py new file mode 100644 index 0000000..7742e56 --- /dev/null +++ b/kiwixbuild/dependencies/zim_testing_suite.py @@ -0,0 +1,19 @@ +from .base import ( + Dependency, + ReleaseDownload, + NoopBuilder +) + +from kiwixbuild.utils import Remotefile + + +class ZimTestingSuite(Dependency): + name = "zim-testing-suite" + dont_skip = True + + class Source(ReleaseDownload): + archive = Remotefile('zim-testing-suite-0.2.tar.gz', + '04a6db258a48a09ebf25fdf5856029f11269190467b46d54e02c26f2236e2b32', + 'https://github.com/openzim/zim-testing-suite/releases/download/v0.2/zim-testing-suite-0.2.tar.gz') + + Builder = NoopBuilder diff --git a/kiwixbuild/versions.py b/kiwixbuild/versions.py index 038fb6e..23cd473 100644 --- a/kiwixbuild/versions.py +++ b/kiwixbuild/versions.py @@ -39,7 +39,7 @@ release_versions = { # This is the "version" of the whole base_deps_versions dict. # Change this when you change base_deps_versions. -base_deps_meta_version = '71' +base_deps_meta_version = '72' base_deps_versions = { 'zlib' : '1.2.8', @@ -60,4 +60,5 @@ base_deps_versions = { 'qt' : '5.10.1', 'qtwebengine' : '5.10.1', 'org.kde' : '5.12', + 'zim-testing-suite': '0.2', }