From 76aa746f845da09f003e74e5d9cb1636330a51f8 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 12 Apr 2021 18:09:52 +0200 Subject: [PATCH 1/6] Add the `dont_skip` attribute on dependency. Instead of explicitly add the target associated to the toolchain if we use `build_nodeps` option let add an attribute base ourself on it to know if we need to add it or not. This way, we may have other dependency we must not skip. --- kiwixbuild/builder.py | 12 ++++-------- kiwixbuild/dependencies/android_ndk.py | 1 + kiwixbuild/dependencies/android_sdk.py | 1 + kiwixbuild/dependencies/armhf.py | 1 + kiwixbuild/dependencies/base.py | 1 + 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/kiwixbuild/builder.py b/kiwixbuild/builder.py index 304d698..78f8ca3 100644 --- a/kiwixbuild/builder.py +++ b/kiwixbuild/builder.py @@ -34,14 +34,10 @@ class Builder: if option('build_nodeps'): # add all platform steps - for pltName in PlatformInfo.all_running_platforms: - plt = PlatformInfo.all_platforms[pltName] - for tlcName in plt.toolchain_names: - tlc = Dependency.all_deps[tlcName] - src_plt_step = ('source', tlcName) - add_target_step(src_plt_step, self._targets[src_plt_step]) - blt_plt_step = ('neutral' if tlc.neutral else pltName, tlcName) - add_target_step(blt_plt_step, self._targets[blt_plt_step]) + for dep in steps: + stepClass = Dependency.all_deps[dep[1]] + if stepClass.dont_skip: + add_target_step(dep, self._targets[dep]) src_targetDef = ('source', targetDef[1]) add_target_step(src_targetDef, self._targets[src_targetDef]) diff --git a/kiwixbuild/dependencies/android_ndk.py b/kiwixbuild/dependencies/android_ndk.py index 57e0792..fb38a03 100644 --- a/kiwixbuild/dependencies/android_ndk.py +++ b/kiwixbuild/dependencies/android_ndk.py @@ -6,6 +6,7 @@ from kiwixbuild.utils import Remotefile, add_execution_right, run_command pj = os.path.join class android_ndk(Dependency): + dont_skip = True neutral = False name = 'android-ndk' gccver = '4.9.x' diff --git a/kiwixbuild/dependencies/android_sdk.py b/kiwixbuild/dependencies/android_sdk.py index 39c1a9e..7319a59 100644 --- a/kiwixbuild/dependencies/android_sdk.py +++ b/kiwixbuild/dependencies/android_sdk.py @@ -7,6 +7,7 @@ from kiwixbuild.utils import Remotefile, run_command pj = os.path.join class android_sdk(Dependency): + dont_skip = True neutral = True name = 'android-sdk' diff --git a/kiwixbuild/dependencies/armhf.py b/kiwixbuild/dependencies/armhf.py index b79735e..35adca3 100644 --- a/kiwixbuild/dependencies/armhf.py +++ b/kiwixbuild/dependencies/armhf.py @@ -2,6 +2,7 @@ from .base import Dependency, ReleaseDownload, NoopBuilder from kiwixbuild.utils import Remotefile class armhf_toolchain(Dependency): + dont_skip = True neutral = True name = 'armhf' diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index 3ed677d..71fc967 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -23,6 +23,7 @@ class Dependency(metaclass=_MetaDependency): all_deps = {} force_build = False force_native_build = False + dont_skip = False @classmethod def version(cls): From b286be1207ca24b69c334650e4c0364d92c3e2ae Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 12 Apr 2021 18:11:05 +0200 Subject: [PATCH 2/6] Introduce new method `get_env` to create the env. --- kiwixbuild/dependencies/base.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index 71fc967..e584f80 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -293,6 +293,9 @@ class Builder: if getattr(self, 'configure_option', ''): module['config-opts'] = self.configure_option.split(' ') + def get_env(self, *, cross_comp_flags, cross_compilers, cross_path): + return self.buildEnv.get_env(cross_comp_flags=cross_comp_flags, cross_compilers=cross_compilers, cross_path=cross_path) + class NoopBuilder(Builder): def build(self): @@ -353,7 +356,7 @@ class MakeBuilder(Builder): configure_script=pj(self.source_path, self.configure_script), configure_option=self.all_configure_option ) - env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) + env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) self.set_configure_env(env) run_command(command, self.build_path, context, env=env) @@ -363,7 +366,7 @@ class MakeBuilder(Builder): make_target=self.make_target, make_option=self.make_option ) - env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) + env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) run_command(command, self.build_path, context, env=env) def _install(self, context): @@ -372,13 +375,13 @@ class MakeBuilder(Builder): make_install_target=self.make_install_target, make_option=self.make_option ) - env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) + env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) run_command(command, self.build_path, context, env=env) def _make_dist(self, context): context.try_skip(self.build_path) command = "make dist" - env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) + env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) run_command(command, self.build_path, context, env=env) @@ -403,7 +406,7 @@ class CMakeBuilder(MakeBuilder): source_path=self.source_path, cross_option=cross_option ) - env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=False, cross_path=True) + env = self.get_env(cross_comp_flags=True, cross_compilers=False, cross_path=True) self.set_configure_env(env) run_command(command, self.build_path, context, env=env) @@ -438,7 +441,7 @@ class QMakeBuilder(MakeBuilder): source_path=self.source_path, cross_option=cross_option ) - env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=False, cross_path=True) + env = self.get_env(cross_comp_flags=True, cross_compilers=False, cross_path=True) self.set_configure_env(env) run_command(command, self.build_path, context, env=env) @@ -497,13 +500,13 @@ class MesonBuilder(Builder): buildEnv=self.buildEnv, cross_option=cross_option ) - env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) + env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) run_command(command, self.source_path, context, env=env) def _compile(self, context): context.try_skip(self.build_path) command = "{} -v".format(neutralEnv('ninja_command')) - env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) + env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) run_command(command, self.build_path, context, env=env) def _test(self, context): @@ -514,18 +517,18 @@ class MesonBuilder(Builder): ): raise SkipCommand() command = "{} --verbose {}".format(neutralEnv('mesontest_command'), self.test_option) - env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) + env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) run_command(command, self.build_path, context, env=env) def _install(self, context): context.try_skip(self.build_path) command = "{} -v install".format(neutralEnv('ninja_command')) - env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) + env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) run_command(command, self.build_path, context, env=env) def _make_dist(self, context): command = "{} -v dist".format(neutralEnv('ninja_command')) - env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) + env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) run_command(command, self.build_path, context, env=env) @@ -552,5 +555,5 @@ class GradleBuilder(Builder): command = command.format( gradle_target=self.gradle_target, gradle_option=self.gradle_option) - env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=True, cross_path=True) + env = self.get_env(cross_comp_flags=False, cross_compilers=True, cross_path=True) run_command(command, self.build_path, context, env=env) From dca64eb5d52fe6db3a1c47a3eb450ae0ed41617e Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 12 Apr 2021 18:14:23 +0200 Subject: [PATCH 3/6] Allow builder to set a build environment for projects using it. --- kiwixbuild/dependencies/base.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index e584f80..d7ba87c 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -5,7 +5,7 @@ import time from kiwixbuild.utils import pj, Context, SkipCommand, WarningMessage, extract_archive, Defaultdict, StopBuild, run_command, colorize from kiwixbuild.versions import main_project_versions, base_deps_versions -from kiwixbuild._global import neutralEnv, option +from kiwixbuild._global import neutralEnv, option, get_target_step SCRIPT_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) @@ -294,7 +294,18 @@ class Builder: module['config-opts'] = self.configure_option.split(' ') def get_env(self, *, cross_comp_flags, cross_compilers, cross_path): - return self.buildEnv.get_env(cross_comp_flags=cross_comp_flags, cross_compilers=cross_compilers, cross_path=cross_path) + env = self.buildEnv.get_env(cross_comp_flags=cross_comp_flags, cross_compilers=cross_compilers, cross_path=cross_path) + for dep in self.get_dependencies(self.buildEnv.platformInfo, False): + try: + builder = get_target_step(dep, self.buildEnv.platformInfo.name) + builder.set_env(env) + except KeyError: + # Some target may be missing (installed by a package, ...) + pass + return env + + def set_env(self, env): + pass class NoopBuilder(Builder): From 11a7ec4020138a5c0a295f4f897a0d576a02d199 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 28 Apr 2021 16:20:32 +0200 Subject: [PATCH 4/6] Use the full_name as for the skip file marker. This way we are sure to download the last archive all the time. --- kiwixbuild/dependencies/base.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index d7ba87c..a4702ca 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -51,6 +51,10 @@ class Source: def name(self): return self.target.name + @property + def full_name(self): + return self.target.full_name() + @property def source_dir(self): return self.target.full_name() @@ -110,7 +114,7 @@ class ReleaseDownload(Source): return pj(neutralEnv('source_dir'), self.source_dir) def _download(self, context): - context.try_skip(neutralEnv('archive_dir'), self.name) + context.try_skip(neutralEnv('archive_dir'), self.full_name) neutralEnv('download')(self.archive) def _extract(self, context): From 2d3c5ed4c7d62e20c674fc7e46dd7c3cada5b704 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 12 Apr 2021 18:15:51 +0200 Subject: [PATCH 5/6] Add zim-testing-suite dependency. --- kiwixbuild/dependencies/__init__.py | 1 + kiwixbuild/dependencies/all_dependencies.py | 2 +- kiwixbuild/dependencies/libzim.py | 2 +- kiwixbuild/dependencies/zim_testing_suite.py | 19 +++++++++++++++++++ kiwixbuild/versions.py | 3 ++- 5 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 kiwixbuild/dependencies/zim_testing_suite.py diff --git a/kiwixbuild/dependencies/__init__.py b/kiwixbuild/dependencies/__init__.py index 3558c53..f6ea52d 100644 --- a/kiwixbuild/dependencies/__init__.py +++ b/kiwixbuild/dependencies/__init__.py @@ -25,6 +25,7 @@ from . import ( uuid, xapian, zim_tools, + zim_testing_suite, zlib, zstd ) diff --git a/kiwixbuild/dependencies/all_dependencies.py b/kiwixbuild/dependencies/all_dependencies.py index 45dacef..feabe5a 100644 --- a/kiwixbuild/dependencies/all_dependencies.py +++ b/kiwixbuild/dependencies/all_dependencies.py @@ -12,7 +12,7 @@ class AllBaseDependencies(Dependency): class Builder(NoopBuilder): @classmethod def get_dependencies(cls, platformInfo, allDeps): - base_deps = ['zlib', 'lzma', 'zstd', 'xapian-core', 'pugixml', 'libcurl', 'icu4c', 'mustache', 'libmicrohttpd'] + base_deps = ['zlib', 'lzma', 'zstd', 'xapian-core', 'pugixml', 'libcurl', 'icu4c', 'mustache', 'libmicrohttpd', 'zim-testing-suite'] # zimtools # We do not build zimtools at all on "android" and "iOS" if platformInfo.build not in ('android', 'iOS'): diff --git a/kiwixbuild/dependencies/libzim.py b/kiwixbuild/dependencies/libzim.py index 8640d2f..f0b8904 100644 --- a/kiwixbuild/dependencies/libzim.py +++ b/kiwixbuild/dependencies/libzim.py @@ -14,7 +14,7 @@ class Libzim(Dependency): class Builder(MesonBuilder): test_option = "-t 8" - dependencies = ['lzma', 'zstd', 'xapian-core', 'icu4c'] + dependencies = ['lzma', 'zstd', 'xapian-core', 'icu4c', 'zim-testing-suite'] strip_option = '' @property diff --git a/kiwixbuild/dependencies/zim_testing_suite.py b/kiwixbuild/dependencies/zim_testing_suite.py new file mode 100644 index 0000000..7742e56 --- /dev/null +++ b/kiwixbuild/dependencies/zim_testing_suite.py @@ -0,0 +1,19 @@ +from .base import ( + Dependency, + ReleaseDownload, + NoopBuilder +) + +from kiwixbuild.utils import Remotefile + + +class ZimTestingSuite(Dependency): + name = "zim-testing-suite" + dont_skip = True + + class Source(ReleaseDownload): + archive = Remotefile('zim-testing-suite-0.2.tar.gz', + '04a6db258a48a09ebf25fdf5856029f11269190467b46d54e02c26f2236e2b32', + 'https://github.com/openzim/zim-testing-suite/releases/download/v0.2/zim-testing-suite-0.2.tar.gz') + + Builder = NoopBuilder diff --git a/kiwixbuild/versions.py b/kiwixbuild/versions.py index 038fb6e..23cd473 100644 --- a/kiwixbuild/versions.py +++ b/kiwixbuild/versions.py @@ -39,7 +39,7 @@ release_versions = { # This is the "version" of the whole base_deps_versions dict. # Change this when you change base_deps_versions. -base_deps_meta_version = '71' +base_deps_meta_version = '72' base_deps_versions = { 'zlib' : '1.2.8', @@ -60,4 +60,5 @@ base_deps_versions = { 'qt' : '5.10.1', 'qtwebengine' : '5.10.1', 'org.kde' : '5.12', + 'zim-testing-suite': '0.2', } From 9d905682040451b3215823ee4a96efde32a3c448 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 28 Apr 2021 16:21:04 +0200 Subject: [PATCH 6/6] Make libzim use the test data dire downloaded with zim-testing-tool. --- kiwixbuild/dependencies/libzim.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/kiwixbuild/dependencies/libzim.py b/kiwixbuild/dependencies/libzim.py index f0b8904..58b166c 100644 --- a/kiwixbuild/dependencies/libzim.py +++ b/kiwixbuild/dependencies/libzim.py @@ -2,7 +2,7 @@ from .base import ( Dependency, GitClone, MesonBuilder) -from kiwixbuild._global import option +from kiwixbuild._global import option, get_target_step class Libzim(Dependency): name = "libzim" @@ -20,12 +20,14 @@ class Libzim(Dependency): @property def configure_option(self): platformInfo = self.buildEnv.platformInfo + zim_testing_suite = get_target_step('zim-testing-suite', platformInfo.name) + config_options = ['-Dtest_data_dir={}'.format(zim_testing_suite.source_path)] if platformInfo.build == 'android': - return "-DUSE_BUFFER_HEADER=false" + config_options.append("-DUSE_BUFFER_HEADER=false") if platformInfo.build == 'iOS': - return "-Db_bitcode=true" + config_options.append("-Db_bitcode=true") if platformInfo.name == 'native_mixed' and option('target') == 'libzim': - return "-Dstatic-linkage=true" + config_options.append("-Dstatic-linkage=true") if platformInfo.name == "flatpak": - return "--wrap-mode=nodownload" - return "" + config_options.append("--wrap-mode=nodownload") + return " ".join(config_options)