From fb07b5881264ccbd78f8a28a9f8d5382e20d9c06 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 6 Apr 2020 17:49:35 +0200 Subject: [PATCH] Fix recompilation of the CI of meson project. We were assuming that meson project correspond to our projects and so we were always building them, even if they were already compiled. (This way, a simple `kiwix-build` is enough to recompile the WIP code of our project). However, on the CI, we do not archive the source code/build directory in the base deps archive. So when we try to compile, the compile step of meson projects fails because the source are not here. We have a small workaround for pugixml but as zstd is also meson, it is time to do something correct. By default, all projects now try to skip if a build is already present. Our projects are marked as `force_build` and so, they do not try to skip. --- .github/scripts/common.py | 6 ------ kiwixbuild/dependencies/base.py | 7 +++++++ kiwixbuild/dependencies/kiwix_desktop.py | 1 + kiwixbuild/dependencies/kiwix_lib.py | 2 ++ kiwixbuild/dependencies/kiwix_tools.py | 1 + kiwixbuild/dependencies/libzim.py | 1 + kiwixbuild/dependencies/zim_tools.py | 1 + kiwixbuild/dependencies/zimwriterfs.py | 1 + kiwixbuild/utils.py | 3 +++ travis/compile_all.py | 4 ---- 10 files changed, 17 insertions(+), 10 deletions(-) diff --git a/.github/scripts/common.py b/.github/scripts/common.py index b47fe5a..ec07ebe 100644 --- a/.github/scripts/common.py +++ b/.github/scripts/common.py @@ -148,12 +148,6 @@ def make_deps_archive(target=None, name=None, full=False): files_to_archive += (HOME / "BUILD_native_static").glob("*/.*_ok") files_to_archive += HOME.glob("BUILD_android*/**/.*_ok") files_to_archive += SOURCE_DIR.glob("*/.*_ok") - files_to_archive += [ - SOURCE_DIR / "pugixml-{}".format(base_deps_versions["pugixml"]) - ] - files_to_archive += HOME.glob( - "BUILD_*/pugixml-{}".format(base_deps_versions["pugixml"]) - ) if PLATFORM_TARGET.startswith("armhf"): files_to_archive += (SOURCE_DIR / "armhf").glob("*") toolchains_subdirs = HOME.glob("BUILD_*/TOOLCHAINS/*/*") diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index e6d0f94..d43a90d 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -21,6 +21,7 @@ class _MetaDependency(type): class Dependency(metaclass=_MetaDependency): all_deps = {} + force_build = False force_native_build = False @classmethod @@ -240,6 +241,8 @@ class Builder: print(" {} {} : ".format(name, self.name), end="", flush=True) log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name)) context = Context(name, log, self.target.force_native_build) + if self.target.force_build: + context.no_skip = True try: start_time = time.time() ret = function(*args, context=context) @@ -493,11 +496,13 @@ class MesonBuilder(Builder): run_command(command, self.source_path, context, env=env) def _compile(self, context): + context.try_skip(self.build_path) command = "{} -v".format(neutralEnv('ninja_command')) env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) run_command(command, self.build_path, context, env=env) def _test(self, context): + context.try_skip(self.build_path) if ( self.buildEnv.platformInfo.build == 'android' or (self.buildEnv.platformInfo.build != 'native' and not self.buildEnv.platformInfo.static) @@ -508,6 +513,7 @@ class MesonBuilder(Builder): run_command(command, self.build_path, context, env=env) def _install(self, context): + context.try_skip(self.build_path) command = "{} -v install".format(neutralEnv('ninja_command')) env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) run_command(command, self.build_path, context, env=env) @@ -536,6 +542,7 @@ class GradleBuilder(Builder): shutil.copytree(self.source_path, self.build_path) def _compile(self, context): + context.try_skip(self.build_path) command = "./gradlew {gradle_target} {gradle_option}" command = command.format( gradle_target=self.gradle_target, diff --git a/kiwixbuild/dependencies/kiwix_desktop.py b/kiwixbuild/dependencies/kiwix_desktop.py index 1b62763..73d8ede 100644 --- a/kiwixbuild/dependencies/kiwix_desktop.py +++ b/kiwixbuild/dependencies/kiwix_desktop.py @@ -5,6 +5,7 @@ from .base import ( class KiwixDesktop(Dependency): name = "kiwix-desktop" + force_build = True class Source(GitClone): git_remote = "https://github.com/kiwix/kiwix-desktop.git" diff --git a/kiwixbuild/dependencies/kiwix_lib.py b/kiwixbuild/dependencies/kiwix_lib.py index 0a66c12..3a64f96 100644 --- a/kiwixbuild/dependencies/kiwix_lib.py +++ b/kiwixbuild/dependencies/kiwix_lib.py @@ -10,6 +10,7 @@ from kiwixbuild._global import option, get_target_step, neutralEnv class Kiwixlib(Dependency): name = "kiwix-lib" + force_build = True class Source(GitClone): git_remote = "https://github.com/kiwix/kiwix-lib.git" @@ -39,6 +40,7 @@ class Kiwixlib(Dependency): class KiwixlibApp(Dependency): name = "kiwix-lib-app" + force_build = True class Source(Kiwixlib.Source): name = "kiwix-lib" diff --git a/kiwixbuild/dependencies/kiwix_tools.py b/kiwixbuild/dependencies/kiwix_tools.py index 58bf12d..2f1500f 100644 --- a/kiwixbuild/dependencies/kiwix_tools.py +++ b/kiwixbuild/dependencies/kiwix_tools.py @@ -5,6 +5,7 @@ from .base import ( class KiwixTools(Dependency): name = "kiwix-tools" + force_build = True class Source(GitClone): git_remote = "https://github.com/kiwix/kiwix-tools.git" diff --git a/kiwixbuild/dependencies/libzim.py b/kiwixbuild/dependencies/libzim.py index 946ef61..cba24ad 100644 --- a/kiwixbuild/dependencies/libzim.py +++ b/kiwixbuild/dependencies/libzim.py @@ -6,6 +6,7 @@ from kiwixbuild._global import option class Libzim(Dependency): name = "libzim" + force_build = True class Source(GitClone): git_remote = "https://github.com/openzim/libzim.git" diff --git a/kiwixbuild/dependencies/zim_tools.py b/kiwixbuild/dependencies/zim_tools.py index 46332e0..7cb7d91 100644 --- a/kiwixbuild/dependencies/zim_tools.py +++ b/kiwixbuild/dependencies/zim_tools.py @@ -5,6 +5,7 @@ from .base import ( class ZimTools(Dependency): name = "zim-tools" + force_build = True class Source(GitClone): git_remote = "https://github.com/openzim/zim-tools.git" diff --git a/kiwixbuild/dependencies/zimwriterfs.py b/kiwixbuild/dependencies/zimwriterfs.py index 55f73a5..cf3898a 100644 --- a/kiwixbuild/dependencies/zimwriterfs.py +++ b/kiwixbuild/dependencies/zimwriterfs.py @@ -5,6 +5,7 @@ from .base import ( class Zimwriterfs(Dependency): name = "zimwriterfs" + force_build = True class Source(GitClone): git_remote = "https://github.com/openzim/zimwriterfs.git" diff --git a/kiwixbuild/utils.py b/kiwixbuild/utils.py index 5070d1e..57ec8c2 100644 --- a/kiwixbuild/utils.py +++ b/kiwixbuild/utils.py @@ -187,8 +187,11 @@ class Context: self.log_file = log_file self.force_native_build = force_native_build self.autoskip_file = None + self.no_skip = False def try_skip(self, path, extra_name=""): + if self.no_skip: + return if extra_name: extra_name = "_{}".format(extra_name) self.autoskip_file = pj(path, ".{}{}_ok".format(self.command_name, extra_name)) diff --git a/travis/compile_all.py b/travis/compile_all.py index 2e7e44c..6264cb9 100755 --- a/travis/compile_all.py +++ b/travis/compile_all.py @@ -236,10 +236,6 @@ def make_deps_archive(target=None, name=None, full=False): files_to_archive += (HOME/"BUILD_native_static").glob('*/.*_ok') files_to_archive += HOME.glob('BUILD_android*/**/.*_ok') files_to_archive += SOURCE_DIR.glob('*/.*_ok') - files_to_archive += [SOURCE_DIR/'pugixml-{}'.format( - base_deps_versions['pugixml'])] - files_to_archive += HOME.glob('BUILD_*/pugixml-{}'.format( - base_deps_versions['pugixml'])) if PLATFORM.startswith('armhf'): files_to_archive += (SOURCE_DIR/'armhf').glob('*') toolchains_subdirs = HOME.glob('**/TOOLCHAINS/*/*')