import subprocess import os import shutil import time import platform from kiwixbuild.utils import ( pj, Context, SkipCommand, WarningMessage, extract_archive, StopBuild, run_command, colorize, copy_tree, ) from kiwixbuild.versions import main_project_versions, base_deps_versions from kiwixbuild._global import neutralEnv, option, get_target_step SCRIPT_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) class _MetaDependency(type): def __new__(cls, name, bases, dct): _class = type.__new__(cls, name, bases, dct) if name != "Dependency": dep_name = dct["name"] Dependency.all_deps[dep_name] = _class return _class class Dependency(metaclass=_MetaDependency): all_deps = {} force_build = False force_native_build = False dont_skip = False @classmethod def version(cls): if cls.name in base_deps_versions: return base_deps_versions[cls.name] elif option("make_release"): return main_project_versions.get(cls.name, None) return None @classmethod def full_name(cls): if cls.version(): return "{}-{}".format(cls.name, cls.version()) return cls.name class Source: """Base Class to the real preparator A source preparator must install source in the self.source_dir attribute inside the neutralEnv.source_dir.""" def __init__(self, target): self.target = target @property def name(self): return self.target.name @property def full_name(self): return self.target.full_name() @property def source_dir(self): return self.target.full_name() @property def source_path(self): return pj(neutralEnv("source_dir"), self.source_dir) @property def _log_dir(self): return neutralEnv("log_dir") def _patch(self, context): context.try_skip(self.source_path) for p in self.patches: patch_file_path = pj(SCRIPT_DIR, "patches", p) patch_command = [*neutralEnv("patch_command"), "-p1", "-i", patch_file_path] run_command(patch_command, self.source_path, context) def command(self, name, function, *args): print(" {} {} : ".format(name, self.name), end="", flush=True) log = pj(self._log_dir, "cmd_{}_{}.log".format(name, self.name)) context = Context(name, log, True) try: start_time = time.time() ret = function(*args, context=context) context._finalise() duration = time.time() - start_time print(colorize("OK"), "({:.1f}s)".format(duration)) return ret except WarningMessage as e: print(e) except SkipCommand as e: print(e) except subprocess.CalledProcessError: print(colorize("ERROR")) try: with open(log, "r") as f: print(f.read()) except: pass raise StopBuild() except: print(colorize("ERROR")) raise class NoopSource(Source): def prepare(self): pass class ReleaseDownload(Source): archive_top_dir = None @property def archives(self): return (self.archive,) @property def extract_path(self): return pj(neutralEnv("source_dir"), self.source_dir) def _download(self, context): context.try_skip(neutralEnv("archive_dir"), self.full_name) archive_iter = iter(self.archives) archive = next(archive_iter, None) while archive: try: neutralEnv("download")(archive) except SkipCommand as e: archive = next(archive_iter, None) if not archive: raise e continue archive = next(archive_iter, None) def _extract(self, context): context.try_skip(self.extract_path) if os.path.exists(self.extract_path): shutil.rmtree(self.extract_path) for archive in self.archives: extract_archive( pj(neutralEnv("archive_dir"), archive.name), neutralEnv("source_dir"), topdir=self.archive_top_dir, name=self.source_dir, ) def prepare(self): self.command("download", self._download) self.command("extract", self._extract) if hasattr(self, "patches"): self.command("patch", self._patch) if hasattr(self, "_post_prepare_script"): self.command("post_prepare_script", self._post_prepare_script) class GitClone(Source): base_git_ref = "main" force_full_clone = False @property def release_git_ref(self): return main_project_versions.get(self.name, self.base_git_ref) @property def source_dir(self): if option("make_release"): return "{}_release".format(self.git_dir) else: return self.git_dir @property def git_path(self): return pj(neutralEnv("source_dir"), self.source_dir) @property def git_ref(self): if option("make_release"): return self.release_git_ref else: return self.base_git_ref def _git_init(self, context): if option("fast_clone") and self.force_full_clone == False: command = [ *neutralEnv("git_command"), "clone", "--depth=1", "--branch", self.git_ref, self.git_remote, self.source_dir, ] run_command(command, neutralEnv("source_dir"), context) else: command = [ *neutralEnv("git_command"), "clone", self.git_remote, self.source_dir, ] run_command(command, neutralEnv("source_dir"), context) command = [*neutralEnv("git_command"), "checkout", self.git_ref] run_command(command, self.git_path, context) def _git_update(self, context): command = [*neutralEnv("git_command"), "fetch", "origin", self.git_ref] run_command(command, self.git_path, context) try: command = [ *neutralEnv("git_command"), "merge", "--ff-only", f"origin/{self.git_ref}", ] run_command(command, self.git_path, context) except subprocess.CalledProcessError: raise WarningMessage("Cannot update, please check log for information") def prepare(self): if not os.path.exists(self.git_path): self.command("gitinit", self._git_init) else: self.command("gitupdate", self._git_update) if hasattr(self, "_post_prepare_script"): self.command("post_prepare_script", self._post_prepare_script) class Builder: subsource_dir = None dependencies = [] def __init__(self, target, source, buildEnv): self.target = target self.source = source self.buildEnv = buildEnv @classmethod def get_dependencies(cls, configInfo, allDeps): return cls.dependencies @property def name(self): return self.target.name @property def source_path(self): base_source_path = self.source.source_path if self.subsource_dir: return pj(base_source_path, self.subsource_dir) return base_source_path @property def build_path(self): return pj(self.buildEnv.build_dir, self.target.full_name()) @property def _log_dir(self): return self.buildEnv.log_dir def command(self, name, function, *args): print(" {} {} : ".format(name, self.name), end="", flush=True) log = pj(self._log_dir, "cmd_{}_{}.log".format(name, self.name)) context = Context(name, log, self.target.force_native_build) if self.target.force_build: context.no_skip = True try: start_time = time.time() ret = function(*args, context=context) context._finalise() duration = time.time() - start_time print(colorize("OK"), "({:.1f}s)".format(duration)) return ret except SkipCommand as e: print(e) except WarningMessage as e: print(e) except subprocess.CalledProcessError: print(colorize("ERROR")) try: with open(log, "r") as f: print(f.read()) except: pass raise StopBuild() except: print(colorize("ERROR")) raise def build(self): if hasattr(self, "_pre_build_script"): self.command("pre_build_script", self._pre_build_script) self.command("configure", self._configure) if hasattr(self, "_post_configure_script"): self.command("post_configure_script", self._post_configure_script) self.command("compile", self._compile) if hasattr(self, "_test"): self.command("test", self._test) self.command("install", self._install) if hasattr(self, "_post_build_script"): self.command("post_build_script", self._post_build_script) def make_dist(self): if hasattr(self, "_pre_build_script"): self.command("pre_build_script", self._pre_build_script) self.command("configure", self._configure) self.command("make_dist", self._make_dist) def set_flatpak_buildsystem(self, module): if getattr(self, "flatpak_buildsystem", None): module["buildsystem"] = self.flatpak_buildsystem if getattr(self, "subsource_dir", None): module["subdir"] = self.subsource_dir if getattr(self, "flatpack_build_options", None): module["build-options"] = self.flatpack_build_options if getattr(self, "configure_options", ""): module["config-opts"] = list(self.configure_options) def get_env(self, *, cross_comp_flags, cross_compilers, cross_path): env = self.buildEnv.get_env( cross_comp_flags=cross_comp_flags, cross_compilers=cross_compilers, cross_path=cross_path, ) for dep in self.get_dependencies(self.buildEnv.configInfo, False): try: builder = get_target_step(dep, self.buildEnv.configInfo.name) builder.set_env(env) except KeyError: # Some target may be missing (installed by a package, ...) pass return env def set_env(self, env): pass class NoopBuilder(Builder): def build(self): pass def make_dist(self): pass class TcCopyBuilder(Builder): src_subdir = None @property def build_path(self): return pj(self.buildEnv.toolchain_dir, self.target.full_name()) def build(self): self.command("copy", self._copy) def _copy(self, context): context.try_skip(self.build_path) if self.src_subdir: source_path = pj(self.source_path, self.src_subdir) else: source_path = self.source_path copy_tree(source_path, self.build_path) def make_dist(self): pass class MakeBuilder(Builder): configure_options = [] dynamic_configure_options = ["--enable-shared", "--disable-static"] static_configure_options = ["--enable-static", "--disable-shared"] make_options = ["-j4"] install_options = [] configure_script = "configure" configure_env = { "_format_CFLAGS": "{env[CFLAGS]} -O3", "_format_CXXFLAGS": "{env[CXXFLAGS]} -O3", } make_targets = [] flatpak_buildsystem = None @property def make_install_targets(self): if self.buildEnv.configInfo.build in ("iOS", "wasm"): yield "install" else: yield "install-strip" @property def all_configure_options(self): yield from self.configure_options if self.buildEnv.configInfo.static: yield from self.static_configure_options else: yield from self.dynamic_configure_options if not self.target.force_native_build: yield from self.buildEnv.configInfo.configure_options yield from ("--prefix", self.buildEnv.install_dir) yield from ("--libdir", pj(self.buildEnv.install_dir, self.buildEnv.libprefix)) def set_configure_env(self, env): dep_conf_env = self.configure_env if not dep_conf_env: return for k, v in dep_conf_env.items(): if k.startswith("_format_"): v = v.format(buildEnv=self.buildEnv, env=env) env[k[8:]] = v else: env[k] = v def _configure(self, context): context.try_skip(self.build_path) command = [ *self.buildEnv.configure_wrapper, pj(self.source_path, self.configure_script), *self.all_configure_options, ] env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) self.set_configure_env(env) run_command(command, self.build_path, context, env=env) def _compile(self, context): context.try_skip(self.build_path) command = [ *self.buildEnv.make_wrapper, *neutralEnv("make_command"), *self.make_targets, *self.make_options, ] env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) run_command(command, self.build_path, context, env=env) def _install(self, context): context.try_skip(self.build_path) command = [ *self.buildEnv.make_wrapper, *neutralEnv("make_command"), *self.make_install_targets, *self.make_options, ] env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) run_command(command, self.build_path, context, env=env) def _make_dist(self, context): context.try_skip(self.build_path) command = [*self.buildEnv.make_wrapper, *neutralEnv("make_command"), "dist"] env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) run_command(command, self.build_path, context, env=env) class CMakeBuilder(MakeBuilder): flatpak_buildsystem = "cmake" def _configure(self, context): context.try_skip(self.build_path) cross_options = [] if not self.target.force_native_build and self.buildEnv.cmake_crossfile: cross_options += [f"-DCMAKE_TOOLCHAIN_FILE={self.buildEnv.cmake_crossfile}"] command = [ *neutralEnv("cmake_command"), *self.configure_options, "-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON", f"-DCMAKE_INSTALL_PREFIX={self.buildEnv.install_dir}", f"-DCMAKE_INSTALL_LIBDIR={self.buildEnv.libprefix}", self.source_path, *cross_options, ] env = self.get_env( cross_comp_flags=True, cross_compilers=False, cross_path=True ) self.set_configure_env(env) run_command(command, self.build_path, context, env=env) def set_flatpak_buildsystem(self, module): super().set_flatpak_buildsystem(module) module["buildir"] = True class QMakeBuilder(MakeBuilder): qmake_targets = [] flatpak_buildsystem = "qmake" @property def make_options(self): if platform.system() == "Windows": return [] return super().make_options @property def env_options(self): if "QMAKE_CC" in os.environ: yield f"QMAKE_CC={os.environ['QMAKE_CC']}" if "QMAKE_CXX" in os.environ: yield f"QMAKE_CXX={os.environ['QMAKE_CXX']}" def _configure(self, context): context.try_skip(self.build_path) command = [ *neutralEnv("qmake_command"), *self.configure_options, *self.env_options, self.source_path, ] env = self.get_env( cross_comp_flags=True, cross_compilers=False, cross_path=True ) self.set_configure_env(env) run_command(command, self.build_path, context, env=env) def _make_dist(self, context): command = [ *neutralEnv("git_command"), "archive", "-o", f"{self.build_path}/{self.target.full_name()}.tar.gz", f"--prefix={self.target.full_name()}/", "HEAD", ] run_command(command, self.source_path, context) class MesonBuilder(Builder): configure_options = [] test_options = [] flatpak_buildsystem = "meson" @property def build_type(self): if platform.system() == "Windows": return "release" return "release" if option("make_release") else "debug" @property def strip_options(self): if option("make_release"): yield "--strip" @property def library_type(self): return "static" if self.buildEnv.configInfo.static else "shared" def _configure(self, context): context.try_skip(self.build_path) if os.path.exists(self.build_path): shutil.rmtree(self.build_path) os.makedirs(self.build_path) cross_options = [] if not self.target.force_native_build and self.buildEnv.meson_crossfile: cross_options += ["--cross-file", self.buildEnv.meson_crossfile] command = [ *neutralEnv("meson_command"), "setup", ".", self.build_path, f"--buildtype={self.build_type}", *self.strip_options, f"--default-library={self.library_type}", *self.configure_options, f"--prefix={self.buildEnv.install_dir}", f"--libdir={self.buildEnv.libprefix}", *cross_options, ] env = self.get_env( cross_comp_flags=False, cross_compilers=False, cross_path=True ) run_command(command, self.source_path, context, env=env) def _compile(self, context): context.try_skip(self.build_path) command = [*neutralEnv("ninja_command"), "-v"] env = self.get_env( cross_comp_flags=False, cross_compilers=False, cross_path=True ) run_command(command, self.build_path, context, env=env) def _test(self, context): context.try_skip(self.build_path) if self.buildEnv.configInfo.build == "android" or ( self.buildEnv.configInfo.build != "native" and not self.buildEnv.configInfo.static ): raise SkipCommand() command = [*neutralEnv("mesontest_command"), "--verbose", *self.test_options] env = self.get_env( cross_comp_flags=False, cross_compilers=False, cross_path=True ) run_command(command, self.build_path, context, env=env) def _install(self, context): context.try_skip(self.build_path) command = [*neutralEnv("ninja_command"), "-v", "install"] env = self.get_env( cross_comp_flags=False, cross_compilers=False, cross_path=True ) run_command(command, self.build_path, context, env=env) def _make_dist(self, context): command = [*neutralEnv("ninja_command"), "-v", "dist"] env = self.get_env( cross_comp_flags=False, cross_compilers=False, cross_path=True ) run_command(command, self.build_path, context, env=env)