From e87835c61d3f02af9569282125e0c7742cd24791 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 19 Jun 2018 10:55:00 +0200 Subject: [PATCH 1/7] Correctly set the environment. When running command without `shell=True`, python's subproccess will look for variable `"PATH"` and `b"PATH"` and will complain if both are set. So the defaultdict should not return something for `b"PATH"`. We also escape space ` ` in the environment variables to not break everything with directory containing space. --- kiwixbuild/buildenv.py | 33 ++++++++++++++++++++--------- kiwixbuild/dependencies/base.py | 2 +- kiwixbuild/dependencies/libmagic.py | 2 +- kiwixbuild/utils.py | 19 ++++++++++++++--- 4 files changed, 41 insertions(+), 15 deletions(-) diff --git a/kiwixbuild/buildenv.py b/kiwixbuild/buildenv.py index 97719cf..2652b41 100644 --- a/kiwixbuild/buildenv.py +++ b/kiwixbuild/buildenv.py @@ -4,7 +4,7 @@ import subprocess import platform import distro -from .utils import pj, download_remote, Defaultdict +from .utils import pj, download_remote, escape_path from ._global import neutralEnv, option @@ -125,21 +125,34 @@ class BuildEnv: pkgconfig_path = pj(self.install_dir, self.libprefix, 'pkgconfig') env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path]) - env['PATH'] = ':'.join([pj(self.install_dir, 'bin'), env['PATH']]) + env['PATH'] = ':'.join([ + escape_path(pj(self.install_dir, 'bin')), + env['PATH'] + ]) env['LD_LIBRARY_PATH'] = ':'.join([env['LD_LIBRARY_PATH'], pj(self.install_dir, 'lib'), pj(self.install_dir, self.libprefix) ]) - env['QMAKE_CXXFLAGS'] = " ".join(['-I'+pj(self.install_dir, 'include'), env['QMAKE_CXXFLAGS']]) - env['CPPFLAGS'] = " ".join(['-I'+pj(self.install_dir, 'include'), env['CPPFLAGS']]) - env['QMAKE_LFLAGS'] = " ".join(['-L'+pj(self.install_dir, 'lib'), - '-L'+pj(self.install_dir, self.libprefix), - env['QMAKE_LFLAGS']]) - env['LDFLAGS'] = " ".join(['-L'+pj(self.install_dir, 'lib'), - '-L'+pj(self.install_dir, self.libprefix), - env['LDFLAGS']]) + env['QMAKE_CXXFLAGS'] = " ".join([ + escape_path('-I'+pj(self.install_dir, 'include')), + env['QMAKE_CXXFLAGS'] + ]) + env['CPPFLAGS'] = " ".join([ + escape_path('-I'+pj(self.install_dir, 'include')), + env['CPPFLAGS'] + ]) + env['QMAKE_LFLAGS'] = " ".join([ + escape_path('-L'+pj(self.install_dir, 'lib')), + escape_path('-L'+pj(self.install_dir, self.libprefix)), + env['QMAKE_LFLAGS'] + ]) + env['LDFLAGS'] = " ".join([ + escape_path('-L'+pj(self.install_dir, 'lib')), + escape_path('-L'+pj(self.install_dir, self.libprefix)), + env['LDFLAGS'] + ]) if cross_comp_flags: self.platformInfo.set_comp_flags(env) diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index d57102c..49c4176 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -3,7 +3,7 @@ import os import shutil import time -from kiwixbuild.utils import pj, Context, SkipCommand, WarningMessage, extract_archive, Defaultdict, StopBuild, run_command, colorize, copy_tree +from kiwixbuild.utils import pj, Context, SkipCommand, WarningMessage, extract_archive, StopBuild, run_command, colorize, copy_tree from kiwixbuild.versions import main_project_versions, base_deps_versions from kiwixbuild._global import neutralEnv, option, get_target_step diff --git a/kiwixbuild/dependencies/libmagic.py b/kiwixbuild/dependencies/libmagic.py index 53f3302..01c72ec 100644 --- a/kiwixbuild/dependencies/libmagic.py +++ b/kiwixbuild/dependencies/libmagic.py @@ -6,7 +6,7 @@ from .base import ( MakeBuilder, ) -from kiwixbuild.utils import Remotefile, pj, Defaultdict, SkipCommand, run_command +from kiwixbuild.utils import Remotefile, pj, SkipCommand, run_command from kiwixbuild._global import get_target_step class LibMagic(Dependency): diff --git a/kiwixbuild/utils.py b/kiwixbuild/utils.py index c0c0b4a..d9feefd 100644 --- a/kiwixbuild/utils.py +++ b/kiwixbuild/utils.py @@ -8,6 +8,7 @@ import urllib.request import urllib.error import ssl import subprocess +import re from collections import namedtuple, defaultdict from kiwixbuild._global import neutralEnv, option @@ -39,13 +40,25 @@ def xrun_find(name): return output[:-1].decode() +regex_space = re.compile(r'((? Date: Tue, 19 Jun 2018 11:27:07 +0200 Subject: [PATCH 2/7] Run the command without using shell=True. It mainly allow to run command in directory containing space. (Hello, `C:\Program Files\...`) It would also allow to work on directory containning spaces (`C:\Users\John Doe`) but xapian configure (at least) expressly doesn't support it :/ - Run the command without shell=True - The command must be a list instead of a string. - All options must also be a list (or an iterable). --- kiwixbuild/buildenv.py | 23 +-- kiwixbuild/dependencies/aria2.py | 4 +- kiwixbuild/dependencies/base.py | 217 +++++++++++----------- kiwixbuild/dependencies/docoptcpp.py | 2 +- kiwixbuild/dependencies/gumbo.py | 2 +- kiwixbuild/dependencies/icu4c.py | 21 ++- kiwixbuild/dependencies/ios_fat_lib.py | 9 +- kiwixbuild/dependencies/kiwix_desktop.py | 19 +- kiwixbuild/dependencies/kiwix_tools.py | 5 +- kiwixbuild/dependencies/libcurl.py | 16 +- kiwixbuild/dependencies/libkiwix.py | 14 +- kiwixbuild/dependencies/libmagic.py | 21 ++- kiwixbuild/dependencies/libmicrohttpd.py | 2 +- kiwixbuild/dependencies/libzim.py | 24 ++- kiwixbuild/dependencies/lzma.py | 20 +- kiwixbuild/dependencies/pugixml.py | 2 +- kiwixbuild/dependencies/qt.py | 32 +++- kiwixbuild/dependencies/tc_android_ndk.py | 14 +- kiwixbuild/dependencies/tc_emsdk.py | 4 +- kiwixbuild/dependencies/tc_flatpak.py | 48 ++--- kiwixbuild/dependencies/uuid.py | 15 +- kiwixbuild/dependencies/xapian.py | 6 +- kiwixbuild/dependencies/zim_tools.py | 8 +- kiwixbuild/dependencies/zlib.py | 40 ++-- kiwixbuild/dependencies/zstd.py | 4 +- kiwixbuild/flatpak_builder.py | 16 +- kiwixbuild/platforms/android.py | 4 +- kiwixbuild/platforms/armhf.py | 4 +- kiwixbuild/platforms/base.py | 2 +- kiwixbuild/platforms/i586.py | 4 +- kiwixbuild/platforms/ios.py | 4 +- kiwixbuild/platforms/musl.py | 4 +- kiwixbuild/platforms/wasm.py | 4 +- kiwixbuild/platforms/win32.py | 4 +- kiwixbuild/platforms/win64.py | 4 +- kiwixbuild/utils.py | 2 +- 36 files changed, 326 insertions(+), 298 deletions(-) diff --git a/kiwixbuild/buildenv.py b/kiwixbuild/buildenv.py index 2652b41..c4bcacb 100644 --- a/kiwixbuild/buildenv.py +++ b/kiwixbuild/buildenv.py @@ -30,7 +30,7 @@ class PlatformNeutralEnv: self.qmake_command = self._detect_qmake() if not self.qmake_command: print("WARNING: qmake command not found.", file=sys.stderr) - self.mesontest_command = "{} test".format(self.meson_command) + self.mesontest_command = [*self.meson_command, "test"] def detect_platform(self): _platform = platform.system() @@ -59,7 +59,7 @@ class PlatformNeutralEnv: # Doesn't exist in PATH or isn't executable continue if retcode == 0: - return n + return [n] def _detect_ninja(self): @@ -165,17 +165,14 @@ class BuildEnv: @property def configure_wrapper(self): - wrapper = getattr(self.platformInfo, "configure_wrapper", "") - if wrapper: - return "{} ".format(wrapper) - else: - return "" + try: + yield self.platformInfo.configure_wrapper + except AttributeError: + pass @property def make_wrapper(self): - wrapper = getattr(self.platformInfo, "make_wrapper", "") - if wrapper: - return "{} ".format(wrapper) - else: - return "" - + try: + yield self.platformInfo.make_wrapper + except AttributeError: + pass diff --git a/kiwixbuild/dependencies/aria2.py b/kiwixbuild/dependencies/aria2.py index d19bcb7..d543be4 100644 --- a/kiwixbuild/dependencies/aria2.py +++ b/kiwixbuild/dependencies/aria2.py @@ -16,9 +16,9 @@ class Aria2(Dependency): def _post_prepare_script(self, context): context.try_skip(self.extract_path) - command = "autoreconf -i" + command = ["autoreconf", "-i"] run_command(command, self.extract_path, context) class Builder(MakeBuilder): dependencies = ['zlib'] - configure_option = "--disable-libaria2 --disable-websocket --without-sqlite3" + configure_options = ["--disable-libaria2", "--disable-websocket", "--without-sqlite3"] diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index 49c4176..55d32b6 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -71,7 +71,7 @@ class Source: context.try_skip(self.source_path) for p in self.patches: patch_file_path = pj(SCRIPT_DIR, 'patches', p) - patch_command = "patch -p1 -i {patch}".format(patch=patch_file_path) + patch_command = ["patch", "-p1", "-i", patch_file_path] run_command(patch_command, self.source_path, context) def command(self, name, function, *args): @@ -170,20 +170,19 @@ class GitClone(Source): def _git_init(self, context): if option('fast_clone') and self.force_full_clone == False: - command = "git clone --depth=1 --branch {} {} {}".format( - self.git_ref, self.git_remote, self.source_dir) + command = ["git", "clone" , "--depth=1", "--branch", self.git_ref, self.git_remote, self.source_dir] run_command(command, neutralEnv('source_dir'), context) else: - command = "git clone {} {}".format(self.git_remote, self.source_dir) + command = ["git", "clone", self.git_remote, self.source_dir] run_command(command, neutralEnv('source_dir'), context) - command = "git checkout {}".format(self.git_ref) + command = ["git", "checkout", self.git_ref] run_command(command, self.git_path, context) def _git_update(self, context): - command = "git fetch origin {}".format(self.git_ref) + command = ["git", "fetch", "origin", self.git_ref] run_command(command, self.git_path, context) try: - command = "git merge --ff-only origin/{}".format(self.git_ref) + command = ["git", "merge", "--ff-only", f"origin/{self.git_ref}"] run_command(command, self.git_path, context) except subprocess.CalledProcessError: raise WarningMessage("Cannot update, please check log for information") @@ -209,7 +208,10 @@ class SvnClone(Source): def _svn_export(self, context): if os.path.exists(self.svn_path): raise SkipCommand() - command = "svn export {} {}".format(self.svn_remote, self.svn_dir) + command = [ + "svn", "export", + self.svn_remote, self.svn_dir + ] run_command(command, neutralEnv('source_dir'), context) def prepare(self): @@ -354,36 +356,37 @@ class TcCopyBuilder(Builder): class MakeBuilder(Builder): - configure_option_template = "{dep_options} {static_option} {env_option} --prefix {install_dir} --libdir {libdir}" - configure_option = "" - dynamic_configure_option = "--enable-shared --disable-static" - static_configure_option = "--enable-static --disable-shared" - make_option = "" - install_option = "" + configure_options = [] + dynamic_configure_options = ["--enable-shared", "--disable-static"] + static_configure_options = ["--enable-static", "--disable-shared"] + make_options = [] + install_options = [] configure_script = "configure" configure_env = { '_format_CFLAGS' : '{env[CFLAGS]} -O3', '_format_CXXFLAGS': '{env[CXXFLAGS]} -O3' } - make_target = "" + make_targets = [] flatpak_buildsystem = None @property - def make_install_target(self): + def make_install_targets(self): if self.buildEnv.platformInfo.build in ('iOS', "wasm"): - return 'install' - return 'install-strip' + yield 'install' + else: + yield 'install-strip' @property - def all_configure_option(self): - option = self.configure_option_template.format( - dep_options=self.configure_option, - static_option=self.static_configure_option if self.buildEnv.platformInfo.static else self.dynamic_configure_option, - env_option=self.buildEnv.platformInfo.configure_option if not self.target.force_native_build else "", - install_dir=self.buildEnv.install_dir, - libdir=pj(self.buildEnv.install_dir, self.buildEnv.libprefix) - ) - return option + def all_configure_options(self): + yield from self.configure_options + if self.buildEnv.platformInfo.static: + yield from self.static_configure_options + else: + yield from self.dynamic_configure_options + if not self.target.force_native_build: + yield from self.buildEnv.platformInfo.configure_options + yield from ('--prefix', self.buildEnv.install_dir) + yield from ('--libdir', pj(self.buildEnv.install_dir, self.buildEnv.libprefix)) def set_configure_env(self, env): dep_conf_env = self.configure_env @@ -399,41 +402,43 @@ class MakeBuilder(Builder): def _configure(self, context): context.try_skip(self.build_path) - command = "{configure_wrapper}{configure_script} {configure_option}" - command = command.format( - configure_wrapper=self.buildEnv.configure_wrapper, - configure_script=pj(self.source_path, self.configure_script), - configure_option=self.all_configure_option - ) + command = [ + *self.buildEnv.configure_wrapper, + pj(self.source_path, self.configure_script), + *self.all_configure_options + ] env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) self.set_configure_env(env) run_command(command, self.build_path, context, env=env) def _compile(self, context): context.try_skip(self.build_path) - command = "{make_wrapper}make -j4 {make_target} {make_option}".format( - make_wrapper=self.buildEnv.make_wrapper, - make_target=self.make_target, - make_option=self.make_option - ) + command = [ + *self.buildEnv.make_wrapper, + "make", "-j4", + *self.make_targets, + *self.make_options + ] env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) run_command(command, self.build_path, context, env=env) def _install(self, context): context.try_skip(self.build_path) - command = "{make_wrapper}make {make_install_target} {make_option}".format( - make_wrapper=self.buildEnv.make_wrapper, - make_install_target=self.make_install_target, - make_option=self.make_option - ) + command = [ + *self.buildEnv.make_wrapper, + "make", + *self.make_install_targets, + *self.make_options + ] env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) run_command(command, self.build_path, context, env=env) def _make_dist(self, context): context.try_skip(self.build_path) - command = "{make_wrapper}make dist".format( - make_wrapper=self.buildEnv.make_wrapper - ) + command = [ + *self.buildEnv.make_wrapper, + "make", "dist" + ] env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) run_command(command, self.build_path, context, env=env) @@ -443,22 +448,18 @@ class CMakeBuilder(MakeBuilder): def _configure(self, context): context.try_skip(self.build_path) - cross_option = "" + cross_options = [] if not self.target.force_native_build and self.buildEnv.cmake_crossfile: - cross_option = "-DCMAKE_TOOLCHAIN_FILE={}".format(self.buildEnv.cmake_crossfile) - command = ("cmake {configure_option}" - " -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON" - " -DCMAKE_INSTALL_PREFIX={install_dir}" - " -DCMAKE_INSTALL_LIBDIR={libdir}" - " {source_path}" - " {cross_option}") - command = command.format( - configure_option=self.configure_option, - install_dir=self.buildEnv.install_dir, - libdir=self.buildEnv.libprefix, - source_path=self.source_path, - cross_option=cross_option - ) + cross_options += [f"-DCMAKE_TOOLCHAIN_FILE={self.buildEnv.cmake_crossfile}"] + command = [ + "cmake", + *self.configure_options, + "-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON", + f"-DCMAKE_INSTALL_PREFIX={self.buildEnv.install_dir}", + f"-DCMAKE_INSTALL_LIBDIR={self.buildEnv.libprefix}", + self.source_path, + *cross_options + ] env = self.get_env(cross_comp_flags=True, cross_compilers=False, cross_path=True) self.set_configure_env(env) run_command(command, self.build_path, context, env=env) @@ -469,49 +470,42 @@ class CMakeBuilder(MakeBuilder): class QMakeBuilder(MakeBuilder): - qmake_target = "" + qmake_targets = [] flatpak_buildsystem = 'qmake' @property - def env_option(self): - options = "" + def env_options(self): if 'QMAKE_CC' in os.environ: - options += 'QMAKE_CC={} '.format(os.environ['QMAKE_CC']) + yield f"QMAKE_CC={os.environ['QMAKE_CC']}" if 'QMAKE_CXX' in os.environ: - options += 'QMAKE_CXX={} '.format(os.environ['QMAKE_CXX']) - return options + yield f"QMAKE_CXX={os.environ['QMAKE_CXX']}" def _configure(self, context): context.try_skip(self.build_path) - cross_option = "" - command = ("{command} {configure_option}" - " {env_option}" - " {source_path}" - " {cross_option}") - command = command.format( - command = neutralEnv('qmake_command'), - configure_option=self.configure_option, - env_option=self.env_option, - source_path=self.source_path, - cross_option=cross_option - ) + command = [ + *neutralEnv('qmake_command'), + *self.configure_options, + *self.env_options, + self.source_path, + ] env = self.get_env(cross_comp_flags=True, cross_compilers=False, cross_path=True) self.set_configure_env(env) run_command(command, self.build_path, context, env=env) def _make_dist(self, context): - command = "git archive -o {build_dir}/{name}.tar.gz --prefix={name}/ HEAD" - command = command.format( - build_dir = self.build_path, - name = self.target.full_name() - ) + command = [ + "git", "archive", + "-o", f"{self.build_path}/{self.target_full_name()}.tar.gz", + f"--prefix={self.target_full_name()}/", + "HEAD" + ] run_command(command, self.source_path, context) class MesonBuilder(Builder): - configure_option = "" - test_option = "" + configure_options = [] + test_options = [] flatpak_buildsystem = 'meson' @property @@ -519,8 +513,9 @@ class MesonBuilder(Builder): return 'release' if option('make_release') else 'debug' @property - def strip_option(self): - return '--strip' if option('make_release') else '' + def strip_options(self): + if option('make_release'): + yield '--strip' @property def library_type(self): @@ -531,34 +526,26 @@ class MesonBuilder(Builder): if os.path.exists(self.build_path): shutil.rmtree(self.build_path) os.makedirs(self.build_path) - configure_option = self.configure_option.format(buildEnv=self.buildEnv) - cross_option = "" + cross_options = [] if not self.target.force_native_build and self.buildEnv.meson_crossfile: - cross_option = "--cross-file {}".format( - self.buildEnv.meson_crossfile) - command = ("{command} . {build_path}" - " --buildtype={build_type} {strip_option}" - " --default-library={library_type}" - " {configure_option}" - " --prefix={buildEnv.install_dir}" - " --libdir={buildEnv.libprefix}" - " {cross_option}") - command = command.format( - command=neutralEnv('meson_command'), - build_type=self.build_type, - strip_option=self.strip_option, - library_type=self.library_type, - configure_option=configure_option, - build_path=self.build_path, - buildEnv=self.buildEnv, - cross_option=cross_option - ) + cross_options += ["--cross-file", self.buildEnv.meson_crossfile] + command = [ + *neutralEnv('meson_command'), + '.', self.build_path, + f'--buildtype={self.build_type}', + *self.strip_options, + f'--default-library={self.library_type}', + *self.configure_options, + f'--prefix={self.buildEnv.install_dir}', + f'--libdir={self.buildEnv.libprefix}', + *cross_options + ] env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) run_command(command, self.source_path, context, env=env) def _compile(self, context): context.try_skip(self.build_path) - command = "{} -v".format(neutralEnv('ninja_command')) + command = [*neutralEnv('ninja_command'), "-v"] env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) run_command(command, self.build_path, context, env=env) @@ -569,17 +556,21 @@ class MesonBuilder(Builder): and not self.buildEnv.platformInfo.static) ): raise SkipCommand() - command = "{} --verbose {}".format(neutralEnv('mesontest_command'), self.test_option) + command = [ + *neutralEnv('mesontest_command'), + '--verbose', + *self.test_options + ] env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) run_command(command, self.build_path, context, env=env) def _install(self, context): context.try_skip(self.build_path) - command = "{} -v install".format(neutralEnv('ninja_command')) + command = [*neutralEnv('ninja_command'), '-v', 'install'] env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) run_command(command, self.build_path, context, env=env) def _make_dist(self, context): - command = "{} -v dist".format(neutralEnv('ninja_command')) + command = [*neutralEnv('ninja_command'), "-v", "dist"] env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) run_command(command, self.build_path, context, env=env) diff --git a/kiwixbuild/dependencies/docoptcpp.py b/kiwixbuild/dependencies/docoptcpp.py index 0c8e64e..73c8f4c 100644 --- a/kiwixbuild/dependencies/docoptcpp.py +++ b/kiwixbuild/dependencies/docoptcpp.py @@ -17,5 +17,5 @@ class docoptcpp(Dependency): git_ref = "3dd23e3280f213bacefdf5fcb04857bf52e90917" class Builder(CMakeBuilder): - make_install_target = 'install' + make_install_targets = ['install'] diff --git a/kiwixbuild/dependencies/gumbo.py b/kiwixbuild/dependencies/gumbo.py index 078f0fd..23693c3 100644 --- a/kiwixbuild/dependencies/gumbo.py +++ b/kiwixbuild/dependencies/gumbo.py @@ -17,7 +17,7 @@ class Gumbo(Dependency): def _post_prepare_script(self, context): context.try_skip(self.extract_path) - command = "./autogen.sh" + command = ["./autogen.sh"] run_command(command, self.extract_path, context) Builder = MakeBuilder diff --git a/kiwixbuild/dependencies/icu4c.py b/kiwixbuild/dependencies/icu4c.py index dfeabcd..55492b6 100644 --- a/kiwixbuild/dependencies/icu4c.py +++ b/kiwixbuild/dependencies/icu4c.py @@ -53,7 +53,7 @@ class Icu(Dependency): class Builder(MakeBuilder): subsource_dir = "source" - make_install_target = "install" + make_install_targets = ["install"] @classmethod def get_dependencies(cls, platformInfo, allDeps): @@ -61,20 +61,23 @@ class Icu(Dependency): return [(plt, 'icu4c')] @property - def configure_option(self): - options = ("--disable-samples --disable-tests --disable-extras " - "--disable-dyload --enable-rpath " - "--disable-icuio --disable-layoutex") + def configure_options(self): + yield "--disable-samples" + yield "--disable-tests" + yield "--disable-extras" + yield "--disable-dyload" + yield "--enable-rpath" + yield "--disable-icuio" + yield "--disable-layoutex" platformInfo = self.buildEnv.platformInfo if platformInfo.build != 'native': icu_native_builder = get_target_step( 'icu4c', 'native_static' if platformInfo.static else 'native_dyn') - options += " --with-cross-build={} --disable-tools".format( - icu_native_builder.build_path) + yield f"--with-cross-build={icu_native_builder.build_path}" + yield "--disable-tools" if platformInfo.build in ('android', 'wasm'): - options += " --with-data-packaging=archive" - return options + yield "--with-data-packaging=archive" def set_env(self, env): env['ICU_DATA_FILTER_FILE'] = pj(os.path.dirname(os.path.realpath(__file__)), "icu4c_data_filter.json") diff --git a/kiwixbuild/dependencies/ios_fat_lib.py b/kiwixbuild/dependencies/ios_fat_lib.py index fddf107..27e6cfe 100644 --- a/kiwixbuild/dependencies/ios_fat_lib.py +++ b/kiwixbuild/dependencies/ios_fat_lib.py @@ -40,11 +40,12 @@ class IOSFatLib(Dependency): if f.endswith('.a') or f.endswith('.dylib'): libs.append(f) os.makedirs(pj(self.buildEnv.install_dir, 'lib'), exist_ok=True) - command_tmp = "lipo -create {input} -output {output}" for l in libs: - command = command_tmp.format( - input=" ".join(pj(d, l) for d in lib_dirs), - output=pj(self.buildEnv.install_dir, 'lib', l)) + command = [ + 'lipo', + '-create', *[pj(d, l) for d in lib_dirs], + '-output', pj(self.buildEnv.install_dir, 'lib', l) + ] run_command(command, self.buildEnv.install_dir, context) def build(self): diff --git a/kiwixbuild/dependencies/kiwix_desktop.py b/kiwixbuild/dependencies/kiwix_desktop.py index 1afac43..51371a1 100644 --- a/kiwixbuild/dependencies/kiwix_desktop.py +++ b/kiwixbuild/dependencies/kiwix_desktop.py @@ -13,7 +13,7 @@ class KiwixDesktop(Dependency): class Builder(QMakeBuilder): dependencies = ["qt", "qtwebengine", "libkiwix", "aria2"] - make_install_target = 'install' + make_install_targets = ['install'] configure_env = None flatpack_build_options = { @@ -23,15 +23,12 @@ class KiwixDesktop(Dependency): } @property - def configure_option(self): + def configure_options(self): if self.buildEnv.platformInfo.name == 'flatpak': - options = [ - 'QMAKE_INCDIR+=/app/include/QtWebEngine', - 'QMAKE_INCDIR+=/app/include/QtWebEngineCore', - 'QMAKE_INCDIR+=/app/include/QtWebEngineWidgets' - ] + yield 'QMAKE_INCDIR+=/app/include/QtWebEngine' + yield 'QMAKE_INCDIR+=/app/include/QtWebEngineCore' + yield 'QMAKE_INCDIR+=/app/include/QtWebEngineWidgets' else: - options = ["PREFIX={}".format(self.buildEnv.install_dir)] - if self.buildEnv.platformInfo.static: - options.append('"CONFIG+=static"') - return " ".join(options) + yield f"PREFIX={self.buildEnv.install_dir}" + if self.buildEnv.platformInfo.static: + yield 'CONFIG+=static' diff --git a/kiwixbuild/dependencies/kiwix_tools.py b/kiwixbuild/dependencies/kiwix_tools.py index a9ea548..2e73488 100644 --- a/kiwixbuild/dependencies/kiwix_tools.py +++ b/kiwixbuild/dependencies/kiwix_tools.py @@ -15,7 +15,6 @@ class KiwixTools(Dependency): dependencies = ["libkiwix"] @property - def configure_option(self): + def configure_options(self): if self.buildEnv.platformInfo.static: - return "-Dstatic-linkage=true" - return "" + yield "-Dstatic-linkage=true" diff --git a/kiwixbuild/dependencies/libcurl.py b/kiwixbuild/dependencies/libcurl.py index 5cdd576..2599180 100644 --- a/kiwixbuild/dependencies/libcurl.py +++ b/kiwixbuild/dependencies/libcurl.py @@ -20,11 +20,11 @@ class LibCurl(Dependency): class Builder(MakeBuilder): dependencies = ['zlib'] - configure_option = " ".join( - ["--without-{}".format(p) - for p in ('libssh2', 'ssl', 'libmetalink', 'librtmp', - 'nghttp2', 'libidn2', 'brotli')] + - ["--disable-{}".format(p) - for p in ('ftp', 'file', 'ldap', 'ldaps', 'rtsp', 'dict', - 'telnet', 'tftp', 'pop3', 'imap', 'smb', 'smtp', - 'gopher', 'manual')]) + configure_options = [ + *[f"--without-{p}" for p in + ('libssh2', 'ssl', 'libmetalink', 'librtmp', 'nghttp2', 'libidn2', 'brotli') + ], + *[f"--disable-{p}" for p in + ('ftp', 'file', 'ldap', 'ldaps', 'rtsp', 'dict', 'telnet', + 'tftp', 'pop3', 'imap', 'smb', 'smtp', 'gopher', 'manual')] + ] diff --git a/kiwixbuild/dependencies/libkiwix.py b/kiwixbuild/dependencies/libkiwix.py index 169e83f..6c52a85 100644 --- a/kiwixbuild/dependencies/libkiwix.py +++ b/kiwixbuild/dependencies/libkiwix.py @@ -17,7 +17,7 @@ class Libkiwix(Dependency): class Builder(MesonBuilder): dependencies = ["pugixml", "libzim", "zlib", "lzma", "libcurl", "libmicrohttpd", "icu4c", "mustache", "xapian-core"] - strip_option = '' + strip_options = [] @property def build_type(self): @@ -26,15 +26,17 @@ class Libkiwix(Dependency): return super().build_type @property - def configure_option(self): + def configure_options(self): platformInfo = self.buildEnv.platformInfo if platformInfo.build == 'android': - return '-Dstatic-linkage=true -Dwerror=false' + yield '-Dstatic-linkage=true' + yield '-Dwerror=false' + if platformInfo.build == 'iOS': + yield '-Db_bitcode=true' if platformInfo.name == 'flatpak': - return '--wrap-mode=nodownload' + yield '--wrap-mode=nodownload' if platformInfo.mixed and option('target') == 'libkiwix': - return "-Dstatic-linkage=true" - return '' + yield "-Dstatic-linkage=true" @property def library_type(self): diff --git a/kiwixbuild/dependencies/libmagic.py b/kiwixbuild/dependencies/libmagic.py index 01c72ec..df66c2e 100644 --- a/kiwixbuild/dependencies/libmagic.py +++ b/kiwixbuild/dependencies/libmagic.py @@ -21,12 +21,11 @@ class LibMagic(Dependency): class Builder(MakeBuilder): @property - def configure_option(self): - return ("--disable-bzlib " - "--disable-xzlib " - "--disable-zstdlib " - "--disable-lzlib " - ) + def configure_options(self): + yield "--disable-bzlib" + yield "--disable-xzlib" + yield "--disable-zstdlib" + yield "--disable-lzlib" @classmethod def get_dependencies(cls, platformInfo, allDeps): @@ -39,10 +38,12 @@ class LibMagic(Dependency): if platformInfo.build == 'native': return super()._compile(context) context.try_skip(self.build_path) - command = "make -j4 {make_target} {make_option}".format( - make_target=self.make_target, - make_option=self.make_option - ) + command = [ + "make", + "-j4", + *self.make_targets, + *self.make_options + ] env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) libmagic_native_builder = get_target_step('libmagic', 'native_static') env['PATH'] = ':'.join([pj(libmagic_native_builder.build_path, 'src'), env['PATH']]) diff --git a/kiwixbuild/dependencies/libmicrohttpd.py b/kiwixbuild/dependencies/libmicrohttpd.py index 9ebe666..3e161a2 100644 --- a/kiwixbuild/dependencies/libmicrohttpd.py +++ b/kiwixbuild/dependencies/libmicrohttpd.py @@ -14,4 +14,4 @@ class MicroHttpd(Dependency): 'https://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.76.tar.gz') class Builder(MakeBuilder): - configure_option = "--disable-https --without-libgcrypt --without-libcurl --disable-doc --disable-examples" + configure_options = ["--disable-https", "--without-libgcrypt", "--without-libcurl", "--disable-doc", "--disable-examples"] diff --git a/kiwixbuild/dependencies/libzim.py b/kiwixbuild/dependencies/libzim.py index b1d92a3..0f62ca0 100644 --- a/kiwixbuild/dependencies/libzim.py +++ b/kiwixbuild/dependencies/libzim.py @@ -13,8 +13,8 @@ class Libzim(Dependency): git_dir = "libzim" class Builder(MesonBuilder): - test_option = "-t 8" - strip_option = '' + test_options = ["-t", "8"] + strip_options = [] @property def build_type(self): @@ -30,24 +30,22 @@ class Libzim(Dependency): return deps @property - def configure_option(self): + def configure_options(self): platformInfo = self.buildEnv.platformInfo - config_options = [] if platformInfo.build == 'android': - config_options.append("-DUSE_BUFFER_HEADER=false") - config_options.append("-Dstatic-linkage=true") + yield "-DUSE_BUFFER_HEADER=false" + yield "-Dstatic-linkage=true" if platformInfo.mixed and option('target') == 'libzim': - config_options.append("-Dstatic-linkage=true") + yield "-Dstatic-linkage=true" if platformInfo.name == "flatpak": - config_options.append("--wrap-mode=nodownload") - config_options.append("-Dtest_data_dir=none") + yield "--wrap-mode=nodownload" + yield "-Dtest_data_dir=none" if platformInfo.name == "wasm": - config_options.append("-Dexamples=false") - config_options.append("-DUSE_MMAP=false") + yield "-Dexamples=false" + yield "-DUSE_MMAP=false" if platformInfo.name not in ("flatpak", "wasm"): zim_testing_suite = get_target_step('zim-testing-suite', 'source') - config_options.append('-Dtest_data_dir={}'.format(zim_testing_suite.source_path)) - return " ".join(config_options) + yield '-Dtest_data_dir={}'.format(zim_testing_suite.source_path) @property def library_type(self): diff --git a/kiwixbuild/dependencies/lzma.py b/kiwixbuild/dependencies/lzma.py index bcfb4f7..f2d3cfa 100644 --- a/kiwixbuild/dependencies/lzma.py +++ b/kiwixbuild/dependencies/lzma.py @@ -16,13 +16,13 @@ class lzma(Dependency): class Builder(MakeBuilder): @property - def configure_option(self): - return ("--disable-xz " - "--disable-xzdec " - "--disable-lzmadec " - "--disable-lzmainfo " - "--disable-lzma-links " - "--disable-scripts " - "--disable-doc " -# "--disable-symbol-versions " - ) + def configure_options(self): + return ["--disable-xz", + "--disable-xzdec", + "--disable-lzmadec", + "--disable-lzmainfo", + "--disable-lzma-links", + "--disable-scripts", + "--disable-doc", +# "--disable-symbol-versions" + ] diff --git a/kiwixbuild/dependencies/pugixml.py b/kiwixbuild/dependencies/pugixml.py index a8310b0..3c4a531 100644 --- a/kiwixbuild/dependencies/pugixml.py +++ b/kiwixbuild/dependencies/pugixml.py @@ -16,4 +16,4 @@ class Pugixml(Dependency): class Builder(MesonBuilder): build_type = 'release' - strip_option = '' + strip_options = [] diff --git a/kiwixbuild/dependencies/qt.py b/kiwixbuild/dependencies/qt.py index 326b0de..efdfd85 100644 --- a/kiwixbuild/dependencies/qt.py +++ b/kiwixbuild/dependencies/qt.py @@ -21,12 +21,23 @@ class Qt(Dependency): class Builder(MakeBuilder): dependencies = ['icu4c', 'zlib'] - configure_option_template = "{dep_options} {static_option} {env_option} -prefix {install_dir} -libdir {libdir}" - dynamic_configure_option = "-shared" - static_configure_option = "-static" + dynamic_configure_options = ["-shared"] + static_configure_options = ["-static"] @property - def configure_option(self): + def all_configure_options(self): + yield from self.configure_options + if self.buildEnv.platformInfo.static: + yield from self.static_configure_options + else: + yield from self.dynamic_configure_options + if not self.target.force_native_build: + yield from self.buildEnv.platformInfo.configure_options + yield from ('-prefix', self.buildEnv.install_dir) + yield from ('-libdir', pj(self.buildEnv.install_dir, self.buildEnv.libprefix)) + + @property + def configure_options(self): skip_modules = [ 'qt3d', 'qtcanvas3d', @@ -55,10 +66,15 @@ class Qt(Dependency): 'qtwebglplugin', 'qtwebsockets', # 'qtwebview', -] - skip_modules = " ".join("-skip {}".format(m) for m in skip_modules) - options = "-recheck -opensource -confirm-license -ccache -make libs {}".format(skip_modules) - return options + ] + yield '-recheck' + yield '-opensource' + yield '-confirm-license' + yield '-ccache' + yield from ('-make', 'libs') + for module in skip_modules: + yield from ('-skip', module) + class QtWebEngine(Dependency): name = "qtwebengine" diff --git a/kiwixbuild/dependencies/tc_android_ndk.py b/kiwixbuild/dependencies/tc_android_ndk.py index 96be642..4b565d4 100644 --- a/kiwixbuild/dependencies/tc_android_ndk.py +++ b/kiwixbuild/dependencies/tc_android_ndk.py @@ -47,13 +47,13 @@ class android_ndk(Dependency): context.try_skip(self.build_path) script = pj(self.source_path, 'build/tools/make_standalone_toolchain.py') add_execution_right(script) - command = '{script} --arch={arch} --api={api} --install-dir={install_dir} --force' - command = command.format( - script=script, - arch=self.arch, - api=self.api, - install_dir=self.install_path - ) + command = [ + script, + f'--arch={self.arch}', + f'--api={self.api}', + f'--install-dir={self.install_path}', + '--force' + ] env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False) run_command(command, self.build_path, context, env=env) diff --git a/kiwixbuild/dependencies/tc_emsdk.py b/kiwixbuild/dependencies/tc_emsdk.py index 265de3b..8303bed 100644 --- a/kiwixbuild/dependencies/tc_emsdk.py +++ b/kiwixbuild/dependencies/tc_emsdk.py @@ -31,12 +31,12 @@ class emsdk(Dependency): def _install(self, context): context.try_skip(self.build_path) - command = "./emsdk install 3.1.24" + command = ["./emsdk", "install", "3.1.24"] run_command(command, self.install_path, context) def _activate(self, context): context.try_skip(self.build_path) - command = "./emsdk activate 3.1.24" + command = ["./emsdk", "activate", "3.1.24"] run_command(command, self.install_path, context) diff --git a/kiwixbuild/dependencies/tc_flatpak.py b/kiwixbuild/dependencies/tc_flatpak.py index 4ff5265..8fe54c7 100644 --- a/kiwixbuild/dependencies/tc_flatpak.py +++ b/kiwixbuild/dependencies/tc_flatpak.py @@ -13,21 +13,23 @@ class org_kde(Dependency): class Builder(Builder): def _setup_remote(self, context): - command = "flatpak --user remote-add --if-not-exists {remote_name} {remote_url}" - command = command.format( - remote_name = 'flathub', - remote_url = 'https://flathub.org/repo/flathub.flatpakrepo' - ) + command = [ + "flatpak", + "--user", "remote-add", "--if-not-exists", + "flathub", + "https://flathub.org/repo/flathub.flatpakrepo" + ] env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False) run_command(command, self.buildEnv.build_dir, context, env=env) def _install_sdk(self, context): - command = "flatpak --user install --noninteractive --verbose -y {remote_name} {name}.Sdk//{version} {name}.Platform//{version}" - command = command.format( - remote_name = 'flathub', - name = self.target.name, - version = self.target.version() - ) + command = [ + "flatpak", + "--user", "install", "--noninteractive", "--verbose", "-y", + "flathub", + f"{self.target.name}.Sdk//{self.target.version()}", + f"{self.target.name}.Platform//{self.target.version()}" + ] env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False) run_command(command, self.buildEnv.build_dir, context, env=env) @@ -44,25 +46,25 @@ class io_qt_qtwebengine(Dependency): class Builder(Builder): def _setup_remote(self, context): - command = "flatpak --user remote-add --if-not-exists {remote_name} {remote_url}" - command = command.format( - remote_name = 'flathub', - remote_url = 'https://flathub.org/repo/flathub.flatpakrepo' - ) + command = [ + "flatpak", + "--user", "remote-add", "--if-not-exists", + "flathub", + "https://flathub.org/repo/flathub.flatpakrepo" + ] env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False) run_command(command, self.buildEnv.build_dir, context, env=env) def _install_sdk(self, context): - command = "flatpak --user install -y {remote_name} {name}.BaseApp//{version}" - command = command.format( - remote_name = 'flathub', - name = self.target.name, - version = self.target.version() - ) + command = [ + "flatpak", + "--user", "install", "-y", + "flathub", + f"{self.target.name}.BaseApp//{self.target.version()}" + ] env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False) run_command(command, self.buildEnv.build_dir, context, env=env) def build(self): self.command('setup_remote', self._setup_remote) self.command('install_sdk', self._install_sdk) - diff --git a/kiwixbuild/dependencies/uuid.py b/kiwixbuild/dependencies/uuid.py index 5034126..245d064 100644 --- a/kiwixbuild/dependencies/uuid.py +++ b/kiwixbuild/dependencies/uuid.py @@ -17,9 +17,14 @@ class UUID(Dependency): extract_dir = 'e2fsprogs-libs-1.43.4' class Builder(MakeBuilder): - configure_option = ("--enable-libuuid --disable-debugfs --disable-imager --disable-resizer --disable-defrag --enable-fsck" - " --disable-uuidd") + configure_options = ["--enable-libuuid", + "--disable-debugfs", + "--disable-imager", + "--disable-resizer", + "--disable-defrag", + "--enable-fsck", + "--disable-uuidd"] configure_env = {'_format_CFLAGS': "{env.CFLAGS} -O3 -fPIC"} - static_configure_option = dynamic_configure_option = "" - make_target = 'libs' - make_install_target = 'install-libs' + static_configure_options = dynamic_configure_options = [] + make_targets = ['libs'] + make_install_targets = ['install-libs'] diff --git a/kiwixbuild/dependencies/xapian.py b/kiwixbuild/dependencies/xapian.py index e71ed89..6857d10 100644 --- a/kiwixbuild/dependencies/xapian.py +++ b/kiwixbuild/dependencies/xapian.py @@ -16,7 +16,11 @@ class Xapian(Dependency): '30d3518172084f310dab86d262b512718a7f9a13635aaa1a188e61dc26b2288c') class Builder(MakeBuilder): - configure_option = "--disable-sse --disable-backend-chert --disable-backend-remote --disable-documentation" + configure_options = [ + "--disable-sse", + "--disable-backend-chert", + "--disable-backend-remote", + "--disable-documentation"] configure_env = {'_format_LDFLAGS': "{env.LDFLAGS} -L{buildEnv.install_dir}/{buildEnv.libprefix}", '_format_CXXFLAGS': "{env.CXXFLAGS} -O3 -I{buildEnv.install_dir}/include"} diff --git a/kiwixbuild/dependencies/zim_tools.py b/kiwixbuild/dependencies/zim_tools.py index 4c67504..8179fbc 100644 --- a/kiwixbuild/dependencies/zim_tools.py +++ b/kiwixbuild/dependencies/zim_tools.py @@ -20,11 +20,9 @@ class ZimTools(Dependency): return base_deps @property - def configure_option(self): - base_option = "" + def configure_options(self): # We don't build zimwriterfs on win32, and so we don't have magic if self.buildEnv.platformInfo.build != 'win32': - base_option += " -Dmagic-install-prefix={buildEnv.install_dir}" + yield f"-Dmagic-install-prefix={self.buildEnv.install_dir}" if self.buildEnv.platformInfo.static: - base_option += " -Dstatic-linkage=true" - return base_option + yield "-Dstatic-linkage=true" diff --git a/kiwixbuild/dependencies/zlib.py b/kiwixbuild/dependencies/zlib.py index 87759e0..1ca6eda 100644 --- a/kiwixbuild/dependencies/zlib.py +++ b/kiwixbuild/dependencies/zlib.py @@ -18,10 +18,9 @@ class zlib(Dependency): patches = ['zlib_std_libname.patch'] class Builder(MakeBuilder): - dynamic_configure_option = "--shared" - static_configure_option = "--static" - make_install_target = 'install' - configure_option_template = "{dep_options} {static_option} --prefix {install_dir} --libdir {libdir}" + dynamic_configure_options = ["--shared"] + static_configure_options = ["--static"] + make_install_targets = ['install'] def _pre_build_script(self, context): context.try_skip(self.build_path) @@ -34,20 +33,27 @@ class zlib(Dependency): return super()._configure(context) @property - def make_option(self): - if self.buildEnv.platformInfo.build == 'win32': - return "--makefile win32/Makefile.gcc PREFIX={host}- SHARED_MODE={static} INCLUDE_PATH={include_path} LIBRARY_PATH={library_path} BINARY_PATH={binary_path}".format( - host='i686-w64-mingw32', - static="0" if self.buildEnv.platformInfo.static else "1", - include_path=pj(self.buildEnv.install_dir, 'include'), - library_path=pj(self.buildEnv.install_dir, self.buildEnv.libprefix), - binary_path=pj(self.buildEnv.install_dir, 'bin'), - ) - return "" + def all_configure_options(self): + yield from self.configure_options + yield from self.static_configure_options if self.buildEnv.platformInfo.static else self.dynamic_configure_options + yield from ('--prefix', self.buildEnv.install_dir) + yield from ('--libdir', pj(self.buildEnv.install_dir, self.buildEnv.libprefix)) @property - def make_target(self): + def make_options(self): + if self.buildEnv.platformInfo.build != 'win32': + return + yield "--makefile" + yield "win32/Makefile.gcc" + yield "PREFIX=i686-w64-mingw32-", + yield "SHARED_MODE={}".format("0" if self.buildEnv.platformInfo.static else "1"), + yield "INCLUDE_PATH={}".format(pj(self.buildEnv.install_dir, 'include')), + yield "LIBRARY_PATH={}".format(pj(self.buildEnv.install_dir, self.buildEnv.libprefix)), + yield "BINARY_PATH={}".format(pj(self.buildEnv.install_dir, 'bin')) + + @property + def make_targets(self): if self.buildEnv.platformInfo.static: - return "static" + return ["static"] else: - return "shared" + return ["shared"] diff --git a/kiwixbuild/dependencies/zstd.py b/kiwixbuild/dependencies/zstd.py index 0428b2c..09c93b7 100644 --- a/kiwixbuild/dependencies/zstd.py +++ b/kiwixbuild/dependencies/zstd.py @@ -18,5 +18,5 @@ class zstd(Dependency): class Builder(MesonBuilder): subsource_dir = 'build/meson' build_type = 'release' - strip_option = '' - configure_option = '-Dbin_programs=false -Dbin_contrib=false' + strip_options = [] + configure_options = ['-Dbin_programs=false', '-Dbin_contrib=false'] diff --git a/kiwixbuild/flatpak_builder.py b/kiwixbuild/flatpak_builder.py index 797bbfc..ef61734 100644 --- a/kiwixbuild/flatpak_builder.py +++ b/kiwixbuild/flatpak_builder.py @@ -223,8 +223,12 @@ class FlatpakBuilder: def build(self): log = pj(self.platform.buildEnv.log_dir, 'cmd_build_flatpak.log') context = Context('build', log, False) - command = "flatpak-builder --user --ccache --force-clean --keep-build-dirs --disable-rofiles-fuse --repo=repo builddir {id}.json" - command = command.format(id = MANIFEST['app-id']) + command = [ + "flatpak-builder", + "--user", "--ccache", "--force-clean", "--keep-build-dirs", + "--disable-rofiles-fuse", "--repo=repo", "builddir", + f"{MANIFEST['app-id']}.json" + ] try: run_command(command, self.platform.buildEnv.build_dir, context, env=self.platform.get_env()) context._finalise() @@ -236,8 +240,12 @@ class FlatpakBuilder: def bundle(self): log = pj(self.platform.buildEnv.log_dir, 'cmd_bundle_flatpak.log') context = Context('bundle', log, False) - command = "flatpak build-bundle repo {id}.flatpak {id}" - command = command.format(id = MANIFEST['app-id']) + app_id = MANIFEST['app-id'] + command = [ + "flatpak", "build-bundle", "repo", + f"{app_id}.flatpak", + app_id + ] try: run_command(command, self.platform.buildEnv.build_dir, context, env=self.platform.get_env()) context._finalise() diff --git a/kiwixbuild/platforms/android.py b/kiwixbuild/platforms/android.py index b5a49b4..2fb6d09 100644 --- a/kiwixbuild/platforms/android.py +++ b/kiwixbuild/platforms/android.py @@ -90,8 +90,8 @@ class AndroidPlatformInfo(PlatformInfo): env[k] = v @property - def configure_option(self): - return '--host={}'.format(self.arch_full) + def configure_options(self): + yield '--host={}'.format(self.arch_full) def finalize_setup(self): super().finalize_setup() diff --git a/kiwixbuild/platforms/armhf.py b/kiwixbuild/platforms/armhf.py index d47e5e0..834db05 100644 --- a/kiwixbuild/platforms/armhf.py +++ b/kiwixbuild/platforms/armhf.py @@ -65,8 +65,8 @@ class ArmPlatformInfo(PlatformInfo): return "exe_wrapper = 'qemu-arm'" @property - def configure_option(self): - return '--host={}'.format(self.arch_full) + def configure_options(self): + yield '--host={}'.format(self.arch_full) def get_bin_dir(self): return [pj(self.root_path, 'bin')] diff --git a/kiwixbuild/platforms/base.py b/kiwixbuild/platforms/base.py index 881bff1..41cbee2 100644 --- a/kiwixbuild/platforms/base.py +++ b/kiwixbuild/platforms/base.py @@ -23,7 +23,7 @@ class PlatformInfo(metaclass=_MetaPlatform): all_platforms = {} all_running_platforms = {} toolchain_names = [] - configure_option = "" + configure_options = [] mixed = False libdir = None diff --git a/kiwixbuild/platforms/i586.py b/kiwixbuild/platforms/i586.py index 7745696..71ae316 100644 --- a/kiwixbuild/platforms/i586.py +++ b/kiwixbuild/platforms/i586.py @@ -31,8 +31,8 @@ class I586PlatformInfo(PlatformInfo): } @property - def configure_option(self): - return '--host={}'.format(self.arch_full) + def configure_options(self): + yield f'--host={self.arch_full}' @property def binaries(self): diff --git a/kiwixbuild/platforms/ios.py b/kiwixbuild/platforms/ios.py index 8256eef..fb16e28 100644 --- a/kiwixbuild/platforms/ios.py +++ b/kiwixbuild/platforms/ios.py @@ -118,8 +118,8 @@ class ApplePlatformInfo(PlatformInfo): } @property - def configure_option(self): - return '--host={}'.format(self.host) + def configure_options(self): + yield f'--host={self.host}' class iOSArm64(ApplePlatformInfo): diff --git a/kiwixbuild/platforms/musl.py b/kiwixbuild/platforms/musl.py index ade0c22..cd5fc46 100644 --- a/kiwixbuild/platforms/musl.py +++ b/kiwixbuild/platforms/musl.py @@ -62,8 +62,8 @@ class MuslPlatformInfo(PlatformInfo): return f"exe_wrapper = '{self.qemu}'" @property - def configure_option(self): - return '--host={}'.format(self.arch_full) + def configure_options(self): + return [f'--host={self.arch_full}'] def get_bin_dir(self): return [pj(self.root_path, 'bin')] diff --git a/kiwixbuild/platforms/wasm.py b/kiwixbuild/platforms/wasm.py index ea2e4cd..7b5cda0 100644 --- a/kiwixbuild/platforms/wasm.py +++ b/kiwixbuild/platforms/wasm.py @@ -59,9 +59,9 @@ class WasmPlatformInfo(PlatformInfo): return binaries @property - def configure_option(self): + def configure_options(self): #return "" - return '--host={}'.format(self.arch_full) + return [f'--host={self.arch_full}'] @property def configure_wrapper(self): diff --git a/kiwixbuild/platforms/win32.py b/kiwixbuild/platforms/win32.py index a88fe36..c5612d9 100644 --- a/kiwixbuild/platforms/win32.py +++ b/kiwixbuild/platforms/win32.py @@ -63,8 +63,8 @@ class Win32PlatformInfo(PlatformInfo): return "exe_wrapper = 'wine'" @property - def configure_option(self): - return '--host={}'.format(self.arch_full) + def configure_options(self): + yield f'--host={self.arch_full}' def set_compiler(self, env): for k, v in self.binaries.items(): diff --git a/kiwixbuild/platforms/win64.py b/kiwixbuild/platforms/win64.py index 62c0669..f7c17c0 100644 --- a/kiwixbuild/platforms/win64.py +++ b/kiwixbuild/platforms/win64.py @@ -63,8 +63,8 @@ class Win64PlatformInfo(PlatformInfo): return "exe_wrapper = 'wine'" @property - def configure_option(self): - return '--host={}'.format(self.arch_full) + def configure_options(self): + return [f'--host={self.arch_full}'] def set_compiler(self, env): for k, v in self.binaries.items(): diff --git a/kiwixbuild/utils.py b/kiwixbuild/utils.py index d9feefd..bdf8550 100644 --- a/kiwixbuild/utils.py +++ b/kiwixbuild/utils.py @@ -294,7 +294,7 @@ def run_command(command, cwd, context, *, env=None, input=None): kwargs = dict() if input: kwargs['stdin'] = subprocess.PIPE - process = subprocess.Popen(command, shell=True, cwd=cwd, env=env, stdout=log or sys.stdout, stderr=subprocess.STDOUT, **kwargs) + process = subprocess.Popen(command, cwd=cwd, env=env, stdout=log or sys.stdout, stderr=subprocess.STDOUT, **kwargs) if input: input = input.encode() while True: From 985643089a91329728f0ee01e0a74c0b6649ca4f Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Fri, 22 Jun 2018 18:14:35 +0200 Subject: [PATCH 3/7] Detect the command instead of hardcoding them. --- kiwixbuild/buildenv.py | 55 ++++++++++++++++++--------------- kiwixbuild/dependencies/base.py | 34 ++++++++++---------- 2 files changed, 48 insertions(+), 41 deletions(-) diff --git a/kiwixbuild/buildenv.py b/kiwixbuild/buildenv.py index c4bcacb..281a8fd 100644 --- a/kiwixbuild/buildenv.py +++ b/kiwixbuild/buildenv.py @@ -21,16 +21,20 @@ class PlatformNeutralEnv: self.log_dir): os.makedirs(d, exist_ok=True) self.detect_platform() - self.ninja_command = self._detect_ninja() - if not self.ninja_command: - sys.exit("ERROR: ninja command not found.") - self.meson_command = self._detect_meson() - if not self.meson_command: - sys.exit("ERROR: meson command not found.") - self.qmake_command = self._detect_qmake() - if not self.qmake_command: - print("WARNING: qmake command not found.", file=sys.stderr) + self.ninja_command = self._detect_command( + 'ninja', + default=[['ninja'], ['ninja-build']]) + self.meson_command = self._detect_command( + 'meson', + default= [['meson.py'], ['meson']]) self.mesontest_command = [*self.meson_command, "test"] + self.patch_command = self._detect_command('patch') + self.git_command = self._detect_command('git') + self.svn_command = self._detect_command('svn') + self.make_command = self._detect_command('make') + self.cmake_command = self._detect_command('cmake') + self.qmake_command = self._detect_command('qmake', required=False) + def detect_platform(self): _platform = platform.system() @@ -49,28 +53,29 @@ class PlatformNeutralEnv: def download(self, what, where=None): where = where or self.archive_dir download_remote(what, where) + - def _detect_binary(self, *bin_variants): - for n in bin_variants: + def _detect_command(self, name, default=None, options=['--version'], required=True): + if default is None: + default = [[name]] + env_key = 'KBUILD_{}_COMMAND'.format(name.upper()) + if env_key in os.environ: + default = [os.environ[env_key].split()] + default + for command in default: try: - retcode = subprocess.check_call([n, '--version'], + retcode = subprocess.check_call(command + options, stdout=subprocess.DEVNULL) - except (FileNotFoundError, PermissionError): + except (FileNotFoundError, PermissionError, OSError): # Doesn't exist in PATH or isn't executable continue if retcode == 0: - return [n] - - - def _detect_ninja(self): - return self._detect_binary('ninja', 'ninja-build') - - def _detect_meson(self): - return self._detect_binary('meson.py', 'meson') - - def _detect_qmake(self): - return self._detect_binary('qmake-qt5', 'qmake') - + return command + else: + if required: + sys.exit("ERROR: {} command not found".format(name)) + else: + print("WARNING: {} command not found".format(name)) + return ["{}_NOT_FOUND".format(name.upper())] class BuildEnv: def __init__(self, platformInfo): diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index 55d32b6..4cae628 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -71,7 +71,7 @@ class Source: context.try_skip(self.source_path) for p in self.patches: patch_file_path = pj(SCRIPT_DIR, 'patches', p) - patch_command = ["patch", "-p1", "-i", patch_file_path] + patch_command = [*neutralEnv('patch_command'), "-p1", "-i", patch_file_path] run_command(patch_command, self.source_path, context) def command(self, name, function, *args): @@ -170,19 +170,19 @@ class GitClone(Source): def _git_init(self, context): if option('fast_clone') and self.force_full_clone == False: - command = ["git", "clone" , "--depth=1", "--branch", self.git_ref, self.git_remote, self.source_dir] + command = [*neutralEnv('git_command'), "clone" , "--depth=1", "--branch", self.git_ref, self.git_remote, self.source_dir] run_command(command, neutralEnv('source_dir'), context) else: - command = ["git", "clone", self.git_remote, self.source_dir] + command = [*neutralEnv('git_command'), "clone", self.git_remote, self.source_dir] run_command(command, neutralEnv('source_dir'), context) - command = ["git", "checkout", self.git_ref] + command = [*neutralEnv('git_command'), "checkout", self.git_ref] run_command(command, self.git_path, context) def _git_update(self, context): - command = ["git", "fetch", "origin", self.git_ref] + command = [*neutralEnv('git_command'), "fetch", "origin", self.git_ref] run_command(command, self.git_path, context) try: - command = ["git", "merge", "--ff-only", f"origin/{self.git_ref}"] + command = [*neutralEnv('git_command'), "merge", "--ff-only", f"origin/{self.git_ref}"] run_command(command, self.git_path, context) except subprocess.CalledProcessError: raise WarningMessage("Cannot update, please check log for information") @@ -209,7 +209,7 @@ class SvnClone(Source): if os.path.exists(self.svn_path): raise SkipCommand() command = [ - "svn", "export", + *neutralEnv('svn_command'), "export", self.svn_remote, self.svn_dir ] run_command(command, neutralEnv('source_dir'), context) @@ -415,7 +415,8 @@ class MakeBuilder(Builder): context.try_skip(self.build_path) command = [ *self.buildEnv.make_wrapper, - "make", "-j4", + *neutralEnv('make_command'), + "-j4", *self.make_targets, *self.make_options ] @@ -426,7 +427,7 @@ class MakeBuilder(Builder): context.try_skip(self.build_path) command = [ *self.buildEnv.make_wrapper, - "make", + *neutralEnv('make_command'), *self.make_install_targets, *self.make_options ] @@ -437,7 +438,8 @@ class MakeBuilder(Builder): context.try_skip(self.build_path) command = [ *self.buildEnv.make_wrapper, - "make", "dist" + *neutralEnv('make_command'), + "dist" ] env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) run_command(command, self.build_path, context, env=env) @@ -452,7 +454,7 @@ class CMakeBuilder(MakeBuilder): if not self.target.force_native_build and self.buildEnv.cmake_crossfile: cross_options += [f"-DCMAKE_TOOLCHAIN_FILE={self.buildEnv.cmake_crossfile}"] command = [ - "cmake", + *neutralEnv('cmake_command'), *self.configure_options, "-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON", f"-DCMAKE_INSTALL_PREFIX={self.buildEnv.install_dir}", @@ -483,10 +485,10 @@ class QMakeBuilder(MakeBuilder): def _configure(self, context): context.try_skip(self.build_path) command = [ - *neutralEnv('qmake_command'), - *self.configure_options, - *self.env_options, - self.source_path, + "qmake", + *self.configure_options, + *self.env_options, + self.source_path ] env = self.get_env(cross_comp_flags=True, cross_compilers=False, cross_path=True) self.set_configure_env(env) @@ -494,7 +496,7 @@ class QMakeBuilder(MakeBuilder): def _make_dist(self, context): command = [ - "git", "archive", + *neutralEnv('git_command'), "archive", "-o", f"{self.build_path}/{self.target_full_name()}.tar.gz", f"--prefix={self.target_full_name()}/", "HEAD" From c753e629137350945c0dcd75a8438b254dc4783e Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 8 Nov 2023 16:17:53 +0100 Subject: [PATCH 4/7] Change base deps_meta version. --- kiwixbuild/versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kiwixbuild/versions.py b/kiwixbuild/versions.py index 6d2581a..626de22 100644 --- a/kiwixbuild/versions.py +++ b/kiwixbuild/versions.py @@ -40,7 +40,7 @@ release_versions = { # This is the "version" of the whole base_deps_versions dict. # Change this when you change base_deps_versions. -base_deps_meta_version = '95' +base_deps_meta_version = '96' base_deps_versions = { 'zlib' : '1.2.12', From a0978a6ab496997b8584cd9058c90d7d5d492645 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 30 Nov 2023 13:45:35 +0100 Subject: [PATCH 5/7] Remove SvnClone. We don't use it since a long time. --- kiwixbuild/buildenv.py | 3 +-- kiwixbuild/dependencies/base.py | 24 ------------------------ 2 files changed, 1 insertion(+), 26 deletions(-) diff --git a/kiwixbuild/buildenv.py b/kiwixbuild/buildenv.py index 281a8fd..458542f 100644 --- a/kiwixbuild/buildenv.py +++ b/kiwixbuild/buildenv.py @@ -30,7 +30,6 @@ class PlatformNeutralEnv: self.mesontest_command = [*self.meson_command, "test"] self.patch_command = self._detect_command('patch') self.git_command = self._detect_command('git') - self.svn_command = self._detect_command('svn') self.make_command = self._detect_command('make') self.cmake_command = self._detect_command('cmake') self.qmake_command = self._detect_command('qmake', required=False) @@ -53,7 +52,7 @@ class PlatformNeutralEnv: def download(self, what, where=None): where = where or self.archive_dir download_remote(what, where) - + def _detect_command(self, name, default=None, options=['--version'], required=True): if default is None: diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index 4cae628..a778c6a 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -196,30 +196,6 @@ class GitClone(Source): self.command('post_prepare_script', self._post_prepare_script) -class SvnClone(Source): - @property - def source_dir(self): - return self.svn_dir - - @property - def svn_path(self): - return pj(neutralEnv('source_dir'), self.svn_dir) - - def _svn_export(self, context): - if os.path.exists(self.svn_path): - raise SkipCommand() - command = [ - *neutralEnv('svn_command'), "export", - self.svn_remote, self.svn_dir - ] - run_command(command, neutralEnv('source_dir'), context) - - def prepare(self): - self.command('svnexport', self._svn_export) - if hasattr(self, 'patches'): - self.command('patch', self._patch) - - class Builder: subsource_dir = None dependencies = [] From b7c4854d83a21513fa34b20e428c44b3a71c9b39 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 30 Nov 2023 13:49:28 +0100 Subject: [PATCH 6/7] Build only libzim dependencies on bionic. We compile only libzim on bionic, for compatibility with old libc. No need to compile all other dependencies (and fight with some of them not compiling) --- kiwixbuild/dependencies/all_dependencies.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kiwixbuild/dependencies/all_dependencies.py b/kiwixbuild/dependencies/all_dependencies.py index f2bfd46..09f29b3 100644 --- a/kiwixbuild/dependencies/all_dependencies.py +++ b/kiwixbuild/dependencies/all_dependencies.py @@ -1,3 +1,5 @@ +from os import environ + from .base import ( Dependency, NoopSource, @@ -12,7 +14,7 @@ class AllBaseDependencies(Dependency): class Builder(NoopBuilder): @classmethod def get_dependencies(cls, platformInfo, allDeps): - if platformInfo.build == "wasm": + if platformInfo.build == "wasm" or environ.get('OS_NAME') == 'bionic': return ['zlib', 'lzma', 'zstd', 'icu4c', 'xapian-core'] base_deps = ['zlib', 'lzma', 'zstd', 'xapian-core', 'pugixml', 'libcurl', 'icu4c', 'mustache', 'libmicrohttpd', 'zim-testing-suite'] From ab273f7d126d345b92f02ba2c61ac8869a3b44f1 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 30 Nov 2023 15:32:31 +0100 Subject: [PATCH 7/7] Run apple command as list. --- kiwixbuild/dependencies/apple_xcframework.py | 32 ++++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/kiwixbuild/dependencies/apple_xcframework.py b/kiwixbuild/dependencies/apple_xcframework.py index 739bbb6..986ece7 100644 --- a/kiwixbuild/dependencies/apple_xcframework.py +++ b/kiwixbuild/dependencies/apple_xcframework.py @@ -56,8 +56,13 @@ class AppleXCFramework(Dependency): static_ars = [str(f) for f in Path(lib_dir).glob("*.a")] # create merged.a from all *.a in install_dir/lib - command = "libtool -static -o merged.a " - command += " ".join(static_ars) + command = [ + "libtool", + "-static", + "-o", + "merged.a", + *static_ars + ] run_command(command, lib_dir, context) # will be included in xcframework @@ -76,21 +81,28 @@ class AppleXCFramework(Dependency): fat_dir = pj(self.buildEnv.build_dir, "macOS_fat") os.makedirs(fat_dir, exist_ok=True) - command = "lipo -create -output {fat_dir}/merged.a ".format(fat_dir=fat_dir) - command += " ".join(macos_libs) + output_merged = pj(fat_dir, "merged.a") + command = [ + "lipo", + "-create", + "-output", + output_merged, + *macos_libs + ] run_command(command, self.buildEnv.build_dir, context) - return [pj(fat_dir, "merged.a")] + return [output_merged] def _build_xcframework(self, xcf_libs, context): # create xcframework ref_plat = PlatformInfo.get_platform(self.macos_subplatforms[0]) - command = "xcodebuild -create-xcframework " + command = ["xcodebuild", "-create-xcframework"] for lib in xcf_libs: - command += " -library {lib} -headers {include}".format( - lib=lib, include=pj(ref_plat.buildEnv.install_dir, "include") - ) - command += " -output {dest}".format(dest=self.final_path) + command += [ + "-library", lib, + "-headers", pj(ref_plat.buildEnv.install_dir, "include") + ] + command += ["-output", self.final_path] run_command(command, self.buildEnv.build_dir, context) def build(self):