From 5a1175cf2d7ea274b40d1d124e3fd11dd71b9e07 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 5 Feb 2024 11:41:09 +0100 Subject: [PATCH 1/6] Format our code with black --- kiwixbuild/__init__.py | 169 ++++++---- kiwixbuild/_global.py | 18 +- kiwixbuild/buildenv.py | 148 +++++---- kiwixbuild/builder.py | 97 +++--- kiwixbuild/dependencies/__init__.py | 3 +- kiwixbuild/dependencies/all_dependencies.py | 39 ++- kiwixbuild/dependencies/aria2.py | 23 +- kiwixbuild/dependencies/base.py | 308 +++++++++++-------- kiwixbuild/dependencies/docoptcpp.py | 11 +- kiwixbuild/dependencies/gumbo.py | 14 +- kiwixbuild/dependencies/icu4c.py | 78 ++--- kiwixbuild/dependencies/ios_fat_lib.py | 39 ++- kiwixbuild/dependencies/kiwix_desktop.py | 24 +- kiwixbuild/dependencies/kiwix_tools.py | 6 +- kiwixbuild/dependencies/libcurl.py | 48 ++- kiwixbuild/dependencies/libkiwix.py | 38 ++- kiwixbuild/dependencies/libmagic.py | 32 +- kiwixbuild/dependencies/libmicrohttpd.py | 22 +- kiwixbuild/dependencies/libzim.py | 24 +- kiwixbuild/dependencies/lzma.py | 36 +-- kiwixbuild/dependencies/mustache.py | 26 +- kiwixbuild/dependencies/pugixml.py | 14 +- kiwixbuild/dependencies/qt.py | 93 +++--- kiwixbuild/dependencies/tc_android_ndk.py | 52 ++-- kiwixbuild/dependencies/tc_armhf.py | 37 ++- kiwixbuild/dependencies/tc_emsdk.py | 20 +- kiwixbuild/dependencies/tc_flatpak.py | 55 ++-- kiwixbuild/dependencies/tc_musl.py | 17 +- kiwixbuild/dependencies/uuid.py | 40 +-- kiwixbuild/dependencies/xapian.py | 32 +- kiwixbuild/dependencies/zim_testing_suite.py | 14 +- kiwixbuild/dependencies/zim_tools.py | 14 +- kiwixbuild/dependencies/zlib.py | 42 +-- kiwixbuild/dependencies/zstd.py | 22 +- kiwixbuild/flatpak_builder.py | 242 ++++++++------- kiwixbuild/packages.py | 212 +++++++------ kiwixbuild/platforms/__init__.py | 14 +- kiwixbuild/platforms/android.py | 145 +++++---- kiwixbuild/platforms/armhf.py | 162 +++++----- kiwixbuild/platforms/base.py | 68 ++-- kiwixbuild/platforms/flatpak.py | 14 +- kiwixbuild/platforms/i586.py | 83 ++--- kiwixbuild/platforms/ios.py | 219 +++++++------ kiwixbuild/platforms/musl.py | 154 +++++----- kiwixbuild/platforms/native.py | 27 +- kiwixbuild/platforms/neutral.py | 7 +- kiwixbuild/platforms/wasm.py | 103 ++++--- kiwixbuild/platforms/win32.py | 87 +++--- kiwixbuild/platforms/win64.py | 91 +++--- kiwixbuild/utils.py | 88 +++--- kiwixbuild/versions.py | 63 ++-- 51 files changed, 1916 insertions(+), 1518 deletions(-) diff --git a/kiwixbuild/__init__.py b/kiwixbuild/__init__.py index cdbd6b9..6bb3e5a 100644 --- a/kiwixbuild/__init__.py +++ b/kiwixbuild/__init__.py @@ -9,76 +9,141 @@ from .builder import Builder from .flatpak_builder import FlatpakBuilder from . import _global + def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument('target', default='kiwix-tools', nargs='?', metavar='TARGET', - choices=Dependency.all_deps.keys()) - parser.add_argument('--working-dir', default=".", - help=("Directory where kiwix-build puts all its files " - "(source, archive and build)\n" - "working-dir can be absolute path or a relative (to cwd) one.")) - parser.add_argument('--build-dir', default=".", - help=("Directory where kiwix-build puts all build files.\n" - "build-dir can be absolute path or a relative (to working-dir) one.")) - parser.add_argument('--libprefix', default=None) - parser.add_argument('--target-platform', choices=PlatformInfo.all_platforms) - parser.add_argument('--verbose', '-v', action="store_true", - help=("Print all logs on stdout instead of in specific" - " log files per commands")) - parser.add_argument('--hide-progress', action='store_false', dest='show_progress', - help="Hide intermediate progress information.") - parser.add_argument('--skip-source-prepare', action='store_true', - help="Skip the source download part") - parser.add_argument('--build-deps-only', action='store_true', - help="Build only the dependencies of the specified target.") - parser.add_argument('--build-nodeps', action='store_true', - help="Build only the target, not its dependencies.") - parser.add_argument('--make-dist', action='store_true', - help="Build distrubution (dist) source archive") - parser.add_argument('--make-release', action='store_true', - help="Build a release version") - subgroup = parser.add_argument_group('advanced') - subgroup.add_argument('--no-cert-check', action='store_true', - help="Skip SSL certificate verification during download") - subgroup.add_argument('--clean-at-end', action='store_true', - help="Clean all intermediate files after the (successfull) build") - subgroup.add_argument('--dont-install-packages', action='store_true', - help="Do not try to install packages before compiling") - subgroup.add_argument('--assume-packages-installed', action='store_true', - help="Assume the package to install to be aleady installed") - subgroup.add_argument('--android-arch', action='append', - help=("Specify the architecture to build for android application/libraries.\n" - "Can be specified several times to build for several architectures.\n" - "If not specified, all architectures will be build.")) - subgroup.add_argument('--ios-arch', action='append', - help=("Specify the architecture to build for ios application/libraries.\n" - "Can be specified several times to build for several architectures.\n" - "If not specified, all architectures will be build.")) - subgroup.add_argument('--fast-clone', action='store_true', - help=("Do not clone the whole repository.\n" - "This is useful for one shot build but it is not recommended if you want " - "to develop with the cloned sources.")) + parser.add_argument( + "target", + default="kiwix-tools", + nargs="?", + metavar="TARGET", + choices=Dependency.all_deps.keys(), + ) + parser.add_argument( + "--working-dir", + default=".", + help=( + "Directory where kiwix-build puts all its files " + "(source, archive and build)\n" + "working-dir can be absolute path or a relative (to cwd) one." + ), + ) + parser.add_argument( + "--build-dir", + default=".", + help=( + "Directory where kiwix-build puts all build files.\n" + "build-dir can be absolute path or a relative (to working-dir) one." + ), + ) + parser.add_argument("--libprefix", default=None) + parser.add_argument("--target-platform", choices=PlatformInfo.all_platforms) + parser.add_argument( + "--verbose", + "-v", + action="store_true", + help=( + "Print all logs on stdout instead of in specific" " log files per commands" + ), + ) + parser.add_argument( + "--hide-progress", + action="store_false", + dest="show_progress", + help="Hide intermediate progress information.", + ) + parser.add_argument( + "--skip-source-prepare", + action="store_true", + help="Skip the source download part", + ) + parser.add_argument( + "--build-deps-only", + action="store_true", + help="Build only the dependencies of the specified target.", + ) + parser.add_argument( + "--build-nodeps", + action="store_true", + help="Build only the target, not its dependencies.", + ) + parser.add_argument( + "--make-dist", + action="store_true", + help="Build distrubution (dist) source archive", + ) + parser.add_argument( + "--make-release", action="store_true", help="Build a release version" + ) + subgroup = parser.add_argument_group("advanced") + subgroup.add_argument( + "--no-cert-check", + action="store_true", + help="Skip SSL certificate verification during download", + ) + subgroup.add_argument( + "--clean-at-end", + action="store_true", + help="Clean all intermediate files after the (successfull) build", + ) + subgroup.add_argument( + "--dont-install-packages", + action="store_true", + help="Do not try to install packages before compiling", + ) + subgroup.add_argument( + "--assume-packages-installed", + action="store_true", + help="Assume the package to install to be aleady installed", + ) + subgroup.add_argument( + "--android-arch", + action="append", + help=( + "Specify the architecture to build for android application/libraries.\n" + "Can be specified several times to build for several architectures.\n" + "If not specified, all architectures will be build." + ), + ) + subgroup.add_argument( + "--ios-arch", + action="append", + help=( + "Specify the architecture to build for ios application/libraries.\n" + "Can be specified several times to build for several architectures.\n" + "If not specified, all architectures will be build." + ), + ) + subgroup.add_argument( + "--fast-clone", + action="store_true", + help=( + "Do not clone the whole repository.\n" + "This is useful for one shot build but it is not recommended if you want " + "to develop with the cloned sources." + ), + ) options = parser.parse_args() if not options.android_arch: - options.android_arch = ['arm', 'arm64', 'x86', 'x86_64'] + options.android_arch = ["arm", "arm64", "x86", "x86_64"] if not options.ios_arch: - options.ios_arch = ['arm64', 'x86_64'] + options.ios_arch = ["arm64", "x86_64"] if not options.target_platform: - options.target_platform = 'native_dyn' + options.target_platform = "native_dyn" return options + def main(): options = parse_args() options.working_dir = os.path.abspath(options.working_dir) _global.set_options(options) neutralEnv = buildenv.PlatformNeutralEnv() _global.set_neutralEnv(neutralEnv) - if options.target_platform == 'flatpak': + if options.target_platform == "flatpak": builder = FlatpakBuilder() else: builder = Builder() builder.run() - diff --git a/kiwixbuild/_global.py b/kiwixbuild/_global.py index c476c09..9e37bd1 100644 --- a/kiwixbuild/_global.py +++ b/kiwixbuild/_global.py @@ -5,23 +5,29 @@ _neutralEnv = None _options = None _target_steps = _OrderedDict() + def set_neutralEnv(env): global _neutralEnv _neutralEnv = env + def neutralEnv(what): return getattr(_neutralEnv, what) + def set_options(options): global _options _options = options + def option(what): return getattr(_options, what) + def add_target_step(key, what): _target_steps[key] = what + def get_target_step(key, default_context=None): if isinstance(key, tuple): context, target = key @@ -29,6 +35,7 @@ def get_target_step(key, default_context=None): context, target = default_context, key return _target_steps[(context, target)] + def target_steps(): return _target_steps @@ -39,14 +46,15 @@ def backend(): return _backend _platform = platform.system() - if _platform == 'Windows': - print('ERROR: kiwix-build is not intented to run on Windows platform.\n' - 'There is no backend for Windows, so we can\'t launch any commands.') + if _platform == "Windows": + print( + "ERROR: kiwix-build is not intented to run on Windows platform.\n" + "There is no backend for Windows, so we can't launch any commands." + ) sys.exit(0) - if _platform == 'Linux': + if _platform == "Linux": _platform, _, _ = platform.linux_distribution() _platform = _platform.lower() _backend = backends.Linux() return _backend - diff --git a/kiwixbuild/buildenv.py b/kiwixbuild/buildenv.py index 458542f..27c8979 100644 --- a/kiwixbuild/buildenv.py +++ b/kiwixbuild/buildenv.py @@ -1,4 +1,3 @@ - import os, sys, shutil import subprocess import platform @@ -10,60 +9,58 @@ from ._global import neutralEnv, option class PlatformNeutralEnv: def __init__(self): - self.working_dir = option('working_dir') + self.working_dir = option("working_dir") self.source_dir = pj(self.working_dir, "SOURCE") self.archive_dir = pj(self.working_dir, "ARCHIVE") self.toolchain_dir = pj(self.working_dir, "TOOLCHAINS") - self.log_dir = pj(self.working_dir, 'LOGS') - for d in (self.source_dir, - self.archive_dir, - self.toolchain_dir, - self.log_dir): + self.log_dir = pj(self.working_dir, "LOGS") + for d in (self.source_dir, self.archive_dir, self.toolchain_dir, self.log_dir): os.makedirs(d, exist_ok=True) self.detect_platform() self.ninja_command = self._detect_command( - 'ninja', - default=[['ninja'], ['ninja-build']]) + "ninja", default=[["ninja"], ["ninja-build"]] + ) self.meson_command = self._detect_command( - 'meson', - default= [['meson.py'], ['meson']]) + "meson", default=[["meson.py"], ["meson"]] + ) self.mesontest_command = [*self.meson_command, "test"] - self.patch_command = self._detect_command('patch') - self.git_command = self._detect_command('git') - self.make_command = self._detect_command('make') - self.cmake_command = self._detect_command('cmake') - self.qmake_command = self._detect_command('qmake', required=False) - + self.patch_command = self._detect_command("patch") + self.git_command = self._detect_command("git") + self.make_command = self._detect_command("make") + self.cmake_command = self._detect_command("cmake") + self.qmake_command = self._detect_command("qmake", required=False) def detect_platform(self): _platform = platform.system() self.distname = _platform - if _platform == 'Windows': - print('ERROR: kiwix-build is not intented to run on Windows platform.\n' - 'It should probably not work, but well, you still can have a try.') - cont = input('Do you want to continue ? [y/N]') - if cont.lower() != 'y': + if _platform == "Windows": + print( + "ERROR: kiwix-build is not intented to run on Windows platform.\n" + "It should probably not work, but well, you still can have a try." + ) + cont = input("Do you want to continue ? [y/N]") + if cont.lower() != "y": sys.exit(0) - if _platform == 'Linux': + if _platform == "Linux": self.distname = distro.id() - if self.distname == 'ubuntu': - self.distname = 'debian' + if self.distname == "ubuntu": + self.distname = "debian" def download(self, what, where=None): where = where or self.archive_dir download_remote(what, where) - - def _detect_command(self, name, default=None, options=['--version'], required=True): + def _detect_command(self, name, default=None, options=["--version"], required=True): if default is None: default = [[name]] - env_key = 'KBUILD_{}_COMMAND'.format(name.upper()) + env_key = "KBUILD_{}_COMMAND".format(name.upper()) if env_key in os.environ: default = [os.environ[env_key].split()] + default for command in default: try: - retcode = subprocess.check_call(command + options, - stdout=subprocess.DEVNULL) + retcode = subprocess.check_call( + command + options, stdout=subprocess.DEVNULL + ) except (FileNotFoundError, PermissionError, OSError): # Doesn't exist in PATH or isn't executable continue @@ -76,22 +73,20 @@ class PlatformNeutralEnv: print("WARNING: {} command not found".format(name)) return ["{}_NOT_FOUND".format(name.upper())] + class BuildEnv: def __init__(self, platformInfo): build_dir = "BUILD_{}".format(platformInfo.name) self.platformInfo = platformInfo - self.base_build_dir = pj(option('working_dir'), option('build_dir')) + self.base_build_dir = pj(option("working_dir"), option("build_dir")) self.build_dir = pj(self.base_build_dir, build_dir) self.install_dir = pj(self.build_dir, "INSTALL") self.toolchain_dir = pj(self.build_dir, "TOOLCHAINS") - self.log_dir = pj(self.build_dir, 'LOGS') - for d in (self.build_dir, - self.install_dir, - self.toolchain_dir, - self.log_dir): + self.log_dir = pj(self.build_dir, "LOGS") + for d in (self.build_dir, self.install_dir, self.toolchain_dir, self.log_dir): os.makedirs(d, exist_ok=True) - self.libprefix = option('libprefix') or self._detect_libdir() + self.libprefix = option("libprefix") or self._detect_libdir() def clean_intermediate_directories(self): for subdir in os.listdir(self.build_dir): @@ -104,69 +99,72 @@ class BuildEnv: os.remove(subpath) def _is_debianlike(self): - return os.path.isfile('/etc/debian_version') + return os.path.isfile("/etc/debian_version") def _detect_libdir(self): if self.platformInfo.libdir is not None: return self.platformInfo.libdir if self._is_debianlike(): try: - pc = subprocess.Popen(['dpkg-architecture', '-qDEB_HOST_MULTIARCH'], - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL) + pc = subprocess.Popen( + ["dpkg-architecture", "-qDEB_HOST_MULTIARCH"], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) (stdo, _) = pc.communicate() if pc.returncode == 0: archpath = stdo.decode().strip() - return 'lib/' + archpath + return "lib/" + archpath except Exception: pass - if os.path.isdir('/usr/lib64') and not os.path.islink('/usr/lib64'): - return 'lib64' - return 'lib' + if os.path.isdir("/usr/lib64") and not os.path.islink("/usr/lib64"): + return "lib64" + return "lib" def get_env(self, *, cross_comp_flags, cross_compilers, cross_path): env = self.platformInfo.get_env() - pkgconfig_path = pj(self.install_dir, self.libprefix, 'pkgconfig') - env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path]) + pkgconfig_path = pj(self.install_dir, self.libprefix, "pkgconfig") + env["PKG_CONFIG_PATH"] = ":".join([env["PKG_CONFIG_PATH"], pkgconfig_path]) - env['PATH'] = ':'.join([ - escape_path(pj(self.install_dir, 'bin')), - env['PATH'] - ]) + env["PATH"] = ":".join([escape_path(pj(self.install_dir, "bin")), env["PATH"]]) - env['LD_LIBRARY_PATH'] = ':'.join([env['LD_LIBRARY_PATH'], - pj(self.install_dir, 'lib'), - pj(self.install_dir, self.libprefix) - ]) + env["LD_LIBRARY_PATH"] = ":".join( + [ + env["LD_LIBRARY_PATH"], + pj(self.install_dir, "lib"), + pj(self.install_dir, self.libprefix), + ] + ) - env['QMAKE_CXXFLAGS'] = " ".join([ - escape_path('-I'+pj(self.install_dir, 'include')), - env['QMAKE_CXXFLAGS'] - ]) - env['CPPFLAGS'] = " ".join([ - escape_path('-I'+pj(self.install_dir, 'include')), - env['CPPFLAGS'] - ]) - env['QMAKE_LFLAGS'] = " ".join([ - escape_path('-L'+pj(self.install_dir, 'lib')), - escape_path('-L'+pj(self.install_dir, self.libprefix)), - env['QMAKE_LFLAGS'] - ]) - env['LDFLAGS'] = " ".join([ - escape_path('-L'+pj(self.install_dir, 'lib')), - escape_path('-L'+pj(self.install_dir, self.libprefix)), - env['LDFLAGS'] - ]) + env["QMAKE_CXXFLAGS"] = " ".join( + [escape_path("-I" + pj(self.install_dir, "include")), env["QMAKE_CXXFLAGS"]] + ) + env["CPPFLAGS"] = " ".join( + [escape_path("-I" + pj(self.install_dir, "include")), env["CPPFLAGS"]] + ) + env["QMAKE_LFLAGS"] = " ".join( + [ + escape_path("-L" + pj(self.install_dir, "lib")), + escape_path("-L" + pj(self.install_dir, self.libprefix)), + env["QMAKE_LFLAGS"], + ] + ) + env["LDFLAGS"] = " ".join( + [ + escape_path("-L" + pj(self.install_dir, "lib")), + escape_path("-L" + pj(self.install_dir, self.libprefix)), + env["LDFLAGS"], + ] + ) if cross_comp_flags: self.platformInfo.set_comp_flags(env) if cross_compilers: self.platformInfo.set_compiler(env) if cross_path: - env['PATH'] = ':'.join(self.platformInfo.get_bin_dir() + [env['PATH']]) + env["PATH"] = ":".join(self.platformInfo.get_bin_dir() + [env["PATH"]]) return env - @property def configure_wrapper(self): try: diff --git a/kiwixbuild/builder.py b/kiwixbuild/builder.py index 78f8ca3..a59461d 100644 --- a/kiwixbuild/builder.py +++ b/kiwixbuild/builder.py @@ -1,4 +1,3 @@ - import sys from collections import OrderedDict from .buildenv import * @@ -8,23 +7,32 @@ from .utils import remove_duplicates, StopBuild, colorize from .dependencies import Dependency from .packages import PACKAGE_NAME_MAPPERS from ._global import ( - neutralEnv, option, - add_target_step, get_target_step, target_steps, - backend) + neutralEnv, + option, + add_target_step, + get_target_step, + target_steps, + backend, +) from . import _global + class Builder: def __init__(self): self._targets = {} - PlatformInfo.get_platform('neutral', self._targets) + PlatformInfo.get_platform("neutral", self._targets) - target_platform = option('target_platform') + target_platform = option("target_platform") platform = PlatformInfo.get_platform(target_platform, self._targets) - if neutralEnv('distname') not in platform.compatible_hosts: - print((colorize('ERROR')+': The target platform {} cannot be build on host {}.\n' - 'Select another target platform or change your host system.' - ).format(platform.name, neutralEnv('distname'))) - self.targetDefs = platform.add_targets(option('target'), self._targets) + if neutralEnv("distname") not in platform.compatible_hosts: + print( + ( + colorize("ERROR") + + ": The target platform {} cannot be build on host {}.\n" + "Select another target platform or change your host system." + ).format(platform.name, neutralEnv("distname")) + ) + self.targetDefs = platform.add_targets(option("target"), self._targets) def finalize_target_steps(self): steps = [] @@ -32,19 +40,19 @@ class Builder: steps += self.order_steps(targetDef) steps = list(remove_duplicates(steps)) - if option('build_nodeps'): + if option("build_nodeps"): # add all platform steps for dep in steps: stepClass = Dependency.all_deps[dep[1]] if stepClass.dont_skip: add_target_step(dep, self._targets[dep]) - src_targetDef = ('source', targetDef[1]) + src_targetDef = ("source", targetDef[1]) add_target_step(src_targetDef, self._targets[src_targetDef]) add_target_step(targetDef, self._targets[targetDef]) else: for dep in steps: - if option('build_deps_only') and dep[1] == targetDef[1]: + if option("build_deps_only") and dep[1] == targetDef[1]: continue add_target_step(dep, self._targets[dep]) self.instanciate_steps() @@ -54,14 +62,14 @@ class Builder: plt = PlatformInfo.all_platforms[pltName] for tlcName in plt.toolchain_names: tlc = Dependency.all_deps[tlcName] - yield('source', tlcName) - yield('neutral' if tlc.neutral else pltName, tlcName) - _targets =dict(self._targets) + yield ("source", tlcName) + yield ("neutral" if tlc.neutral else pltName, tlcName) + _targets = dict(self._targets) yield from self.order_dependencies(targetDef, _targets) def order_dependencies(self, targetDef, targets): targetPlatformName, targetName = targetDef - if targetPlatformName == 'source': + if targetPlatformName == "source": # Do not try to order sources, they will be added as dep by the # build step two lines later. return @@ -75,38 +83,40 @@ class Builder: depPlatform, depName = targetPlatform.get_fully_qualified_dep(dep) if (depPlatform, depName) in targets: yield from self.order_dependencies((depPlatform, depName), targets) - yield ('source', targetName) + yield ("source", targetName) yield targetDef def instanciate_steps(self): for stepDef in list(target_steps()): stepPlatform, stepName = stepDef stepClass = Dependency.all_deps[stepName] - if stepPlatform == 'source': + if stepPlatform == "source": source = get_target_step(stepDef)(stepClass) add_target_step(stepDef, source) else: - source = get_target_step(stepName, 'source') + source = get_target_step(stepName, "source") env = PlatformInfo.get_platform(stepPlatform).buildEnv builder = get_target_step(stepDef)(stepClass, source, env) add_target_step(stepDef, builder) def prepare_sources(self): - if option('skip_source_prepare'): + if option("skip_source_prepare"): print(colorize("SKIP")) return - sourceDefs = remove_duplicates(tDef for tDef in target_steps() if tDef[0]=='source') + sourceDefs = remove_duplicates( + tDef for tDef in target_steps() if tDef[0] == "source" + ) for sourceDef in sourceDefs: print("prepare sources {} :".format(sourceDef[1])) source = get_target_step(sourceDef) source.prepare() def build(self): - builderDefs = (tDef for tDef in target_steps() if tDef[0] != 'source') + builderDefs = (tDef for tDef in target_steps() if tDef[0] != "source") for builderDef in builderDefs: builder = get_target_step(builderDef) - if option('make_dist') and builderDef[1] == option('target'): + if option("make_dist") and builderDef[1] == option("target"): print("make dist {} ({}):".format(builder.name, builderDef[0])) builder.make_dist() continue @@ -118,17 +128,17 @@ class Builder: packages_list = [] for platform in PlatformInfo.all_running_platforms.values(): mapper_name = "{host}_{target}".format( - host=neutralEnv('distname'), - target=platform) + host=neutralEnv("distname"), target=platform + ) package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {}) - packages_list += package_name_mapper.get('COMMON', []) + packages_list += package_name_mapper.get("COMMON", []) to_drop = [] for builderDef in self._targets: platformName, builderName = builderDef mapper_name = "{host}_{target}".format( - host=neutralEnv('distname'), - target=platformName) + host=neutralEnv("distname"), target=platformName + ) package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {}) packages = package_name_mapper.get(builderName) if packages: @@ -144,20 +154,20 @@ class Builder: packages_to_have = self._get_packages() packages_to_have = remove_duplicates(packages_to_have) - if option('assume_packages_installed'): + if option("assume_packages_installed"): print(colorize("SKIP") + ", Assume package installed") return - distname = neutralEnv('distname') - if distname in ('fedora', 'redhat', 'centos'): - package_installer = 'sudo dnf install {}' - package_checker = 'rpm -q --quiet {}' - elif distname in ('debian', 'Ubuntu'): - package_installer = 'sudo apt-get install {}' + distname = neutralEnv("distname") + if distname in ("fedora", "redhat", "centos"): + package_installer = "sudo dnf install {}" + package_checker = "rpm -q --quiet {}" + elif distname in ("debian", "Ubuntu"): + package_installer = "sudo apt-get install {}" package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1' - elif distname == 'Darwin': - package_installer = 'brew install {}' - package_checker = 'brew ls --version {} > /dev/null' + elif distname == "Darwin": + package_installer = "brew install {}" + package_checker = "brew ls --version {} > /dev/null" packages_to_install = [] for package in packages_to_have: @@ -176,12 +186,12 @@ class Builder: print(command) subprocess.check_call(command, shell=True) else: - print(colorize("SKIP")+ ", No package to install.") + print(colorize("SKIP") + ", No package to install.") def run(self): try: print("[INSTALL PACKAGES]") - if option('dont_install_packages'): + if option("dont_install_packages"): print(colorize("SKIP")) else: self.install_packages() @@ -195,7 +205,7 @@ class Builder: self.build() # No error, clean intermediate file at end of build if needed. print("[CLEAN]") - if option('clean_at_end'): + if option("clean_at_end"): for platform in PlatformInfo.all_running_platforms.values(): platform.clean_intermediate_directories() else: @@ -203,4 +213,3 @@ class Builder: except StopBuild as e: print(e) sys.exit("Stopping build due to errors") - diff --git a/kiwixbuild/dependencies/__init__.py b/kiwixbuild/dependencies/__init__.py index bdede5b..7a45018 100644 --- a/kiwixbuild/dependencies/__init__.py +++ b/kiwixbuild/dependencies/__init__.py @@ -1,4 +1,3 @@ - from .base import * from . import ( all_dependencies, @@ -28,5 +27,5 @@ from . import ( zim_tools, zim_testing_suite, zlib, - zstd + zstd, ) diff --git a/kiwixbuild/dependencies/all_dependencies.py b/kiwixbuild/dependencies/all_dependencies.py index 09f29b3..ab0a889 100644 --- a/kiwixbuild/dependencies/all_dependencies.py +++ b/kiwixbuild/dependencies/all_dependencies.py @@ -1,32 +1,45 @@ from os import environ -from .base import ( - Dependency, - NoopSource, - NoopBuilder) +from .base import Dependency, NoopSource, NoopBuilder from kiwixbuild._global import neutralEnv + class AllBaseDependencies(Dependency): name = "alldependencies" Source = NoopSource + class Builder(NoopBuilder): @classmethod def get_dependencies(cls, platformInfo, allDeps): - if platformInfo.build == "wasm" or environ.get('OS_NAME') == 'bionic': - return ['zlib', 'lzma', 'zstd', 'icu4c', 'xapian-core'] + if platformInfo.build == "wasm" or environ.get("OS_NAME") == "bionic": + return ["zlib", "lzma", "zstd", "icu4c", "xapian-core"] - base_deps = ['zlib', 'lzma', 'zstd', 'xapian-core', 'pugixml', 'libcurl', 'icu4c', 'mustache', 'libmicrohttpd', 'zim-testing-suite'] + base_deps = [ + "zlib", + "lzma", + "zstd", + "xapian-core", + "pugixml", + "libcurl", + "icu4c", + "mustache", + "libmicrohttpd", + "zim-testing-suite", + ] # Add specific dependencies depending of the platform - if platformInfo.build not in ('android', 'iOS'): + if platformInfo.build not in ("android", "iOS"): # For zimtools - base_deps += ['docoptcpp'] - if platformInfo.build != 'win32': + base_deps += ["docoptcpp"] + if platformInfo.build != "win32": # zimwriterfs - base_deps += ['libmagic', 'gumbo'] - if platformInfo.build == 'native' and neutralEnv('distname') != 'Darwin': + base_deps += ["libmagic", "gumbo"] + if ( + platformInfo.build == "native" + and neutralEnv("distname") != "Darwin" + ): # We compile kiwix-desktop only on native and not on `Darwin` # So we need aria2 only there - base_deps += ['aria2'] + base_deps += ["aria2"] return base_deps diff --git a/kiwixbuild/dependencies/aria2.py b/kiwixbuild/dependencies/aria2.py index d543be4..b2b8ee7 100644 --- a/kiwixbuild/dependencies/aria2.py +++ b/kiwixbuild/dependencies/aria2.py @@ -1,18 +1,17 @@ -from .base import ( - Dependency, - ReleaseDownload, - MakeBuilder -) +from .base import Dependency, ReleaseDownload, MakeBuilder from kiwixbuild.utils import Remotefile, run_command + class Aria2(Dependency): name = "aria2" class Source(ReleaseDownload): - archive = Remotefile('aria2-1.37.0.tar.xz', - '60a420ad7085eb616cb6e2bdf0a7206d68ff3d37fb5a956dc44242eb2f79b66b', - 'https://github.com/aria2/aria2/releases/download/release-1.37.0/aria2-1.37.0.tar.xz') + archive = Remotefile( + "aria2-1.37.0.tar.xz", + "60a420ad7085eb616cb6e2bdf0a7206d68ff3d37fb5a956dc44242eb2f79b66b", + "https://github.com/aria2/aria2/releases/download/release-1.37.0/aria2-1.37.0.tar.xz", + ) def _post_prepare_script(self, context): context.try_skip(self.extract_path) @@ -20,5 +19,9 @@ class Aria2(Dependency): run_command(command, self.extract_path, context) class Builder(MakeBuilder): - dependencies = ['zlib'] - configure_options = ["--disable-libaria2", "--disable-websocket", "--without-sqlite3"] + dependencies = ["zlib"] + configure_options = [ + "--disable-libaria2", + "--disable-websocket", + "--without-sqlite3", + ] diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index a778c6a..91e4a21 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -3,7 +3,17 @@ import os import shutil import time -from kiwixbuild.utils import pj, Context, SkipCommand, WarningMessage, extract_archive, StopBuild, run_command, colorize, copy_tree +from kiwixbuild.utils import ( + pj, + Context, + SkipCommand, + WarningMessage, + extract_archive, + StopBuild, + run_command, + colorize, + copy_tree, +) from kiwixbuild.versions import main_project_versions, base_deps_versions from kiwixbuild._global import neutralEnv, option, get_target_step @@ -13,8 +23,8 @@ SCRIPT_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) class _MetaDependency(type): def __new__(cls, name, bases, dct): _class = type.__new__(cls, name, bases, dct) - if name != 'Dependency': - dep_name = dct['name'] + if name != "Dependency": + dep_name = dct["name"] Dependency.all_deps[dep_name] = _class return _class @@ -29,7 +39,7 @@ class Dependency(metaclass=_MetaDependency): def version(cls): if cls.name in base_deps_versions: return base_deps_versions[cls.name] - elif option('make_release'): + elif option("make_release"): return main_project_versions.get(cls.name, None) return None @@ -42,8 +52,9 @@ class Dependency(metaclass=_MetaDependency): class Source: """Base Class to the real preparator - A source preparator must install source in the self.source_dir attribute - inside the neutralEnv.source_dir.""" + A source preparator must install source in the self.source_dir attribute + inside the neutralEnv.source_dir.""" + def __init__(self, target): self.target = target @@ -61,22 +72,22 @@ class Source: @property def source_path(self): - return pj(neutralEnv('source_dir'), self.source_dir) + return pj(neutralEnv("source_dir"), self.source_dir) @property def _log_dir(self): - return neutralEnv('log_dir') + return neutralEnv("log_dir") def _patch(self, context): context.try_skip(self.source_path) for p in self.patches: - patch_file_path = pj(SCRIPT_DIR, 'patches', p) - patch_command = [*neutralEnv('patch_command'), "-p1", "-i", patch_file_path] + patch_file_path = pj(SCRIPT_DIR, "patches", p) + patch_command = [*neutralEnv("patch_command"), "-p1", "-i", patch_file_path] run_command(patch_command, self.source_path, context) def command(self, name, function, *args): print(" {} {} : ".format(name, self.name), end="", flush=True) - log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name)) + log = pj(self._log_dir, "cmd_{}_{}.log".format(name, self.name)) context = Context(name, log, True) try: start_time = time.time() @@ -92,7 +103,7 @@ class Source: except subprocess.CalledProcessError: print(colorize("ERROR")) try: - with open(log, 'r') as f: + with open(log, "r") as f: print(f.read()) except: pass @@ -112,34 +123,36 @@ class ReleaseDownload(Source): @property def archives(self): - return (self.archive, ) + return (self.archive,) @property def extract_path(self): - return pj(neutralEnv('source_dir'), self.source_dir) + return pj(neutralEnv("source_dir"), self.source_dir) def _download(self, context): - context.try_skip(neutralEnv('archive_dir'), self.full_name) + context.try_skip(neutralEnv("archive_dir"), self.full_name) for archive in self.archives: - neutralEnv('download')(archive) + neutralEnv("download")(archive) def _extract(self, context): context.try_skip(self.extract_path) if os.path.exists(self.extract_path): shutil.rmtree(self.extract_path) for archive in self.archives: - extract_archive(pj(neutralEnv('archive_dir'), archive.name), - neutralEnv('source_dir'), - topdir=self.archive_top_dir, - name=self.source_dir) + extract_archive( + pj(neutralEnv("archive_dir"), archive.name), + neutralEnv("source_dir"), + topdir=self.archive_top_dir, + name=self.source_dir, + ) def prepare(self): - self.command('download', self._download) - self.command('extract', self._extract) - if hasattr(self, 'patches'): - self.command('patch', self._patch) - if hasattr(self, '_post_prepare_script'): - self.command('post_prepare_script', self._post_prepare_script) + self.command("download", self._download) + self.command("extract", self._extract) + if hasattr(self, "patches"): + self.command("patch", self._patch) + if hasattr(self, "_post_prepare_script"): + self.command("post_prepare_script", self._post_prepare_script) class GitClone(Source): @@ -152,48 +165,66 @@ class GitClone(Source): @property def source_dir(self): - if option('make_release'): + if option("make_release"): return "{}_release".format(self.git_dir) else: return self.git_dir @property def git_path(self): - return pj(neutralEnv('source_dir'), self.source_dir) + return pj(neutralEnv("source_dir"), self.source_dir) @property def git_ref(self): - if option('make_release'): + if option("make_release"): return self.release_git_ref else: return self.base_git_ref def _git_init(self, context): - if option('fast_clone') and self.force_full_clone == False: - command = [*neutralEnv('git_command'), "clone" , "--depth=1", "--branch", self.git_ref, self.git_remote, self.source_dir] - run_command(command, neutralEnv('source_dir'), context) + if option("fast_clone") and self.force_full_clone == False: + command = [ + *neutralEnv("git_command"), + "clone", + "--depth=1", + "--branch", + self.git_ref, + self.git_remote, + self.source_dir, + ] + run_command(command, neutralEnv("source_dir"), context) else: - command = [*neutralEnv('git_command'), "clone", self.git_remote, self.source_dir] - run_command(command, neutralEnv('source_dir'), context) - command = [*neutralEnv('git_command'), "checkout", self.git_ref] + command = [ + *neutralEnv("git_command"), + "clone", + self.git_remote, + self.source_dir, + ] + run_command(command, neutralEnv("source_dir"), context) + command = [*neutralEnv("git_command"), "checkout", self.git_ref] run_command(command, self.git_path, context) def _git_update(self, context): - command = [*neutralEnv('git_command'), "fetch", "origin", self.git_ref] + command = [*neutralEnv("git_command"), "fetch", "origin", self.git_ref] run_command(command, self.git_path, context) try: - command = [*neutralEnv('git_command'), "merge", "--ff-only", f"origin/{self.git_ref}"] + command = [ + *neutralEnv("git_command"), + "merge", + "--ff-only", + f"origin/{self.git_ref}", + ] run_command(command, self.git_path, context) except subprocess.CalledProcessError: raise WarningMessage("Cannot update, please check log for information") def prepare(self): if not os.path.exists(self.git_path): - self.command('gitinit', self._git_init) + self.command("gitinit", self._git_init) else: - self.command('gitupdate', self._git_update) - if hasattr(self, '_post_prepare_script'): - self.command('post_prepare_script', self._post_prepare_script) + self.command("gitupdate", self._git_update) + if hasattr(self, "_post_prepare_script"): + self.command("post_prepare_script", self._post_prepare_script) class Builder: @@ -230,7 +261,7 @@ class Builder: def command(self, name, function, *args): print(" {} {} : ".format(name, self.name), end="", flush=True) - log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name)) + log = pj(self._log_dir, "cmd_{}_{}.log".format(name, self.name)) context = Context(name, log, self.target.force_native_build) if self.target.force_build: context.no_skip = True @@ -248,7 +279,7 @@ class Builder: except subprocess.CalledProcessError: print(colorize("ERROR")) try: - with open(log, 'r') as f: + with open(log, "r") as f: print(f.read()) except: pass @@ -258,36 +289,40 @@ class Builder: raise def build(self): - if hasattr(self, '_pre_build_script'): - self.command('pre_build_script', self._pre_build_script) - self.command('configure', self._configure) - if hasattr(self, '_post_configure_script'): - self.command('post_configure_script', self._post_configure_script) - self.command('compile', self._compile) - if hasattr(self, '_test'): - self.command('test', self._test) - self.command('install', self._install) - if hasattr(self, '_post_build_script'): - self.command('post_build_script', self._post_build_script) + if hasattr(self, "_pre_build_script"): + self.command("pre_build_script", self._pre_build_script) + self.command("configure", self._configure) + if hasattr(self, "_post_configure_script"): + self.command("post_configure_script", self._post_configure_script) + self.command("compile", self._compile) + if hasattr(self, "_test"): + self.command("test", self._test) + self.command("install", self._install) + if hasattr(self, "_post_build_script"): + self.command("post_build_script", self._post_build_script) def make_dist(self): - if hasattr(self, '_pre_build_script'): - self.command('pre_build_script', self._pre_build_script) - self.command('configure', self._configure) - self.command('make_dist', self._make_dist) + if hasattr(self, "_pre_build_script"): + self.command("pre_build_script", self._pre_build_script) + self.command("configure", self._configure) + self.command("make_dist", self._make_dist) def set_flatpak_buildsystem(self, module): - if getattr(self, 'flatpak_buildsystem', None): - module['buildsystem'] = self.flatpak_buildsystem - if getattr(self, 'subsource_dir', None): - module['subdir'] = self.subsource_dir - if getattr(self, 'flatpack_build_options', None): - module['build-options'] = self.flatpack_build_options - if getattr(self, 'configure_option', ''): - module['config-opts'] = self.configure_option.split(' ') + if getattr(self, "flatpak_buildsystem", None): + module["buildsystem"] = self.flatpak_buildsystem + if getattr(self, "subsource_dir", None): + module["subdir"] = self.subsource_dir + if getattr(self, "flatpack_build_options", None): + module["build-options"] = self.flatpack_build_options + if getattr(self, "configure_option", ""): + module["config-opts"] = self.configure_option.split(" ") def get_env(self, *, cross_comp_flags, cross_compilers, cross_path): - env = self.buildEnv.get_env(cross_comp_flags=cross_comp_flags, cross_compilers=cross_compilers, cross_path=cross_path) + env = self.buildEnv.get_env( + cross_comp_flags=cross_comp_flags, + cross_compilers=cross_compilers, + cross_path=cross_path, + ) for dep in self.get_dependencies(self.buildEnv.platformInfo, False): try: builder = get_target_step(dep, self.buildEnv.platformInfo.name) @@ -317,7 +352,7 @@ class TcCopyBuilder(Builder): return pj(self.buildEnv.toolchain_dir, self.target.full_name()) def build(self): - self.command('copy', self._copy) + self.command("copy", self._copy) def _copy(self, context): context.try_skip(self.build_path) @@ -339,18 +374,18 @@ class MakeBuilder(Builder): install_options = [] configure_script = "configure" configure_env = { - '_format_CFLAGS' : '{env[CFLAGS]} -O3', - '_format_CXXFLAGS': '{env[CXXFLAGS]} -O3' + "_format_CFLAGS": "{env[CFLAGS]} -O3", + "_format_CXXFLAGS": "{env[CXXFLAGS]} -O3", } make_targets = [] flatpak_buildsystem = None @property def make_install_targets(self): - if self.buildEnv.platformInfo.build in ('iOS', "wasm"): - yield 'install' + if self.buildEnv.platformInfo.build in ("iOS", "wasm"): + yield "install" else: - yield 'install-strip' + yield "install-strip" @property def all_configure_options(self): @@ -361,27 +396,26 @@ class MakeBuilder(Builder): yield from self.dynamic_configure_options if not self.target.force_native_build: yield from self.buildEnv.platformInfo.configure_options - yield from ('--prefix', self.buildEnv.install_dir) - yield from ('--libdir', pj(self.buildEnv.install_dir, self.buildEnv.libprefix)) + yield from ("--prefix", self.buildEnv.install_dir) + yield from ("--libdir", pj(self.buildEnv.install_dir, self.buildEnv.libprefix)) def set_configure_env(self, env): dep_conf_env = self.configure_env if not dep_conf_env: return for k, v in dep_conf_env.items(): - if k.startswith('_format_'): + if k.startswith("_format_"): v = v.format(buildEnv=self.buildEnv, env=env) env[k[8:]] = v else: env[k] = v - def _configure(self, context): context.try_skip(self.build_path) command = [ *self.buildEnv.configure_wrapper, pj(self.source_path, self.configure_script), - *self.all_configure_options + *self.all_configure_options, ] env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) self.set_configure_env(env) @@ -391,10 +425,10 @@ class MakeBuilder(Builder): context.try_skip(self.build_path) command = [ *self.buildEnv.make_wrapper, - *neutralEnv('make_command'), + *neutralEnv("make_command"), "-j4", *self.make_targets, - *self.make_options + *self.make_options, ] env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) run_command(command, self.build_path, context, env=env) @@ -403,26 +437,22 @@ class MakeBuilder(Builder): context.try_skip(self.build_path) command = [ *self.buildEnv.make_wrapper, - *neutralEnv('make_command'), + *neutralEnv("make_command"), *self.make_install_targets, - *self.make_options + *self.make_options, ] env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) run_command(command, self.build_path, context, env=env) def _make_dist(self, context): context.try_skip(self.build_path) - command = [ - *self.buildEnv.make_wrapper, - *neutralEnv('make_command'), - "dist" - ] + command = [*self.buildEnv.make_wrapper, *neutralEnv("make_command"), "dist"] env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) run_command(command, self.build_path, context, env=env) class CMakeBuilder(MakeBuilder): - flatpak_buildsystem = 'cmake' + flatpak_buildsystem = "cmake" def _configure(self, context): context.try_skip(self.build_path) @@ -430,74 +460,79 @@ class CMakeBuilder(MakeBuilder): if not self.target.force_native_build and self.buildEnv.cmake_crossfile: cross_options += [f"-DCMAKE_TOOLCHAIN_FILE={self.buildEnv.cmake_crossfile}"] command = [ - *neutralEnv('cmake_command'), + *neutralEnv("cmake_command"), *self.configure_options, "-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON", f"-DCMAKE_INSTALL_PREFIX={self.buildEnv.install_dir}", f"-DCMAKE_INSTALL_LIBDIR={self.buildEnv.libprefix}", self.source_path, - *cross_options + *cross_options, ] - env = self.get_env(cross_comp_flags=True, cross_compilers=False, cross_path=True) + env = self.get_env( + cross_comp_flags=True, cross_compilers=False, cross_path=True + ) self.set_configure_env(env) run_command(command, self.build_path, context, env=env) def set_flatpak_buildsystem(self, module): - super().set_flatpak_buildsystem( module) - module['buildir'] = True + super().set_flatpak_buildsystem(module) + module["buildir"] = True class QMakeBuilder(MakeBuilder): qmake_targets = [] - flatpak_buildsystem = 'qmake' + flatpak_buildsystem = "qmake" @property def env_options(self): - if 'QMAKE_CC' in os.environ: + if "QMAKE_CC" in os.environ: yield f"QMAKE_CC={os.environ['QMAKE_CC']}" - if 'QMAKE_CXX' in os.environ: + if "QMAKE_CXX" in os.environ: yield f"QMAKE_CXX={os.environ['QMAKE_CXX']}" def _configure(self, context): context.try_skip(self.build_path) command = [ "qmake", - *self.configure_options, - *self.env_options, - self.source_path + *self.configure_options, + *self.env_options, + self.source_path, ] - env = self.get_env(cross_comp_flags=True, cross_compilers=False, cross_path=True) + env = self.get_env( + cross_comp_flags=True, cross_compilers=False, cross_path=True + ) self.set_configure_env(env) run_command(command, self.build_path, context, env=env) def _make_dist(self, context): command = [ - *neutralEnv('git_command'), "archive", - "-o", f"{self.build_path}/{self.target_full_name()}.tar.gz", + *neutralEnv("git_command"), + "archive", + "-o", + f"{self.build_path}/{self.target_full_name()}.tar.gz", f"--prefix={self.target_full_name()}/", - "HEAD" + "HEAD", ] run_command(command, self.source_path, context) - class MesonBuilder(Builder): configure_options = [] test_options = [] - flatpak_buildsystem = 'meson' + flatpak_buildsystem = "meson" @property def build_type(self): - return 'release' if option('make_release') else 'debug' + return "release" if option("make_release") else "debug" @property def strip_options(self): - if option('make_release'): - yield '--strip' + if option("make_release"): + yield "--strip" @property def library_type(self): - return 'static' if self.buildEnv.platformInfo.static else 'shared' + return "static" if self.buildEnv.platformInfo.static else "shared" def _configure(self, context): context.try_skip(self.build_path) @@ -508,47 +543,54 @@ class MesonBuilder(Builder): if not self.target.force_native_build and self.buildEnv.meson_crossfile: cross_options += ["--cross-file", self.buildEnv.meson_crossfile] command = [ - *neutralEnv('meson_command'), - '.', self.build_path, - f'--buildtype={self.build_type}', + *neutralEnv("meson_command"), + ".", + self.build_path, + f"--buildtype={self.build_type}", *self.strip_options, - f'--default-library={self.library_type}', + f"--default-library={self.library_type}", *self.configure_options, - f'--prefix={self.buildEnv.install_dir}', - f'--libdir={self.buildEnv.libprefix}', - *cross_options + f"--prefix={self.buildEnv.install_dir}", + f"--libdir={self.buildEnv.libprefix}", + *cross_options, ] - env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) + env = self.get_env( + cross_comp_flags=False, cross_compilers=False, cross_path=True + ) run_command(command, self.source_path, context, env=env) def _compile(self, context): context.try_skip(self.build_path) - command = [*neutralEnv('ninja_command'), "-v"] - env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) + command = [*neutralEnv("ninja_command"), "-v"] + env = self.get_env( + cross_comp_flags=False, cross_compilers=False, cross_path=True + ) run_command(command, self.build_path, context, env=env) def _test(self, context): context.try_skip(self.build_path) - if ( self.buildEnv.platformInfo.build == 'android' - or (self.buildEnv.platformInfo.build != 'native' - and not self.buildEnv.platformInfo.static) - ): + if self.buildEnv.platformInfo.build == "android" or ( + self.buildEnv.platformInfo.build != "native" + and not self.buildEnv.platformInfo.static + ): raise SkipCommand() - command = [ - *neutralEnv('mesontest_command'), - '--verbose', - *self.test_options - ] - env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) + command = [*neutralEnv("mesontest_command"), "--verbose", *self.test_options] + env = self.get_env( + cross_comp_flags=False, cross_compilers=False, cross_path=True + ) run_command(command, self.build_path, context, env=env) def _install(self, context): context.try_skip(self.build_path) - command = [*neutralEnv('ninja_command'), '-v', 'install'] - env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) + command = [*neutralEnv("ninja_command"), "-v", "install"] + env = self.get_env( + cross_comp_flags=False, cross_compilers=False, cross_path=True + ) run_command(command, self.build_path, context, env=env) def _make_dist(self, context): - command = [*neutralEnv('ninja_command'), "-v", "dist"] - env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) + command = [*neutralEnv("ninja_command"), "-v", "dist"] + env = self.get_env( + cross_comp_flags=False, cross_compilers=False, cross_path=True + ) run_command(command, self.build_path, context, env=env) diff --git a/kiwixbuild/dependencies/docoptcpp.py b/kiwixbuild/dependencies/docoptcpp.py index 73c8f4c..b5bf4e1 100644 --- a/kiwixbuild/dependencies/docoptcpp.py +++ b/kiwixbuild/dependencies/docoptcpp.py @@ -1,14 +1,10 @@ -from .base import ( - Dependency, - GitClone, - CMakeBuilder) +from .base import Dependency, GitClone, CMakeBuilder from kiwixbuild.utils import Remotefile - class docoptcpp(Dependency): - name = 'docoptcpp' + name = "docoptcpp" class Source(GitClone): git_remote = "https://github.com/docopt/docopt.cpp.git" @@ -17,5 +13,4 @@ class docoptcpp(Dependency): git_ref = "3dd23e3280f213bacefdf5fcb04857bf52e90917" class Builder(CMakeBuilder): - make_install_targets = ['install'] - + make_install_targets = ["install"] diff --git a/kiwixbuild/dependencies/gumbo.py b/kiwixbuild/dependencies/gumbo.py index 23693c3..76412fa 100644 --- a/kiwixbuild/dependencies/gumbo.py +++ b/kiwixbuild/dependencies/gumbo.py @@ -1,8 +1,4 @@ -from .base import ( - Dependency, - ReleaseDownload, - MakeBuilder -) +from .base import Dependency, ReleaseDownload, MakeBuilder from kiwixbuild.utils import Remotefile, run_command @@ -11,9 +7,11 @@ class Gumbo(Dependency): name = "gumbo" class Source(ReleaseDownload): - archive = Remotefile('gumbo-0.10.1.tar.gz', - '28463053d44a5dfbc4b77bcf49c8cee119338ffa636cc17fc3378421d714efad', - 'https://github.com/google/gumbo-parser/archive/v0.10.1.tar.gz') + archive = Remotefile( + "gumbo-0.10.1.tar.gz", + "28463053d44a5dfbc4b77bcf49c8cee119338ffa636cc17fc3378421d714efad", + "https://github.com/google/gumbo-parser/archive/v0.10.1.tar.gz", + ) def _post_prepare_script(self, context): context.try_skip(self.extract_path) diff --git a/kiwixbuild/dependencies/icu4c.py b/kiwixbuild/dependencies/icu4c.py index 55492b6..816d4e3 100644 --- a/kiwixbuild/dependencies/icu4c.py +++ b/kiwixbuild/dependencies/icu4c.py @@ -1,25 +1,25 @@ -from .base import ( - Dependency, - ReleaseDownload, - MakeBuilder -) +from .base import Dependency, ReleaseDownload, MakeBuilder from kiwixbuild.utils import pj, SkipCommand, Remotefile, extract_archive from kiwixbuild._global import get_target_step, neutralEnv import os, shutil import fileinput + class Icu(Dependency): name = "icu4c" class Source(ReleaseDownload): - archive_src = Remotefile('icu4c-73_2-src.tgz', - '818a80712ed3caacd9b652305e01afc7fa167e6f2e94996da44b90c2ab604ce1', - 'https://github.com/unicode-org/icu/releases/download/release-73-2/icu4c-73_2-src.tgz') - archive_data = Remotefile('icu4c-73_2-data.zip', - 'ca1ee076163b438461e484421a7679fc33a64cd0a54f9d4b401893fa1eb42701', - 'https://github.com/unicode-org/icu/releases/download/release-73-2/icu4c-73_2-data.zip') - + archive_src = Remotefile( + "icu4c-73_2-src.tgz", + "818a80712ed3caacd9b652305e01afc7fa167e6f2e94996da44b90c2ab604ce1", + "https://github.com/unicode-org/icu/releases/download/release-73-2/icu4c-73_2-src.tgz", + ) + archive_data = Remotefile( + "icu4c-73_2-data.zip", + "ca1ee076163b438461e484421a7679fc33a64cd0a54f9d4b401893fa1eb42701", + "https://github.com/unicode-org/icu/releases/download/release-73-2/icu4c-73_2-data.zip", + ) archives = [archive_src, archive_data] @@ -28,28 +28,30 @@ class Icu(Dependency): if os.path.exists(self.extract_path): shutil.rmtree(self.extract_path) extract_archive( - pj(neutralEnv('archive_dir'), self.archive_src.name), - neutralEnv('source_dir'), + pj(neutralEnv("archive_dir"), self.archive_src.name), + neutralEnv("source_dir"), topdir=None, - name=self.source_dir) - shutil.rmtree(pj(neutralEnv('source_dir'), self.source_dir, 'source', 'data')) + name=self.source_dir, + ) + shutil.rmtree( + pj(neutralEnv("source_dir"), self.source_dir, "source", "data") + ) extract_archive( - pj(neutralEnv('archive_dir'), self.archive_data.name), - pj(neutralEnv('source_dir'), self.source_dir, 'source'), - topdir='data', - name='data' + pj(neutralEnv("archive_dir"), self.archive_data.name), + pj(neutralEnv("source_dir"), self.source_dir, "source"), + topdir="data", + name="data", ) patches = [ - "icu4c_fix_static_lib_name_mingw.patch", - # "icu4c_android_elf64_st_info.patch", - # "icu4c_custom_data.patch", - # "icu4c_noxlocale.patch", - "icu4c_rpath.patch", - # "icu4c_build_config.patch", - "icu4c_wasm.patch" - ] - + "icu4c_fix_static_lib_name_mingw.patch", + # "icu4c_android_elf64_st_info.patch", + # "icu4c_custom_data.patch", + # "icu4c_noxlocale.patch", + "icu4c_rpath.patch", + # "icu4c_build_config.patch", + "icu4c_wasm.patch", + ] class Builder(MakeBuilder): subsource_dir = "source" @@ -57,8 +59,8 @@ class Icu(Dependency): @classmethod def get_dependencies(cls, platformInfo, allDeps): - plt = 'native_static' if platformInfo.static else 'native_dyn' - return [(plt, 'icu4c')] + plt = "native_static" if platformInfo.static else "native_dyn" + return [(plt, "icu4c")] @property def configure_options(self): @@ -70,23 +72,25 @@ class Icu(Dependency): yield "--disable-icuio" yield "--disable-layoutex" platformInfo = self.buildEnv.platformInfo - if platformInfo.build != 'native': + if platformInfo.build != "native": icu_native_builder = get_target_step( - 'icu4c', - 'native_static' if platformInfo.static else 'native_dyn') + "icu4c", "native_static" if platformInfo.static else "native_dyn" + ) yield f"--with-cross-build={icu_native_builder.build_path}" yield "--disable-tools" - if platformInfo.build in ('android', 'wasm'): + if platformInfo.build in ("android", "wasm"): yield "--with-data-packaging=archive" def set_env(self, env): - env['ICU_DATA_FILTER_FILE'] = pj(os.path.dirname(os.path.realpath(__file__)), "icu4c_data_filter.json") + env["ICU_DATA_FILTER_FILE"] = pj( + os.path.dirname(os.path.realpath(__file__)), "icu4c_data_filter.json" + ) def _post_configure_script(self, context): if self.buildEnv.platformInfo.build != "wasm": context.skip() context.try_skip(self.build_path) - for line in fileinput.input(pj(self.build_path, 'Makefile'), inplace=True): + for line in fileinput.input(pj(self.build_path, "Makefile"), inplace=True): if line == "#DATASUBDIR = data\n": print("DATASUBDIR = data") else: diff --git a/kiwixbuild/dependencies/ios_fat_lib.py b/kiwixbuild/dependencies/ios_fat_lib.py index 27e6cfe..742e339 100644 --- a/kiwixbuild/dependencies/ios_fat_lib.py +++ b/kiwixbuild/dependencies/ios_fat_lib.py @@ -3,11 +3,7 @@ import os from kiwixbuild.platforms import PlatformInfo from kiwixbuild.utils import pj, copy_tree, run_command from kiwixbuild._global import option -from .base import ( - Dependency, - NoopSource, - Builder as BaseBuilder) - +from .base import Dependency, NoopSource, Builder as BaseBuilder class IOSFatLib(Dependency): @@ -16,38 +12,39 @@ class IOSFatLib(Dependency): Source = NoopSource class Builder(BaseBuilder): - @classmethod def get_dependencies(self, platfomInfo, alldeps): - base_target = option('target') - return [('iOS_{}'.format(arch), base_target) for arch in option('ios_arch')] + base_target = option("target") + return [("iOS_{}".format(arch), base_target) for arch in option("ios_arch")] def _copy_headers(self, context): - plt = PlatformInfo.get_platform('iOS_{}'.format(option('ios_arch')[0])) - include_src = pj(plt.buildEnv.install_dir, 'include') - include_dst = pj(self.buildEnv.install_dir, 'include') + plt = PlatformInfo.get_platform("iOS_{}".format(option("ios_arch")[0])) + include_src = pj(plt.buildEnv.install_dir, "include") + include_dst = pj(self.buildEnv.install_dir, "include") copy_tree(include_src, include_dst) def _merge_libs(self, context): lib_dirs = [] - for arch in option('ios_arch'): - plt = PlatformInfo.get_platform('iOS_{}'.format(arch)) - lib_dirs.append(pj(plt.buildEnv.install_dir, 'lib')) + for arch in option("ios_arch"): + plt = PlatformInfo.get_platform("iOS_{}".format(arch)) + lib_dirs.append(pj(plt.buildEnv.install_dir, "lib")) libs = [] for f in os.listdir(lib_dirs[0]): if os.path.islink(pj(lib_dirs[0], f)): continue - if f.endswith('.a') or f.endswith('.dylib'): + if f.endswith(".a") or f.endswith(".dylib"): libs.append(f) - os.makedirs(pj(self.buildEnv.install_dir, 'lib'), exist_ok=True) + os.makedirs(pj(self.buildEnv.install_dir, "lib"), exist_ok=True) for l in libs: command = [ - 'lipo', - '-create', *[pj(d, l) for d in lib_dirs], - '-output', pj(self.buildEnv.install_dir, 'lib', l) + "lipo", + "-create", + *[pj(d, l) for d in lib_dirs], + "-output", + pj(self.buildEnv.install_dir, "lib", l), ] run_command(command, self.buildEnv.install_dir, context) def build(self): - self.command('copy_headers', self._copy_headers) - self.command('merge_libs', self._merge_libs) + self.command("copy_headers", self._copy_headers) + self.command("merge_libs", self._merge_libs) diff --git a/kiwixbuild/dependencies/kiwix_desktop.py b/kiwixbuild/dependencies/kiwix_desktop.py index 51371a1..5b1eab5 100644 --- a/kiwixbuild/dependencies/kiwix_desktop.py +++ b/kiwixbuild/dependencies/kiwix_desktop.py @@ -1,7 +1,5 @@ -from .base import ( - Dependency, - GitClone, - QMakeBuilder) +from .base import Dependency, GitClone, QMakeBuilder + class KiwixDesktop(Dependency): name = "kiwix-desktop" @@ -13,22 +11,18 @@ class KiwixDesktop(Dependency): class Builder(QMakeBuilder): dependencies = ["qt", "qtwebengine", "libkiwix", "aria2"] - make_install_targets = ['install'] + make_install_targets = ["install"] configure_env = None - flatpack_build_options = { - "env": [ - "QMAKEPATH=/app/lib" - ] - } + flatpack_build_options = {"env": ["QMAKEPATH=/app/lib"]} @property def configure_options(self): - if self.buildEnv.platformInfo.name == 'flatpak': - yield 'QMAKE_INCDIR+=/app/include/QtWebEngine' - yield 'QMAKE_INCDIR+=/app/include/QtWebEngineCore' - yield 'QMAKE_INCDIR+=/app/include/QtWebEngineWidgets' + if self.buildEnv.platformInfo.name == "flatpak": + yield "QMAKE_INCDIR+=/app/include/QtWebEngine" + yield "QMAKE_INCDIR+=/app/include/QtWebEngineCore" + yield "QMAKE_INCDIR+=/app/include/QtWebEngineWidgets" else: yield f"PREFIX={self.buildEnv.install_dir}" if self.buildEnv.platformInfo.static: - yield 'CONFIG+=static' + yield "CONFIG+=static" diff --git a/kiwixbuild/dependencies/kiwix_tools.py b/kiwixbuild/dependencies/kiwix_tools.py index 2e73488..8895b08 100644 --- a/kiwixbuild/dependencies/kiwix_tools.py +++ b/kiwixbuild/dependencies/kiwix_tools.py @@ -1,7 +1,5 @@ -from .base import ( - Dependency, - GitClone, - MesonBuilder) +from .base import Dependency, GitClone, MesonBuilder + class KiwixTools(Dependency): name = "kiwix-tools" diff --git a/kiwixbuild/dependencies/libcurl.py b/kiwixbuild/dependencies/libcurl.py index 2599180..f63ef7b 100644 --- a/kiwixbuild/dependencies/libcurl.py +++ b/kiwixbuild/dependencies/libcurl.py @@ -9,22 +9,50 @@ from .base import ( from kiwixbuild.utils import Remotefile, pj, Defaultdict, SkipCommand, run_command from kiwixbuild._global import get_target_step + class LibCurl(Dependency): name = "libcurl" class Source(ReleaseDownload): name = "libcurl" - archive = Remotefile('curl-7.67.0.tar.xz', - 'f5d2e7320379338c3952dcc7566a140abb49edb575f9f99272455785c40e536c', - 'https://curl.haxx.se/download/curl-7.67.0.tar.xz') + archive = Remotefile( + "curl-7.67.0.tar.xz", + "f5d2e7320379338c3952dcc7566a140abb49edb575f9f99272455785c40e536c", + "https://curl.haxx.se/download/curl-7.67.0.tar.xz", + ) class Builder(MakeBuilder): - dependencies = ['zlib'] + dependencies = ["zlib"] configure_options = [ - *[f"--without-{p}" for p in - ('libssh2', 'ssl', 'libmetalink', 'librtmp', 'nghttp2', 'libidn2', 'brotli') - ], - *[f"--disable-{p}" for p in - ('ftp', 'file', 'ldap', 'ldaps', 'rtsp', 'dict', 'telnet', - 'tftp', 'pop3', 'imap', 'smb', 'smtp', 'gopher', 'manual')] + *[ + f"--without-{p}" + for p in ( + "libssh2", + "ssl", + "libmetalink", + "librtmp", + "nghttp2", + "libidn2", + "brotli", + ) + ], + *[ + f"--disable-{p}" + for p in ( + "ftp", + "file", + "ldap", + "ldaps", + "rtsp", + "dict", + "telnet", + "tftp", + "pop3", + "imap", + "smb", + "smtp", + "gopher", + "manual", + ) + ], ] diff --git a/kiwixbuild/dependencies/libkiwix.py b/kiwixbuild/dependencies/libkiwix.py index 6c52a85..e97dbff 100644 --- a/kiwixbuild/dependencies/libkiwix.py +++ b/kiwixbuild/dependencies/libkiwix.py @@ -1,12 +1,10 @@ import shutil, os -from .base import ( - Dependency, - GitClone, - MesonBuilder) +from .base import Dependency, GitClone, MesonBuilder from kiwixbuild.utils import pj, copy_tree from kiwixbuild._global import option, get_target_step, neutralEnv + class Libkiwix(Dependency): name = "libkiwix" force_build = True @@ -16,7 +14,17 @@ class Libkiwix(Dependency): git_dir = "libkiwix" class Builder(MesonBuilder): - dependencies = ["pugixml", "libzim", "zlib", "lzma", "libcurl", "libmicrohttpd", "icu4c", "mustache", "xapian-core"] + dependencies = [ + "pugixml", + "libzim", + "zlib", + "lzma", + "libcurl", + "libmicrohttpd", + "icu4c", + "mustache", + "xapian-core", + ] strip_options = [] @property @@ -28,18 +36,18 @@ class Libkiwix(Dependency): @property def configure_options(self): platformInfo = self.buildEnv.platformInfo - if platformInfo.build == 'android': - yield '-Dstatic-linkage=true' - yield '-Dwerror=false' - if platformInfo.build == 'iOS': - yield '-Db_bitcode=true' - if platformInfo.name == 'flatpak': - yield '--wrap-mode=nodownload' - if platformInfo.mixed and option('target') == 'libkiwix': + if platformInfo.build == "android": + yield "-Dstatic-linkage=true" + yield "-Dwerror=false" + if platformInfo.build == "iOS": + yield "-Db_bitcode=true" + if platformInfo.name == "flatpak": + yield "--wrap-mode=nodownload" + if platformInfo.mixed and option("target") == "libkiwix": yield "-Dstatic-linkage=true" @property def library_type(self): - if self.buildEnv.platformInfo.build == 'android': - return 'shared' + if self.buildEnv.platformInfo.build == "android": + return "shared" return super().library_type diff --git a/kiwixbuild/dependencies/libmagic.py b/kiwixbuild/dependencies/libmagic.py index df66c2e..165f5e9 100644 --- a/kiwixbuild/dependencies/libmagic.py +++ b/kiwixbuild/dependencies/libmagic.py @@ -9,15 +9,18 @@ from .base import ( from kiwixbuild.utils import Remotefile, pj, SkipCommand, run_command from kiwixbuild._global import get_target_step + class LibMagic(Dependency): name = "libmagic" class Source(ReleaseDownload): name = "libmagic" source_dir = "libmagic" - archive_top_dir = 'file-5.44' - archive = Remotefile('file-5.44.tar.gz', - '3751c7fba8dbc831cb8d7cc8aff21035459b8ce5155ef8b0880a27d028475f3b') + archive_top_dir = "file-5.44" + archive = Remotefile( + "file-5.44.tar.gz", + "3751c7fba8dbc831cb8d7cc8aff21035459b8ce5155ef8b0880a27d028475f3b", + ) class Builder(MakeBuilder): @property @@ -29,22 +32,21 @@ class LibMagic(Dependency): @classmethod def get_dependencies(cls, platformInfo, allDeps): - if platformInfo.build != 'native': - return [('native_static', 'libmagic')] + if platformInfo.build != "native": + return [("native_static", "libmagic")] return [] def _compile(self, context): platformInfo = self.buildEnv.platformInfo - if platformInfo.build == 'native': + if platformInfo.build == "native": return super()._compile(context) context.try_skip(self.build_path) - command = [ - "make", - "-j4", - *self.make_targets, - *self.make_options - ] - env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) - libmagic_native_builder = get_target_step('libmagic', 'native_static') - env['PATH'] = ':'.join([pj(libmagic_native_builder.build_path, 'src'), env['PATH']]) + command = ["make", "-j4", *self.make_targets, *self.make_options] + env = self.buildEnv.get_env( + cross_comp_flags=True, cross_compilers=True, cross_path=True + ) + libmagic_native_builder = get_target_step("libmagic", "native_static") + env["PATH"] = ":".join( + [pj(libmagic_native_builder.build_path, "src"), env["PATH"]] + ) run_command(command, self.build_path, context, env=env) diff --git a/kiwixbuild/dependencies/libmicrohttpd.py b/kiwixbuild/dependencies/libmicrohttpd.py index 3e161a2..ddc0178 100644 --- a/kiwixbuild/dependencies/libmicrohttpd.py +++ b/kiwixbuild/dependencies/libmicrohttpd.py @@ -1,17 +1,23 @@ -from .base import ( - Dependency, - ReleaseDownload, - MakeBuilder) +from .base import Dependency, ReleaseDownload, MakeBuilder from kiwixbuild.utils import Remotefile + class MicroHttpd(Dependency): name = "libmicrohttpd" class Source(ReleaseDownload): - archive = Remotefile('libmicrohttpd-0.9.76.tar.gz', - 'f0b1547b5a42a6c0f724e8e1c1cb5ce9c4c35fb495e7d780b9930d35011ceb4c', - 'https://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.76.tar.gz') + archive = Remotefile( + "libmicrohttpd-0.9.76.tar.gz", + "f0b1547b5a42a6c0f724e8e1c1cb5ce9c4c35fb495e7d780b9930d35011ceb4c", + "https://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.76.tar.gz", + ) class Builder(MakeBuilder): - configure_options = ["--disable-https", "--without-libgcrypt", "--without-libcurl", "--disable-doc", "--disable-examples"] + configure_options = [ + "--disable-https", + "--without-libgcrypt", + "--without-libcurl", + "--disable-doc", + "--disable-examples", + ] diff --git a/kiwixbuild/dependencies/libzim.py b/kiwixbuild/dependencies/libzim.py index 0f62ca0..48dee52 100644 --- a/kiwixbuild/dependencies/libzim.py +++ b/kiwixbuild/dependencies/libzim.py @@ -1,9 +1,7 @@ -from .base import ( - Dependency, - GitClone, - MesonBuilder) +from .base import Dependency, GitClone, MesonBuilder from kiwixbuild._global import option, get_target_step + class Libzim(Dependency): name = "libzim" force_build = True @@ -24,18 +22,18 @@ class Libzim(Dependency): @classmethod def get_dependencies(cls, platformInfo, allDeps): - deps = ['lzma', 'zstd', 'xapian-core', 'icu4c'] - if platformInfo.name not in ('flatpak', 'wasm'): - deps.append('zim-testing-suite') + deps = ["lzma", "zstd", "xapian-core", "icu4c"] + if platformInfo.name not in ("flatpak", "wasm"): + deps.append("zim-testing-suite") return deps @property def configure_options(self): platformInfo = self.buildEnv.platformInfo - if platformInfo.build == 'android': + if platformInfo.build == "android": yield "-DUSE_BUFFER_HEADER=false" yield "-Dstatic-linkage=true" - if platformInfo.mixed and option('target') == 'libzim': + if platformInfo.mixed and option("target") == "libzim": yield "-Dstatic-linkage=true" if platformInfo.name == "flatpak": yield "--wrap-mode=nodownload" @@ -44,11 +42,11 @@ class Libzim(Dependency): yield "-Dexamples=false" yield "-DUSE_MMAP=false" if platformInfo.name not in ("flatpak", "wasm"): - zim_testing_suite = get_target_step('zim-testing-suite', 'source') - yield '-Dtest_data_dir={}'.format(zim_testing_suite.source_path) + zim_testing_suite = get_target_step("zim-testing-suite", "source") + yield "-Dtest_data_dir={}".format(zim_testing_suite.source_path) @property def library_type(self): - if self.buildEnv.platformInfo.build == 'android': - return 'shared' + if self.buildEnv.platformInfo.build == "android": + return "shared" return super().library_type diff --git a/kiwixbuild/dependencies/lzma.py b/kiwixbuild/dependencies/lzma.py index f2d3cfa..62947f0 100644 --- a/kiwixbuild/dependencies/lzma.py +++ b/kiwixbuild/dependencies/lzma.py @@ -1,28 +1,28 @@ -from .base import ( - Dependency, - ReleaseDownload, - MakeBuilder) +from .base import Dependency, ReleaseDownload, MakeBuilder from kiwixbuild.utils import Remotefile + class lzma(Dependency): - name = 'lzma' + name = "lzma" class Source(ReleaseDownload): - archive = Remotefile('xz-5.2.6.tar.gz', - 'a2105abee17bcd2ebd15ced31b4f5eda6e17efd6b10f921a01cda4a44c91b3a0', - 'https://altushost-swe.dl.sourceforge.net/project/lzmautils/xz-5.2.6.tar.gz' - ) + archive = Remotefile( + "xz-5.2.6.tar.gz", + "a2105abee17bcd2ebd15ced31b4f5eda6e17efd6b10f921a01cda4a44c91b3a0", + "https://altushost-swe.dl.sourceforge.net/project/lzmautils/xz-5.2.6.tar.gz", + ) class Builder(MakeBuilder): @property def configure_options(self): - return ["--disable-xz", - "--disable-xzdec", - "--disable-lzmadec", - "--disable-lzmainfo", - "--disable-lzma-links", - "--disable-scripts", - "--disable-doc", -# "--disable-symbol-versions" - ] + return [ + "--disable-xz", + "--disable-xzdec", + "--disable-lzmadec", + "--disable-lzmainfo", + "--disable-lzma-links", + "--disable-scripts", + "--disable-doc", + # "--disable-symbol-versions" + ] diff --git a/kiwixbuild/dependencies/mustache.py b/kiwixbuild/dependencies/mustache.py index 2c07cdf..08eb947 100644 --- a/kiwixbuild/dependencies/mustache.py +++ b/kiwixbuild/dependencies/mustache.py @@ -1,28 +1,30 @@ -from .base import ( - Dependency, - ReleaseDownload, - Builder as BaseBuilder) +from .base import Dependency, ReleaseDownload, Builder as BaseBuilder from kiwixbuild.utils import Remotefile, pj from shutil import copy2 + class Mustache(Dependency): name = "mustache" class Source(ReleaseDownload): - archive = Remotefile('Mustache-4.1.tar.gz', - 'acd66359feb4318b421f9574cfc5a511133a77d916d0b13c7caa3783c0bfe167', - 'https://github.com/kainjow/Mustache/archive/v4.1.tar.gz') + archive = Remotefile( + "Mustache-4.1.tar.gz", + "acd66359feb4318b421f9574cfc5a511133a77d916d0b13c7caa3783c0bfe167", + "https://github.com/kainjow/Mustache/archive/v4.1.tar.gz", + ) class Builder(BaseBuilder): def build(self): - self.command('copy_header', self._copy_header) + self.command("copy_header", self._copy_header) def _copy_header(self, context): context.try_skip(self.build_path) - copy2(pj(self.source_path, 'mustache.hpp'), - pj(self.buildEnv.install_dir, 'include')) + copy2( + pj(self.source_path, "mustache.hpp"), + pj(self.buildEnv.install_dir, "include"), + ) def set_flatpak_buildsystem(self, module): - module['buildsystem'] = 'simple' - module['build-commands'] = ['cp mustache.hpp /app/include'] + module["buildsystem"] = "simple" + module["build-commands"] = ["cp mustache.hpp /app/include"] diff --git a/kiwixbuild/dependencies/pugixml.py b/kiwixbuild/dependencies/pugixml.py index 3c4a531..30bac3d 100644 --- a/kiwixbuild/dependencies/pugixml.py +++ b/kiwixbuild/dependencies/pugixml.py @@ -1,19 +1,19 @@ -from .base import ( - Dependency, - ReleaseDownload, - MesonBuilder) +from .base import Dependency, ReleaseDownload, MesonBuilder from kiwixbuild.utils import Remotefile + class Pugixml(Dependency): name = "pugixml" class Source(ReleaseDownload): - archive = Remotefile('pugixml-1.2.tar.gz', - '0f422dad86da0a2e56a37fb2a88376aae6e931f22cc8b956978460c9db06136b') + archive = Remotefile( + "pugixml-1.2.tar.gz", + "0f422dad86da0a2e56a37fb2a88376aae6e931f22cc8b956978460c9db06136b", + ) patches = ["pugixml_meson.patch"] flatpak_dest = "src" class Builder(MesonBuilder): - build_type = 'release' + build_type = "release" strip_options = [] diff --git a/kiwixbuild/dependencies/qt.py b/kiwixbuild/dependencies/qt.py index efdfd85..69e86ef 100644 --- a/kiwixbuild/dependencies/qt.py +++ b/kiwixbuild/dependencies/qt.py @@ -1,26 +1,24 @@ import shutil -from .base import ( - Dependency, - ReleaseDownload, - MakeBuilder, - QMakeBuilder) +from .base import Dependency, ReleaseDownload, MakeBuilder, QMakeBuilder from kiwixbuild.utils import Remotefile, pj, SkipCommand class Qt(Dependency): - name = 'qt' + name = "qt" class Source(ReleaseDownload): name = "qt" source_dir = "qt-5.10.1" - archive = Remotefile('qt-everywhere-src-5.10.1.tar.xz', - '', - 'http://ftp.oregonstate.edu/.1/blfs/conglomeration/qt5/qt-everywhere-src-5.10.1.tar.xz') + archive = Remotefile( + "qt-everywhere-src-5.10.1.tar.xz", + "", + "http://ftp.oregonstate.edu/.1/blfs/conglomeration/qt5/qt-everywhere-src-5.10.1.tar.xz", + ) class Builder(MakeBuilder): - dependencies = ['icu4c', 'zlib'] + dependencies = ["icu4c", "zlib"] dynamic_configure_options = ["-shared"] static_configure_options = ["-static"] @@ -33,47 +31,50 @@ class Qt(Dependency): yield from self.dynamic_configure_options if not self.target.force_native_build: yield from self.buildEnv.platformInfo.configure_options - yield from ('-prefix', self.buildEnv.install_dir) - yield from ('-libdir', pj(self.buildEnv.install_dir, self.buildEnv.libprefix)) + yield from ("-prefix", self.buildEnv.install_dir) + yield from ( + "-libdir", + pj(self.buildEnv.install_dir, self.buildEnv.libprefix), + ) @property def configure_options(self): skip_modules = [ - 'qt3d', - 'qtcanvas3d', - 'qtcharts', - 'qtconnectivity', - 'qtdatavis3d', - # 'qtdeclarative', - 'qtdoc', - 'qtgamepad', - 'qtgraphicaleffects', - 'qtlocation', - 'qtmultimedia', - 'qtnetworkauth', - 'qtpurchasing', - # 'qtquickcontrols', - 'qtquickcontrols2', - 'qtremoteobjects', - 'qtscript', - 'qtscxml', - 'qtsensors', - 'qtserialbus', - 'qtserialport', - 'qtspeech', - 'qtvirtualkeyboard', - 'qtwayland', - 'qtwebglplugin', - 'qtwebsockets', -# 'qtwebview', + "qt3d", + "qtcanvas3d", + "qtcharts", + "qtconnectivity", + "qtdatavis3d", + # 'qtdeclarative', + "qtdoc", + "qtgamepad", + "qtgraphicaleffects", + "qtlocation", + "qtmultimedia", + "qtnetworkauth", + "qtpurchasing", + # 'qtquickcontrols', + "qtquickcontrols2", + "qtremoteobjects", + "qtscript", + "qtscxml", + "qtsensors", + "qtserialbus", + "qtserialport", + "qtspeech", + "qtvirtualkeyboard", + "qtwayland", + "qtwebglplugin", + "qtwebsockets", + # 'qtwebview', ] - yield '-recheck' - yield '-opensource' - yield '-confirm-license' - yield '-ccache' - yield from ('-make', 'libs') + yield "-recheck" + yield "-opensource" + yield "-confirm-license" + yield "-ccache" + yield from ("-make", "libs") for module in skip_modules: - yield from ('-skip', module) + yield from ("-skip", module) class QtWebEngine(Dependency): @@ -82,5 +83,5 @@ class QtWebEngine(Dependency): Source = Qt.Source class Builder(QMakeBuilder): - dependencies = ['qt'] + dependencies = ["qt"] subsource_dir = "qtwebengine" diff --git a/kiwixbuild/dependencies/tc_android_ndk.py b/kiwixbuild/dependencies/tc_android_ndk.py index 4b565d4..542fca9 100644 --- a/kiwixbuild/dependencies/tc_android_ndk.py +++ b/kiwixbuild/dependencies/tc_android_ndk.py @@ -5,23 +5,25 @@ from kiwixbuild.utils import Remotefile, add_execution_right, run_command pj = os.path.join + class android_ndk(Dependency): dont_skip = True neutral = False - name = 'android-ndk' - gccver = '4.9.x' - api = '24' + name = "android-ndk" + gccver = "4.9.x" + api = "24" class Source(ReleaseDownload): - archive = Remotefile('android-ndk-r21e-linux-x86_64.zip', - 'ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e', - 'https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip') + archive = Remotefile( + "android-ndk-r21e-linux-x86_64.zip", + "ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e", + "https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip", + ) @property def source_dir(self): return self.target.full_name() - class Builder(Builder): @property def install_path(self): @@ -33,7 +35,7 @@ class android_ndk(Dependency): @property def platform(self): - return 'android-'+self.api + return "android-" + self.api @property def arch(self): @@ -45,24 +47,33 @@ class android_ndk(Dependency): def _build_platform(self, context): context.try_skip(self.build_path) - script = pj(self.source_path, 'build/tools/make_standalone_toolchain.py') + script = pj(self.source_path, "build/tools/make_standalone_toolchain.py") add_execution_right(script) command = [ script, - f'--arch={self.arch}', - f'--api={self.api}', - f'--install-dir={self.install_path}', - '--force' + f"--arch={self.arch}", + f"--api={self.api}", + f"--install-dir={self.install_path}", + "--force", ] - env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False) + env = self.buildEnv.get_env( + cross_comp_flags=False, cross_compilers=False, cross_path=False + ) run_command(command, self.build_path, context, env=env) def _fix_permission_right(self, context): context.try_skip(self.build_path) - bin_dirs = [pj(self.install_path, 'bin'), - pj(self.install_path, self.arch_full, 'bin'), - pj(self.install_path, 'libexec', 'gcc', self.arch_full, self.target.gccver) - ] + bin_dirs = [ + pj(self.install_path, "bin"), + pj(self.install_path, self.arch_full, "bin"), + pj( + self.install_path, + "libexec", + "gcc", + self.arch_full, + self.target.gccver, + ), + ] for root, dirs, files in os.walk(self.install_path): if not root in bin_dirs: continue @@ -74,6 +85,5 @@ class android_ndk(Dependency): add_execution_right(file_path) def build(self): - self.command('build_platform', self._build_platform) - self.command('fix_permission_right', self._fix_permission_right) - + self.command("build_platform", self._build_platform) + self.command("fix_permission_right", self._fix_permission_right) diff --git a/kiwixbuild/dependencies/tc_armhf.py b/kiwixbuild/dependencies/tc_armhf.py index 507aee1..a40b640 100644 --- a/kiwixbuild/dependencies/tc_armhf.py +++ b/kiwixbuild/dependencies/tc_armhf.py @@ -4,21 +4,25 @@ from kiwixbuild.utils import Remotefile # The arm toolchains # This is based on toolchains published here : https://github.com/tttapa/docker-arm-cross-toolchain -base_url = "https://github.com/tttapa/docker-arm-cross-toolchain/releases/download/0.1.0/" +base_url = ( + "https://github.com/tttapa/docker-arm-cross-toolchain/releases/download/0.1.0/" +) -aarch_base_url = 'https://master.dl.sourceforge.net/project/raspberry-pi-cross-compilers/Bonus%20Raspberry%20Pi%20GCC%2064-Bit%20Toolchains/Raspberry%20Pi%20GCC%2064-Bit%20Cross-Compiler%20Toolchains/Stretch/GCC%206.3.0/' +aarch_base_url = "https://master.dl.sourceforge.net/project/raspberry-pi-cross-compilers/Bonus%20Raspberry%20Pi%20GCC%2064-Bit%20Toolchains/Raspberry%20Pi%20GCC%2064-Bit%20Cross-Compiler%20Toolchains/Stretch/GCC%206.3.0/" + class armv6_toolchain(Dependency): dont_skip = True neutral = True - name = 'armv6' + name = "armv6" class Source(ReleaseDownload): - archive = Remotefile('x-tools-armv6-rpi-linux-gnueabihf.tar.xz', - '4c371c4c5b55ebd1f3d7dd26b14703632d9ba47423f901bcd9303d83ad444434', - base_url + 'x-tools-armv6-rpi-linux-gnueabihf.tar.xz') - + archive = Remotefile( + "x-tools-armv6-rpi-linux-gnueabihf.tar.xz", + "4c371c4c5b55ebd1f3d7dd26b14703632d9ba47423f901bcd9303d83ad444434", + base_url + "x-tools-armv6-rpi-linux-gnueabihf.tar.xz", + ) class Builder(TcCopyBuilder): src_subdir = "armv6-rpi-linux-gnueabihf" @@ -27,24 +31,29 @@ class armv6_toolchain(Dependency): class armv8_toolchain(Dependency): dont_skip = True neutral = True - name = 'armv8' + name = "armv8" class Source(ReleaseDownload): - archive = Remotefile('x-tools-armv8-rpi-linux-gnueabihf.tar.xz', - 'cc28f5c3f6a3e7d9985f98779c4e72224b4eb5a7e4dc2bcdefd90cb241fb94a5', - base_url + 'x-tools-armv8-rpi3-linux-gnueabihf.tar.xz') + archive = Remotefile( + "x-tools-armv8-rpi-linux-gnueabihf.tar.xz", + "cc28f5c3f6a3e7d9985f98779c4e72224b4eb5a7e4dc2bcdefd90cb241fb94a5", + base_url + "x-tools-armv8-rpi3-linux-gnueabihf.tar.xz", + ) class Builder(TcCopyBuilder): src_subdir = "armv8-rpi3-linux-gnueabihf" + class aarch64_toolchain(Dependency): dont_skip = True neutral = True name = "aarch64" class Source(ReleaseDownload): - archive = Remotefile('cross-gcc-6.3.0-pi_64.tar.gz', - '1b048bb8886ad63d21797cd9129fc37b9ea0dfaac7e3c36f888aa16fbec1d320', - aarch_base_url + 'cross-gcc-6.3.0-pi_64.tar.gz') + archive = Remotefile( + "cross-gcc-6.3.0-pi_64.tar.gz", + "1b048bb8886ad63d21797cd9129fc37b9ea0dfaac7e3c36f888aa16fbec1d320", + aarch_base_url + "cross-gcc-6.3.0-pi_64.tar.gz", + ) Builder = TcCopyBuilder diff --git a/kiwixbuild/dependencies/tc_emsdk.py b/kiwixbuild/dependencies/tc_emsdk.py index 8303bed..5b26c3b 100644 --- a/kiwixbuild/dependencies/tc_emsdk.py +++ b/kiwixbuild/dependencies/tc_emsdk.py @@ -5,21 +5,23 @@ from kiwixbuild.utils import Remotefile, run_command, copy_tree pj = os.path.join + class emsdk(Dependency): dont_skip = True neutral = False - name = 'emsdk' + name = "emsdk" class Source(ReleaseDownload): - archive = Remotefile('emsdk-3.1.41.tar.gz', - '147a2d72df34227bdb4ffedc587a8cb674a42269c40458f3f69ae37e8966cdc6', - 'https://codeload.github.com/emscripten-core/emsdk/tar.gz/refs/tags/3.1.41') + archive = Remotefile( + "emsdk-3.1.41.tar.gz", + "147a2d72df34227bdb4ffedc587a8cb674a42269c40458f3f69ae37e8966cdc6", + "https://codeload.github.com/emscripten-core/emsdk/tar.gz/refs/tags/3.1.41", + ) @property def source_dir(self): return self.target.full_name() - class Builder(Builder): @property def install_path(self): @@ -39,9 +41,7 @@ class emsdk(Dependency): command = ["./emsdk", "activate", "3.1.24"] run_command(command, self.install_path, context) - def build(self): - self.command('copy_source', self._copy_source) - self.command('install', self._install) - self.command('activate', self._activate) - + self.command("copy_source", self._copy_source) + self.command("install", self._install) + self.command("activate", self._activate) diff --git a/kiwixbuild/dependencies/tc_flatpak.py b/kiwixbuild/dependencies/tc_flatpak.py index 8fe54c7..21b0fd5 100644 --- a/kiwixbuild/dependencies/tc_flatpak.py +++ b/kiwixbuild/dependencies/tc_flatpak.py @@ -5,9 +5,10 @@ from kiwixbuild.utils import Remotefile, add_execution_right, run_command pj = os.path.join + class org_kde(Dependency): neutral = False - name = 'org.kde' + name = "org.kde" Source = NoopSource @@ -15,32 +16,42 @@ class org_kde(Dependency): def _setup_remote(self, context): command = [ "flatpak", - "--user", "remote-add", "--if-not-exists", + "--user", + "remote-add", + "--if-not-exists", "flathub", - "https://flathub.org/repo/flathub.flatpakrepo" + "https://flathub.org/repo/flathub.flatpakrepo", ] - env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False) + env = self.buildEnv.get_env( + cross_comp_flags=False, cross_compilers=False, cross_path=False + ) run_command(command, self.buildEnv.build_dir, context, env=env) def _install_sdk(self, context): command = [ "flatpak", - "--user", "install", "--noninteractive", "--verbose", "-y", + "--user", + "install", + "--noninteractive", + "--verbose", + "-y", "flathub", f"{self.target.name}.Sdk//{self.target.version()}", - f"{self.target.name}.Platform//{self.target.version()}" + f"{self.target.name}.Platform//{self.target.version()}", ] - env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False) + env = self.buildEnv.get_env( + cross_comp_flags=False, cross_compilers=False, cross_path=False + ) run_command(command, self.buildEnv.build_dir, context, env=env) def build(self): - self.command('setup_remote', self._setup_remote) - self.command('install_sdk', self._install_sdk) + self.command("setup_remote", self._setup_remote) + self.command("install_sdk", self._install_sdk) class io_qt_qtwebengine(Dependency): neutral = False - name = 'io.qt.qtwebengine' + name = "io.qt.qtwebengine" Source = NoopSource @@ -48,23 +59,31 @@ class io_qt_qtwebengine(Dependency): def _setup_remote(self, context): command = [ "flatpak", - "--user", "remote-add", "--if-not-exists", + "--user", + "remote-add", + "--if-not-exists", "flathub", - "https://flathub.org/repo/flathub.flatpakrepo" + "https://flathub.org/repo/flathub.flatpakrepo", ] - env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False) + env = self.buildEnv.get_env( + cross_comp_flags=False, cross_compilers=False, cross_path=False + ) run_command(command, self.buildEnv.build_dir, context, env=env) def _install_sdk(self, context): command = [ "flatpak", - "--user", "install", "-y", + "--user", + "install", + "-y", "flathub", - f"{self.target.name}.BaseApp//{self.target.version()}" + f"{self.target.name}.BaseApp//{self.target.version()}", ] - env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False) + env = self.buildEnv.get_env( + cross_comp_flags=False, cross_compilers=False, cross_path=False + ) run_command(command, self.buildEnv.build_dir, context, env=env) def build(self): - self.command('setup_remote', self._setup_remote) - self.command('install_sdk', self._install_sdk) + self.command("setup_remote", self._setup_remote) + self.command("install_sdk", self._install_sdk) diff --git a/kiwixbuild/dependencies/tc_musl.py b/kiwixbuild/dependencies/tc_musl.py index 4ca68f5..103c846 100644 --- a/kiwixbuild/dependencies/tc_musl.py +++ b/kiwixbuild/dependencies/tc_musl.py @@ -1,15 +1,18 @@ from .base import Dependency, ReleaseDownload, TcCopyBuilder from kiwixbuild.utils import Remotefile + class aarch64_musl_toolchain(Dependency): dont_skip = True neutral = True name = "aarch64_musl" class Source(ReleaseDownload): - archive = Remotefile('aarch64-linux-musl-cross.tgz', - '0f18a885b161815520bbb5757a4b4ab40d0898c29bebee58d0cddd6112e59cc6', - 'https://more.musl.cc/10/x86_64-linux-musl/aarch64-linux-musl-cross.tgz') + archive = Remotefile( + "aarch64-linux-musl-cross.tgz", + "0f18a885b161815520bbb5757a4b4ab40d0898c29bebee58d0cddd6112e59cc6", + "https://more.musl.cc/10/x86_64-linux-musl/aarch64-linux-musl-cross.tgz", + ) Builder = TcCopyBuilder @@ -20,8 +23,10 @@ class x86_64_musl_toolchain(Dependency): name = "x86-64_musl" class Source(ReleaseDownload): - archive = Remotefile('x86_64-linux-musl-cross.tgz', - 'a3d55de8105739fcfb8b10eaa72cdb5d779319726bacff24149388d7608d1ed8', - 'https://more.musl.cc/10/x86_64-linux-musl/x86_64-linux-musl-cross.tgz') + archive = Remotefile( + "x86_64-linux-musl-cross.tgz", + "a3d55de8105739fcfb8b10eaa72cdb5d779319726bacff24149388d7608d1ed8", + "https://more.musl.cc/10/x86_64-linux-musl/x86_64-linux-musl-cross.tgz", + ) Builder = TcCopyBuilder diff --git a/kiwixbuild/dependencies/uuid.py b/kiwixbuild/dependencies/uuid.py index 245d064..b0b8976 100644 --- a/kiwixbuild/dependencies/uuid.py +++ b/kiwixbuild/dependencies/uuid.py @@ -1,30 +1,30 @@ -from .base import ( - Dependency, - ReleaseDownload, - MakeBuilder -) +from .base import Dependency, ReleaseDownload, MakeBuilder from kiwixbuild.utils import Remotefile class UUID(Dependency): - name = 'uuid' + name = "uuid" class Source(ReleaseDownload): - archive = Remotefile('e2fsprogs-libs-1.43.4.tar.gz', - 'eed4516325768255c9745e7b82c9d7d0393abce302520a5b2cde693204b0e419', - 'https://www.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v1.43.4/e2fsprogs-libs-1.43.4.tar.gz') - extract_dir = 'e2fsprogs-libs-1.43.4' + archive = Remotefile( + "e2fsprogs-libs-1.43.4.tar.gz", + "eed4516325768255c9745e7b82c9d7d0393abce302520a5b2cde693204b0e419", + "https://www.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v1.43.4/e2fsprogs-libs-1.43.4.tar.gz", + ) + extract_dir = "e2fsprogs-libs-1.43.4" class Builder(MakeBuilder): - configure_options = ["--enable-libuuid", - "--disable-debugfs", - "--disable-imager", - "--disable-resizer", - "--disable-defrag", - "--enable-fsck", - "--disable-uuidd"] - configure_env = {'_format_CFLAGS': "{env.CFLAGS} -O3 -fPIC"} + configure_options = [ + "--enable-libuuid", + "--disable-debugfs", + "--disable-imager", + "--disable-resizer", + "--disable-defrag", + "--enable-fsck", + "--disable-uuidd", + ] + configure_env = {"_format_CFLAGS": "{env.CFLAGS} -O3 -fPIC"} static_configure_options = dynamic_configure_options = [] - make_targets = ['libs'] - make_install_targets = ['install-libs'] + make_targets = ["libs"] + make_install_targets = ["install-libs"] diff --git a/kiwixbuild/dependencies/xapian.py b/kiwixbuild/dependencies/xapian.py index 6857d10..a337f68 100644 --- a/kiwixbuild/dependencies/xapian.py +++ b/kiwixbuild/dependencies/xapian.py @@ -1,8 +1,4 @@ -from .base import ( - Dependency, - ReleaseDownload, - MakeBuilder -) +from .base import Dependency, ReleaseDownload, MakeBuilder from kiwixbuild.utils import Remotefile from kiwixbuild._global import neutralEnv @@ -12,23 +8,29 @@ class Xapian(Dependency): name = "xapian-core" class Source(ReleaseDownload): - archive = Remotefile('xapian-core-1.4.23.tar.xz', - '30d3518172084f310dab86d262b512718a7f9a13635aaa1a188e61dc26b2288c') + archive = Remotefile( + "xapian-core-1.4.23.tar.xz", + "30d3518172084f310dab86d262b512718a7f9a13635aaa1a188e61dc26b2288c", + ) class Builder(MakeBuilder): configure_options = [ "--disable-sse", "--disable-backend-chert", "--disable-backend-remote", - "--disable-documentation"] - configure_env = {'_format_LDFLAGS': "{env.LDFLAGS} -L{buildEnv.install_dir}/{buildEnv.libprefix}", - '_format_CXXFLAGS': "{env.CXXFLAGS} -O3 -I{buildEnv.install_dir}/include"} - + "--disable-documentation", + ] + configure_env = { + "_format_LDFLAGS": "{env.LDFLAGS} -L{buildEnv.install_dir}/{buildEnv.libprefix}", + "_format_CXXFLAGS": "{env.CXXFLAGS} -O3 -I{buildEnv.install_dir}/include", + } @classmethod def get_dependencies(cls, platformInfo, allDeps): - deps = ['zlib', 'lzma'] - if (platformInfo.build in ('win32', 'wasm') - or neutralEnv('distname') == 'Darwin'): + deps = ["zlib", "lzma"] + if ( + platformInfo.build in ("win32", "wasm") + or neutralEnv("distname") == "Darwin" + ): return deps - return deps + ['uuid'] + return deps + ["uuid"] diff --git a/kiwixbuild/dependencies/zim_testing_suite.py b/kiwixbuild/dependencies/zim_testing_suite.py index 9f0f536..dd9224d 100644 --- a/kiwixbuild/dependencies/zim_testing_suite.py +++ b/kiwixbuild/dependencies/zim_testing_suite.py @@ -1,8 +1,4 @@ -from .base import ( - Dependency, - ReleaseDownload, - NoopBuilder -) +from .base import Dependency, ReleaseDownload, NoopBuilder from kiwixbuild.utils import Remotefile @@ -12,8 +8,10 @@ class ZimTestingSuite(Dependency): dont_skip = True class Source(ReleaseDownload): - archive = Remotefile('zim-testing-suite-0.3.tar.gz', - 'cd7d1ccc48af3783af9156cb6bf3c18d9a3319a73fdeefe65f0b4cae402d3d66', - 'https://github.com/openzim/zim-testing-suite/releases/download/v0.3/zim-testing-suite-0.3.tar.gz') + archive = Remotefile( + "zim-testing-suite-0.3.tar.gz", + "cd7d1ccc48af3783af9156cb6bf3c18d9a3319a73fdeefe65f0b4cae402d3d66", + "https://github.com/openzim/zim-testing-suite/releases/download/v0.3/zim-testing-suite-0.3.tar.gz", + ) Builder = NoopBuilder diff --git a/kiwixbuild/dependencies/zim_tools.py b/kiwixbuild/dependencies/zim_tools.py index 8179fbc..f44d94c 100644 --- a/kiwixbuild/dependencies/zim_tools.py +++ b/kiwixbuild/dependencies/zim_tools.py @@ -1,7 +1,5 @@ -from .base import ( - Dependency, - GitClone, - MesonBuilder) +from .base import Dependency, GitClone, MesonBuilder + class ZimTools(Dependency): name = "zim-tools" @@ -14,15 +12,15 @@ class ZimTools(Dependency): class Builder(MesonBuilder): @classmethod def get_dependencies(cls, platformInfo, allDeps): - base_deps = ['libzim', 'docoptcpp', 'mustache'] - if platformInfo.build != 'win32': - base_deps += ['libmagic', 'gumbo'] + base_deps = ["libzim", "docoptcpp", "mustache"] + if platformInfo.build != "win32": + base_deps += ["libmagic", "gumbo"] return base_deps @property def configure_options(self): # We don't build zimwriterfs on win32, and so we don't have magic - if self.buildEnv.platformInfo.build != 'win32': + if self.buildEnv.platformInfo.build != "win32": yield f"-Dmagic-install-prefix={self.buildEnv.install_dir}" if self.buildEnv.platformInfo.static: yield "-Dstatic-linkage=true" diff --git a/kiwixbuild/dependencies/zlib.py b/kiwixbuild/dependencies/zlib.py index 1ca6eda..7f8539c 100644 --- a/kiwixbuild/dependencies/zlib.py +++ b/kiwixbuild/dependencies/zlib.py @@ -1,34 +1,31 @@ import shutil -from .base import ( - Dependency, - ReleaseDownload, - MakeBuilder) +from .base import Dependency, ReleaseDownload, MakeBuilder from kiwixbuild.utils import Remotefile, pj, SkipCommand - class zlib(Dependency): - name = 'zlib' + name = "zlib" class Source(ReleaseDownload): - archive = Remotefile('zlib-1.2.12.tar.gz', - '91844808532e5ce316b3c010929493c0244f3d37593afd6de04f71821d5136d9') - patches = ['zlib_std_libname.patch'] + archive = Remotefile( + "zlib-1.2.12.tar.gz", + "91844808532e5ce316b3c010929493c0244f3d37593afd6de04f71821d5136d9", + ) + patches = ["zlib_std_libname.patch"] class Builder(MakeBuilder): dynamic_configure_options = ["--shared"] static_configure_options = ["--static"] - make_install_targets = ['install'] + make_install_targets = ["install"] def _pre_build_script(self, context): context.try_skip(self.build_path) shutil.copytree(self.source_path, self.build_path) - def _configure(self, context): - if self.buildEnv.platformInfo.build == 'win32': + if self.buildEnv.platformInfo.build == "win32": raise SkipCommand() return super()._configure(context) @@ -36,20 +33,27 @@ class zlib(Dependency): def all_configure_options(self): yield from self.configure_options yield from self.static_configure_options if self.buildEnv.platformInfo.static else self.dynamic_configure_options - yield from ('--prefix', self.buildEnv.install_dir) - yield from ('--libdir', pj(self.buildEnv.install_dir, self.buildEnv.libprefix)) + yield from ("--prefix", self.buildEnv.install_dir) + yield from ( + "--libdir", + pj(self.buildEnv.install_dir, self.buildEnv.libprefix), + ) @property def make_options(self): - if self.buildEnv.platformInfo.build != 'win32': + if self.buildEnv.platformInfo.build != "win32": return yield "--makefile" yield "win32/Makefile.gcc" yield "PREFIX=i686-w64-mingw32-", - yield "SHARED_MODE={}".format("0" if self.buildEnv.platformInfo.static else "1"), - yield "INCLUDE_PATH={}".format(pj(self.buildEnv.install_dir, 'include')), - yield "LIBRARY_PATH={}".format(pj(self.buildEnv.install_dir, self.buildEnv.libprefix)), - yield "BINARY_PATH={}".format(pj(self.buildEnv.install_dir, 'bin')) + yield "SHARED_MODE={}".format( + "0" if self.buildEnv.platformInfo.static else "1" + ), + yield "INCLUDE_PATH={}".format(pj(self.buildEnv.install_dir, "include")), + yield "LIBRARY_PATH={}".format( + pj(self.buildEnv.install_dir, self.buildEnv.libprefix) + ), + yield "BINARY_PATH={}".format(pj(self.buildEnv.install_dir, "bin")) @property def make_targets(self): diff --git a/kiwixbuild/dependencies/zstd.py b/kiwixbuild/dependencies/zstd.py index 09c93b7..87daf75 100644 --- a/kiwixbuild/dependencies/zstd.py +++ b/kiwixbuild/dependencies/zstd.py @@ -1,22 +1,20 @@ -from .base import ( - Dependency, - ReleaseDownload, - MesonBuilder) +from .base import Dependency, ReleaseDownload, MesonBuilder from kiwixbuild.utils import Remotefile - class zstd(Dependency): - name = 'zstd' + name = "zstd" class Source(ReleaseDownload): - archive = Remotefile('zstd-1.5.2.tar.gz', - 'f7de13462f7a82c29ab865820149e778cbfe01087b3a55b5332707abf9db4a6e', - 'https://github.com/facebook/zstd/archive/refs/tags/v1.5.2.tar.gz') + archive = Remotefile( + "zstd-1.5.2.tar.gz", + "f7de13462f7a82c29ab865820149e778cbfe01087b3a55b5332707abf9db4a6e", + "https://github.com/facebook/zstd/archive/refs/tags/v1.5.2.tar.gz", + ) class Builder(MesonBuilder): - subsource_dir = 'build/meson' - build_type = 'release' + subsource_dir = "build/meson" + build_type = "release" strip_options = [] - configure_options = ['-Dbin_programs=false', '-Dbin_contrib=false'] + configure_options = ["-Dbin_programs=false", "-Dbin_contrib=false"] diff --git a/kiwixbuild/flatpak_builder.py b/kiwixbuild/flatpak_builder.py index ef61734..6cc6de7 100644 --- a/kiwixbuild/flatpak_builder.py +++ b/kiwixbuild/flatpak_builder.py @@ -1,4 +1,3 @@ - import sys from collections import OrderedDict from .buildenv import * @@ -9,9 +8,13 @@ from .dependencies import Dependency from .packages import PACKAGE_NAME_MAPPERS from .versions import base_deps_versions from ._global import ( - neutralEnv, option, - add_target_step, get_target_step, target_steps, - backend) + neutralEnv, + option, + add_target_step, + get_target_step, + target_steps, + backend, +) from . import _global from .dependencies.base import ( Source, @@ -22,7 +25,8 @@ from .dependencies.base import ( CMakeBuilder, QMakeBuilder, MakeBuilder, - SCRIPT_DIR) + SCRIPT_DIR, +) import json from shutil import copyfile from urllib.parse import urlparse @@ -30,54 +34,60 @@ from urllib.request import urlopen import json MANIFEST = { - 'app-id': 'org.kiwix.desktop', - 'runtime': 'org.kde.Platform', - 'runtime-version': base_deps_versions['org.kde'], - 'base': 'io.qt.qtwebengine.BaseApp', - 'base-version': base_deps_versions['org.kde'], # keep BaseApp (qwebengine) in sync with org.kde - 'sdk': 'org.kde.Sdk', - 'command': 'kiwix-desktop', - 'rename-icon': 'kiwix-desktop', - 'finish-args': [ - '--socket=wayland', - '--socket=x11', - '--share=network', - '--share=ipc', - '--device=dri', - '--socket=pulseaudio', + "app-id": "org.kiwix.desktop", + "runtime": "org.kde.Platform", + "runtime-version": base_deps_versions["org.kde"], + "base": "io.qt.qtwebengine.BaseApp", + "base-version": base_deps_versions[ + "org.kde" + ], # keep BaseApp (qwebengine) in sync with org.kde + "sdk": "org.kde.Sdk", + "command": "kiwix-desktop", + "rename-icon": "kiwix-desktop", + "finish-args": [ + "--socket=wayland", + "--socket=x11", + "--share=network", + "--share=ipc", + "--device=dri", + "--socket=pulseaudio", + ], + "cleanup": [ + "/include", + "/lib/pkgconfig", + "/lib/cmake", + "/lib/*.la", + "/bin/curl", + "/bin/copydatabase", + "/bin/kiwix-compile-resources", + "/bin/kiwix-manage", + "/bin/kiwix-read", + "/bin/kiwix-search", + "/bin/quest", + "/bin/simple*", + "/bin/xapian-*", + "/share/aclocal", + "/share/doc", + "/share/man", ], - 'cleanup': [ - '/include', - '/lib/pkgconfig', - '/lib/cmake', - '/lib/*.la', - '/bin/curl', - '/bin/copydatabase', - '/bin/kiwix-compile-resources', - '/bin/kiwix-manage', - '/bin/kiwix-read', - '/bin/kiwix-search', - '/bin/quest', - '/bin/simple*', - '/bin/xapian-*', - '/share/aclocal', - '/share/doc', - '/share/man' - ] } -GET_REF_URL_API_TEMPLATE = 'https://api.github.com/repos{repo}/git/refs/tags/{ref}' +GET_REF_URL_API_TEMPLATE = "https://api.github.com/repos{repo}/git/refs/tags/{ref}" + class FlatpakBuilder: def __init__(self): self._targets = {} - PlatformInfo.get_platform('neutral', self._targets) - self.platform = PlatformInfo.get_platform('flatpak', self._targets) - if neutralEnv('distname') not in self.platform.compatible_hosts: - print(('ERROR: The target platform {} cannot be build on host {}.\n' - 'Select another target platform or change your host system.' - ).format(self.platform.name, neutralEnv('distname'))) - self.targetDefs = self.platform.add_targets(option('target'), self._targets) + PlatformInfo.get_platform("neutral", self._targets) + self.platform = PlatformInfo.get_platform("flatpak", self._targets) + if neutralEnv("distname") not in self.platform.compatible_hosts: + print( + ( + "ERROR: The target platform {} cannot be build on host {}.\n" + "Select another target platform or change your host system." + ).format(self.platform.name, neutralEnv("distname")) + ) + self.targetDefs = self.platform.add_targets(option("target"), self._targets) def finalize_target_steps(self): steps = [] @@ -89,9 +99,9 @@ class FlatpakBuilder: plt = PlatformInfo.all_platforms[pltName] for tlcName in plt.toolchain_names: tlc = Dependency.all_deps[tlcName] - src_plt_step = ('source', tlcName) + src_plt_step = ("source", tlcName) add_target_step(src_plt_step, self._targets[src_plt_step]) - blt_plt_step = ('neutral' if tlc.neutral else pltName, tlcName) + blt_plt_step = ("neutral" if tlc.neutral else pltName, tlcName) add_target_step(blt_plt_step, self._targets[blt_plt_step]) for dep in steps: @@ -104,7 +114,7 @@ class FlatpakBuilder: def order_dependencies(self, targetDef, targets): targetPlatformName, targetName = targetDef - if targetPlatformName == 'source': + if targetPlatformName == "source": # Do not try to order sources, they will be added as dep by the # build step two lines later. return @@ -121,18 +131,18 @@ class FlatpakBuilder: depPlatform, depName = targetPlatformName, dep if (depPlatform, depName) in targets: yield from self.order_dependencies((depPlatform, depName), targets) - yield ('source', targetName) + yield ("source", targetName) yield targetDef def instanciate_steps(self): for stepDef in list(target_steps()): stepPlatform, stepName = stepDef stepClass = Dependency.all_deps[stepName] - if stepPlatform == 'source': + if stepPlatform == "source": source = get_target_step(stepDef)(stepClass) add_target_step(stepDef, source) else: - source = get_target_step(stepName, 'source') + source = get_target_step(stepName, "source") env = PlatformInfo.get_platform(stepPlatform).buildEnv builder = get_target_step(stepDef)(stepClass, source, env) add_target_step(stepDef, builder) @@ -142,121 +152,126 @@ class FlatpakBuilder: modules = OrderedDict() for stepDef in steps: module = modules.setdefault(stepDef[1], {}) - module['name'] = stepDef[1] - if stepDef[0] == 'source': + module["name"] = stepDef[1] + if stepDef[0] == "source": source = get_target_step(stepDef) - if getattr(source, 'flatpak_no_autogen', False): - module['no-autogen'] = True - module_sources = module.setdefault('sources', []) + if getattr(source, "flatpak_no_autogen", False): + module["no-autogen"] = True + module_sources = module.setdefault("sources", []) if isinstance(source, ReleaseDownload): src = { - 'type': 'archive', - 'sha256': source.archive.sha256, - 'url': source.archive.url + "type": "archive", + "sha256": source.archive.sha256, + "url": source.archive.url, } - if hasattr(source, 'flatpak_dest'): - src['dest'] = source.flatpak_dest + if hasattr(source, "flatpak_dest"): + src["dest"] = source.flatpak_dest module_sources.append(src) elif isinstance(source, GitClone): src = { - 'type': 'git', - 'url': source.git_remote, - 'tag': source.git_ref + "type": "git", + "url": source.git_remote, + "tag": source.git_ref, } try: parsed = urlparse(source.git_remote) - if parsed.hostname == 'github.com': + if parsed.hostname == "github.com": repo = parsed.path[:-4] api_url = GET_REF_URL_API_TEMPLATE.format( - repo = repo, - ref = source.git_ref) + repo=repo, ref=source.git_ref + ) with urlopen(api_url) as r: ret = json.loads(r.read()) - src['commit'] = ret['object']['sha'] + src["commit"] = ret["object"]["sha"] except: pass module_sources.append(src) - for p in getattr(source, 'patches', []): - patch = { - 'type': 'patch', - 'path': 'patches/' + p - } + for p in getattr(source, "patches", []): + patch = {"type": "patch", "path": "patches/" + p} module_sources.append(patch) - if hasattr(source, 'flatpak_command'): - command = { - 'type': 'shell', - 'commands': [source.flatpak_command] - } + if hasattr(source, "flatpak_command"): + command = {"type": "shell", "commands": [source.flatpak_command]} module_sources.append(command) else: builder = get_target_step(stepDef) builder.set_flatpak_buildsystem(module) - print(module['name']) + print(module["name"]) manifest = MANIFEST.copy() - modules = [m for m in modules.values() if m.get('sources')] + modules = [m for m in modules.values() if m.get("sources")] for m in modules: - temp = m['sources'] - del m['sources'] - m['sources'] = temp - manifest['modules'] = modules - manifest_name = "{}.json".format(MANIFEST['app-id']) + temp = m["sources"] + del m["sources"] + m["sources"] = temp + manifest["modules"] = modules + manifest_name = "{}.json".format(MANIFEST["app-id"]) manifest_path = pj(self.platform.buildEnv.build_dir, manifest_name) - with open(manifest_path, 'w') as f: + with open(manifest_path, "w") as f: f.write(json.dumps(manifest, indent=4)) def copy_patches(self): - sourceDefs = (tDef for tDef in target_steps() if tDef[0] == 'source') + sourceDefs = (tDef for tDef in target_steps() if tDef[0] == "source") for sourceDef in sourceDefs: source = get_target_step(sourceDef) - if not hasattr(source, 'patches'): + if not hasattr(source, "patches"): continue for p in source.patches: - path = pj(SCRIPT_DIR, 'patches', p) - os.makedirs(pj(self.platform.buildEnv.build_dir, 'patches'), exist_ok=True) - dest = pj(self.platform.buildEnv.build_dir, 'patches', p) + path = pj(SCRIPT_DIR, "patches", p) + os.makedirs( + pj(self.platform.buildEnv.build_dir, "patches"), exist_ok=True + ) + dest = pj(self.platform.buildEnv.build_dir, "patches", p) copyfile(path, dest) - def build(self): - log = pj(self.platform.buildEnv.log_dir, 'cmd_build_flatpak.log') - context = Context('build', log, False) + log = pj(self.platform.buildEnv.log_dir, "cmd_build_flatpak.log") + context = Context("build", log, False) command = [ "flatpak-builder", - "--user", "--ccache", "--force-clean", "--keep-build-dirs", - "--disable-rofiles-fuse", "--repo=repo", "builddir", - f"{MANIFEST['app-id']}.json" + "--user", + "--ccache", + "--force-clean", + "--keep-build-dirs", + "--disable-rofiles-fuse", + "--repo=repo", + "builddir", + f"{MANIFEST['app-id']}.json", ] try: - run_command(command, self.platform.buildEnv.build_dir, context, env=self.platform.get_env()) + run_command( + command, + self.platform.buildEnv.build_dir, + context, + env=self.platform.get_env(), + ) context._finalise() except subprocess.CalledProcessError: - with open(log, 'r') as f: + with open(log, "r") as f: print(f.read()) raise StopBuild() def bundle(self): - log = pj(self.platform.buildEnv.log_dir, 'cmd_bundle_flatpak.log') - context = Context('bundle', log, False) - app_id = MANIFEST['app-id'] - command = [ - "flatpak", "build-bundle", "repo", - f"{app_id}.flatpak", - app_id - ] + log = pj(self.platform.buildEnv.log_dir, "cmd_bundle_flatpak.log") + context = Context("bundle", log, False) + app_id = MANIFEST["app-id"] + command = ["flatpak", "build-bundle", "repo", f"{app_id}.flatpak", app_id] try: - run_command(command, self.platform.buildEnv.build_dir, context, env=self.platform.get_env()) + run_command( + command, + self.platform.buildEnv.build_dir, + context, + env=self.platform.get_env(), + ) context._finalise() except subprocess.CalledProcessError: - with open(log, 'r') as f: + with open(log, "r") as f: print(f.read()) raise StopBuild() - def _get_packages(self): - package_name_mapper = PACKAGE_NAME_MAPPERS.get('flatpak', {}) + package_name_mapper = PACKAGE_NAME_MAPPERS.get("flatpak", {}) to_drop = [] for builderDef in self._targets: @@ -295,11 +310,10 @@ class FlatpakBuilder: self.bundle() # No error, clean intermediate file at end of build if needed. print("[CLEAN]") - if option('clean_at_end'): + if option("clean_at_end"): for platform in PlatformInfo.all_running_platforms.values(): platform.clean_intermediate_directories() else: print("SKIP") except StopBuild: sys.exit("Stopping build due to errors") - diff --git a/kiwixbuild/packages.py b/kiwixbuild/packages.py index 5a229f3..e7364ff 100644 --- a/kiwixbuild/packages.py +++ b/kiwixbuild/packages.py @@ -1,107 +1,147 @@ - - -_fedora_common = ['automake', 'libtool', 'cmake', 'git', 'subversion', 'ccache', 'pkgconf-pkg-config', 'gcc-c++', 'gettext-devel'] -_debian_common = ['automake', 'libtool', 'cmake', 'git', 'subversion', 'ccache', 'pkg-config', 'gcc', 'autopoint'] +_fedora_common = [ + "automake", + "libtool", + "cmake", + "git", + "subversion", + "ccache", + "pkgconf-pkg-config", + "gcc-c++", + "gettext-devel", +] +_debian_common = [ + "automake", + "libtool", + "cmake", + "git", + "subversion", + "ccache", + "pkg-config", + "gcc", + "autopoint", +] PACKAGE_NAME_MAPPERS = { - 'flatpak': { - 'zlib': True, - 'lzma': True, - 'icu4c': True, - 'qt': True, - 'qtwebengine': True, - 'uuid': True, - 'libxml2': True, - 'libssl': True, - 'libcurl': True, + "flatpak": { + "zlib": True, + "lzma": True, + "icu4c": True, + "qt": True, + "qtwebengine": True, + "uuid": True, + "libxml2": True, + "libssl": True, + "libcurl": True, }, - 'fedora_native_dyn': { - 'COMMON': _fedora_common, - 'uuid': ['libuuid-devel'], - 'xapian-core': None, # Not the right version on fedora 25 - 'pugixml': None, # ['pugixml-devel'] but package doesn't provide pkg-config file - 'libmicrohttpd': ['libmicrohttpd-devel'], - 'zlib': ['zlib-devel'], - 'lzma': ['xz-devel'], - 'icu4c': None, - 'zimlib': None, - 'file' : ['file-devel'], - 'gumbo' : ['gumbo-parser-devel'], - 'aria2': ['aria2'], - 'qt': ['qt5-qtbase-devel', 'qt5-qtsvg'], - 'qtwebengine': ['qt5-qtwebengine-devel'] + "fedora_native_dyn": { + "COMMON": _fedora_common, + "uuid": ["libuuid-devel"], + "xapian-core": None, # Not the right version on fedora 25 + "pugixml": None, # ['pugixml-devel'] but package doesn't provide pkg-config file + "libmicrohttpd": ["libmicrohttpd-devel"], + "zlib": ["zlib-devel"], + "lzma": ["xz-devel"], + "icu4c": None, + "zimlib": None, + "file": ["file-devel"], + "gumbo": ["gumbo-parser-devel"], + "aria2": ["aria2"], + "qt": ["qt5-qtbase-devel", "qt5-qtsvg"], + "qtwebengine": ["qt5-qtwebengine-devel"], }, - 'fedora_native_static': { - 'COMMON': _fedora_common + ['glibc-static', 'libstdc++-static'], - 'lzma': ['xz-devel', 'xz-static'] + "fedora_native_static": { + "COMMON": _fedora_common + ["glibc-static", "libstdc++-static"], + "lzma": ["xz-devel", "xz-static"] # Either there is no packages, or no static or too old }, - 'fedora_i586_dyn': { - 'COMMON': _fedora_common + ['glibc-devel.i686', 'libstdc++-devel.i686'], + "fedora_i586_dyn": { + "COMMON": _fedora_common + ["glibc-devel.i686", "libstdc++-devel.i686"], }, - 'fedora_i586_static': { - 'COMMON': _fedora_common + ['glibc-devel.i686'], + "fedora_i586_static": { + "COMMON": _fedora_common + ["glibc-devel.i686"], }, - 'fedora_win32_dyn': { - 'COMMON': _fedora_common + ['mingw32-gcc-c++', 'mingw32-bzip2', 'mingw32-win-iconv', 'mingw32-winpthreads', 'wine'], - 'zlib': ['mingw32-zlib'], - 'lzma': ['mingw32-xz-libs'], - 'libmicrohttpd': ['mingw32-libmicrohttpd'], + "fedora_win32_dyn": { + "COMMON": _fedora_common + + [ + "mingw32-gcc-c++", + "mingw32-bzip2", + "mingw32-win-iconv", + "mingw32-winpthreads", + "wine", + ], + "zlib": ["mingw32-zlib"], + "lzma": ["mingw32-xz-libs"], + "libmicrohttpd": ["mingw32-libmicrohttpd"], }, - 'fedora_win32_static': { - 'COMMON': _fedora_common + ['mingw32-gcc-c++', 'mingw32-bzip2-static', 'mingw32-win-iconv-static', 'mingw32-winpthreads-static', 'wine'], - 'zlib': ['mingw32-zlib-static'], - 'lzma': ['mingw32-xz-libs-static'], - 'libmicrohttpd': None, # ['mingw32-libmicrohttpd-static'] packaging dependecy seems buggy, and some static lib are name libfoo.dll.a and - # gcc cannot found them. + "fedora_win32_static": { + "COMMON": _fedora_common + + [ + "mingw32-gcc-c++", + "mingw32-bzip2-static", + "mingw32-win-iconv-static", + "mingw32-winpthreads-static", + "wine", + ], + "zlib": ["mingw32-zlib-static"], + "lzma": ["mingw32-xz-libs-static"], + "libmicrohttpd": None, # ['mingw32-libmicrohttpd-static'] packaging dependecy seems buggy, and some static lib are name libfoo.dll.a and + # gcc cannot found them. }, - 'fedora_armhf_static': { - 'COMMON': _fedora_common + "fedora_armhf_static": {"COMMON": _fedora_common}, + "fedora_armhf_dyn": {"COMMON": _fedora_common}, + "fedora_android": {"COMMON": _fedora_common}, + "debian_native_dyn": { + "COMMON": _debian_common + ["libbz2-dev", "libmagic-dev"], + "zlib": ["zlib1g-dev"], + "uuid": ["uuid-dev"], + "libmicrohttpd": ["libmicrohttpd-dev", "ccache"], + "qt": ["libqt5gui5", "qtbase5-dev", "qt5-default"], + "qtwebengine": ["qtwebengine5-dev"], + "aria2": ["aria2"], }, - 'fedora_armhf_dyn': { - 'COMMON': _fedora_common + "debian_native_static": { + "COMMON": _debian_common + ["libbz2-dev", "libmagic-dev"], }, - 'fedora_android': { - 'COMMON': _fedora_common + "debian_i586_dyn": { + "COMMON": _debian_common + + ["libc6-dev-i386", "lib32stdc++6", "gcc-multilib", "g++-multilib"], }, - 'debian_native_dyn': { - 'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'], - 'zlib': ['zlib1g-dev'], - 'uuid': ['uuid-dev'], - 'libmicrohttpd': ['libmicrohttpd-dev', 'ccache'], - 'qt' : ['libqt5gui5', 'qtbase5-dev', 'qt5-default'], - 'qtwebengine' : ['qtwebengine5-dev'], - 'aria2': ['aria2'], + "debian_i586_static": { + "COMMON": _debian_common + + ["libc6-dev-i386", "lib32stdc++6", "gcc-multilib", "g++-multilib"], }, - 'debian_native_static': { - 'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'], + "debian_win32_dyn": { + "COMMON": _debian_common + + [ + "g++-mingw-w64-i686", + "gcc-mingw-w64-i686", + "gcc-mingw-w64-base", + "mingw-w64-tools", + ], }, - 'debian_i586_dyn': { - 'COMMON': _debian_common + ['libc6-dev-i386', 'lib32stdc++6', 'gcc-multilib', 'g++-multilib'], + "debian_win32_static": { + "COMMON": _debian_common + + [ + "g++-mingw-w64-i686", + "gcc-mingw-w64-i686", + "gcc-mingw-w64-base", + "mingw-w64-tools", + ], }, - 'debian_i586_static': { - 'COMMON': _debian_common + ['libc6-dev-i386', 'lib32stdc++6', 'gcc-multilib', 'g++-multilib'], + "debian_armhf_static": { + "COMMON": _debian_common, }, - 'debian_win32_dyn': { - 'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'], + "debian_armhf_dyn": { + "COMMON": _debian_common, }, - 'debian_win32_static': { - 'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'], + "debian_android": { + "COMMON": _debian_common, }, - 'debian_armhf_static': { - 'COMMON': _debian_common, + "Darwin_native_dyn": { + "COMMON": ["autoconf", "automake", "libtool", "cmake", "pkg-config"], + "file": ["libmagic"], }, - 'debian_armhf_dyn': { - 'COMMON': _debian_common, - }, - 'debian_android': { - 'COMMON': _debian_common, - }, - 'Darwin_native_dyn': { - 'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'], - 'file': ['libmagic'] - }, - 'Darwin_iOS': { - 'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'], - 'file': ['libmagic'] + "Darwin_iOS": { + "COMMON": ["autoconf", "automake", "libtool", "cmake", "pkg-config"], + "file": ["libmagic"], }, } diff --git a/kiwixbuild/platforms/__init__.py b/kiwixbuild/platforms/__init__.py index fac9e94..e82de4d 100644 --- a/kiwixbuild/platforms/__init__.py +++ b/kiwixbuild/platforms/__init__.py @@ -1,15 +1,3 @@ - from .base import * -from . import ( - android, - armhf, - musl, - flatpak, - i586, - ios, - native, - neutral, - win32, - wasm -) +from . import android, armhf, musl, flatpak, i586, ios, native, neutral, win32, wasm diff --git a/kiwixbuild/platforms/android.py b/kiwixbuild/platforms/android.py index 2fb6d09..679a6d7 100644 --- a/kiwixbuild/platforms/android.py +++ b/kiwixbuild/platforms/android.py @@ -4,137 +4,152 @@ from kiwixbuild._global import get_target_step, option class AndroidPlatformInfo(PlatformInfo): - build = 'android' + build = "android" static = True - toolchain_names = ['android-ndk'] - compatible_hosts = ['fedora', 'debian'] + toolchain_names = ["android-ndk"] + compatible_hosts = ["fedora", "debian"] def __str__(self): return "android" @property def libdir(self): - return 'lib/{}'.format(self.arch_full) + return "lib/{}".format(self.arch_full) @property def binaries_name(self): arch_full = self.arch_full return { - 'CC': '{}-{}'.format(arch_full, 'clang'), - 'CXX': '{}-{}'.format(arch_full, 'clang++'), - 'AR': '{}-{}'.format(arch_full, 'ar'), - 'STRIP': '{}-{}'.format(arch_full, 'strip'), - 'RANLIB': '{}-{}'.format(arch_full, 'ranlib'), - 'LD': '{}-{}'.format(arch_full, 'ld') + "CC": "{}-{}".format(arch_full, "clang"), + "CXX": "{}-{}".format(arch_full, "clang++"), + "AR": "{}-{}".format(arch_full, "ar"), + "STRIP": "{}-{}".format(arch_full, "strip"), + "RANLIB": "{}-{}".format(arch_full, "ranlib"), + "LD": "{}-{}".format(arch_full, "ld"), } def binaries(self): install_path = self.install_path - binaries = {k:pj(install_path, 'bin', v) - for k,v in self.binaries_name.items()} - binaries['PKGCONFIG'] = 'pkg-config' + binaries = { + k: pj(install_path, "bin", v) for k, v in self.binaries_name.items() + } + binaries["PKGCONFIG"] = "pkg-config" return binaries @property def ndk_builder(self): - return get_target_step('android-ndk', self.name) + return get_target_step("android-ndk", self.name) @property def install_path(self): return self.ndk_builder.install_path def get_cross_config(self): - extra_libs = ['-llog'] - extra_cflags = ['-I{}'.format(include_dir) for include_dir in self.get_include_dirs()] - if hasattr(self, 'march'): - extra_libs.append('-march={}'.format(self.march)) - extra_cflags.append('-march={}'.format(self.march)) + extra_libs = ["-llog"] + extra_cflags = [ + "-I{}".format(include_dir) for include_dir in self.get_include_dirs() + ] + if hasattr(self, "march"): + extra_libs.append("-march={}".format(self.march)) + extra_cflags.append("-march={}".format(self.march)) return { - 'exe_wrapper_def': '', - 'install_path': self.install_path, - 'binaries': self.binaries(), - 'root_path': pj(self.install_path, 'sysroot'), - 'extra_libs': extra_libs, - 'extra_cflags': extra_cflags, - 'host_machine': { - 'system': 'Android', - 'lsystem': 'android', - 'cpu_family': self.arch, - 'cpu': self.cpu, - 'endian': 'little', - 'abi': self.abi - } + "exe_wrapper_def": "", + "install_path": self.install_path, + "binaries": self.binaries(), + "root_path": pj(self.install_path, "sysroot"), + "extra_libs": extra_libs, + "extra_cflags": extra_cflags, + "host_machine": { + "system": "Android", + "lsystem": "android", + "cpu_family": self.arch, + "cpu": self.cpu, + "endian": "little", + "abi": self.abi, + }, } def get_env(self): env = super().get_env() - root_path = pj(self.install_path, 'sysroot') - env['PKG_CONFIG_LIBDIR'] = pj(root_path, 'lib', 'pkgconfig') - env['NDK_DEBUG'] = '0' + root_path = pj(self.install_path, "sysroot") + env["PKG_CONFIG_LIBDIR"] = pj(root_path, "lib", "pkgconfig") + env["NDK_DEBUG"] = "0" return env def get_bin_dir(self): - return [pj(self.install_path, 'bin')] + return [pj(self.install_path, "bin")] def set_comp_flags(self, env): super().set_comp_flags(env) - root_path = pj(self.install_path, 'sysroot') - march = '-march={}'.format(self.march) if hasattr(self,'march') else '' - env['CFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} {} '.format(root_path, march) + env['CFLAGS'] - env['CXXFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} {} '.format(root_path, march) + env['CXXFLAGS'] - env['LDFLAGS'] = '--sysroot={} {} '.format(root_path, march) + env['LDFLAGS'] + root_path = pj(self.install_path, "sysroot") + march = "-march={}".format(self.march) if hasattr(self, "march") else "" + env["CFLAGS"] = ( + "-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} {} ".format( + root_path, march + ) + + env["CFLAGS"] + ) + env["CXXFLAGS"] = ( + "-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} {} ".format( + root_path, march + ) + + env["CXXFLAGS"] + ) + env["LDFLAGS"] = "--sysroot={} {} ".format(root_path, march) + env["LDFLAGS"] def set_compiler(self, env): binaries = self.binaries() - for k,v in binaries.items(): + for k, v in binaries.items(): env[k] = v @property def configure_options(self): - yield '--host={}'.format(self.arch_full) + yield "--host={}".format(self.arch_full) def finalize_setup(self): super().finalize_setup() - self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_android_cross_file.txt', 'cmake_cross_file.txt') - self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') + self.buildEnv.cmake_crossfile = self._gen_crossfile( + "cmake_android_cross_file.txt", "cmake_cross_file.txt" + ) + self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt") class AndroidArm(AndroidPlatformInfo): - name = 'android_arm' - arch = cpu = 'arm' - arch_full = 'arm-linux-androideabi' - abi = 'armeabi-v7a' - march = 'armv7-a' + name = "android_arm" + arch = cpu = "arm" + arch_full = "arm-linux-androideabi" + abi = "armeabi-v7a" + march = "armv7-a" class AndroidArm64(AndroidPlatformInfo): - name = 'android_arm64' - arch = 'arm64' - arch_full = 'aarch64-linux-android' - cpu = 'aarch64' - abi = 'arm64-v8a' + name = "android_arm64" + arch = "arm64" + arch_full = "aarch64-linux-android" + cpu = "aarch64" + abi = "arm64-v8a" class AndroidX86(AndroidPlatformInfo): - name = 'android_x86' - arch = abi = 'x86' - arch_full = 'i686-linux-android' - cpu = 'i686' + name = "android_x86" + arch = abi = "x86" + arch_full = "i686-linux-android" + cpu = "i686" class AndroidX8664(AndroidPlatformInfo): - name = 'android_x86_64' - arch = cpu = abi = 'x86_64' - arch_full = 'x86_64-linux-android' + name = "android_x86_64" + arch = cpu = abi = "x86_64" + arch_full = "x86_64-linux-android" class Android(MetaPlatformInfo): name = "android" - compatible_hosts = ['fedora', 'debian'] + compatible_hosts = ["fedora", "debian"] @property def subPlatformNames(self): - return ['android_{}'.format(arch) for arch in option('android_arch')] + return ["android_{}".format(arch) for arch in option("android_arch")] def add_targets(self, targetName, targets): return super().add_targets(targetName, targets) diff --git a/kiwixbuild/platforms/armhf.py b/kiwixbuild/platforms/armhf.py index 834db05..07cf001 100644 --- a/kiwixbuild/platforms/armhf.py +++ b/kiwixbuild/platforms/armhf.py @@ -6,23 +6,25 @@ from kiwixbuild._global import get_target_step # Base platform class ArmPlatformInfo(PlatformInfo): - compatible_hosts = ['fedora', 'debian'] + compatible_hosts = ["fedora", "debian"] def get_cross_config(self): return { - 'binaries': self.binaries, - 'exe_wrapper_def': '', - 'root_path': self.root_path, - 'extra_libs': [], - 'extra_cflags': ['-I{}'.format(include_dir) for include_dir in self.get_include_dirs()], - 'host_machine': { - 'system': 'linux', - 'lsystem': 'linux', - 'cpu_family': self.cpu_family, - 'cpu': self.cpu, - 'endian': 'little', - 'abi': '' - } + "binaries": self.binaries, + "exe_wrapper_def": "", + "root_path": self.root_path, + "extra_libs": [], + "extra_cflags": [ + "-I{}".format(include_dir) for include_dir in self.get_include_dirs() + ], + "host_machine": { + "system": "linux", + "lsystem": "linux", + "cpu_family": self.cpu_family, + "cpu": self.cpu, + "endian": "little", + "abi": "", + }, } @property @@ -31,7 +33,7 @@ class ArmPlatformInfo(PlatformInfo): @property def toolchain(self): - return get_target_step(self.build, 'neutral') + return get_target_step(self.build, "neutral") @property def root_path(self): @@ -39,26 +41,27 @@ class ArmPlatformInfo(PlatformInfo): @property def binaries(self): - binaries = ((k,'{}-{}'.format(self.arch_full, v)) - for k, v in (('CC', 'gcc'), - ('CXX', 'g++'), - ('AR', 'ar'), - ('STRIP', 'strip'), - ('WINDRES', 'windres'), - ('RANLIB', 'ranlib'), - ('LD', 'ld'), - ('LDSHARED', 'g++ -shared') - ) - ) - binaries = {k:pj(self.root_path, 'bin', v) - for k,v in binaries} - binaries['PKGCONFIG'] = 'pkg-config' + binaries = ( + (k, "{}-{}".format(self.arch_full, v)) + for k, v in ( + ("CC", "gcc"), + ("CXX", "g++"), + ("AR", "ar"), + ("STRIP", "strip"), + ("WINDRES", "windres"), + ("RANLIB", "ranlib"), + ("LD", "ld"), + ("LDSHARED", "g++ -shared"), + ) + ) + binaries = {k: pj(self.root_path, "bin", v) for k, v in binaries} + binaries["PKGCONFIG"] = "pkg-config" return binaries @property def exe_wrapper_def(self): try: - which('qemu-arm') + which("qemu-arm") except subprocess.CalledProcessError: return "" else: @@ -66,31 +69,39 @@ class ArmPlatformInfo(PlatformInfo): @property def configure_options(self): - yield '--host={}'.format(self.arch_full) + yield "--host={}".format(self.arch_full) def get_bin_dir(self): - return [pj(self.root_path, 'bin')] + return [pj(self.root_path, "bin")] def get_env(self): env = super().get_env() - env['LD_LIBRARY_PATH'] = ':'.join([ - pj(self.root_path, self.arch_full, 'lib64'), - pj(self.root_path, 'lib'), - env['LD_LIBRARY_PATH'] - ]) - env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') - env['QEMU_LD_PREFIX'] = pj(self.root_path, self.arch_full, "libc") - env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format( - ':'.join([ - pj(self.root_path, self.arch_full, "lib"), - env['LD_LIBRARY_PATH'] - ])) + env["LD_LIBRARY_PATH"] = ":".join( + [ + pj(self.root_path, self.arch_full, "lib64"), + pj(self.root_path, "lib"), + env["LD_LIBRARY_PATH"], + ] + ) + env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig") + env["QEMU_LD_PREFIX"] = pj(self.root_path, self.arch_full, "libc") + env["QEMU_SET_ENV"] = "LD_LIBRARY_PATH={}".format( + ":".join( + [pj(self.root_path, self.arch_full, "lib"), env["LD_LIBRARY_PATH"]] + ) + ) return env def set_comp_flags(self, env): super().set_comp_flags(env) - env['CFLAGS'] = " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS'] - env['CXXFLAGS'] = " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS'] + env["CFLAGS"] = ( + " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 " + + env["CFLAGS"] + ) + env["CXXFLAGS"] = ( + " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 " + + env["CXXFLAGS"] + ) def set_compiler(self, env): for k, v in self.binaries.items(): @@ -98,69 +109,78 @@ class ArmPlatformInfo(PlatformInfo): def finalize_setup(self): super().finalize_setup() - self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') - self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') + self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt") + self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt") + class Armv6(ArmPlatformInfo): build = "armv6" arch_full = "armv6-rpi-linux-gnueabihf" - toolchain_names = ['armv6'] - cpu_family = 'arm' - cpu = 'armv6' + toolchain_names = ["armv6"] + cpu_family = "arm" + cpu = "armv6" + class Armv6Dyn(Armv6): - name = 'armv6_dyn' + name = "armv6_dyn" static = False + class Armv6Static(Armv6): - name = 'armv6_static' + name = "armv6_static" static = True -class Armv6Mixed(MixedMixin('armv6_static'), Armv6): - name = 'armv6_mixed' + +class Armv6Mixed(MixedMixin("armv6_static"), Armv6): + name = "armv6_mixed" static = False class Armv8(ArmPlatformInfo): build = "armv8" arch_full = "armv8-rpi3-linux-gnueabihf" - toolchain_names = ['armv8'] - cpu_family = 'arm' - cpu = 'armv8' + toolchain_names = ["armv8"] + cpu_family = "arm" + cpu = "armv8" + class Armv8Dyn(Armv8): - name = 'armv8_dyn' + name = "armv8_dyn" static = False + class Armv8Static(Armv8): - name = 'armv8_static' + name = "armv8_static" static = True -class Armv8Mixed(MixedMixin('armv8_static'), Armv8): - name = 'armv8_mixed' + +class Armv8Mixed(MixedMixin("armv8_static"), Armv8): + name = "armv8_mixed" static = False class Aarch64(ArmPlatformInfo): - build = 'aarch64' - arch_full = 'aarch64-linux-gnu' - toolchain_names = ['aarch64'] - cpu_family = 'aarch64' - cpu = 'aarch64' + build = "aarch64" + arch_full = "aarch64-linux-gnu" + toolchain_names = ["aarch64"] + cpu_family = "aarch64" + cpu = "aarch64" @property def root_path(self): return self.toolchain.build_path + class Aarch64Dyn(Aarch64): - name = 'aarch64_dyn' + name = "aarch64_dyn" static = False + class Aarch64Static(Aarch64): - name = 'aarch64_static' + name = "aarch64_static" static = True -class Aarch64Mixed(MixedMixin('aarch64_static'), Aarch64): - name = 'aarch64_mixed' +class Aarch64Mixed(MixedMixin("aarch64_static"), Aarch64): + name = "aarch64_mixed" static = False diff --git a/kiwixbuild/platforms/base.py b/kiwixbuild/platforms/base.py index 41cbee2..1210380 100644 --- a/kiwixbuild/platforms/base.py +++ b/kiwixbuild/platforms/base.py @@ -1,4 +1,3 @@ - import os, sys import subprocess @@ -8,13 +7,14 @@ from kiwixbuild.buildenv import BuildEnv from kiwixbuild._global import neutralEnv, option, target_steps _SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -TEMPLATES_DIR = pj(os.path.dirname(_SCRIPT_DIR), 'templates') +TEMPLATES_DIR = pj(os.path.dirname(_SCRIPT_DIR), "templates") + class _MetaPlatform(type): def __new__(cls, name, bases, dct): _class = type.__new__(cls, name, bases, dct) - if name not in ('PlatformInfo', 'MetaPlatformInfo') and 'name' in dct: - dep_name = dct['name'] + if name not in ("PlatformInfo", "MetaPlatformInfo") and "name" in dct: + dep_name = dct["name"] PlatformInfo.all_platforms[dep_name] = _class return _class @@ -43,20 +43,20 @@ class PlatformInfo(metaclass=_MetaPlatform): self.setup_toolchains(targets) def __str__(self): - return "{}_{}".format(self.build, 'static' if self.static else 'dyn') + return "{}_{}".format(self.build, "static" if self.static else "dyn") def setup_toolchains(self, targets): for tlc_name in self.toolchain_names: ToolchainClass = Dependency.all_deps[tlc_name] - targets[('source', tlc_name)] = ToolchainClass.Source - plt_name = 'neutral' if ToolchainClass.neutral else self.name + targets[("source", tlc_name)] = ToolchainClass.Source + plt_name = "neutral" if ToolchainClass.neutral else self.name targets[(plt_name, tlc_name)] = ToolchainClass.Builder def add_targets(self, targetName, targets): if (self.name, targetName) in targets: return [] targetClass = Dependency.all_deps[targetName] - targets[('source', targetName)] = targetClass.Source + targets[("source", targetName)] = targetClass.Source targets[(self.name, targetName)] = targetClass.Builder for dep in targetClass.Builder.get_dependencies(self, False): if isinstance(dep, tuple): @@ -68,47 +68,40 @@ class PlatformInfo(metaclass=_MetaPlatform): return [(self.name, targetName)] def get_fully_qualified_dep(self, dep): - if isinstance(dep, tuple): + if isinstance(dep, tuple): return dep - else: + else: return self.name, dep - def get_cross_config(self): return {} def get_include_dirs(self): - return [pj(self.buildEnv.install_dir, 'include')] + return [pj(self.buildEnv.install_dir, "include")] def get_env(self): return DefaultEnv() - def get_bin_dir(self): return [] - def set_compiler(self, env): pass - def set_comp_flags(self, env): if self.static: - env['CFLAGS'] = env['CFLAGS'] + ' -fPIC' - env['CXXFLAGS'] = env['CXXFLAGS'] + ' -fPIC' - + env["CFLAGS"] = env["CFLAGS"] + " -fPIC" + env["CXXFLAGS"] = env["CXXFLAGS"] + " -fPIC" def _gen_crossfile(self, name, outname=None): if outname is None: outname = name crossfile = pj(self.buildEnv.build_dir, outname) template_file = pj(TEMPLATES_DIR, name) - with open(template_file, 'r') as f: + with open(template_file, "r") as f: template = f.read() - content = template.format( - **self.get_cross_config() - ) - with open(crossfile, 'w') as outfile: + content = template.format(**self.get_cross_config()) + with open(crossfile, "w") as outfile: outfile.write(content) return crossfile @@ -121,7 +114,6 @@ class PlatformInfo(metaclass=_MetaPlatform): self.buildEnv.clean_intermediate_directories() - class MetaPlatformInfo(PlatformInfo): subPlatformNames = [] @@ -133,7 +125,6 @@ class MetaPlatformInfo(PlatformInfo): return targetDefs - def MixedMixin(static_name): class MixedMixinClass: mixed = True @@ -141,7 +132,7 @@ def MixedMixin(static_name): def add_targets(self, targetName, targets): print(targetName) - if option('target') == targetName: + if option("target") == targetName: return super().add_targets(targetName, targets) else: static_platform = self.get_platform(static_name, targets) @@ -150,7 +141,7 @@ def MixedMixin(static_name): def get_fully_qualified_dep(self, dep): if isinstance(dep, tuple): return dep - if option('target') == dep: + if option("target") == dep: return self.name, dep return static_name, dep @@ -161,16 +152,27 @@ def MixedMixin(static_name): def get_include_dirs(self): return [ - pj(self.buildEnv.install_dir, 'include'), - pj(self.static_buildEnv.install_dir, 'include') + pj(self.buildEnv.install_dir, "include"), + pj(self.static_buildEnv.install_dir, "include"), ] def get_env(self): env = super().get_env() - env['PATH'] = ':'.join([pj(self.static_buildEnv.install_dir, 'bin')] + [env['PATH']]) - pkgconfig_path = pj(self.static_buildEnv.install_dir, self.static_buildEnv.libprefix, 'pkgconfig') - env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path]) - env['CPPFLAGS'] = " ".join(['-I'+pj(self.static_buildEnv.install_dir, 'include'), env['CPPFLAGS']]) + env["PATH"] = ":".join( + [pj(self.static_buildEnv.install_dir, "bin")] + [env["PATH"]] + ) + pkgconfig_path = pj( + self.static_buildEnv.install_dir, + self.static_buildEnv.libprefix, + "pkgconfig", + ) + env["PKG_CONFIG_PATH"] = ":".join([env["PKG_CONFIG_PATH"], pkgconfig_path]) + env["CPPFLAGS"] = " ".join( + [ + "-I" + pj(self.static_buildEnv.install_dir, "include"), + env["CPPFLAGS"], + ] + ) return env return MixedMixinClass diff --git a/kiwixbuild/platforms/flatpak.py b/kiwixbuild/platforms/flatpak.py index 4cd8342..be57bca 100644 --- a/kiwixbuild/platforms/flatpak.py +++ b/kiwixbuild/platforms/flatpak.py @@ -1,18 +1,18 @@ from .base import PlatformInfo from kiwixbuild._global import option, neutralEnv + class FlatpakPlatformInfo(PlatformInfo): - name = 'flatpak' - build = 'flatpak' - static = '' - toolchain_names = ['org.kde', 'io.qt.qtwebengine'] - compatible_hosts = ['debian', 'fedora'] + name = "flatpak" + build = "flatpak" + static = "" + toolchain_names = ["org.kde", "io.qt.qtwebengine"] + compatible_hosts = ["debian", "fedora"] def __str__(self): return "flatpak" def get_env(self): env = super().get_env() - env['FLATPAK_USER_DIR'] = self.buildEnv.build_dir + env["FLATPAK_USER_DIR"] = self.buildEnv.build_dir return env - diff --git a/kiwixbuild/platforms/i586.py b/kiwixbuild/platforms/i586.py index 71ae316..341047d 100644 --- a/kiwixbuild/platforms/i586.py +++ b/kiwixbuild/platforms/i586.py @@ -5,66 +5,75 @@ from kiwixbuild.utils import which, pj class I586PlatformInfo(PlatformInfo): - build = 'i586' - arch_full = 'i586-linux-gnu' - compatible_hosts = ['fedora', 'debian'] + build = "i586" + arch_full = "i586-linux-gnu" + compatible_hosts = ["fedora", "debian"] def get_cross_config(self): return { - 'binaries': self.binaries, - 'exe_wrapper_def': '', - 'extra_libs': ['-m32', '-march=i586', '-mno-sse'], - 'extra_cflags': [ - '-m32', - '-march=i586', - '-mno-sse', - *('-I{}'.format(include_dir) for include_dir in self.get_include_dirs()) + "binaries": self.binaries, + "exe_wrapper_def": "", + "extra_libs": ["-m32", "-march=i586", "-mno-sse"], + "extra_cflags": [ + "-m32", + "-march=i586", + "-mno-sse", + *( + "-I{}".format(include_dir) + for include_dir in self.get_include_dirs() + ), ], - 'host_machine': { - 'system': 'linux', - 'lsystem': 'linux', - 'cpu_family': 'x86', - 'cpu': 'i586', - 'endian': 'little', - 'abi': '' - } + "host_machine": { + "system": "linux", + "lsystem": "linux", + "cpu_family": "x86", + "cpu": "i586", + "endian": "little", + "abi": "", + }, } @property def configure_options(self): - yield f'--host={self.arch_full}' + yield f"--host={self.arch_full}" @property def binaries(self): - return {k:which(v) - for k, v in (('CC', os.environ.get('CC', 'gcc')), - ('CXX', os.environ.get('CXX', 'g++')), - ('AR', 'ar'), - ('STRIP', 'strip'), - ('RANLIB', 'ranlib'), - ('LD', 'ld'), - ('PKGCONFIG', 'pkg-config')) - } + return { + k: which(v) + for k, v in ( + ("CC", os.environ.get("CC", "gcc")), + ("CXX", os.environ.get("CXX", "g++")), + ("AR", "ar"), + ("STRIP", "strip"), + ("RANLIB", "ranlib"), + ("LD", "ld"), + ("PKGCONFIG", "pkg-config"), + ) + } def set_comp_flags(self, env): super().set_comp_flags(env) - env['CFLAGS'] = "-m32 -march=i586 -mno-sse "+env['CFLAGS'] - env['CXXFLAGS'] = "-m32 -march=i586 -mno-sse "+env['CXXFLAGS'] - env['LDFLAGS'] = "-m32 -march=i586 -mno-sse "+env['LDFLAGS'] + env["CFLAGS"] = "-m32 -march=i586 -mno-sse " + env["CFLAGS"] + env["CXXFLAGS"] = "-m32 -march=i586 -mno-sse " + env["CXXFLAGS"] + env["LDFLAGS"] = "-m32 -march=i586 -mno-sse " + env["LDFLAGS"] def get_bin_dir(self): return [] - def finalize_setup(self): super().finalize_setup() - self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_i586_cross_file.txt', 'cmake_cross_file.txt') - self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') + self.buildEnv.cmake_crossfile = self._gen_crossfile( + "cmake_i586_cross_file.txt", "cmake_cross_file.txt" + ) + self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt") + class I586Dyn(I586PlatformInfo): - name = 'i586_dyn' + name = "i586_dyn" static = False + class I586Static(I586PlatformInfo): - name = 'i586_static' + name = "i586_static" static = True diff --git a/kiwixbuild/platforms/ios.py b/kiwixbuild/platforms/ios.py index 89964b9..5fb8d7d 100644 --- a/kiwixbuild/platforms/ios.py +++ b/kiwixbuild/platforms/ios.py @@ -6,12 +6,13 @@ from .base import PlatformInfo, MetaPlatformInfo, MixedMixin from kiwixbuild.dependencies.apple_xcframework import AppleXCFramework -MIN_MACOS_VERSION = '12.0' +MIN_MACOS_VERSION = "12.0" + class ApplePlatformInfo(PlatformInfo): - build = 'iOS' + build = "iOS" static = True - compatible_hosts = ['Darwin'] + compatible_hosts = ["Darwin"] arch = None host = None target = None @@ -35,168 +36,194 @@ class ApplePlatformInfo(PlatformInfo): def finalize_setup(self): super().finalize_setup() - self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_ios_cross_file.txt', 'cmake_cross_file.txt') - self.buildEnv.meson_crossfile = self._gen_crossfile('meson_ios_cross_file.txt', 'meson_cross_file.txt') + self.buildEnv.cmake_crossfile = self._gen_crossfile( + "cmake_ios_cross_file.txt", "cmake_cross_file.txt" + ) + self.buildEnv.meson_crossfile = self._gen_crossfile( + "meson_ios_cross_file.txt", "meson_cross_file.txt" + ) def get_cross_config(self): config = { - 'root_path': self.root_path, - 'binaries': self.binaries, - 'exe_wrapper_def': '', - 'extra_libs': [ - '-isysroot', self.root_path, - '-arch', self.arch, - '-target', self.target, + "root_path": self.root_path, + "binaries": self.binaries, + "exe_wrapper_def": "", + "extra_libs": [ + "-isysroot", + self.root_path, + "-arch", + self.arch, + "-target", + self.target, ], - 'extra_cflags': [ - '-isysroot', self.root_path, - '-arch', self.arch, - '-target', self.target, - *('-I{}'.format(include_dir) for include_dir in self.get_include_dirs()) + "extra_cflags": [ + "-isysroot", + self.root_path, + "-arch", + self.arch, + "-target", + self.target, + *( + "-I{}".format(include_dir) + for include_dir in self.get_include_dirs() + ), ], - 'host_machine': { - 'system': 'Darwin', - 'lsystem': 'darwin', - 'cpu_family': self.arch, - 'cpu': self.cpu, - 'endian': '', - 'abi': '' - } + "host_machine": { + "system": "Darwin", + "lsystem": "darwin", + "cpu_family": self.arch, + "cpu": self.cpu, + "endian": "", + "abi": "", + }, } if self.min_iphoneos_version: - config['extra_libs'].append('-miphoneos-version-min={}'.format(self.min_iphoneos_version)) - config['extra_cflags'].append('-miphoneos-version-min={}'.format(self.min_iphoneos_version)) + config["extra_libs"].append( + "-miphoneos-version-min={}".format(self.min_iphoneos_version) + ) + config["extra_cflags"].append( + "-miphoneos-version-min={}".format(self.min_iphoneos_version) + ) if self.min_macos_version: - config['extra_libs'].append('-mmacosx-version-min={}'.format(self.min_macos_version)) - config['extra_cflags'].append('-mmacosx-version-min={}'.format(self.min_macos_version)) + config["extra_libs"].append( + "-mmacosx-version-min={}".format(self.min_macos_version) + ) + config["extra_cflags"].append( + "-mmacosx-version-min={}".format(self.min_macos_version) + ) return config def get_env(self): env = super().get_env() - cflags = [env['CFLAGS']] + cflags = [env["CFLAGS"]] if self.min_iphoneos_version: - cflags.append('-miphoneos-version-min={}'.format(self.min_iphoneos_version)) + cflags.append("-miphoneos-version-min={}".format(self.min_iphoneos_version)) if self.min_macos_version: - cflags.append('-mmacosx-version-min={}'.format(self.min_macos_version)) - env['CFLAGS'] = ' '.join(cflags) + cflags.append("-mmacosx-version-min={}".format(self.min_macos_version)) + env["CFLAGS"] = " ".join(cflags) return env def set_comp_flags(self, env): super().set_comp_flags(env) cflags = [ - '-isysroot {}'.format(self.root_path), - '-arch {}'.format(self.arch), - '-target {}'.format(self.target), - env['CFLAGS'], + "-isysroot {}".format(self.root_path), + "-arch {}".format(self.arch), + "-target {}".format(self.target), + env["CFLAGS"], ] if self.min_iphoneos_version: - cflags.append('-miphoneos-version-min={}'.format(self.min_iphoneos_version)) - env['CFLAGS'] = ' '.join(cflags) - env['CXXFLAGS'] = ' '.join([ - env['CFLAGS'], - '-std=c++11', - env['CXXFLAGS'], - ]) - env['LDFLAGS'] = ' '.join([ - ' -arch {}'.format(self.arch), - '-isysroot {}'.format(self.root_path), - ]) + cflags.append("-miphoneos-version-min={}".format(self.min_iphoneos_version)) + env["CFLAGS"] = " ".join(cflags) + env["CXXFLAGS"] = " ".join( + [ + env["CFLAGS"], + "-std=c++11", + env["CXXFLAGS"], + ] + ) + env["LDFLAGS"] = " ".join( + [ + " -arch {}".format(self.arch), + "-isysroot {}".format(self.root_path), + ] + ) def get_bin_dir(self): - return [pj(self.root_path, 'bin')] + return [pj(self.root_path, "bin")] @property def binaries(self): return { - 'CC': xrun_find('clang'), - 'CXX': xrun_find('clang++'), - 'AR': xrun_find('ar'), - 'STRIP': xrun_find('strip'), - 'RANLIB': xrun_find('ranlib'), - 'LD': xrun_find('ld'), - 'PKGCONFIG': 'pkg-config', + "CC": xrun_find("clang"), + "CXX": xrun_find("clang++"), + "AR": xrun_find("ar"), + "STRIP": xrun_find("strip"), + "RANLIB": xrun_find("ranlib"), + "LD": xrun_find("ld"), + "PKGCONFIG": "pkg-config", } @property def configure_options(self): - yield f'--host={self.host}' + yield f"--host={self.host}" class iOSArm64(ApplePlatformInfo): - name = 'iOS_arm64' - arch = cpu = 'arm64' - host = 'arm-apple-darwin' - target = 'aarch64-apple-ios' - sdk_name = 'iphoneos' - min_iphoneos_version = '15.0' + name = "iOS_arm64" + arch = cpu = "arm64" + host = "arm-apple-darwin" + target = "aarch64-apple-ios" + sdk_name = "iphoneos" + min_iphoneos_version = "15.0" class iOSx64Simulator(ApplePlatformInfo): - name = 'iOSSimulator_x86_64' - arch = cpu = 'x86_64' - host = 'x86_64-apple-darwin' - target = 'x86-apple-ios-simulator' - sdk_name = 'iphonesimulator' - min_iphoneos_version = '15.0' + name = "iOSSimulator_x86_64" + arch = cpu = "x86_64" + host = "x86_64-apple-darwin" + target = "x86-apple-ios-simulator" + sdk_name = "iphonesimulator" + min_iphoneos_version = "15.0" class iOSArm64Simulator(ApplePlatformInfo): - name = 'iOSSimulator_arm64' - arch = cpu = 'arm64' - host = 'arm-apple-darwin' - target = 'aarch64-apple-ios-simulator' - sdk_name = 'iphonesimulator' - min_iphoneos_version = '15.0' + name = "iOSSimulator_arm64" + arch = cpu = "arm64" + host = "arm-apple-darwin" + target = "aarch64-apple-ios-simulator" + sdk_name = "iphonesimulator" + min_iphoneos_version = "15.0" class macOSArm64(ApplePlatformInfo): - name = 'macOS_arm64_static' - arch = cpu = 'arm64' - host = 'aarch64-apple-darwin' - target = 'arm64-apple-macos' - sdk_name = 'macosx' + name = "macOS_arm64_static" + arch = cpu = "arm64" + host = "aarch64-apple-darwin" + target = "arm64-apple-macos" + sdk_name = "macosx" min_iphoneos_version = None min_macos_version = MIN_MACOS_VERSION -class macOSArm64Mixed(MixedMixin('macOS_arm64_static'), ApplePlatformInfo): - name = 'macOS_arm64_mixed' - arch = cpu = 'arm64' - host = 'aarch64-apple-darwin' - target = 'arm64-apple-macos' - sdk_name = 'macosx' +class macOSArm64Mixed(MixedMixin("macOS_arm64_static"), ApplePlatformInfo): + name = "macOS_arm64_mixed" + arch = cpu = "arm64" + host = "aarch64-apple-darwin" + target = "arm64-apple-macos" + sdk_name = "macosx" min_iphoneos_version = None min_macos_version = MIN_MACOS_VERSION class macOSx64(ApplePlatformInfo): - name = 'macOS_x86_64' - arch = cpu = 'x86_64' - host = 'x86_64-apple-darwin' - target = 'x86_64-apple-macos' - sdk_name = 'macosx' + name = "macOS_x86_64" + arch = cpu = "x86_64" + host = "x86_64-apple-darwin" + target = "x86_64-apple-macos" + sdk_name = "macosx" min_iphoneos_version = None min_macos_version = MIN_MACOS_VERSION class IOS(MetaPlatformInfo): name = "iOS_multi" - compatible_hosts = ['Darwin'] + compatible_hosts = ["Darwin"] @property def subPlatformNames(self): - return ['iOS_{}'.format(arch) for arch in option('ios_arch')] + return ["iOS_{}".format(arch) for arch in option("ios_arch")] def add_targets(self, targetName, targets): super().add_targets(targetName, targets) - return PlatformInfo.add_targets(self, '_ios_fat_lib', targets) + return PlatformInfo.add_targets(self, "_ios_fat_lib", targets) def __str__(self): return self.name + class AppleStaticAll(MetaPlatformInfo): name = "apple_all_static" - compatible_hosts = ['Darwin'] + compatible_hosts = ["Darwin"] @property def subPlatformNames(self): @@ -204,7 +231,7 @@ class AppleStaticAll(MetaPlatformInfo): def add_targets(self, targetName, targets): super().add_targets(targetName, targets) - return PlatformInfo.add_targets(self, 'apple_xcframework', targets) + return PlatformInfo.add_targets(self, "apple_xcframework", targets) def __str__(self): return self.name diff --git a/kiwixbuild/platforms/musl.py b/kiwixbuild/platforms/musl.py index cd5fc46..c4405c6 100644 --- a/kiwixbuild/platforms/musl.py +++ b/kiwixbuild/platforms/musl.py @@ -5,28 +5,30 @@ from kiwixbuild._global import get_target_step class MuslPlatformInfo(PlatformInfo): - compatible_hosts = ['fedora', 'debian'] + compatible_hosts = ["fedora", "debian"] def get_cross_config(self): return { - 'binaries': self.binaries, - 'exe_wrapper_def': '', - 'root_path': self.root_path, - 'extra_libs': [], - 'extra_cflags': ['-I{}'.format(include_dir) for include_dir in self.get_include_dirs()], - 'host_machine': { - 'system': 'linux', - 'lsystem': 'linux', - 'cpu_family': self.cpu_family, - 'cpu': self.cpu, - 'endian': 'little', - 'abi': '' - } + "binaries": self.binaries, + "exe_wrapper_def": "", + "root_path": self.root_path, + "extra_libs": [], + "extra_cflags": [ + "-I{}".format(include_dir) for include_dir in self.get_include_dirs() + ], + "host_machine": { + "system": "linux", + "lsystem": "linux", + "cpu_family": self.cpu_family, + "cpu": self.cpu, + "endian": "little", + "abi": "", + }, } @property def toolchain(self): - return get_target_step(self.build, 'neutral') + return get_target_step(self.build, "neutral") @property def root_path(self): @@ -34,20 +36,21 @@ class MuslPlatformInfo(PlatformInfo): @property def binaries(self): - binaries = ((k,'{}-{}'.format(self.arch_full, v)) - for k, v in (('CC', 'gcc'), - ('CXX', 'g++'), - ('AR', 'ar'), - ('STRIP', 'strip'), - ('WINDRES', 'windres'), - ('RANLIB', 'ranlib'), - ('LD', 'ld'), - ('LDSHARED', 'g++ -shared') - ) - ) - binaries = {k:pj(self.root_path, 'bin', v) - for k,v in binaries} - binaries['PKGCONFIG'] = 'pkg-config' + binaries = ( + (k, "{}-{}".format(self.arch_full, v)) + for k, v in ( + ("CC", "gcc"), + ("CXX", "g++"), + ("AR", "ar"), + ("STRIP", "strip"), + ("WINDRES", "windres"), + ("RANLIB", "ranlib"), + ("LD", "ld"), + ("LDSHARED", "g++ -shared"), + ) + ) + binaries = {k: pj(self.root_path, "bin", v) for k, v in binaries} + binaries["PKGCONFIG"] = "pkg-config" return binaries @property @@ -63,35 +66,42 @@ class MuslPlatformInfo(PlatformInfo): @property def configure_options(self): - return [f'--host={self.arch_full}'] + return [f"--host={self.arch_full}"] def get_bin_dir(self): - return [pj(self.root_path, 'bin')] + return [pj(self.root_path, "bin")] def get_env(self): env = super().get_env() - env['LD_LIBRARY_PATH'] = ':'.join([ - pj(self.root_path, self.arch_full, 'lib64'), - pj(self.root_path, 'lib'), - env['LD_LIBRARY_PATH'] - ]) - env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') - env['QEMU_LD_PREFIX'] = pj(self.root_path, self.arch_full, "libc") - env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format( - ':'.join([ - pj(self.root_path, self.arch_full, "lib"), - env['LD_LIBRARY_PATH'] - ])) + env["LD_LIBRARY_PATH"] = ":".join( + [ + pj(self.root_path, self.arch_full, "lib64"), + pj(self.root_path, "lib"), + env["LD_LIBRARY_PATH"], + ] + ) + env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig") + env["QEMU_LD_PREFIX"] = pj(self.root_path, self.arch_full, "libc") + env["QEMU_SET_ENV"] = "LD_LIBRARY_PATH={}".format( + ":".join( + [pj(self.root_path, self.arch_full, "lib"), env["LD_LIBRARY_PATH"]] + ) + ) return env def set_comp_flags(self, env): super().set_comp_flags(env) - env['LD_LIBRARY_PATH'] = ':'.join([ - pj(self.root_path, self.arch_full, 'lib'), - env['LD_LIBRARY_PATH'] - ]) - env['CFLAGS'] = " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS'] - env['CXXFLAGS'] = " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS'] + env["LD_LIBRARY_PATH"] = ":".join( + [pj(self.root_path, self.arch_full, "lib"), env["LD_LIBRARY_PATH"]] + ) + env["CFLAGS"] = ( + " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 " + + env["CFLAGS"] + ) + env["CXXFLAGS"] = ( + " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 " + + env["CXXFLAGS"] + ) def set_compiler(self, env): for k, v in self.binaries.items(): @@ -99,50 +109,54 @@ class MuslPlatformInfo(PlatformInfo): def finalize_setup(self): super().finalize_setup() - self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') - self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') + self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt") + self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt") class Aarch64MuslPlatformInfo(MuslPlatformInfo): - build = 'aarch64_musl' - arch_full = 'aarch64-linux-musl' - toolchain_names = ['aarch64_musl'] + build = "aarch64_musl" + arch_full = "aarch64-linux-musl" + toolchain_names = ["aarch64_musl"] libdir = "lib/aarch64-linux-musl" - cpu_family = 'arm' - cpu = 'armhf' - qemu = 'qemu-arm' + cpu_family = "arm" + cpu = "armhf" + qemu = "qemu-arm" class Aarch64MuslDyn(Aarch64MuslPlatformInfo): - name = 'aarch64_musl_dyn' + name = "aarch64_musl_dyn" static = False + class Aarch64MuslStatic(Aarch64MuslPlatformInfo): - name = 'aarch64_musl_static' + name = "aarch64_musl_static" static = True -class Aarch64MuslMixed(MixedMixin('aarch64_musl_static'), Aarch64MuslPlatformInfo): - name = 'aarch64_musl_mixed' + +class Aarch64MuslMixed(MixedMixin("aarch64_musl_static"), Aarch64MuslPlatformInfo): + name = "aarch64_musl_mixed" static = False class X86_64MuslPlatformInfo(MuslPlatformInfo): - build = 'x86-64_musl' - arch_full = 'x86_64-linux-musl' - toolchain_names = ['x86-64_musl'] + build = "x86-64_musl" + arch_full = "x86_64-linux-musl" + toolchain_names = ["x86-64_musl"] libdir = "lib/x86_64-linux-musl" - cpu_family = 'x86_64' - cpu = 'x86_64' + cpu_family = "x86_64" + cpu = "x86_64" class X86_64MuslDyn(X86_64MuslPlatformInfo): - name = 'x86-64_musl_dyn' + name = "x86-64_musl_dyn" static = False + class X86_64MuslStatic(X86_64MuslPlatformInfo): - name = 'x86-64_musl_static' + name = "x86-64_musl_static" static = True -class x86_64MuslMixed(MixedMixin('x86-64_musl_static'), X86_64MuslPlatformInfo): - name = 'x86-64_musl_mixed' + +class x86_64MuslMixed(MixedMixin("x86-64_musl_static"), X86_64MuslPlatformInfo): + name = "x86-64_musl_mixed" static = False diff --git a/kiwixbuild/platforms/native.py b/kiwixbuild/platforms/native.py index deac809..83efe07 100644 --- a/kiwixbuild/platforms/native.py +++ b/kiwixbuild/platforms/native.py @@ -4,29 +4,32 @@ from kiwixbuild.utils import pj from kiwixbuild._global import option, neutralEnv from kiwixbuild.platforms.ios import MIN_MACOS_VERSION + class NativePlatformInfo(PlatformInfo): - build = 'native' + build = "native" def get_env(self): env = super().get_env() - if neutralEnv('distname') == 'fedora': - env['QT_SELECT'] = "5-64" - if neutralEnv('distname') == 'Darwin': - env['CFLAGS'] += f'-mmacosx-version-min={MIN_MACOS_VERSION}' + if neutralEnv("distname") == "fedora": + env["QT_SELECT"] = "5-64" + if neutralEnv("distname") == "Darwin": + env["CFLAGS"] += f"-mmacosx-version-min={MIN_MACOS_VERSION}" return env class NativeDyn(NativePlatformInfo): - name = 'native_dyn' + name = "native_dyn" static = False - compatible_hosts = ['fedora', 'debian', 'Darwin'] + compatible_hosts = ["fedora", "debian", "Darwin"] + class NativeStatic(NativePlatformInfo): - name = 'native_static' + name = "native_static" static = True - compatible_hosts = ['fedora', 'debian'] + compatible_hosts = ["fedora", "debian"] -class NativeMixed(MixedMixin('native_static'), NativePlatformInfo): - name = 'native_mixed' + +class NativeMixed(MixedMixin("native_static"), NativePlatformInfo): + name = "native_mixed" static = False - compatible_hosts = ['fedora', 'debian', 'Darwin'] + compatible_hosts = ["fedora", "debian", "Darwin"] diff --git a/kiwixbuild/platforms/neutral.py b/kiwixbuild/platforms/neutral.py index bbe339c..7315c3a 100644 --- a/kiwixbuild/platforms/neutral.py +++ b/kiwixbuild/platforms/neutral.py @@ -1,9 +1,10 @@ from .base import PlatformInfo + class NeutralPlatformInfo(PlatformInfo): - name = 'neutral' - static = '' - compatible_hosts = ['fedora', 'debian', 'Darwin'] + name = "neutral" + static = "" + compatible_hosts = ["fedora", "debian", "Darwin"] def __str__(self): return "neutral" diff --git a/kiwixbuild/platforms/wasm.py b/kiwixbuild/platforms/wasm.py index 7b5cda0..de743d7 100644 --- a/kiwixbuild/platforms/wasm.py +++ b/kiwixbuild/platforms/wasm.py @@ -5,36 +5,36 @@ from kiwixbuild._global import get_target_step class WasmPlatformInfo(PlatformInfo): - name = 'wasm' + name = "wasm" static = True - build = 'wasm' - arch_full = 'wasm64-emscripten' + build = "wasm" + arch_full = "wasm64-emscripten" libdir = "lib" - #arch_full = 'wasm64-linux' - toolchain_names = ['emsdk'] - compatible_hosts = ['fedora', 'debian'] + # arch_full = 'wasm64-linux' + toolchain_names = ["emsdk"] + compatible_hosts = ["fedora", "debian"] exe_wrapper_def = "" def get_cross_config(self): return { - 'binaries': self.binaries, - 'exe_wrapper_def': '', - 'root_path': self.root_path, - 'extra_libs': [], - 'extra_cflags': [], - 'host_machine': { - 'system': 'emscripten', - 'lsystem': 'emscripten', - 'cpu_family': 'wasm64', - 'cpu': 'wasm64', - 'endian': 'little', - 'abi': '' - } + "binaries": self.binaries, + "exe_wrapper_def": "", + "root_path": self.root_path, + "extra_libs": [], + "extra_cflags": [], + "host_machine": { + "system": "emscripten", + "lsystem": "emscripten", + "cpu_family": "wasm64", + "cpu": "wasm64", + "endian": "little", + "abi": "", + }, } @property def wasm_sdk(self): - return get_target_step('emsdk', self.name) + return get_target_step("emsdk", self.name) @property def install_path(self): @@ -46,22 +46,25 @@ class WasmPlatformInfo(PlatformInfo): @property def binaries(self): - binaries = (('CC', 'emcc'), - ('CXX', 'em++'), - ('AR', 'emar'), - ('STRIP', 'emstrip'), - ('WINDRES', 'windres'), - ('RANLIB', 'emranlib'), - ('LD', 'wasm-ld')) - binaries = {k:pj(self.install_path, 'upstream', 'emscripten', v) - for k,v in binaries} - binaries['PKGCONFIG'] = 'pkg-config' + binaries = ( + ("CC", "emcc"), + ("CXX", "em++"), + ("AR", "emar"), + ("STRIP", "emstrip"), + ("WINDRES", "windres"), + ("RANLIB", "emranlib"), + ("LD", "wasm-ld"), + ) + binaries = { + k: pj(self.install_path, "upstream", "emscripten", v) for k, v in binaries + } + binaries["PKGCONFIG"] = "pkg-config" return binaries @property def configure_options(self): - #return "" - return [f'--host={self.arch_full}'] + # return "" + return [f"--host={self.arch_full}"] @property def configure_wrapper(self): @@ -72,24 +75,34 @@ class WasmPlatformInfo(PlatformInfo): return "emmake" def get_bin_dir(self): - return [pj(self.install_path, 'bin')] + return [pj(self.install_path, "bin")] def get_env(self): env = super().get_env() - env['PATH'] = ':'.join([ - env['PATH'], - self.install_path, - pj(self.install_path, 'upstream', 'emscripten'), - pj(self.install_path, 'node', '14.18.2_64bit', 'bin') - ]) - env['EMSDK'] = self.install_path - env['EMSDK_NODE'] = pj(self.install_path, 'node', '14.18.2_64bit', 'bin', 'node') + env["PATH"] = ":".join( + [ + env["PATH"], + self.install_path, + pj(self.install_path, "upstream", "emscripten"), + pj(self.install_path, "node", "14.18.2_64bit", "bin"), + ] + ) + env["EMSDK"] = self.install_path + env["EMSDK_NODE"] = pj( + self.install_path, "node", "14.18.2_64bit", "bin", "node" + ) return env def set_comp_flags(self, env): super().set_comp_flags(env) - env['CFLAGS'] = " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS'] - env['CXXFLAGS'] = " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS'] + env["CFLAGS"] = ( + " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 " + + env["CFLAGS"] + ) + env["CXXFLAGS"] = ( + " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 " + + env["CXXFLAGS"] + ) def set_compiler(self, env): for k, v in self.binaries.items(): @@ -97,5 +110,5 @@ class WasmPlatformInfo(PlatformInfo): def finalize_setup(self): super().finalize_setup() - self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') - self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') + self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt") + self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt") diff --git a/kiwixbuild/platforms/win32.py b/kiwixbuild/platforms/win32.py index c5612d9..4e41ddd 100644 --- a/kiwixbuild/platforms/win32.py +++ b/kiwixbuild/platforms/win32.py @@ -6,57 +6,66 @@ from kiwixbuild._global import neutralEnv class Win32PlatformInfo(PlatformInfo): - build = 'win32' - compatible_hosts = ['fedora', 'debian'] - arch_full = 'i686-w64-mingw32' - extra_libs = ['-lwinmm', '-lshlwapi', '-lws2_32', '-lssp'] + build = "win32" + compatible_hosts = ["fedora", "debian"] + arch_full = "i686-w64-mingw32" + extra_libs = ["-lwinmm", "-lshlwapi", "-lws2_32", "-lssp"] def get_cross_config(self): return { - 'exe_wrapper_def': self.exe_wrapper_def, - 'binaries': self.binaries, - 'root_path': self.root_path, - 'extra_libs': self.extra_libs, - 'extra_cflags': ['-DWIN32', *('-I{}'.format(include_dir) for include_dir in self.get_include_dirs())], - 'host_machine': { - 'system': 'Windows', - 'lsystem': 'windows', - 'cpu_family': 'x86', - 'cpu': 'i686', - 'endian': 'little', - 'abi': '' - } + "exe_wrapper_def": self.exe_wrapper_def, + "binaries": self.binaries, + "root_path": self.root_path, + "extra_libs": self.extra_libs, + "extra_cflags": [ + "-DWIN32", + *( + "-I{}".format(include_dir) + for include_dir in self.get_include_dirs() + ), + ], + "host_machine": { + "system": "Windows", + "lsystem": "windows", + "cpu_family": "x86", + "cpu": "i686", + "endian": "little", + "abi": "", + }, } def finalize_setup(self): super().finalize_setup() - self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') - self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') + self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt") + self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt") @property def root_path(self): root_paths = { - 'fedora': '/usr/i686-w64-mingw32/sys-root/mingw', - 'debian': '/usr/i686-w64-mingw32' + "fedora": "/usr/i686-w64-mingw32/sys-root/mingw", + "debian": "/usr/i686-w64-mingw32", } - return root_paths[neutralEnv('distname')] + return root_paths[neutralEnv("distname")] @property def binaries(self): - return {k:which('{}-{}'.format(self.arch_full, v)) - for k, v in (('CC', 'gcc'), - ('CXX', 'g++'), - ('AR', 'ar'), - ('STRIP', 'strip'), - ('WINDRES', 'windres'), - ('RANLIB', 'ranlib'), - ('PKGCONFIG', 'pkg-config')) - } + return { + k: which("{}-{}".format(self.arch_full, v)) + for k, v in ( + ("CC", "gcc"), + ("CXX", "g++"), + ("AR", "ar"), + ("STRIP", "strip"), + ("WINDRES", "windres"), + ("RANLIB", "ranlib"), + ("PKGCONFIG", "pkg-config"), + ) + } @property def exe_wrapper_def(self): try: - which('wine') + which("wine") except subprocess.CalledProcessError: return "" else: @@ -64,25 +73,27 @@ class Win32PlatformInfo(PlatformInfo): @property def configure_options(self): - yield f'--host={self.arch_full}' + yield f"--host={self.arch_full}" def set_compiler(self, env): for k, v in self.binaries.items(): env[k] = v def get_bin_dir(self): - return [pj(self.root_path, 'bin')] + return [pj(self.root_path, "bin")] def get_env(self): env = super().get_env() - env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') - env['LIBS'] = " ".join(self.extra_libs) + " " +env['LIBS'] + env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig") + env["LIBS"] = " ".join(self.extra_libs) + " " + env["LIBS"] return env + class Win32Dyn(Win32PlatformInfo): - name = 'win32_dyn' + name = "win32_dyn" static = False + class Win32Static(Win32PlatformInfo): - name = 'win32_static' + name = "win32_static" static = True diff --git a/kiwixbuild/platforms/win64.py b/kiwixbuild/platforms/win64.py index f7c17c0..62d9d35 100644 --- a/kiwixbuild/platforms/win64.py +++ b/kiwixbuild/platforms/win64.py @@ -6,57 +6,70 @@ from kiwixbuild._global import neutralEnv class Win64PlatformInfo(PlatformInfo): - extra_libs = ['-lmingw32', '-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4', '-lmsvcr100', '-liphlpapi', '-lshell32', '-lkernel32'] - build = 'win64' - compatible_hosts = ['fedora', 'debian'] - arch_full = 'x86_64-w64-mingw32' + extra_libs = [ + "-lmingw32", + "-lwinmm", + "-lws2_32", + "-lshlwapi", + "-lrpcrt4", + "-lmsvcr100", + "-liphlpapi", + "-lshell32", + "-lkernel32", + ] + build = "win64" + compatible_hosts = ["fedora", "debian"] + arch_full = "x86_64-w64-mingw32" def get_cross_config(self): return { - 'exe_wrapper_def': self.exe_wrapper_def, - 'binaries': self.binaries, - 'root_path': self.root_path, - 'extra_libs': self.extra_libs, - 'extra_cflags': ['-DWIN32'], - 'host_machine': { - 'system': 'Windows', - 'lsystem': 'windows', - 'cpu_family': 'x86_64', - 'cpu': 'x86_64', - 'endian': 'little', - 'abi': '' - } + "exe_wrapper_def": self.exe_wrapper_def, + "binaries": self.binaries, + "root_path": self.root_path, + "extra_libs": self.extra_libs, + "extra_cflags": ["-DWIN32"], + "host_machine": { + "system": "Windows", + "lsystem": "windows", + "cpu_family": "x86_64", + "cpu": "x86_64", + "endian": "little", + "abi": "", + }, } def finalize_setup(self): super().finalize_setup() - self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') - self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') + self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt") + self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt") @property def root_path(self): root_paths = { - 'fedora': '/usr/x86_64-w64-mingw32/sys-root/mingw', - 'debian': '/usr/x86_64-w64-mingw32' + "fedora": "/usr/x86_64-w64-mingw32/sys-root/mingw", + "debian": "/usr/x86_64-w64-mingw32", } - return root_paths[neutralEnv('distname')] + return root_paths[neutralEnv("distname")] @property def binaries(self): - return {k:which('{}-{}'.format(self.arch_full, v)) - for k, v in (('CC', 'gcc'), - ('CXX', 'g++'), - ('AR', 'ar'), - ('STRIP', 'strip'), - ('WINDRES', 'windres'), - ('RANLIB', 'ranlib'), - ('PKGCONFIG', 'pkg-config')) - } + return { + k: which("{}-{}".format(self.arch_full, v)) + for k, v in ( + ("CC", "gcc"), + ("CXX", "g++"), + ("AR", "ar"), + ("STRIP", "strip"), + ("WINDRES", "windres"), + ("RANLIB", "ranlib"), + ("PKGCONFIG", "pkg-config"), + ) + } @property def exe_wrapper_def(self): try: - which('wine') + which("wine") except subprocess.CalledProcessError: return "" else: @@ -64,25 +77,27 @@ class Win64PlatformInfo(PlatformInfo): @property def configure_options(self): - return [f'--host={self.arch_full}'] + return [f"--host={self.arch_full}"] def set_compiler(self, env): for k, v in self.binaries.items(): env[k] = v def get_bin_dir(self): - return [pj(self.root_path, 'bin')] + return [pj(self.root_path, "bin")] def get_env(self): env = super().get_env() - env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') - env['LIBS'] = " ".join(self.extra_libs) + " " +env['LIBS'] + env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig") + env["LIBS"] = " ".join(self.extra_libs) + " " + env["LIBS"] return env + class Win64Dyn(Win64PlatformInfo): - name = 'win64_dyn' + name = "win64_dyn" static = False + class Win64Static(Win64PlatformInfo): - name = 'win64_static' + name = "win64_static" static = True diff --git a/kiwixbuild/utils.py b/kiwixbuild/utils.py index bdf8550..1cdc990 100644 --- a/kiwixbuild/utils.py +++ b/kiwixbuild/utils.py @@ -17,16 +17,16 @@ pj = os.path.join COLORS = { - 'OK': '\033[92m', - 'WARNING': '\033[93m', - 'NEEDED': '\033[93m', - 'SKIP': '\033[34m', - 'ERROR': '\033[91m', - '': '\033[0m', + "OK": "\033[92m", + "WARNING": "\033[93m", + "NEEDED": "\033[93m", + "SKIP": "\033[34m", + "ERROR": "\033[91m", + "": "\033[0m", } -REMOTE_PREFIX = 'http://mirror.download.kiwix.org/dev/kiwix-build/' +REMOTE_PREFIX = "http://mirror.download.kiwix.org/dev/kiwix-build/" def which(name): @@ -34,16 +34,19 @@ def which(name): output = subprocess.check_output(command, shell=True) return output[:-1].decode() + def xrun_find(name): command = "xcrun -find {}".format(name) output = subprocess.check_output(command, shell=True) return output[:-1].decode() -regex_space = re.compile(r'((?= (3, 4, 3) else {} + extra_args = {"context": context} if sys.version_info >= (3, 4, 3) else {} progress_chars = "/-\|" try: - with urllib.request.urlopen(what.url, **extra_args) as resource, open(file_path, 'wb') as file: - tsize = resource.info().get('Content-Length', None) + with urllib.request.urlopen(what.url, **extra_args) as resource, open( + file_path, "wb" + ) as file: + tsize = resource.info().get("Content-Length", None) if tsize is not None: tsize = int(tsize) current = 0 @@ -147,17 +152,17 @@ def download_remote(what, where): break if tsize: current += batch_size - print_progress("{:.2%}".format(current/tsize)) + print_progress("{:.2%}".format(current / tsize)) else: - print_progress(progress_chars[current]) - current = (current+1)%4 + print_progress(progress_chars[current]) + current = (current + 1) % 4 file.write(batch) except urllib.error.URLError as e: print("Cannot download url {}:\n{}".format(what.url, e.reason)) raise StopBuild() if not what.sha256: - print('Sha256 for {} not set, do no verify download'.format(what.name)) + print("Sha256 for {} not set, do no verify download".format(what.name)) elif what.sha256 != get_sha256(file_path): os.remove(file_path) raise StopBuild("Sha 256 doesn't correspond") @@ -187,7 +192,7 @@ class StopBuild(BaseCommandResult): pass -class Remotefile(namedtuple('Remotefile', ('name', 'sha256', 'url'))): +class Remotefile(namedtuple("Remotefile", ("name", "sha256", "url"))): def __new__(cls, name, sha256, url=None): if url is None: url = REMOTE_PREFIX + name @@ -217,28 +222,28 @@ class Context: def _finalise(self): if self.autoskip_file is not None: os.makedirs(os.path.dirname(self.autoskip_file), exist_ok=True) - with open(self.autoskip_file, 'w'): + with open(self.autoskip_file, "w"): pass def extract_archive(archive_path, dest_dir, topdir=None, name=None): - is_zip_archive = archive_path.endswith('.zip') + is_zip_archive = archive_path.endswith(".zip") archive = None try: if is_zip_archive: archive = zipfile.ZipFile(archive_path) members = archive.infolist() - getname = lambda info : info.filename - isdir = lambda info: info.filename.endswith('/') + getname = lambda info: info.filename + isdir = lambda info: info.filename.endswith("/") else: archive = tarfile.open(archive_path) members = archive.getmembers() - getname = lambda info : getattr(info, 'name') + getname = lambda info: getattr(info, "name") isdir = lambda info: info.isdir() if not topdir: for d in (m for m in members if isdir(m)): _name = getname(d) - if _name.endswith('/'): + if _name.endswith("/"): _name = _name[:-1] if not os.path.dirname(_name): if topdir: @@ -249,9 +254,13 @@ def extract_archive(archive_path, dest_dir, topdir=None, name=None): break topdir = _name if topdir: - members_to_extract = [m for m in members if getname(m).startswith(topdir+'/')] + members_to_extract = [ + m for m in members if getname(m).startswith(topdir + "/") + ] os.makedirs(dest_dir, exist_ok=True) - with tempfile.TemporaryDirectory(prefix=os.path.basename(archive_path), dir=dest_dir) as tmpdir: + with tempfile.TemporaryDirectory( + prefix=os.path.basename(archive_path), dir=dest_dir + ) as tmpdir: if is_zip_archive: _members_to_extract = [getname(m) for m in members_to_extract] else: @@ -281,8 +290,8 @@ def run_command(command, cwd, context, *, env=None, input=None): env = DefaultEnv() log = None try: - if not option('verbose'): - log = open(context.log_file, 'w') + if not option("verbose"): + log = open(context.log_file, "w") print("run command '{}'".format(command), file=log) print("current directory is '{}'".format(cwd), file=log) print("env is :", file=log) @@ -293,8 +302,15 @@ def run_command(command, cwd, context, *, env=None, input=None): log.flush() kwargs = dict() if input: - kwargs['stdin'] = subprocess.PIPE - process = subprocess.Popen(command, cwd=cwd, env=env, stdout=log or sys.stdout, stderr=subprocess.STDOUT, **kwargs) + kwargs["stdin"] = subprocess.PIPE + process = subprocess.Popen( + command, + cwd=cwd, + env=env, + stdout=log or sys.stdout, + stderr=subprocess.STDOUT, + **kwargs + ) if input: input = input.encode() while True: @@ -308,7 +324,7 @@ def run_command(command, cwd, context, *, env=None, input=None): # `communicate` timeout (and we must set `input` to None # to not communicate again). input = None - print('.', end='', flush=True) + print(".", end="", flush=True) else: break if process.returncode: diff --git a/kiwixbuild/versions.py b/kiwixbuild/versions.py index c6c6aaa..588e828 100644 --- a/kiwixbuild/versions.py +++ b/kiwixbuild/versions.py @@ -1,12 +1,11 @@ - # This file reference all the versions of the depedencies we use in kiwix-build. main_project_versions = { - 'libzim': '9.1.0', - 'libkiwix': '13.0.0', - 'kiwix-tools': '3.6.0', - 'zim-tools': '3.4.0', - 'kiwix-desktop': '2.3.1' # Also change KIWIX_DESKTOP_VERSION and KIWIX_DESKTOP_RELEASE in appveyor.yml + "libzim": "9.1.0", + "libkiwix": "13.0.0", + "kiwix-tools": "3.6.0", + "zim-tools": "3.4.0", + "kiwix-desktop": "2.3.1", # Also change KIWIX_DESKTOP_VERSION and KIWIX_DESKTOP_RELEASE in appveyor.yml } # This dictionnary specify what we need to build at each release process. @@ -30,37 +29,37 @@ main_project_versions = { # - set KIWIX_DESKTOP_RELEASE to 0 release_versions = { - 'libzim': None, # Depends of base deps (was 0) - 'libkiwix': None, # Depends of libzim (was 1) - 'kiwix-tools': None, # Depends of libkiwix and libzim (was 0) - 'zim-tools': 0, # Depends of libzim (was None) - 'kiwix-desktop': None # Depends of libkiwix and libzim (was 4) + "libzim": None, # Depends of base deps (was 0) + "libkiwix": None, # Depends of libzim (was 1) + "kiwix-tools": None, # Depends of libkiwix and libzim (was 0) + "zim-tools": 0, # Depends of libzim (was None) + "kiwix-desktop": None, # Depends of libkiwix and libzim (was 4) } # This is the "version" of the whole base_deps_versions dict. # Change this when you change base_deps_versions. -base_deps_meta_version = '96' +base_deps_meta_version = "96" base_deps_versions = { - 'zlib' : '1.2.12', - 'lzma' : '5.2.6', - 'zstd' : '1.5.2', - 'docoptcpp' : '0.6.2', - 'uuid' : '1.43.4', - 'xapian-core' : '1.4.23', - 'mustache' : '4.1', - 'pugixml' : '1.2', - 'libmicrohttpd' : '0.9.76', - 'gumbo' : '0.10.1', - 'icu4c' : '73.2', - 'libaria2' : '1.37.0', - 'libmagic' : '5.44', - 'android-ndk' : 'r21e', - 'qt' : '5.10.1', - 'qtwebengine' : '5.10.1', - 'org.kde' : '5.15-21.08', - 'io.qt.qtwebengine' : '5.15-21.08', - 'zim-testing-suite': '0.3', - 'emsdk': '3.1.41', + "zlib": "1.2.12", + "lzma": "5.2.6", + "zstd": "1.5.2", + "docoptcpp": "0.6.2", + "uuid": "1.43.4", + "xapian-core": "1.4.23", + "mustache": "4.1", + "pugixml": "1.2", + "libmicrohttpd": "0.9.76", + "gumbo": "0.10.1", + "icu4c": "73.2", + "libaria2": "1.37.0", + "libmagic": "5.44", + "android-ndk": "r21e", + "qt": "5.10.1", + "qtwebengine": "5.10.1", + "org.kde": "5.15-21.08", + "io.qt.qtwebengine": "5.15-21.08", + "zim-testing-suite": "0.3", + "emsdk": "3.1.41", } From 20ca39145dcb90fe7585228311abc0ce9a1eef82 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 5 Feb 2024 11:42:04 +0100 Subject: [PATCH 2/6] Remove `backend` (internal) option. This is not used and it is a good things as it is broken.. --- kiwixbuild/_global.py | 20 -------------------- kiwixbuild/builder.py | 1 - kiwixbuild/flatpak_builder.py | 1 - 3 files changed, 22 deletions(-) diff --git a/kiwixbuild/_global.py b/kiwixbuild/_global.py index 9e37bd1..971555c 100644 --- a/kiwixbuild/_global.py +++ b/kiwixbuild/_global.py @@ -38,23 +38,3 @@ def get_target_step(key, default_context=None): def target_steps(): return _target_steps - - -def backend(): - global _backend - if _backend is not None: - return _backend - - _platform = platform.system() - if _platform == "Windows": - print( - "ERROR: kiwix-build is not intented to run on Windows platform.\n" - "There is no backend for Windows, so we can't launch any commands." - ) - sys.exit(0) - if _platform == "Linux": - _platform, _, _ = platform.linux_distribution() - _platform = _platform.lower() - _backend = backends.Linux() - - return _backend diff --git a/kiwixbuild/builder.py b/kiwixbuild/builder.py index a59461d..7909cd1 100644 --- a/kiwixbuild/builder.py +++ b/kiwixbuild/builder.py @@ -12,7 +12,6 @@ from ._global import ( add_target_step, get_target_step, target_steps, - backend, ) from . import _global diff --git a/kiwixbuild/flatpak_builder.py b/kiwixbuild/flatpak_builder.py index 6cc6de7..cb07f62 100644 --- a/kiwixbuild/flatpak_builder.py +++ b/kiwixbuild/flatpak_builder.py @@ -13,7 +13,6 @@ from ._global import ( add_target_step, get_target_step, target_steps, - backend, ) from . import _global from .dependencies.base import ( From 6dfb0da943f8df57afd4bb368f26fa5f6249d93d Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 5 Feb 2024 17:46:35 +0100 Subject: [PATCH 3/6] =?UTF-8?q?Cleanup=20README=C2=A0about=20libkiwix-app.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 4381815..6675fa4 100644 --- a/README.md +++ b/README.md @@ -108,12 +108,6 @@ Android `kiwix-android` (https://github.com/kiwix/kiwix-android) depends of the `libkiwix` project. -It uses a special `.aar` file that represent (and embed) the libkiwix for -all supported android arch. This is a kind of fat archive we have for MacOs. - -The `.aar` file is build using the `libkiwix-app` project. -`libkiwix-app` itself is architecture independent (it is just a packaging of -other archives) but it use `libkiwix` who is architecture dependent. When building `libkiwix`, you should directly use the target-platform `android_`: @@ -124,17 +118,14 @@ kiwix-build libkiwix --target-platform android_arm But, `libkiwix-app` is mainly multi arch. To compile `libkiwix-app`, you must use the `android` platform: ```bash -kiwix-build --target-platform android libkiwix-app -kiwix-build libkiwix-app # because `android` platform is the default for `libkiwix-app` +kiwix-build --target-platform android libkiwix ``` By default, when using platform `android`, `libkiwix` will be build for all architectures. This can be changed by using the option `--android-arch`: ```bash -kiwix-build libkiwix-app # aar with all architectures -kiwix-build libkiwix-app --android-arch arm # aar with arm architecture -kiwix-build libkiwix-app --android-arch arm --android-arch arm64 # aan with arm and arm64 architectures -``` +kiwix-build libkiwix --android-arch arm # aar with arm architecture +kiwix-build libkiwix --android-arch arm --android-arch arm64 # aan with arm and arm64 architectures To build `kiwix-android` itself, you should see the documentation of `kiwix-android`. From c0ec9c44b8eef1dd0efd0b71dc34b739387e234d Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 5 Feb 2024 17:46:13 +0100 Subject: [PATCH 4/6] Rename option `--target-platform` to `--config` The target_platform option has always be wrongly named. This is not the platform we are targeted but how we compile. This was ok at beginning as specifying how we compile somehow define for what we compile but this is not a one to one mapping. --- README.md | 43 +++++---- kiwixbuild/__init__.py | 13 ++- kiwixbuild/buildenv.py | 24 ++--- kiwixbuild/builder.py | 71 +++++++------- kiwixbuild/{platforms => configs}/__init__.py | 0 kiwixbuild/{platforms => configs}/android.py | 16 ++-- kiwixbuild/{platforms => configs}/armhf.py | 12 +-- kiwixbuild/{platforms => configs}/base.py | 54 +++++------ kiwixbuild/{platforms => configs}/flatpak.py | 4 +- kiwixbuild/{platforms => configs}/i586.py | 8 +- kiwixbuild/{platforms => configs}/ios.py | 30 +++--- kiwixbuild/{platforms => configs}/musl.py | 20 ++-- kiwixbuild/{platforms => configs}/native.py | 12 +-- kiwixbuild/{platforms => configs}/neutral.py | 4 +- kiwixbuild/{platforms => configs}/wasm.py | 4 +- kiwixbuild/{platforms => configs}/win32.py | 8 +- kiwixbuild/{platforms => configs}/win64.py | 8 +- kiwixbuild/dependencies/all_dependencies.py | 15 ++- kiwixbuild/dependencies/apple_xcframework.py | 52 +++++------ kiwixbuild/dependencies/base.py | 20 ++-- kiwixbuild/dependencies/icu4c.py | 14 +-- kiwixbuild/dependencies/ios_fat_lib.py | 6 +- kiwixbuild/dependencies/kiwix_desktop.py | 4 +- kiwixbuild/dependencies/kiwix_tools.py | 2 +- kiwixbuild/dependencies/libkiwix.py | 14 +-- kiwixbuild/dependencies/libmagic.py | 8 +- kiwixbuild/dependencies/libzim.py | 20 ++-- kiwixbuild/dependencies/qt.py | 4 +- kiwixbuild/dependencies/tc_android_ndk.py | 10 +- kiwixbuild/dependencies/xapian.py | 4 +- kiwixbuild/dependencies/zim_tools.py | 8 +- kiwixbuild/dependencies/zlib.py | 10 +- kiwixbuild/flatpak_builder.py | 92 +++++++++---------- 33 files changed, 305 insertions(+), 309 deletions(-) rename kiwixbuild/{platforms => configs}/__init__.py (100%) rename kiwixbuild/{platforms => configs}/android.py (93%) rename kiwixbuild/{platforms => configs}/armhf.py (96%) rename kiwixbuild/{platforms => configs}/base.py (76%) rename kiwixbuild/{platforms => configs}/flatpak.py (84%) rename kiwixbuild/{platforms => configs}/i586.py (93%) rename kiwixbuild/{platforms => configs}/ios.py (89%) rename kiwixbuild/{platforms => configs}/musl.py (91%) rename kiwixbuild/{platforms => configs}/native.py (70%) rename kiwixbuild/{platforms => configs}/neutral.py (66%) rename kiwixbuild/{platforms => configs}/wasm.py (97%) rename kiwixbuild/{platforms => configs}/win32.py (94%) rename kiwixbuild/{platforms => configs}/win64.py (94%) diff --git a/README.md b/README.md index 6675fa4..1ba27ed 100644 --- a/README.md +++ b/README.md @@ -78,12 +78,12 @@ invalid choice: 'not-existing-target' (choose from 'alldependencies', 'android-n ... ``` -#### Target platform +#### Config -If no target platform is specified, the default will be `native_dyn`. +If no config is specified, the default will be `native_dyn`. -You can select another target platform using the option -`--target-platform`. For now, there is ten different supported +You can select another config using the option +`--config`. For now, there is ten different supported platforms: - native_dyn @@ -98,9 +98,12 @@ platforms: - android_x86_64 - flatpak -So, if you want to compile `kiwix-tools` for win32 using static linkage: +All `native_*` config means using the native compiler without any cross-compilation option. +Other may simply use cross-compilation or may download a specific toolchain to use. + +If you want to compile `kiwix-tools` for win32 using static linkage: ```bash -kiwix-build --target-platform win32_dyn +kiwix-build --config win32_dyn ``` Android @@ -112,20 +115,20 @@ the `libkiwix` project. When building `libkiwix`, you should directly use the target-platform `android_`: ```bash -kiwix-build libkiwix --target-platform android_arm +kiwix-build libkiwix --config android_arm ``` -But, `libkiwix-app` is mainly multi arch. -To compile `libkiwix-app`, you must use the `android` platform: +You may directly use the special config `android` which will build different android architectures ```bash -kiwix-build --target-platform android libkiwix +kiwix-build --config android libkiwix ``` -By default, when using platform `android`, `libkiwix` will be build for -all architectures. This can be changed by using the option `--android-arch`: +By default, it will build for all android architecture, +you can limit this with option `--android-arch`: ```bash -kiwix-build libkiwix --android-arch arm # aar with arm architecture -kiwix-build libkiwix --android-arch arm --android-arch arm64 # aan with arm and arm64 architectures +kiwix-build libkiwix --config android --android-arch arm # aar with arm architecture +kiwix-build libkiwix --config android --android-arch arm --android-arch arm64 # aan with arm and arm64 architectures +``` To build `kiwix-android` itself, you should see the documentation of `kiwix-android`. @@ -139,13 +142,13 @@ To do so, you should directly use the target-platfrom `ios_multi`. As for `android`, `kiwix-build` will build the library several times (once for each platform) and then create the fat library. ```bash -kiwix-build --target-platform iOS_multi libkiwix +kiwix-build --config iOS_multi libkiwix ``` You can specify the supported architectures with the option `--ios-arch`: ```bash -kiwix-build --target-platform iOS_multi libkiwix # all architetures -kiwix-build --target-platform iOS_multi --ios-arch arm --ios-arch arm64 # arm and arm64 arch only +kiwix-build --config iOS_multi libkiwix # all architetures +kiwix-build --config iOS_multi --ios-arch arm --ios-arch arm64 # arm and arm64 arch only ``` Outputs @@ -154,9 +157,9 @@ Outputs Kiwix-build.py will create several directories: - `ARCHIVES`: All the downloaded archives go there. - `SOURCES`: All the sources (extracted from archives and patched) go there. -- `BUILD_`: All the build files go there. -- `BUILD_/INSTALL`: The installed files go there. -- `BUILD_/LOGS`: The logs files of the build. +- `BUILD_`: All the build files go there. +- `BUILD_/INSTALL`: The installed files go there. +- `BUILD_/LOGS`: The logs files of the build. If you want to install all those directories elsewhere, you can pass the `--working-dir` option to `kiwix-build`: diff --git a/kiwixbuild/__init__.py b/kiwixbuild/__init__.py index 6bb3e5a..33f2926 100644 --- a/kiwixbuild/__init__.py +++ b/kiwixbuild/__init__.py @@ -4,7 +4,7 @@ import os, sys import argparse from .dependencies import Dependency -from .platforms import PlatformInfo +from .configs import ConfigInfo from .builder import Builder from .flatpak_builder import FlatpakBuilder from . import _global @@ -37,7 +37,9 @@ def parse_args(): ), ) parser.add_argument("--libprefix", default=None) - parser.add_argument("--target-platform", choices=PlatformInfo.all_platforms) + parser.add_argument( + "--config", choices=ConfigInfo.all_configs, default="native_dyn" + ) parser.add_argument( "--verbose", "-v", @@ -130,9 +132,6 @@ def parse_args(): if not options.ios_arch: options.ios_arch = ["arm64", "x86_64"] - if not options.target_platform: - options.target_platform = "native_dyn" - return options @@ -140,9 +139,9 @@ def main(): options = parse_args() options.working_dir = os.path.abspath(options.working_dir) _global.set_options(options) - neutralEnv = buildenv.PlatformNeutralEnv() + neutralEnv = buildenv.NeutralEnv() _global.set_neutralEnv(neutralEnv) - if options.target_platform == "flatpak": + if options.config == "flatpak": builder = FlatpakBuilder() else: builder = Builder() diff --git a/kiwixbuild/buildenv.py b/kiwixbuild/buildenv.py index 27c8979..37fe7fc 100644 --- a/kiwixbuild/buildenv.py +++ b/kiwixbuild/buildenv.py @@ -7,7 +7,7 @@ from .utils import pj, download_remote, escape_path from ._global import neutralEnv, option -class PlatformNeutralEnv: +class NeutralEnv: def __init__(self): self.working_dir = option("working_dir") self.source_dir = pj(self.working_dir, "SOURCE") @@ -75,9 +75,9 @@ class PlatformNeutralEnv: class BuildEnv: - def __init__(self, platformInfo): - build_dir = "BUILD_{}".format(platformInfo.name) - self.platformInfo = platformInfo + def __init__(self, configInfo): + build_dir = "BUILD_{}".format(configInfo.name) + self.configInfo = configInfo self.base_build_dir = pj(option("working_dir"), option("build_dir")) self.build_dir = pj(self.base_build_dir, build_dir) self.install_dir = pj(self.build_dir, "INSTALL") @@ -102,8 +102,8 @@ class BuildEnv: return os.path.isfile("/etc/debian_version") def _detect_libdir(self): - if self.platformInfo.libdir is not None: - return self.platformInfo.libdir + if self.configInfo.libdir is not None: + return self.configInfo.libdir if self._is_debianlike(): try: pc = subprocess.Popen( @@ -122,7 +122,7 @@ class BuildEnv: return "lib" def get_env(self, *, cross_comp_flags, cross_compilers, cross_path): - env = self.platformInfo.get_env() + env = self.configInfo.get_env() pkgconfig_path = pj(self.install_dir, self.libprefix, "pkgconfig") env["PKG_CONFIG_PATH"] = ":".join([env["PKG_CONFIG_PATH"], pkgconfig_path]) @@ -158,23 +158,23 @@ class BuildEnv: ) if cross_comp_flags: - self.platformInfo.set_comp_flags(env) + self.configInfo.set_comp_flags(env) if cross_compilers: - self.platformInfo.set_compiler(env) + self.configInfo.set_compiler(env) if cross_path: - env["PATH"] = ":".join(self.platformInfo.get_bin_dir() + [env["PATH"]]) + env["PATH"] = ":".join(self.configInfo.get_bin_dir() + [env["PATH"]]) return env @property def configure_wrapper(self): try: - yield self.platformInfo.configure_wrapper + yield self.configInfo.configure_wrapper except AttributeError: pass @property def make_wrapper(self): try: - yield self.platformInfo.make_wrapper + yield self.configInfo.make_wrapper except AttributeError: pass diff --git a/kiwixbuild/builder.py b/kiwixbuild/builder.py index 7909cd1..587a3a7 100644 --- a/kiwixbuild/builder.py +++ b/kiwixbuild/builder.py @@ -2,7 +2,7 @@ import sys from collections import OrderedDict from .buildenv import * -from .platforms import PlatformInfo +from .configs import ConfigInfo from .utils import remove_duplicates, StopBuild, colorize from .dependencies import Dependency from .packages import PACKAGE_NAME_MAPPERS @@ -19,19 +19,18 @@ from . import _global class Builder: def __init__(self): self._targets = {} - PlatformInfo.get_platform("neutral", self._targets) + ConfigInfo.get_config("neutral", self._targets) - target_platform = option("target_platform") - platform = PlatformInfo.get_platform(target_platform, self._targets) - if neutralEnv("distname") not in platform.compatible_hosts: + config_name = option("config") + config = ConfigInfo.get_config(config_name, self._targets) + if neutralEnv("distname") not in config.compatible_hosts: print( ( - colorize("ERROR") - + ": The target platform {} cannot be build on host {}.\n" - "Select another target platform or change your host system." - ).format(platform.name, neutralEnv("distname")) + colorize("ERROR") + ": The config {} cannot be build on host {}.\n" + "Select another config or change your host system." + ).format(config.name, neutralEnv("distname")) ) - self.targetDefs = platform.add_targets(option("target"), self._targets) + self.targetDefs = config.add_targets(option("target"), self._targets) def finalize_target_steps(self): steps = [] @@ -40,7 +39,7 @@ class Builder: steps = list(remove_duplicates(steps)) if option("build_nodeps"): - # add all platform steps + # add all config steps for dep in steps: stepClass = Dependency.all_deps[dep[1]] if stepClass.dont_skip: @@ -57,18 +56,18 @@ class Builder: self.instanciate_steps() def order_steps(self, targetDef): - for pltName in PlatformInfo.all_running_platforms: - plt = PlatformInfo.all_platforms[pltName] - for tlcName in plt.toolchain_names: + for cfgName in ConfigInfo.all_running_configs: + cfg = ConfigInfo.all_configs[cfgName] + for tlcName in cfg.toolchain_names: tlc = Dependency.all_deps[tlcName] yield ("source", tlcName) - yield ("neutral" if tlc.neutral else pltName, tlcName) + yield ("neutral" if tlc.neutral else cfgName, tlcName) _targets = dict(self._targets) yield from self.order_dependencies(targetDef, _targets) def order_dependencies(self, targetDef, targets): - targetPlatformName, targetName = targetDef - if targetPlatformName == "source": + targetConfigName, targetName = targetDef + if targetConfigName == "source": # Do not try to order sources, they will be added as dep by the # build step two lines later. return @@ -77,24 +76,24 @@ class Builder: except KeyError: return - targetPlatform = PlatformInfo.get_platform(targetPlatformName) - for dep in target.get_dependencies(targetPlatform, True): - depPlatform, depName = targetPlatform.get_fully_qualified_dep(dep) - if (depPlatform, depName) in targets: - yield from self.order_dependencies((depPlatform, depName), targets) + targetConfig = ConfigInfo.get_config(targetConfigName) + for dep in target.get_dependencies(targetConfig, True): + depConfig, depName = targetConfig.get_fully_qualified_dep(dep) + if (depConfig, depName) in targets: + yield from self.order_dependencies((depConfig, depName), targets) yield ("source", targetName) yield targetDef def instanciate_steps(self): for stepDef in list(target_steps()): - stepPlatform, stepName = stepDef + stepConfig, stepName = stepDef stepClass = Dependency.all_deps[stepName] - if stepPlatform == "source": + if stepConfig == "source": source = get_target_step(stepDef)(stepClass) add_target_step(stepDef, source) else: source = get_target_step(stepName, "source") - env = PlatformInfo.get_platform(stepPlatform).buildEnv + env = ConfigInfo.get_config(stepConfig).buildEnv builder = get_target_step(stepDef)(stepClass, source, env) add_target_step(stepDef, builder) @@ -125,18 +124,18 @@ class Builder: def _get_packages(self): packages_list = [] - for platform in PlatformInfo.all_running_platforms.values(): - mapper_name = "{host}_{target}".format( - host=neutralEnv("distname"), target=platform + for config in ConfigInfo.all_running_configs.values(): + mapper_name = "{host}_{config}".format( + host=neutralEnv("distname"), config=config ) package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {}) packages_list += package_name_mapper.get("COMMON", []) to_drop = [] for builderDef in self._targets: - platformName, builderName = builderDef - mapper_name = "{host}_{target}".format( - host=neutralEnv("distname"), target=platformName + configName, builderName = builderDef + mapper_name = "{host}_{config}".format( + host=neutralEnv("distname"), config=configName ) package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {}) packages = package_name_mapper.get(builderName) @@ -195,9 +194,9 @@ class Builder: else: self.install_packages() self.finalize_target_steps() - print("[SETUP PLATFORMS]") - for platform in PlatformInfo.all_running_platforms.values(): - platform.finalize_setup() + print("[SETUP TOOLCHAINS]") + for config in ConfigInfo.all_running_configs.values(): + config.finalize_setup() print("[PREPARE]") self.prepare_sources() print("[BUILD]") @@ -205,8 +204,8 @@ class Builder: # No error, clean intermediate file at end of build if needed. print("[CLEAN]") if option("clean_at_end"): - for platform in PlatformInfo.all_running_platforms.values(): - platform.clean_intermediate_directories() + for config in ConfigInfo.all_running_configs.values(): + config.clean_intermediate_directories() else: print(colorize("SKIP")) except StopBuild as e: diff --git a/kiwixbuild/platforms/__init__.py b/kiwixbuild/configs/__init__.py similarity index 100% rename from kiwixbuild/platforms/__init__.py rename to kiwixbuild/configs/__init__.py diff --git a/kiwixbuild/platforms/android.py b/kiwixbuild/configs/android.py similarity index 93% rename from kiwixbuild/platforms/android.py rename to kiwixbuild/configs/android.py index 679a6d7..51d6d91 100644 --- a/kiwixbuild/platforms/android.py +++ b/kiwixbuild/configs/android.py @@ -1,9 +1,9 @@ -from .base import PlatformInfo, MetaPlatformInfo +from .base import ConfigInfo, MetaConfigInfo from kiwixbuild.utils import pj from kiwixbuild._global import get_target_step, option -class AndroidPlatformInfo(PlatformInfo): +class AndroidConfigInfo(ConfigInfo): build = "android" static = True toolchain_names = ["android-ndk"] @@ -114,7 +114,7 @@ class AndroidPlatformInfo(PlatformInfo): self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt") -class AndroidArm(AndroidPlatformInfo): +class AndroidArm(AndroidConfigInfo): name = "android_arm" arch = cpu = "arm" arch_full = "arm-linux-androideabi" @@ -122,7 +122,7 @@ class AndroidArm(AndroidPlatformInfo): march = "armv7-a" -class AndroidArm64(AndroidPlatformInfo): +class AndroidArm64(AndroidConfigInfo): name = "android_arm64" arch = "arm64" arch_full = "aarch64-linux-android" @@ -130,25 +130,25 @@ class AndroidArm64(AndroidPlatformInfo): abi = "arm64-v8a" -class AndroidX86(AndroidPlatformInfo): +class AndroidX86(AndroidConfigInfo): name = "android_x86" arch = abi = "x86" arch_full = "i686-linux-android" cpu = "i686" -class AndroidX8664(AndroidPlatformInfo): +class AndroidX8664(AndroidConfigInfo): name = "android_x86_64" arch = cpu = abi = "x86_64" arch_full = "x86_64-linux-android" -class Android(MetaPlatformInfo): +class Android(MetaConfigInfo): name = "android" compatible_hosts = ["fedora", "debian"] @property - def subPlatformNames(self): + def subConfigNames(self): return ["android_{}".format(arch) for arch in option("android_arch")] def add_targets(self, targetName, targets): diff --git a/kiwixbuild/platforms/armhf.py b/kiwixbuild/configs/armhf.py similarity index 96% rename from kiwixbuild/platforms/armhf.py rename to kiwixbuild/configs/armhf.py index 07cf001..de666a3 100644 --- a/kiwixbuild/platforms/armhf.py +++ b/kiwixbuild/configs/armhf.py @@ -1,11 +1,11 @@ -from .base import PlatformInfo, MixedMixin +from .base import ConfigInfo, MixedMixin from kiwixbuild.utils import pj from kiwixbuild._global import get_target_step -# Base platform -class ArmPlatformInfo(PlatformInfo): +# Base config for arm +class ArmConfigInfo(ConfigInfo): compatible_hosts = ["fedora", "debian"] def get_cross_config(self): @@ -113,7 +113,7 @@ class ArmPlatformInfo(PlatformInfo): self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt") -class Armv6(ArmPlatformInfo): +class Armv6(ArmConfigInfo): build = "armv6" arch_full = "armv6-rpi-linux-gnueabihf" toolchain_names = ["armv6"] @@ -136,7 +136,7 @@ class Armv6Mixed(MixedMixin("armv6_static"), Armv6): static = False -class Armv8(ArmPlatformInfo): +class Armv8(ArmConfigInfo): build = "armv8" arch_full = "armv8-rpi3-linux-gnueabihf" toolchain_names = ["armv8"] @@ -159,7 +159,7 @@ class Armv8Mixed(MixedMixin("armv8_static"), Armv8): static = False -class Aarch64(ArmPlatformInfo): +class Aarch64(ArmConfigInfo): build = "aarch64" arch_full = "aarch64-linux-gnu" toolchain_names = ["aarch64"] diff --git a/kiwixbuild/platforms/base.py b/kiwixbuild/configs/base.py similarity index 76% rename from kiwixbuild/platforms/base.py rename to kiwixbuild/configs/base.py index 1210380..43798f2 100644 --- a/kiwixbuild/platforms/base.py +++ b/kiwixbuild/configs/base.py @@ -10,35 +10,35 @@ _SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) TEMPLATES_DIR = pj(os.path.dirname(_SCRIPT_DIR), "templates") -class _MetaPlatform(type): +class _MetaConfig(type): def __new__(cls, name, bases, dct): _class = type.__new__(cls, name, bases, dct) - if name not in ("PlatformInfo", "MetaPlatformInfo") and "name" in dct: + if name not in ("ConfigInfo", "MetaConfigInfo") and "name" in dct: dep_name = dct["name"] - PlatformInfo.all_platforms[dep_name] = _class + ConfigInfo.all_configs[dep_name] = _class return _class -class PlatformInfo(metaclass=_MetaPlatform): - all_platforms = {} - all_running_platforms = {} +class ConfigInfo(metaclass=_MetaConfig): + all_configs = {} + all_running_configs = {} toolchain_names = [] configure_options = [] mixed = False libdir = None @classmethod - def get_platform(cls, name, targets=None): - if name not in cls.all_running_platforms: + def get_config(cls, name, targets=None): + if name not in cls.all_running_configs: if targets is None: print("Should not got there.") - print(cls.all_running_platforms) + print(cls.all_running_configs) raise KeyError(name) - cls.all_running_platforms[name] = cls.all_platforms[name](targets) - return cls.all_running_platforms[name] + cls.all_running_configs[name] = cls.all_configs[name](targets) + return cls.all_running_configs[name] def __init__(self, targets): - self.all_running_platforms[self.name] = self + self.all_running_configs[self.name] = self self.buildEnv = BuildEnv(self) self.setup_toolchains(targets) @@ -49,8 +49,8 @@ class PlatformInfo(metaclass=_MetaPlatform): for tlc_name in self.toolchain_names: ToolchainClass = Dependency.all_deps[tlc_name] targets[("source", tlc_name)] = ToolchainClass.Source - plt_name = "neutral" if ToolchainClass.neutral else self.name - targets[(plt_name, tlc_name)] = ToolchainClass.Builder + cfg_name = "neutral" if ToolchainClass.neutral else self.name + targets[(cfg_name, tlc_name)] = ToolchainClass.Builder def add_targets(self, targetName, targets): if (self.name, targetName) in targets: @@ -60,11 +60,11 @@ class PlatformInfo(metaclass=_MetaPlatform): targets[(self.name, targetName)] = targetClass.Builder for dep in targetClass.Builder.get_dependencies(self, False): if isinstance(dep, tuple): - depPlatformName, depName = dep + depConfigName, depName = dep else: - depPlatformName, depName = self.name, dep - depPlatform = self.get_platform(depPlatformName, targets) - depPlatform.add_targets(depName, targets) + depConfigName, depName = self.name, dep + depConfig = self.get_config(depConfigName, targets) + depConfig.add_targets(depName, targets) return [(self.name, targetName)] def get_fully_qualified_dep(self, dep): @@ -114,14 +114,14 @@ class PlatformInfo(metaclass=_MetaPlatform): self.buildEnv.clean_intermediate_directories() -class MetaPlatformInfo(PlatformInfo): - subPlatformNames = [] +class MetaConfigInfo(ConfigInfo): + subConfigNames = [] def add_targets(self, targetName, targets): targetDefs = [] - for platformName in self.subPlatformNames: - platform = self.get_platform(platformName, targets) - targetDefs += platform.add_targets(targetName, targets) + for configName in self.subConfigNames: + config = self.get_config(configName, targets) + targetDefs += config.add_targets(targetName, targets) return targetDefs @@ -135,8 +135,8 @@ def MixedMixin(static_name): if option("target") == targetName: return super().add_targets(targetName, targets) else: - static_platform = self.get_platform(static_name, targets) - return static_platform.add_targets(targetName, targets) + static_config = self.get_config(static_name, targets) + return static_config.add_targets(targetName, targets) def get_fully_qualified_dep(self, dep): if isinstance(dep, tuple): @@ -147,8 +147,8 @@ def MixedMixin(static_name): @property def static_buildEnv(self): - static_platform = self.get_platform(static_name) - return static_platform.buildEnv + static_config = self.get_config(static_name) + return static_config.buildEnv def get_include_dirs(self): return [ diff --git a/kiwixbuild/platforms/flatpak.py b/kiwixbuild/configs/flatpak.py similarity index 84% rename from kiwixbuild/platforms/flatpak.py rename to kiwixbuild/configs/flatpak.py index be57bca..a49acdd 100644 --- a/kiwixbuild/platforms/flatpak.py +++ b/kiwixbuild/configs/flatpak.py @@ -1,8 +1,8 @@ -from .base import PlatformInfo +from .base import ConfigInfo from kiwixbuild._global import option, neutralEnv -class FlatpakPlatformInfo(PlatformInfo): +class FlatpakConfigInfo(ConfigInfo): name = "flatpak" build = "flatpak" static = "" diff --git a/kiwixbuild/platforms/i586.py b/kiwixbuild/configs/i586.py similarity index 93% rename from kiwixbuild/platforms/i586.py rename to kiwixbuild/configs/i586.py index 341047d..8c402a3 100644 --- a/kiwixbuild/platforms/i586.py +++ b/kiwixbuild/configs/i586.py @@ -1,10 +1,10 @@ import os -from .base import PlatformInfo +from .base import ConfigInfo from kiwixbuild.utils import which, pj -class I586PlatformInfo(PlatformInfo): +class I586ConfigInfo(ConfigInfo): build = "i586" arch_full = "i586-linux-gnu" compatible_hosts = ["fedora", "debian"] @@ -69,11 +69,11 @@ class I586PlatformInfo(PlatformInfo): self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt") -class I586Dyn(I586PlatformInfo): +class I586Dyn(I586ConfigInfo): name = "i586_dyn" static = False -class I586Static(I586PlatformInfo): +class I586Static(I586ConfigInfo): name = "i586_static" static = True diff --git a/kiwixbuild/platforms/ios.py b/kiwixbuild/configs/ios.py similarity index 89% rename from kiwixbuild/platforms/ios.py rename to kiwixbuild/configs/ios.py index 5fb8d7d..bce93a3 100644 --- a/kiwixbuild/platforms/ios.py +++ b/kiwixbuild/configs/ios.py @@ -2,14 +2,14 @@ import subprocess from kiwixbuild._global import option from kiwixbuild.utils import pj, xrun_find -from .base import PlatformInfo, MetaPlatformInfo, MixedMixin +from .base import ConfigInfo, MetaConfigInfo, MixedMixin from kiwixbuild.dependencies.apple_xcframework import AppleXCFramework MIN_MACOS_VERSION = "12.0" -class ApplePlatformInfo(PlatformInfo): +class AppleConfigInfo(ConfigInfo): build = "iOS" static = True compatible_hosts = ["Darwin"] @@ -148,7 +148,7 @@ class ApplePlatformInfo(PlatformInfo): yield f"--host={self.host}" -class iOSArm64(ApplePlatformInfo): +class iOSArm64(AppleConfigInfo): name = "iOS_arm64" arch = cpu = "arm64" host = "arm-apple-darwin" @@ -157,7 +157,7 @@ class iOSArm64(ApplePlatformInfo): min_iphoneos_version = "15.0" -class iOSx64Simulator(ApplePlatformInfo): +class iOSx64Simulator(AppleConfigInfo): name = "iOSSimulator_x86_64" arch = cpu = "x86_64" host = "x86_64-apple-darwin" @@ -166,7 +166,7 @@ class iOSx64Simulator(ApplePlatformInfo): min_iphoneos_version = "15.0" -class iOSArm64Simulator(ApplePlatformInfo): +class iOSArm64Simulator(AppleConfigInfo): name = "iOSSimulator_arm64" arch = cpu = "arm64" host = "arm-apple-darwin" @@ -175,7 +175,7 @@ class iOSArm64Simulator(ApplePlatformInfo): min_iphoneos_version = "15.0" -class macOSArm64(ApplePlatformInfo): +class macOSArm64(AppleConfigInfo): name = "macOS_arm64_static" arch = cpu = "arm64" host = "aarch64-apple-darwin" @@ -185,7 +185,7 @@ class macOSArm64(ApplePlatformInfo): min_macos_version = MIN_MACOS_VERSION -class macOSArm64Mixed(MixedMixin("macOS_arm64_static"), ApplePlatformInfo): +class macOSArm64Mixed(MixedMixin("macOS_arm64_static"), AppleConfigInfo): name = "macOS_arm64_mixed" arch = cpu = "arm64" host = "aarch64-apple-darwin" @@ -195,7 +195,7 @@ class macOSArm64Mixed(MixedMixin("macOS_arm64_static"), ApplePlatformInfo): min_macos_version = MIN_MACOS_VERSION -class macOSx64(ApplePlatformInfo): +class macOSx64(AppleConfigInfo): name = "macOS_x86_64" arch = cpu = "x86_64" host = "x86_64-apple-darwin" @@ -205,33 +205,33 @@ class macOSx64(ApplePlatformInfo): min_macos_version = MIN_MACOS_VERSION -class IOS(MetaPlatformInfo): +class IOS(MetaConfigInfo): name = "iOS_multi" compatible_hosts = ["Darwin"] @property - def subPlatformNames(self): + def subConfigNames(self): return ["iOS_{}".format(arch) for arch in option("ios_arch")] def add_targets(self, targetName, targets): super().add_targets(targetName, targets) - return PlatformInfo.add_targets(self, "_ios_fat_lib", targets) + return ConfigInfo.add_targets(self, "_ios_fat_lib", targets) def __str__(self): return self.name -class AppleStaticAll(MetaPlatformInfo): +class AppleStaticAll(MetaConfigInfo): name = "apple_all_static" compatible_hosts = ["Darwin"] @property - def subPlatformNames(self): - return AppleXCFramework.subPlatformNames + def subConfigNames(self): + return AppleXCFramework.subConfigNames def add_targets(self, targetName, targets): super().add_targets(targetName, targets) - return PlatformInfo.add_targets(self, "apple_xcframework", targets) + return ConfigInfo.add_targets(self, "apple_xcframework", targets) def __str__(self): return self.name diff --git a/kiwixbuild/platforms/musl.py b/kiwixbuild/configs/musl.py similarity index 91% rename from kiwixbuild/platforms/musl.py rename to kiwixbuild/configs/musl.py index c4405c6..c1eebd8 100644 --- a/kiwixbuild/platforms/musl.py +++ b/kiwixbuild/configs/musl.py @@ -1,10 +1,10 @@ -from .base import PlatformInfo, MixedMixin +from .base import ConfigInfo, MixedMixin from kiwixbuild.utils import pj from kiwixbuild._global import get_target_step -class MuslPlatformInfo(PlatformInfo): +class MuslConfigInfo(ConfigInfo): compatible_hosts = ["fedora", "debian"] def get_cross_config(self): @@ -113,7 +113,7 @@ class MuslPlatformInfo(PlatformInfo): self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt") -class Aarch64MuslPlatformInfo(MuslPlatformInfo): +class Aarch64MuslConfigInfo(MuslConfigInfo): build = "aarch64_musl" arch_full = "aarch64-linux-musl" toolchain_names = ["aarch64_musl"] @@ -123,22 +123,22 @@ class Aarch64MuslPlatformInfo(MuslPlatformInfo): qemu = "qemu-arm" -class Aarch64MuslDyn(Aarch64MuslPlatformInfo): +class Aarch64MuslDyn(Aarch64MuslConfigInfo): name = "aarch64_musl_dyn" static = False -class Aarch64MuslStatic(Aarch64MuslPlatformInfo): +class Aarch64MuslStatic(Aarch64MuslConfigInfo): name = "aarch64_musl_static" static = True -class Aarch64MuslMixed(MixedMixin("aarch64_musl_static"), Aarch64MuslPlatformInfo): +class Aarch64MuslMixed(MixedMixin("aarch64_musl_static"), Aarch64MuslConfigInfo): name = "aarch64_musl_mixed" static = False -class X86_64MuslPlatformInfo(MuslPlatformInfo): +class X86_64MuslConfigInfo(MuslConfigInfo): build = "x86-64_musl" arch_full = "x86_64-linux-musl" toolchain_names = ["x86-64_musl"] @@ -147,16 +147,16 @@ class X86_64MuslPlatformInfo(MuslPlatformInfo): cpu = "x86_64" -class X86_64MuslDyn(X86_64MuslPlatformInfo): +class X86_64MuslDyn(X86_64MuslConfigInfo): name = "x86-64_musl_dyn" static = False -class X86_64MuslStatic(X86_64MuslPlatformInfo): +class X86_64MuslStatic(X86_64MuslConfigInfo): name = "x86-64_musl_static" static = True -class x86_64MuslMixed(MixedMixin("x86-64_musl_static"), X86_64MuslPlatformInfo): +class x86_64MuslMixed(MixedMixin("x86-64_musl_static"), X86_64MuslConfigInfo): name = "x86-64_musl_mixed" static = False diff --git a/kiwixbuild/platforms/native.py b/kiwixbuild/configs/native.py similarity index 70% rename from kiwixbuild/platforms/native.py rename to kiwixbuild/configs/native.py index 83efe07..6bf3e36 100644 --- a/kiwixbuild/platforms/native.py +++ b/kiwixbuild/configs/native.py @@ -1,11 +1,11 @@ -from .base import PlatformInfo, MixedMixin +from .base import ConfigInfo, MixedMixin from kiwixbuild.utils import pj from kiwixbuild._global import option, neutralEnv -from kiwixbuild.platforms.ios import MIN_MACOS_VERSION +from kiwixbuild.configs.ios import MIN_MACOS_VERSION -class NativePlatformInfo(PlatformInfo): +class NativeConfigInfo(ConfigInfo): build = "native" def get_env(self): @@ -17,19 +17,19 @@ class NativePlatformInfo(PlatformInfo): return env -class NativeDyn(NativePlatformInfo): +class NativeDyn(NativeConfigInfo): name = "native_dyn" static = False compatible_hosts = ["fedora", "debian", "Darwin"] -class NativeStatic(NativePlatformInfo): +class NativeStatic(NativeConfigInfo): name = "native_static" static = True compatible_hosts = ["fedora", "debian"] -class NativeMixed(MixedMixin("native_static"), NativePlatformInfo): +class NativeMixed(MixedMixin("native_static"), NativeConfigInfo): name = "native_mixed" static = False compatible_hosts = ["fedora", "debian", "Darwin"] diff --git a/kiwixbuild/platforms/neutral.py b/kiwixbuild/configs/neutral.py similarity index 66% rename from kiwixbuild/platforms/neutral.py rename to kiwixbuild/configs/neutral.py index 7315c3a..f5ae53e 100644 --- a/kiwixbuild/platforms/neutral.py +++ b/kiwixbuild/configs/neutral.py @@ -1,7 +1,7 @@ -from .base import PlatformInfo +from .base import ConfigInfo -class NeutralPlatformInfo(PlatformInfo): +class NeutralConfigInfo(ConfigInfo): name = "neutral" static = "" compatible_hosts = ["fedora", "debian", "Darwin"] diff --git a/kiwixbuild/platforms/wasm.py b/kiwixbuild/configs/wasm.py similarity index 97% rename from kiwixbuild/platforms/wasm.py rename to kiwixbuild/configs/wasm.py index de743d7..ff0503e 100644 --- a/kiwixbuild/platforms/wasm.py +++ b/kiwixbuild/configs/wasm.py @@ -1,10 +1,10 @@ -from .base import PlatformInfo +from .base import ConfigInfo from kiwixbuild.utils import pj from kiwixbuild._global import get_target_step -class WasmPlatformInfo(PlatformInfo): +class WasmConfigInfo(ConfigInfo): name = "wasm" static = True build = "wasm" diff --git a/kiwixbuild/platforms/win32.py b/kiwixbuild/configs/win32.py similarity index 94% rename from kiwixbuild/platforms/win32.py rename to kiwixbuild/configs/win32.py index 4e41ddd..6b665e7 100644 --- a/kiwixbuild/platforms/win32.py +++ b/kiwixbuild/configs/win32.py @@ -1,11 +1,11 @@ import subprocess -from .base import PlatformInfo +from .base import ConfigInfo from kiwixbuild.utils import which, pj from kiwixbuild._global import neutralEnv -class Win32PlatformInfo(PlatformInfo): +class Win32ConfigInfo(ConfigInfo): build = "win32" compatible_hosts = ["fedora", "debian"] arch_full = "i686-w64-mingw32" @@ -89,11 +89,11 @@ class Win32PlatformInfo(PlatformInfo): return env -class Win32Dyn(Win32PlatformInfo): +class Win32Dyn(Win32ConfigInfo): name = "win32_dyn" static = False -class Win32Static(Win32PlatformInfo): +class Win32Static(Win32ConfigInfo): name = "win32_static" static = True diff --git a/kiwixbuild/platforms/win64.py b/kiwixbuild/configs/win64.py similarity index 94% rename from kiwixbuild/platforms/win64.py rename to kiwixbuild/configs/win64.py index 62d9d35..10f215a 100644 --- a/kiwixbuild/platforms/win64.py +++ b/kiwixbuild/configs/win64.py @@ -1,11 +1,11 @@ import subprocess -from .base import PlatformInfo +from .base import ConfigInfo from kiwixbuild.utils import which, pj from kiwixbuild._global import neutralEnv -class Win64PlatformInfo(PlatformInfo): +class Win64ConfigInfo(ConfigInfo): extra_libs = [ "-lmingw32", "-lwinmm", @@ -93,11 +93,11 @@ class Win64PlatformInfo(PlatformInfo): return env -class Win64Dyn(Win64PlatformInfo): +class Win64Dyn(Win64ConfigInfo): name = "win64_dyn" static = False -class Win64Static(Win64PlatformInfo): +class Win64Static(Win64ConfigInfo): name = "win64_static" static = True diff --git a/kiwixbuild/dependencies/all_dependencies.py b/kiwixbuild/dependencies/all_dependencies.py index ab0a889..18cbeb7 100644 --- a/kiwixbuild/dependencies/all_dependencies.py +++ b/kiwixbuild/dependencies/all_dependencies.py @@ -12,8 +12,8 @@ class AllBaseDependencies(Dependency): class Builder(NoopBuilder): @classmethod - def get_dependencies(cls, platformInfo, allDeps): - if platformInfo.build == "wasm" or environ.get("OS_NAME") == "bionic": + def get_dependencies(cls, configInfo, allDeps): + if configInfo.build == "wasm" or environ.get("OS_NAME") == "bionic": return ["zlib", "lzma", "zstd", "icu4c", "xapian-core"] base_deps = [ @@ -28,17 +28,14 @@ class AllBaseDependencies(Dependency): "libmicrohttpd", "zim-testing-suite", ] - # Add specific dependencies depending of the platform - if platformInfo.build not in ("android", "iOS"): + # Add specific dependencies depending of the config + if configInfo.build not in ("android", "iOS"): # For zimtools base_deps += ["docoptcpp"] - if platformInfo.build != "win32": + if configInfo.build != "win32": # zimwriterfs base_deps += ["libmagic", "gumbo"] - if ( - platformInfo.build == "native" - and neutralEnv("distname") != "Darwin" - ): + if configInfo.build == "native" and neutralEnv("distname") != "Darwin": # We compile kiwix-desktop only on native and not on `Darwin` # So we need aria2 only there base_deps += ["aria2"] diff --git a/kiwixbuild/dependencies/apple_xcframework.py b/kiwixbuild/dependencies/apple_xcframework.py index 33c179c..7f8b7f7 100644 --- a/kiwixbuild/dependencies/apple_xcframework.py +++ b/kiwixbuild/dependencies/apple_xcframework.py @@ -2,14 +2,14 @@ import os import shutil from pathlib import Path -from kiwixbuild.platforms import PlatformInfo +from kiwixbuild.configs import ConfigInfo from kiwixbuild.utils import pj, run_command from .base import Dependency, NoopSource, Builder as BaseBuilder class AppleXCFramework(Dependency): name = "apple_xcframework" - subPlatformNames = [ + subConfigNames = [ "macOS_x86_64", "macOS_arm64_static", "iOS_arm64", @@ -20,34 +20,32 @@ class AppleXCFramework(Dependency): class Builder(BaseBuilder): @property - def all_subplatforms(self): - return self.buildEnv.platformInfo.subPlatformNames + def all_subconfigs(self): + return self.buildEnv.configInfo.subConfigNames @property - def macos_subplatforms(self): + def macos_subconfigs(self): return [ - target for target in self.all_subplatforms if target.startswith("macOS") + target for target in self.all_subconfigs if target.startswith("macOS") ] @property - def iossimulator_subplatforms(self): + def iossimulator_subconfigs(self): return [ target - for target in self.all_subplatforms + for target in self.all_subconfigs if target.startswith("iOSSimulator") ] @property - def ios_subplatforms(self): + def ios_subconfigs(self): return [ - target for target in self.all_subplatforms if target.startswith("iOS_") + target for target in self.all_subconfigs if target.startswith("iOS_") ] @classmethod - def get_dependencies(cls, platfomInfo, alldeps): - return [ - (target, "libkiwix") for target in AppleXCFramework.subPlatformNames - ] + def get_dependencies(cls, configInfo, alldeps): + return [(target, "libkiwix") for target in AppleXCFramework.subConfigNames] @property def final_path(self): @@ -62,11 +60,11 @@ class AppleXCFramework(Dependency): def _merge_libs(self, context): """create merged.a in all targets to bundle all static archives""" xcf_libs = [] - for target in self.all_subplatforms: + for target in self.all_subconfigs: static_ars = [] - plt = PlatformInfo.get_platform(target) - lib_dir = pj(plt.buildEnv.install_dir, "lib") + cfg = ConfigInfo.get_config(target) + lib_dir = pj(cfg.buildEnv.install_dir, "lib") static_ars = [str(f) for f in Path(lib_dir).glob("*.a")] # create merged.a from all *.a in install_dir/lib @@ -74,17 +72,17 @@ class AppleXCFramework(Dependency): run_command(command, lib_dir, context) # will be included in xcframework - if target in self.ios_subplatforms: + if target in self.ios_subconfigs: xcf_libs.append(pj(lib_dir, "merged.a")) return xcf_libs - def make_fat_with(self, platforms, folder_name, context): - """create fat merged.a in {folder_name} install/lib with {platforms} archs""" + def make_fat_with(self, configs, folder_name, context): + """create fat merged.a in {folder_name} install/lib with {configs}""" libs = [] - for target in platforms: - plt = PlatformInfo.get_platform(target) - libs.append(pj(plt.buildEnv.install_dir, "lib", "merged.a")) + for target in configs: + cfg = ConfigInfo.get_config(target) + libs.append(pj(cfg.buildEnv.install_dir, "lib", "merged.a")) fat_dir = pj(self.buildEnv.build_dir, folder_name) os.makedirs(fat_dir, exist_ok=True) @@ -97,14 +95,14 @@ class AppleXCFramework(Dependency): def _build_xcframework(self, xcf_libs, context): # create xcframework - ref_plat = PlatformInfo.get_platform(self.macos_subplatforms[0]) + ref_conf = ConfigInfo.get_config(self.macos_subconfigs[0]) command = ["xcodebuild", "-create-xcframework"] for lib in xcf_libs: command += [ "-library", lib, "-headers", - pj(ref_plat.buildEnv.install_dir, "include"), + pj(ref_conf.buildEnv.install_dir, "include"), ] command += ["-output", self.final_path] run_command(command, self.buildEnv.build_dir, context) @@ -116,13 +114,13 @@ class AppleXCFramework(Dependency): xcf_libs += self.command( "make_macos_fat", self.make_fat_with, - self.macos_subplatforms, + self.macos_subconfigs, "macOS_fat", ) xcf_libs += self.command( "make_simulator_fat", self.make_fat_with, - self.iossimulator_subplatforms, + self.iossimulator_subconfigs, "iOS-simulator_fat", ) self.command("build_xcframework", self._build_xcframework, xcf_libs) diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index 91e4a21..dbe5ef2 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -237,7 +237,7 @@ class Builder: self.buildEnv = buildEnv @classmethod - def get_dependencies(cls, platformInfo, allDeps): + def get_dependencies(cls, configInfo, allDeps): return cls.dependencies @property @@ -323,9 +323,9 @@ class Builder: cross_compilers=cross_compilers, cross_path=cross_path, ) - for dep in self.get_dependencies(self.buildEnv.platformInfo, False): + for dep in self.get_dependencies(self.buildEnv.configInfo, False): try: - builder = get_target_step(dep, self.buildEnv.platformInfo.name) + builder = get_target_step(dep, self.buildEnv.configInfo.name) builder.set_env(env) except KeyError: # Some target may be missing (installed by a package, ...) @@ -382,7 +382,7 @@ class MakeBuilder(Builder): @property def make_install_targets(self): - if self.buildEnv.platformInfo.build in ("iOS", "wasm"): + if self.buildEnv.configInfo.build in ("iOS", "wasm"): yield "install" else: yield "install-strip" @@ -390,12 +390,12 @@ class MakeBuilder(Builder): @property def all_configure_options(self): yield from self.configure_options - if self.buildEnv.platformInfo.static: + if self.buildEnv.configInfo.static: yield from self.static_configure_options else: yield from self.dynamic_configure_options if not self.target.force_native_build: - yield from self.buildEnv.platformInfo.configure_options + yield from self.buildEnv.configInfo.configure_options yield from ("--prefix", self.buildEnv.install_dir) yield from ("--libdir", pj(self.buildEnv.install_dir, self.buildEnv.libprefix)) @@ -532,7 +532,7 @@ class MesonBuilder(Builder): @property def library_type(self): - return "static" if self.buildEnv.platformInfo.static else "shared" + return "static" if self.buildEnv.configInfo.static else "shared" def _configure(self, context): context.try_skip(self.build_path) @@ -569,9 +569,9 @@ class MesonBuilder(Builder): def _test(self, context): context.try_skip(self.build_path) - if self.buildEnv.platformInfo.build == "android" or ( - self.buildEnv.platformInfo.build != "native" - and not self.buildEnv.platformInfo.static + if self.buildEnv.configInfo.build == "android" or ( + self.buildEnv.configInfo.build != "native" + and not self.buildEnv.configInfo.static ): raise SkipCommand() command = [*neutralEnv("mesontest_command"), "--verbose", *self.test_options] diff --git a/kiwixbuild/dependencies/icu4c.py b/kiwixbuild/dependencies/icu4c.py index 816d4e3..20993f4 100644 --- a/kiwixbuild/dependencies/icu4c.py +++ b/kiwixbuild/dependencies/icu4c.py @@ -58,8 +58,8 @@ class Icu(Dependency): make_install_targets = ["install"] @classmethod - def get_dependencies(cls, platformInfo, allDeps): - plt = "native_static" if platformInfo.static else "native_dyn" + def get_dependencies(cls, configInfo, allDeps): + plt = "native_static" if configInfo.static else "native_dyn" return [(plt, "icu4c")] @property @@ -71,14 +71,14 @@ class Icu(Dependency): yield "--enable-rpath" yield "--disable-icuio" yield "--disable-layoutex" - platformInfo = self.buildEnv.platformInfo - if platformInfo.build != "native": + configInfo = self.buildEnv.configInfo + if configInfo.build != "native": icu_native_builder = get_target_step( - "icu4c", "native_static" if platformInfo.static else "native_dyn" + "icu4c", "native_static" if configInfo.static else "native_dyn" ) yield f"--with-cross-build={icu_native_builder.build_path}" yield "--disable-tools" - if platformInfo.build in ("android", "wasm"): + if configInfo.build in ("android", "wasm"): yield "--with-data-packaging=archive" def set_env(self, env): @@ -87,7 +87,7 @@ class Icu(Dependency): ) def _post_configure_script(self, context): - if self.buildEnv.platformInfo.build != "wasm": + if self.buildEnv.configInfo.build != "wasm": context.skip() context.try_skip(self.build_path) for line in fileinput.input(pj(self.build_path, "Makefile"), inplace=True): diff --git a/kiwixbuild/dependencies/ios_fat_lib.py b/kiwixbuild/dependencies/ios_fat_lib.py index 742e339..a0b74f2 100644 --- a/kiwixbuild/dependencies/ios_fat_lib.py +++ b/kiwixbuild/dependencies/ios_fat_lib.py @@ -1,6 +1,6 @@ import os -from kiwixbuild.platforms import PlatformInfo +from kiwixbuild.configs import ConfigInfo from kiwixbuild.utils import pj, copy_tree, run_command from kiwixbuild._global import option from .base import Dependency, NoopSource, Builder as BaseBuilder @@ -18,7 +18,7 @@ class IOSFatLib(Dependency): return [("iOS_{}".format(arch), base_target) for arch in option("ios_arch")] def _copy_headers(self, context): - plt = PlatformInfo.get_platform("iOS_{}".format(option("ios_arch")[0])) + plt = ConfigInfo.get_config("iOS_{}".format(option("ios_arch")[0])) include_src = pj(plt.buildEnv.install_dir, "include") include_dst = pj(self.buildEnv.install_dir, "include") copy_tree(include_src, include_dst) @@ -26,7 +26,7 @@ class IOSFatLib(Dependency): def _merge_libs(self, context): lib_dirs = [] for arch in option("ios_arch"): - plt = PlatformInfo.get_platform("iOS_{}".format(arch)) + plt = ConfigInfo.get_config("iOS_{}".format(arch)) lib_dirs.append(pj(plt.buildEnv.install_dir, "lib")) libs = [] for f in os.listdir(lib_dirs[0]): diff --git a/kiwixbuild/dependencies/kiwix_desktop.py b/kiwixbuild/dependencies/kiwix_desktop.py index 5b1eab5..c72ff59 100644 --- a/kiwixbuild/dependencies/kiwix_desktop.py +++ b/kiwixbuild/dependencies/kiwix_desktop.py @@ -18,11 +18,11 @@ class KiwixDesktop(Dependency): @property def configure_options(self): - if self.buildEnv.platformInfo.name == "flatpak": + if self.buildEnv.configInfo.name == "flatpak": yield "QMAKE_INCDIR+=/app/include/QtWebEngine" yield "QMAKE_INCDIR+=/app/include/QtWebEngineCore" yield "QMAKE_INCDIR+=/app/include/QtWebEngineWidgets" else: yield f"PREFIX={self.buildEnv.install_dir}" - if self.buildEnv.platformInfo.static: + if self.buildEnv.configInfo.static: yield "CONFIG+=static" diff --git a/kiwixbuild/dependencies/kiwix_tools.py b/kiwixbuild/dependencies/kiwix_tools.py index 8895b08..615b749 100644 --- a/kiwixbuild/dependencies/kiwix_tools.py +++ b/kiwixbuild/dependencies/kiwix_tools.py @@ -14,5 +14,5 @@ class KiwixTools(Dependency): @property def configure_options(self): - if self.buildEnv.platformInfo.static: + if self.buildEnv.configInfo.static: yield "-Dstatic-linkage=true" diff --git a/kiwixbuild/dependencies/libkiwix.py b/kiwixbuild/dependencies/libkiwix.py index e97dbff..573cb23 100644 --- a/kiwixbuild/dependencies/libkiwix.py +++ b/kiwixbuild/dependencies/libkiwix.py @@ -29,25 +29,25 @@ class Libkiwix(Dependency): @property def build_type(self): - if self.buildEnv.platformInfo.build == "android": + if self.buildEnv.configInfo.build == "android": return "debug" return super().build_type @property def configure_options(self): - platformInfo = self.buildEnv.platformInfo - if platformInfo.build == "android": + configInfo = self.buildEnv.configInfo + if configInfo.build == "android": yield "-Dstatic-linkage=true" yield "-Dwerror=false" - if platformInfo.build == "iOS": + if configInfo.build == "iOS": yield "-Db_bitcode=true" - if platformInfo.name == "flatpak": + if configInfo.name == "flatpak": yield "--wrap-mode=nodownload" - if platformInfo.mixed and option("target") == "libkiwix": + if configInfo.mixed and option("target") == "libkiwix": yield "-Dstatic-linkage=true" @property def library_type(self): - if self.buildEnv.platformInfo.build == "android": + if self.buildEnv.configInfo.build == "android": return "shared" return super().library_type diff --git a/kiwixbuild/dependencies/libmagic.py b/kiwixbuild/dependencies/libmagic.py index 165f5e9..18d2f9d 100644 --- a/kiwixbuild/dependencies/libmagic.py +++ b/kiwixbuild/dependencies/libmagic.py @@ -31,14 +31,14 @@ class LibMagic(Dependency): yield "--disable-lzlib" @classmethod - def get_dependencies(cls, platformInfo, allDeps): - if platformInfo.build != "native": + def get_dependencies(cls, configInfo, allDeps): + if configInfo.build != "native": return [("native_static", "libmagic")] return [] def _compile(self, context): - platformInfo = self.buildEnv.platformInfo - if platformInfo.build == "native": + configInfo = self.buildEnv.configInfo + if configInfo.build == "native": return super()._compile(context) context.try_skip(self.build_path) command = ["make", "-j4", *self.make_targets, *self.make_options] diff --git a/kiwixbuild/dependencies/libzim.py b/kiwixbuild/dependencies/libzim.py index 48dee52..804cd38 100644 --- a/kiwixbuild/dependencies/libzim.py +++ b/kiwixbuild/dependencies/libzim.py @@ -16,37 +16,37 @@ class Libzim(Dependency): @property def build_type(self): - if self.buildEnv.platformInfo.build == "android": + if self.buildEnv.configInfo.build == "android": return "debug" return super().build_type @classmethod - def get_dependencies(cls, platformInfo, allDeps): + def get_dependencies(cls, configInfo, allDeps): deps = ["lzma", "zstd", "xapian-core", "icu4c"] - if platformInfo.name not in ("flatpak", "wasm"): + if configInfo.name not in ("flatpak", "wasm"): deps.append("zim-testing-suite") return deps @property def configure_options(self): - platformInfo = self.buildEnv.platformInfo - if platformInfo.build == "android": + configInfo = self.buildEnv.configInfo + if configInfo.build == "android": yield "-DUSE_BUFFER_HEADER=false" yield "-Dstatic-linkage=true" - if platformInfo.mixed and option("target") == "libzim": + if configInfo.mixed and option("target") == "libzim": yield "-Dstatic-linkage=true" - if platformInfo.name == "flatpak": + if configInfo.name == "flatpak": yield "--wrap-mode=nodownload" yield "-Dtest_data_dir=none" - if platformInfo.name == "wasm": + if configInfo.name == "wasm": yield "-Dexamples=false" yield "-DUSE_MMAP=false" - if platformInfo.name not in ("flatpak", "wasm"): + if configInfo.name not in ("flatpak", "wasm"): zim_testing_suite = get_target_step("zim-testing-suite", "source") yield "-Dtest_data_dir={}".format(zim_testing_suite.source_path) @property def library_type(self): - if self.buildEnv.platformInfo.build == "android": + if self.buildEnv.configInfo.build == "android": return "shared" return super().library_type diff --git a/kiwixbuild/dependencies/qt.py b/kiwixbuild/dependencies/qt.py index 69e86ef..1a55ae3 100644 --- a/kiwixbuild/dependencies/qt.py +++ b/kiwixbuild/dependencies/qt.py @@ -25,12 +25,12 @@ class Qt(Dependency): @property def all_configure_options(self): yield from self.configure_options - if self.buildEnv.platformInfo.static: + if self.buildEnv.configInfo.static: yield from self.static_configure_options else: yield from self.dynamic_configure_options if not self.target.force_native_build: - yield from self.buildEnv.platformInfo.configure_options + yield from self.buildEnv.configInfo.configure_options yield from ("-prefix", self.buildEnv.install_dir) yield from ( "-libdir", diff --git a/kiwixbuild/dependencies/tc_android_ndk.py b/kiwixbuild/dependencies/tc_android_ndk.py index 542fca9..3774976 100644 --- a/kiwixbuild/dependencies/tc_android_ndk.py +++ b/kiwixbuild/dependencies/tc_android_ndk.py @@ -34,18 +34,18 @@ class android_ndk(Dependency): return self.target.api @property - def platform(self): + def config(self): return "android-" + self.api @property def arch(self): - return self.buildEnv.platformInfo.arch + return self.buildEnv.configInfo.arch @property def arch_full(self): - return self.buildEnv.platformInfo.arch_full + return self.buildEnv.configInfo.arch_full - def _build_platform(self, context): + def _build_toolchain(self, context): context.try_skip(self.build_path) script = pj(self.source_path, "build/tools/make_standalone_toolchain.py") add_execution_right(script) @@ -85,5 +85,5 @@ class android_ndk(Dependency): add_execution_right(file_path) def build(self): - self.command("build_platform", self._build_platform) + self.command("build_toolchain", self._build_toolchain) self.command("fix_permission_right", self._fix_permission_right) diff --git a/kiwixbuild/dependencies/xapian.py b/kiwixbuild/dependencies/xapian.py index a337f68..aac4fd7 100644 --- a/kiwixbuild/dependencies/xapian.py +++ b/kiwixbuild/dependencies/xapian.py @@ -26,10 +26,10 @@ class Xapian(Dependency): } @classmethod - def get_dependencies(cls, platformInfo, allDeps): + def get_dependencies(cls, configInfo, allDeps): deps = ["zlib", "lzma"] if ( - platformInfo.build in ("win32", "wasm") + configInfo.build in ("win32", "wasm") or neutralEnv("distname") == "Darwin" ): return deps diff --git a/kiwixbuild/dependencies/zim_tools.py b/kiwixbuild/dependencies/zim_tools.py index f44d94c..9e677c8 100644 --- a/kiwixbuild/dependencies/zim_tools.py +++ b/kiwixbuild/dependencies/zim_tools.py @@ -11,16 +11,16 @@ class ZimTools(Dependency): class Builder(MesonBuilder): @classmethod - def get_dependencies(cls, platformInfo, allDeps): + def get_dependencies(cls, configInfo, allDeps): base_deps = ["libzim", "docoptcpp", "mustache"] - if platformInfo.build != "win32": + if configInfo.build != "win32": base_deps += ["libmagic", "gumbo"] return base_deps @property def configure_options(self): # We don't build zimwriterfs on win32, and so we don't have magic - if self.buildEnv.platformInfo.build != "win32": + if self.buildEnv.configInfo.build != "win32": yield f"-Dmagic-install-prefix={self.buildEnv.install_dir}" - if self.buildEnv.platformInfo.static: + if self.buildEnv.configInfo.static: yield "-Dstatic-linkage=true" diff --git a/kiwixbuild/dependencies/zlib.py b/kiwixbuild/dependencies/zlib.py index 7f8539c..b0d712f 100644 --- a/kiwixbuild/dependencies/zlib.py +++ b/kiwixbuild/dependencies/zlib.py @@ -25,14 +25,14 @@ class zlib(Dependency): shutil.copytree(self.source_path, self.build_path) def _configure(self, context): - if self.buildEnv.platformInfo.build == "win32": + if self.buildEnv.configInfo.build == "win32": raise SkipCommand() return super()._configure(context) @property def all_configure_options(self): yield from self.configure_options - yield from self.static_configure_options if self.buildEnv.platformInfo.static else self.dynamic_configure_options + yield from self.static_configure_options if self.buildEnv.configInfo.static else self.dynamic_configure_options yield from ("--prefix", self.buildEnv.install_dir) yield from ( "--libdir", @@ -41,13 +41,13 @@ class zlib(Dependency): @property def make_options(self): - if self.buildEnv.platformInfo.build != "win32": + if self.buildEnv.configInfo.build != "win32": return yield "--makefile" yield "win32/Makefile.gcc" yield "PREFIX=i686-w64-mingw32-", yield "SHARED_MODE={}".format( - "0" if self.buildEnv.platformInfo.static else "1" + "0" if self.buildEnv.configInfo.static else "1" ), yield "INCLUDE_PATH={}".format(pj(self.buildEnv.install_dir, "include")), yield "LIBRARY_PATH={}".format( @@ -57,7 +57,7 @@ class zlib(Dependency): @property def make_targets(self): - if self.buildEnv.platformInfo.static: + if self.buildEnv.configInfo.static: return ["static"] else: return ["shared"] diff --git a/kiwixbuild/flatpak_builder.py b/kiwixbuild/flatpak_builder.py index cb07f62..ea0d934 100644 --- a/kiwixbuild/flatpak_builder.py +++ b/kiwixbuild/flatpak_builder.py @@ -2,7 +2,7 @@ import sys from collections import OrderedDict from .buildenv import * -from .platforms import PlatformInfo +from .configs import ConfigInfo from .utils import remove_duplicates, run_command, StopBuild, Context from .dependencies import Dependency from .packages import PACKAGE_NAME_MAPPERS @@ -77,16 +77,16 @@ GET_REF_URL_API_TEMPLATE = "https://api.github.com/repos{repo}/git/refs/tags/{re class FlatpakBuilder: def __init__(self): self._targets = {} - PlatformInfo.get_platform("neutral", self._targets) - self.platform = PlatformInfo.get_platform("flatpak", self._targets) - if neutralEnv("distname") not in self.platform.compatible_hosts: + ConfigInfo.get_config("neutral", self._targets) + self.config = ConfigInfo.get_config("flatpak", self._targets) + if neutralEnv("distname") not in self.config.compatible_hosts: print( ( - "ERROR: The target platform {} cannot be build on host {}.\n" - "Select another target platform or change your host system." - ).format(self.platform.name, neutralEnv("distname")) + "ERROR: The config {} cannot be build on host {}.\n" + "Select another config or change your host system." + ).format(self.config.name, neutralEnv("distname")) ) - self.targetDefs = self.platform.add_targets(option("target"), self._targets) + self.targetDefs = self.config.add_targets(option("target"), self._targets) def finalize_target_steps(self): steps = [] @@ -94,14 +94,14 @@ class FlatpakBuilder: steps += self.order_steps(targetDef) steps = list(remove_duplicates(steps)) - for pltName in PlatformInfo.all_running_platforms: - plt = PlatformInfo.all_platforms[pltName] - for tlcName in plt.toolchain_names: + for cfgName in ConfigInfo.all_running_configs: + cfg = ConfigInfo.all_configs[cfgName] + for tlcName in cfg.toolchain_names: tlc = Dependency.all_deps[tlcName] - src_plt_step = ("source", tlcName) - add_target_step(src_plt_step, self._targets[src_plt_step]) - blt_plt_step = ("neutral" if tlc.neutral else pltName, tlcName) - add_target_step(blt_plt_step, self._targets[blt_plt_step]) + src_cfg_step = ("source", tlcName) + add_target_step(src_cfg_step, self._targets[src_cfg_step]) + blt_cfg_step = ("neutral" if tlc.neutral else cfgName, tlcName) + add_target_step(blt_cfg_step, self._targets[blt_cfg_step]) for dep in steps: add_target_step(dep, self._targets[dep]) @@ -112,8 +112,8 @@ class FlatpakBuilder: yield from self.order_dependencies(targetDef, _targets) def order_dependencies(self, targetDef, targets): - targetPlatformName, targetName = targetDef - if targetPlatformName == "source": + targetConfigName, targetName = targetDef + if targetConfigName == "source": # Do not try to order sources, they will be added as dep by the # build step two lines later. return @@ -122,27 +122,27 @@ class FlatpakBuilder: except KeyError: return - targetPlatform = PlatformInfo.get_platform(targetPlatformName) - for dep in target.get_dependencies(targetPlatform, True): + targetConfig = ConfigInfo.get_config(targetConfigName) + for dep in target.get_dependencies(targetConfig, True): if isinstance(dep, tuple): - depPlatform, depName = dep + depConfig, depName = dep else: - depPlatform, depName = targetPlatformName, dep - if (depPlatform, depName) in targets: - yield from self.order_dependencies((depPlatform, depName), targets) + depConfig, depName = targetConfigName, dep + if (depConfig, depName) in targets: + yield from self.order_dependencies((depConfig, depName), targets) yield ("source", targetName) yield targetDef def instanciate_steps(self): for stepDef in list(target_steps()): - stepPlatform, stepName = stepDef + stepConfig, stepName = stepDef stepClass = Dependency.all_deps[stepName] - if stepPlatform == "source": + if stepConfig == "source": source = get_target_step(stepDef)(stepClass) add_target_step(stepDef, source) else: source = get_target_step(stepName, "source") - env = PlatformInfo.get_platform(stepPlatform).buildEnv + env = ConfigInfo.get_config(stepConfig).buildEnv builder = get_target_step(stepDef)(stepClass, source, env) add_target_step(stepDef, builder) @@ -206,7 +206,7 @@ class FlatpakBuilder: m["sources"] = temp manifest["modules"] = modules manifest_name = "{}.json".format(MANIFEST["app-id"]) - manifest_path = pj(self.platform.buildEnv.build_dir, manifest_name) + manifest_path = pj(self.config.buildEnv.build_dir, manifest_name) with open(manifest_path, "w") as f: f.write(json.dumps(manifest, indent=4)) @@ -219,13 +219,13 @@ class FlatpakBuilder: for p in source.patches: path = pj(SCRIPT_DIR, "patches", p) os.makedirs( - pj(self.platform.buildEnv.build_dir, "patches"), exist_ok=True + pj(self.config.buildEnv.build_dir, "patches"), exist_ok=True ) - dest = pj(self.platform.buildEnv.build_dir, "patches", p) + dest = pj(self.config.buildEnv.build_dir, "patches", p) copyfile(path, dest) def build(self): - log = pj(self.platform.buildEnv.log_dir, "cmd_build_flatpak.log") + log = pj(self.config.buildEnv.log_dir, "cmd_build_flatpak.log") context = Context("build", log, False) command = [ "flatpak-builder", @@ -241,9 +241,9 @@ class FlatpakBuilder: try: run_command( command, - self.platform.buildEnv.build_dir, + self.config.buildEnv.build_dir, context, - env=self.platform.get_env(), + env=self.config.get_env(), ) context._finalise() except subprocess.CalledProcessError: @@ -252,16 +252,16 @@ class FlatpakBuilder: raise StopBuild() def bundle(self): - log = pj(self.platform.buildEnv.log_dir, "cmd_bundle_flatpak.log") + log = pj(self.config.buildEnv.log_dir, "cmd_bundle_flatpak.log") context = Context("bundle", log, False) app_id = MANIFEST["app-id"] command = ["flatpak", "build-bundle", "repo", f"{app_id}.flatpak", app_id] try: run_command( command, - self.platform.buildEnv.build_dir, + self.config.buildEnv.build_dir, context, - env=self.platform.get_env(), + env=self.config.get_env(), ) context._finalise() except subprocess.CalledProcessError: @@ -274,7 +274,7 @@ class FlatpakBuilder: to_drop = [] for builderDef in self._targets: - platformName, builderName = builderDef + configName, builderName = builderDef packages = package_name_mapper.get(builderName) if packages: to_drop.append(builderDef) @@ -288,16 +288,16 @@ class FlatpakBuilder: # dependencies we already have in the sdk. self._get_packages() self.finalize_target_steps() - print("[SETUP PLATFORMS]") - for platform in PlatformInfo.all_running_platforms.values(): - platform.finalize_setup() - for pltName in PlatformInfo.all_running_platforms: - plt = PlatformInfo.all_platforms[pltName] - for tlcName in plt.toolchain_names: + print("[SETUP TOOLCHAINS]") + for config in ConfigInfo.all_running_configs.values(): + config.finalize_setup() + for cfgName in ConfigInfo.all_running_configs: + cfg = ConfigInfo.all_configs[cfgName] + for tlcName in cfg.toolchain_names: tlc = Dependency.all_deps[tlcName] - builderDef = (pltName, tlcName) + builderDef = (cfgName, tlcName) builder = get_target_step(builderDef) - print("build {} ({}):".format(builder.name, pltName[0])) + print("build {} ({}):".format(builder.name, cfgName[0])) add_target_step(builderDef, builder) builder.build() print("[GENERATE FLATPAK MANIFEST]") @@ -310,8 +310,8 @@ class FlatpakBuilder: # No error, clean intermediate file at end of build if needed. print("[CLEAN]") if option("clean_at_end"): - for platform in PlatformInfo.all_running_platforms.values(): - platform.clean_intermediate_directories() + for config in ConfigInfo.all_running_configs.values(): + config.clean_intermediate_directories() else: print("SKIP") except StopBuild: From c25a2e63e3d65ff4b8bf9ba6e0cf057eb07c3009 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 5 Feb 2024 18:00:35 +0100 Subject: [PATCH 5/6] [CI/CD] Use config instead of target in the CI. --- .github/scripts/build_definition.py | 22 +-- .github/scripts/build_projects.py | 6 +- .github/scripts/build_release_nightly.py | 18 +-- .github/scripts/common.py | 183 +++++++++++++---------- .github/scripts/compile_all_deps.py | 4 +- .github/scripts/ensure_base_deps.py | 14 +- .github/scripts/upload_failure_logs.sh | 2 +- .github/workflows/ci.yml | 30 ++-- .github/workflows/releaseNigthly.yml | 26 ++-- 9 files changed, 167 insertions(+), 138 deletions(-) diff --git a/.github/scripts/build_definition.py b/.github/scripts/build_definition.py index 01fb906..e3ecd80 100644 --- a/.github/scripts/build_definition.py +++ b/.github/scripts/build_definition.py @@ -3,7 +3,7 @@ import csv, io, re # Definition of what to build. # Array is read line by line. -# Empty cells under (OS_NAME, PLATFORM_TARGET) mean "always match" (catch all, or `.*` regex) +# Empty cells under (OS_NAME, COMPILE_CONFIG) mean "always match" (catch all, or `.*` regex) # Once a cell doesn't match, skip to the next line. # Once a line matches, other lines are not read, so put more specific combination first. # Lines composed of `-` , or `=`, or starting by `#` are ignored. @@ -15,7 +15,7 @@ import csv, io, re # 'D' letter means we trigger the docker forkflow to build the docker image. # If a cell contains several letters, all are done. BUILD_DEF = """ - | OS_NAME | PLATFORM_TARGET | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | platform_name | + | OS_NAME | COMPILE_CONFIG | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | platform_name | ===================================================================================================================== # Bionic is a special case as we need to compile libzim on old arch for python | bionic | native_mixed | BP | | | | | linux-x86_64-bionic | @@ -92,10 +92,10 @@ def selector_match(selector, value): class Context(NamedTuple): OS_NAME: str - PLATFORM_TARGET: str + COMPILE_CONFIG: str def match(self, row): - for key in ["OS_NAME", "PLATFORM_TARGET"]: + for key in ["OS_NAME", "COMPILE_CONFIG"]: context_value = getattr(self, key) selector = row[key] if not selector_match(selector, context_value): @@ -109,10 +109,11 @@ SOURCE_PUBLISH = "S" DEPS = "d" DOCKER = "D" -def select_build_targets(criteria): - from common import PLATFORM_TARGET, OS_NAME - context = Context(PLATFORM_TARGET=PLATFORM_TARGET, OS_NAME=OS_NAME) +def select_build_targets(criteria): + from common import COMPILE_CONFIG, OS_NAME + + context = Context(COMPILE_CONFIG=COMPILE_CONFIG, OS_NAME=OS_NAME) reader = csv.DictReader(strip_array(BUILD_DEF), dialect=TableDialect()) for row in reader: @@ -133,10 +134,11 @@ def select_build_targets(criteria): raise ValueError("No definition match with current context.") -def get_platform_name(): - from common import PLATFORM_TARGET, OS_NAME - context = Context(PLATFORM_TARGET=PLATFORM_TARGET, OS_NAME=OS_NAME) +def get_platform_name(): + from common import COMPILE_CONFIG, OS_NAME + + context = Context(COMPILE_CONFIG=COMPILE_CONFIG, OS_NAME=OS_NAME) reader = csv.DictReader(strip_array(BUILD_DEF), dialect=TableDialect()) for row in reader: diff --git a/.github/scripts/build_projects.py b/.github/scripts/build_projects.py index b267b64..68cdce9 100755 --- a/.github/scripts/build_projects.py +++ b/.github/scripts/build_projects.py @@ -8,16 +8,16 @@ from common import ( fix_macos_rpath, upload_archive, OS_NAME, - PLATFORM_TARGET, + COMPILE_CONFIG, DEV_BRANCH, ) for target in select_build_targets(BUILD): - run_kiwix_build(target, platform=PLATFORM_TARGET) + run_kiwix_build(target, config=COMPILE_CONFIG) if target == "kiwix-desktop": archive = create_desktop_image(make_release=False) else: - if PLATFORM_TARGET == "native_mixed" and OS_NAME == "macos": + if COMPILE_CONFIG == "native_mixed" and OS_NAME == "macos": fix_macos_rpath(target) archive = make_archive(target, make_release=False) if archive and DEV_BRANCH: diff --git a/.github/scripts/build_release_nightly.py b/.github/scripts/build_release_nightly.py index e97cda6..5ddf3c9 100755 --- a/.github/scripts/build_release_nightly.py +++ b/.github/scripts/build_release_nightly.py @@ -13,7 +13,7 @@ from common import ( fix_macos_rpath, BASE_DIR, OS_NAME, - PLATFORM_TARGET, + COMPILE_CONFIG, MAKE_RELEASE, notarize_macos_build, ) @@ -21,23 +21,23 @@ from common import ( from build_definition import select_build_targets, BUILD, PUBLISH, SOURCE_PUBLISH - - # Filter what to build if we are doing a release. if MAKE_RELEASE: TARGETS = select_build_targets(PUBLISH) + def release_filter(project): return release_versions.get(project) is not None + TARGETS = tuple(filter(release_filter, TARGETS)) else: TARGETS = select_build_targets(BUILD) for target in TARGETS: - run_kiwix_build(target, platform=PLATFORM_TARGET, make_release=MAKE_RELEASE) + run_kiwix_build(target, config=COMPILE_CONFIG, make_release=MAKE_RELEASE) if target == "kiwix-desktop": archive = create_desktop_image(make_release=MAKE_RELEASE) else: - if OS_NAME == "macos" and PLATFORM_TARGET.endswith("_mixed"): + if OS_NAME == "macos" and COMPILE_CONFIG.endswith("_mixed"): fix_macos_rpath(target) notarize_macos_build(target) archive = make_archive(target, make_release=MAKE_RELEASE) @@ -57,13 +57,11 @@ if MAKE_RELEASE: if target not in source_published_targets: continue run_kiwix_build( - target, platform=PLATFORM_TARGET, make_release=MAKE_RELEASE, make_dist=True + target, config=COMPILE_CONFIG, make_release=MAKE_RELEASE, make_dist=True ) full_target_name = "{}-{}".format(target, main_project_versions[target]) if target == "kiwix-desktop": - archive = ( - BASE_DIR / full_target_name / "{}.tar.gz".format(full_target_name) - ) + archive = BASE_DIR / full_target_name / "{}.tar.gz".format(full_target_name) else: archive = ( BASE_DIR @@ -74,5 +72,5 @@ if MAKE_RELEASE: upload_archive(archive, target, make_release=MAKE_RELEASE) # Publish flathub - if PLATFORM_TARGET == "flatpak" and "kiwix-desktop" in TARGETS: + if COMPILE_CONFIG == "flatpak" and "kiwix-desktop" in TARGETS: update_flathub_git() diff --git a/.github/scripts/common.py b/.github/scripts/common.py index bf635cf..360f193 100644 --- a/.github/scripts/common.py +++ b/.github/scripts/common.py @@ -20,12 +20,11 @@ from kiwixbuild.versions import ( ) - -PLATFORM_TARGET = _environ["PLATFORM_TARGET"] +COMPILE_CONFIG = _environ["COMPILE_CONFIG"] OS_NAME = _environ["OS_NAME"] HOME = Path(os.path.expanduser("~")) -BASE_DIR = HOME / "BUILD_{}".format(PLATFORM_TARGET) +BASE_DIR = HOME / "BUILD_{}".format(COMPILE_CONFIG) SOURCE_DIR = HOME / "SOURCE" ARCHIVE_DIR = HOME / "ARCHIVE" TOOLCHAIN_DIR = BASE_DIR / "TOOLCHAINS" @@ -35,7 +34,7 @@ KBUILD_SOURCE_DIR = HOME / "kiwix-build" _ref = _environ.get("GITHUB_REF", "").split("/")[-1] MAKE_RELEASE = re.fullmatch(r"r_[0-9]+", _ref) is not None -MAKE_RELEASE = MAKE_RELEASE and (_environ.get('GITHUB_EVENT_NAME') != 'schedule') +MAKE_RELEASE = MAKE_RELEASE and (_environ.get("GITHUB_EVENT_NAME") != "schedule") if not MAKE_RELEASE and _ref != "main": DEV_BRANCH = _ref @@ -45,19 +44,18 @@ else: FLATPAK_HTTP_GIT_REMOTE = "https://github.com/flathub/org.kiwix.desktop.git" FLATPAK_GIT_REMOTE = "git@github.com:flathub/org.kiwix.desktop.git" -BIN_EXT = ".exe" if PLATFORM_TARGET.startswith("win32_") else "" +BIN_EXT = ".exe" if COMPILE_CONFIG.startswith("win32_") else "" + def major_version(version: str) -> str: return version.split(".")[0] + # We have build everything. Now create archives for public deployement. EXPORT_FILES = { "kiwix-tools": ( INSTALL_DIR / "bin", - [ - f + BIN_EXT - for f in ("kiwix-manage", "kiwix-search", "kiwix-serve") - ], + [f + BIN_EXT for f in ("kiwix-manage", "kiwix-search", "kiwix-serve")], ), "zim-tools": ( INSTALL_DIR / "bin", @@ -72,7 +70,7 @@ EXPORT_FILES = { "zimpatch", "zimsplit", "zimwriterfs", - "zimrecreate" + "zimrecreate", ) ], ), @@ -80,11 +78,9 @@ EXPORT_FILES = { INSTALL_DIR, ( # We need to package all dependencies (`*.a`) on wasm - "lib/*/libzim.a" if PLATFORM_TARGET != "wasm" else "lib/*.a", + "lib/*/libzim.a" if COMPILE_CONFIG != "wasm" else "lib/*.a", "lib/*/libzim.so", - "lib/*/libzim.so.{version}".format( - version=main_project_versions["libzim"] - ), + "lib/*/libzim.so.{version}".format(version=main_project_versions["libzim"]), "lib/*/libzim.so.{version}".format( version=major_version(main_project_versions["libzim"]) ), @@ -95,9 +91,8 @@ EXPORT_FILES = { "lib/*/libzim.pc", "include/zim/**/*.h", "share/icu/{}/icudt{}l.dat".format( - base_deps_versions["icu4c"], - major_version(base_deps_versions["icu4c"]) - ) + base_deps_versions["icu4c"], major_version(base_deps_versions["icu4c"]) + ), ), ), "libkiwix": ( @@ -118,15 +113,15 @@ EXPORT_FILES = { "lib/*/libkiwix.pc", "include/kiwix/**/*.h", "share/icu/{}/icudt{}l.dat".format( - base_deps_versions["icu4c"], - major_version(base_deps_versions["icu4c"]) - ) + base_deps_versions["icu4c"], major_version(base_deps_versions["icu4c"]) + ), ), ), } DATE = date.today().isoformat() + def print_message(message, *args, **kwargs): message = message.format(*args, **kwargs) message = "{0} {1} {0}".format("-" * 3, message) @@ -136,23 +131,26 @@ def print_message(message, *args, **kwargs): MANIFEST_TEMPLATE = """{archive_name} *************************** -Dependencies archive for {target} on platform {platform} +Dependencies archive for {target} using config {config} Generated at {date} """ -def write_manifest(manifest_file, archive_name, target, platform): +def write_manifest(manifest_file, archive_name, target, config): with manifest_file.open(mode="w") as f: f.write( MANIFEST_TEMPLATE.format( - archive_name=archive_name, target=target, platform=platform, date=DATE, + archive_name=archive_name, + target=target, + config=config, + date=DATE, ) ) def run_kiwix_build( target, - platform, + config, build_deps_only=False, target_only=False, make_release=False, @@ -163,7 +161,7 @@ def run_kiwix_build( command.append("--hide-progress") command.append("--fast-clone") command.append("--assume-packages-installed") - command.extend(["--target-platform", platform]) + command.extend(["--config", config]) if build_deps_only: command.append("--build-deps-only") if target_only: @@ -240,7 +238,7 @@ def upload_archive(archive, project, make_release, dev_branch=None): print_message("No archive {} to upload!", archive) return - if project.startswith("kiwix-") or project in ['libkiwix']: + if project.startswith("kiwix-") or project in ["libkiwix"]: host = "ci@master.download.kiwix.org:30022" dest_path = "/data/download/" else: @@ -264,36 +262,39 @@ def upload_archive(archive, project, make_release, dev_branch=None): # This remove "share/doc" and "share/man" from the thing to copy in the deps archive def filter_install_dir(path): - for dir in path.glob('*'): - if dir.name not in ['share']: + for dir in path.glob("*"): + if dir.name not in ["share"]: yield dir else: - for sub_dir in dir.glob('*'): - if sub_dir.name not in ['doc', 'man']: + for sub_dir in dir.glob("*"): + if sub_dir.name not in ["doc", "man"]: yield sub_dir -# Full: True if we are creating a full archive to be used as cache by kiwix-build (base_deps2_{os}_{platform}_{base_deps_version}.tar.xz) -# Full: False if we are creating a archive to be used as pre-cached dependencies for project's CI (deps2_{os}_{platform}_{target}.tar.xz) + +# Full: True if we are creating a full archive to be used as cache by kiwix-build (base_deps2_{os}_{config}_{base_deps_version}.tar.xz) +# Full: False if we are creating a archive to be used as pre-cached dependencies for project's CI (deps2_{os}_{config}_{target}.tar.xz) def make_deps_archive(target=None, name=None, full=False): archive_name = name or "deps2_{}_{}_{}.tar.xz".format( - OS_NAME, PLATFORM_TARGET, target + OS_NAME, COMPILE_CONFIG, target ) print_message("Create archive {}.", archive_name) files_to_archive = list(filter_install_dir(INSTALL_DIR)) files_to_archive += HOME.glob("BUILD_*/LOGS") - if PLATFORM_TARGET == "apple_all_static": - for subplatform in AppleXCFramework.subPlatformNames: - base_dir = HOME / "BUILD_{}".format(subplatform) + if COMPILE_CONFIG == "apple_all_static": + for subconfig in AppleXCFramework.subConfigNames: + base_dir = HOME / "BUILD_{}".format(subconfig) files_to_archive += filter_install_dir(base_dir / "INSTALL") if (base_dir / "meson_cross_file.txt").exists(): files_to_archive.append(base_dir / "meson_cross_file.txt") - if PLATFORM_TARGET.endswith("_mixed"): - static_platform = PLATFORM_TARGET.replace("_mixed", "_static") - files_to_archive += filter_install_dir(HOME / ("BUILD_" + static_platform) / "INSTALL") - if PLATFORM_TARGET.startswith("android_"): + if COMPILE_CONFIG.endswith("_mixed"): + static_config = COMPILE_CONFIG.replace("_mixed", "_static") + files_to_archive += filter_install_dir( + HOME / ("BUILD_" + static_config) / "INSTALL" + ) + if COMPILE_CONFIG.startswith("android_"): files_to_archive += filter_install_dir(HOME / "BUILD_neutral" / "INSTALL") - base_dir = HOME / "BUILD_{}".format(PLATFORM_TARGET) + base_dir = HOME / "BUILD_{}".format(COMPILE_CONFIG) if (base_dir / "meson_cross_file.txt").exists(): files_to_archive.append(base_dir / "meson_cross_file.txt") # Copy any toolchain @@ -303,7 +304,7 @@ def make_deps_archive(target=None, name=None, full=False): files_to_archive.append(BASE_DIR / "meson_cross_file.txt") manifest_file = BASE_DIR / "manifest.txt" - write_manifest(manifest_file, archive_name, target, PLATFORM_TARGET) + write_manifest(manifest_file, archive_name, target, COMPILE_CONFIG) files_to_archive.append(manifest_file) relative_path = HOME @@ -311,9 +312,9 @@ def make_deps_archive(target=None, name=None, full=False): files_to_archive += ARCHIVE_DIR.glob(".*_ok") files_to_archive += BASE_DIR.glob("*/.*_ok") # Add also static build for mixed target - if PLATFORM_TARGET.endswith("_mixed"): - static_platform = PLATFORM_TARGET.replace("_mixed", "_static") - files_to_archive += (HOME / ("BUILD_" + static_platform)).glob("*/.*_ok") + if COMPILE_CONFIG.endswith("_mixed"): + static_config = COMPILE_CONFIG.replace("_mixed", "_static") + files_to_archive += (HOME / ("BUILD_" + static_config)).glob("*/.*_ok") # Native dyn and static is needed for potential cross compilation that use native tools (icu) files_to_archive += (HOME / "BUILD_native_dyn").glob("*/.*_ok") files_to_archive += (HOME / "BUILD_native_static").glob("*/.*_ok") @@ -391,7 +392,7 @@ def create_desktop_image(make_release): postfix = DATE src_dir = SOURCE_DIR / "kiwix-desktop" - if PLATFORM_TARGET == "flatpak": + if COMPILE_CONFIG == "flatpak": build_path = BASE_DIR / "org.kiwix.desktop.flatpak" app_name = "org.kiwix.desktop.{}.flatpak".format(postfix) print_message("archive is {}", build_path) @@ -459,7 +460,6 @@ def update_flathub_git(): def fix_macos_rpath(project): - base_dir, export_files = EXPORT_FILES[project] for file in filter(lambda f: f.endswith(".dylib"), export_files): lib = base_dir / file @@ -478,22 +478,35 @@ def trigger_workflow(repo, workflow="docker.yml", ref="main", inputs=None): ref: branch or tag name inputs: dict of inputs to pass to the workflow""" print_message( - "triggering workflow `{workflow}` on {repo}@{ref} " - "with inputs={inputs}", workflow=workflow, repo=repo, ref=ref, inputs=inputs) + "triggering workflow `{workflow}` on {repo}@{ref} " "with inputs={inputs}", + workflow=workflow, + repo=repo, + ref=ref, + inputs=inputs, + ) url = "{base_url}/repos/{repo}/actions/workflows/{workflow}/dispatches".format( base_url=os.getenv("GITHUB_API_URL", "https://api.github.com"), - repo=repo, workflow=workflow) + repo=repo, + workflow=workflow, + ) - resp = requests.post(url, headers={ + resp = requests.post( + url, + headers={ "Content-Type": "application/json", - "Authorization": "token {token}".format( - token=os.getenv('GITHUB_PAT', '')), + "Authorization": "token {token}".format(token=os.getenv("GITHUB_PAT", "")), "Accept": "application/vnd.github.v3+json", - }, json={"ref": ref, "inputs": inputs}, timeout=5) + }, + json={"ref": ref, "inputs": inputs}, + timeout=5, + ) if resp.status_code != 204: - raise ValueError("Unexpected HTTP {code}: {reason}".format( - code=resp.status_code, reason=resp.reason)) + raise ValueError( + "Unexpected HTTP {code}: {reason}".format( + code=resp.status_code, reason=resp.reason + ) + ) def trigger_docker_publish(target): @@ -501,13 +514,14 @@ def trigger_docker_publish(target): return version = get_postfix(target) - repo = { - "zim-tools": "openzim/zim-tools", - "kiwix-tools": "kiwix/kiwix-tools"}.get(target) + repo = {"zim-tools": "openzim/zim-tools", "kiwix-tools": "kiwix/kiwix-tools"}.get( + target + ) try: - trigger_workflow(repo, workflow="docker.yml", ref="main", - inputs={"version": version}) + trigger_workflow( + repo, workflow="docker.yml", ref="main", inputs={"version": version} + ) print_message("triggered docker workflow on {repo}", repo=repo) except Exception as exc: print_message("Error triggering workflow: {exc}", exc=exc) @@ -515,39 +529,52 @@ def trigger_docker_publish(target): def notarize_macos_build(project): - """ sign and notarize files for macOS + """sign and notarize files for macOS - Expects the following environment: - - `SIGNING_IDENTITY` environ with Certificate name/identity - - `KEYCHAIN` environ with path to the keychain storing credentials - - `KEYCHAIN_PROFILE` environ with name of the profile in that keychain - - `KEYCHAIN_PASSWORD` environ with password to unlock the keychain - """ + Expects the following environment: + - `SIGNING_IDENTITY` environ with Certificate name/identity + - `KEYCHAIN` environ with path to the keychain storing credentials + - `KEYCHAIN_PROFILE` environ with name of the profile in that keychain + - `KEYCHAIN_PASSWORD` environ with password to unlock the keychain + """ if project != "libzim": return # currently only supports libzim use case: sign every dylib base_dir, export_files = EXPORT_FILES[project] - filepaths = [base_dir.joinpath(file) - for file in filter(lambda f: f.endswith(".dylib"), export_files) - if not base_dir.joinpath(file).is_symlink()] + filepaths = [ + base_dir.joinpath(file) + for file in filter(lambda f: f.endswith(".dylib"), export_files) + if not base_dir.joinpath(file).is_symlink() + ] if not filepaths: return for filepath in filepaths: - subprocess.check_call(["/usr/bin/codesign", "--force", "--sign", - os.getenv("SIGNING_IDENTITY", "no-signing-ident"), - "--keychain", - os.getenv("KEYCHAIN", "no-keychain-path"), - str(filepath), "--deep", "--timestamp"], env=os.environ) + subprocess.check_call( + [ + "/usr/bin/codesign", + "--force", + "--sign", + os.getenv("SIGNING_IDENTITY", "no-signing-ident"), + "--keychain", + os.getenv("KEYCHAIN", "no-keychain-path"), + str(filepath), + "--deep", + "--timestamp", + ], + env=os.environ, + ) # create a zip of the dylibs and upload for notarization zip_name = "{}.zip".format(project) subprocess.check_call( ["/usr/bin/ditto", "-c", "-k", "--keepParent"] - + [str(f) for f in filepaths] + [zip_name], - env=os.environ) + + [str(f) for f in filepaths] + + [zip_name], + env=os.environ, + ) # make sure keychain is unlocked subprocess.check_call( diff --git a/.github/scripts/compile_all_deps.py b/.github/scripts/compile_all_deps.py index 0912763..8a62c73 100755 --- a/.github/scripts/compile_all_deps.py +++ b/.github/scripts/compile_all_deps.py @@ -6,13 +6,13 @@ from common import ( run_kiwix_build, make_deps_archive, upload, - PLATFORM_TARGET, + COMPILE_CONFIG, DEV_BRANCH, ) from build_definition import select_build_targets, DEPS for target in select_build_targets(DEPS): - run_kiwix_build(target, platform=PLATFORM_TARGET, build_deps_only=True) + run_kiwix_build(target, config=COMPILE_CONFIG, build_deps_only=True) archive_file = make_deps_archive(target=target) if DEV_BRANCH: destination = "/data/tmp/ci/dev_preview/" + DEV_BRANCH diff --git a/.github/scripts/ensure_base_deps.py b/.github/scripts/ensure_base_deps.py index ffa012e..0995f9a 100755 --- a/.github/scripts/ensure_base_deps.py +++ b/.github/scripts/ensure_base_deps.py @@ -13,10 +13,11 @@ from common import ( upload, make_deps_archive, HOME, - PLATFORM_TARGET, + COMPILE_CONFIG, OS_NAME, ) + def download_base_archive(base_name): url = "http://tmp.kiwix.org/ci/{}".format(base_name) file_path = str(HOME / base_name) @@ -30,14 +31,15 @@ def download_base_archive(base_name): file.write(batch) return file_path -ARCHIVE_NAME_TEMPLATE = "base_deps2_{os}_{platform}_{version}.tar.xz" -if PLATFORM_TARGET == 'flatpak': +ARCHIVE_NAME_TEMPLATE = "base_deps2_{os}_{config}_{version}.tar.xz" + +if COMPILE_CONFIG == "flatpak": base_dep_archive_name = "base_deps2_flatpak.tar.xz" else: base_dep_archive_name = ARCHIVE_NAME_TEMPLATE.format( os=OS_NAME, - platform=PLATFORM_TARGET, + config=COMPILE_CONFIG, version=base_deps_meta_version, ) @@ -48,11 +50,11 @@ try: f.extractall(str(HOME)) os.remove(str(local_filename)) except URLError: - if PLATFORM_TARGET == "flatpak": + if COMPILE_CONFIG == "flatpak": print_message("Cannot get archive. Move on") else: print_message("Cannot get archive. Build dependencies") - run_kiwix_build("alldependencies", platform=PLATFORM_TARGET) + run_kiwix_build("alldependencies", config=COMPILE_CONFIG) archive_file = make_deps_archive(name=base_dep_archive_name, full=True) upload(archive_file, "ci@tmp.kiwix.org:30022", "/data/tmp/ci") os.remove(str(archive_file)) diff --git a/.github/scripts/upload_failure_logs.sh b/.github/scripts/upload_failure_logs.sh index a284afb..0714d37 100755 --- a/.github/scripts/upload_failure_logs.sh +++ b/.github/scripts/upload_failure_logs.sh @@ -4,7 +4,7 @@ set -e cd $HOME -ARCHIVE_NAME=fail_log_${OS_NAME}_${PLATFORM_TARGET}.tar.gz +ARCHIVE_NAME=fail_log_${OS_NAME}_${COMPILE_CONFIG}.tar.gz tar -czf ${ARCHIVE_NAME} $HOME/BUILD_* $HOME/SOURCE $HOME/LOGS $HOME/TOOLCHAINS echo "Uploading archive $ARCHIVE_NAME" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da9d298..65da67b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,7 @@ jobs: strategy: fail-fast: false matrix: - target: + config: - native_static - native_dyn - native_mixed @@ -37,13 +37,13 @@ jobs: - android_x86_64 image_variant: ['focal'] include: - - target: native_mixed + - config: native_mixed image_variant: bionic - - target: aarch64_mixed + - config: aarch64_mixed image_variant: bionic - - target: win32_static + - config: win32_static image_variant: f35 - - target: win32_dyn + - config: win32_dyn image_variant: f35 env: HOME: /home/runner @@ -74,26 +74,26 @@ jobs: cd $HOME kiwix-build/.github/scripts/ensure_base_deps.py env: - PLATFORM_TARGET: ${{matrix.target}} + COMPILE_CONFIG: ${{matrix.config}} - name: Compile all deps shell: bash run: | cd $HOME kiwix-build/.github/scripts/compile_all_deps.py env: - PLATFORM_TARGET: ${{matrix.target}} + COMPILE_CONFIG: ${{matrix.config}} - name: Build projects shell: bash run: | cd $HOME kiwix-build/.github/scripts/build_projects.py env: - PLATFORM_TARGET: ${{matrix.target}} + COMPILE_CONFIG: ${{matrix.config}} - name: Upload failure logs if: failure() run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh env: - PLATFORM_TARGET: ${{matrix.target}} + COMPILE_CONFIG: ${{matrix.config}} Flatpak: strategy: @@ -101,7 +101,7 @@ jobs: env: HOME: /home/runner SSH_KEY: /tmp/id_rsa - PLATFORM_TARGET: flatpak + COMPILE_CONFIG: flatpak OS_NAME: focal runs-on: ubuntu-22.04 steps: @@ -142,7 +142,7 @@ jobs: strategy: fail-fast: false matrix: - target: + config: - native_dyn - native_static - native_mixed @@ -185,23 +185,23 @@ jobs: cd $HOME $GITHUB_WORKSPACE/.github/scripts/ensure_base_deps.py env: - PLATFORM_TARGET: ${{matrix.target}} + COMPILE_CONFIG: ${{matrix.config}} - name: Compile all deps shell: bash run: | cd $HOME $GITHUB_WORKSPACE/.github/scripts/compile_all_deps.py env: - PLATFORM_TARGET: ${{matrix.target}} + COMPILE_CONFIG: ${{matrix.config}} - name: Build projects shell: bash run: | cd $HOME $GITHUB_WORKSPACE/.github/scripts/build_projects.py env: - PLATFORM_TARGET: ${{matrix.target}} + COMPILE_CONFIG: ${{matrix.config}} - name: Upload failure logs if: failure() run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.sh env: - PLATFORM_TARGET: ${{matrix.target}} + COMPILE_CONFIG: ${{matrix.config}} diff --git a/.github/workflows/releaseNigthly.yml b/.github/workflows/releaseNigthly.yml index 32c48d2..7d786ff 100644 --- a/.github/workflows/releaseNigthly.yml +++ b/.github/workflows/releaseNigthly.yml @@ -12,7 +12,7 @@ jobs: strategy: fail-fast: false matrix: - target: + config: - native_static - native_mixed - native_dyn @@ -34,11 +34,11 @@ jobs: - android_x86_64 image_variant: ['focal'] include: - - target: native_mixed + - config: native_mixed image_variant: bionic - - target: aarch64_mixed + - config: aarch64_mixed image_variant: bionic - - target: win32_static + - config: win32_static image_variant: f35 env: HOME: /home/runner @@ -69,21 +69,21 @@ jobs: cd $HOME kiwix-build/.github/scripts/ensure_base_deps.py env: - PLATFORM_TARGET: ${{matrix.target}} + COMPILE_CONFIG: ${{matrix.config}} - name: Build release shell: bash run: | cd $HOME kiwix-build/.github/scripts/build_release_nightly.py env: - PLATFORM_TARGET: ${{matrix.target}} + COMPILE_CONFIG: ${{matrix.config}} BINTRAY_USER: kiwix BINTRAY_PASS: ${{secrets.bintray_pass}} - name: Upload failure logs if: failure() run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh env: - PLATFORM_TARGET: ${{matrix.target}} + COMPILE_CONFIG: ${{matrix.config}} Flatpak: strategy: @@ -91,7 +91,7 @@ jobs: env: HOME: /home/runner SSH_KEY: /tmp/id_rsa - PLATFORM_TARGET: flatpak + COMPILE_CONFIG: flatpak OS_NAME: focal runs-on: ubuntu-22.04 steps: @@ -132,7 +132,7 @@ jobs: strategy: fail-fast: false matrix: - target: + config: - native_dyn - native_static - native_mixed @@ -195,26 +195,26 @@ jobs: cd $HOME $GITHUB_WORKSPACE/.github/scripts/ensure_base_deps.py env: - PLATFORM_TARGET: ${{matrix.target}} + COMPILE_CONFIG: ${{matrix.config}} - name: Build release shell: bash run: | cd $HOME $GITHUB_WORKSPACE/.github/scripts/build_release_nightly.py env: - PLATFORM_TARGET: ${{matrix.target}} + COMPILE_CONFIG: ${{matrix.config}} - name: Upload failure logs if: failure() run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.sh env: - PLATFORM_TARGET: ${{matrix.target}} + COMPILE_CONFIG: ${{matrix.config}} Trigger_Docker: needs: [Linux] runs-on: ubuntu-22.04 env: - PLATFORM_TARGET: native_static + COMPILE_CONFIG: native_static OS_NAME: linux steps: - name: Checkout code From d4acd108caf60fcfc816db19bfb8c735cd3d3fa5 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 8 Feb 2024 16:12:13 +0100 Subject: [PATCH 6/6] Bump base_deps_meta_version. --- kiwixbuild/versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kiwixbuild/versions.py b/kiwixbuild/versions.py index 588e828..5da3c41 100644 --- a/kiwixbuild/versions.py +++ b/kiwixbuild/versions.py @@ -39,7 +39,7 @@ release_versions = { # This is the "version" of the whole base_deps_versions dict. # Change this when you change base_deps_versions. -base_deps_meta_version = "96" +base_deps_meta_version = "97" base_deps_versions = { "zlib": "1.2.12",