From 696242a0b14c6d22fa08129576fe6af2447254fa Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 25 Mar 2024 15:17:10 +0100 Subject: [PATCH 01/12] Introduce helper `get_column_name()` --- .github/scripts/build_definition.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/scripts/build_definition.py b/.github/scripts/build_definition.py index e3ecd80..e953cee 100644 --- a/.github/scripts/build_definition.py +++ b/.github/scripts/build_definition.py @@ -135,7 +135,7 @@ def select_build_targets(criteria): raise ValueError("No definition match with current context.") -def get_platform_name(): +def get_column_value(column_name): from common import COMPILE_CONFIG, OS_NAME context = Context(COMPILE_CONFIG=COMPILE_CONFIG, OS_NAME=OS_NAME) @@ -143,7 +143,11 @@ def get_platform_name(): reader = csv.DictReader(strip_array(BUILD_DEF), dialect=TableDialect()) for row in reader: if context.match(row): - name = row["platform_name"] + name = row[column_name] return name or None raise ValueError("No definition match with current context.") + + +def get_platform_name(): + return get_column_value("platform_name") From c6936ebc724d3ddf6084b6ea52cab019fb9b95e6 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 25 Mar 2024 15:17:57 +0100 Subject: [PATCH 02/12] Introduce new dependency_name in build_definition --- .github/scripts/build_definition.py | 90 +++++++++++++++-------------- 1 file changed, 47 insertions(+), 43 deletions(-) diff --git a/.github/scripts/build_definition.py b/.github/scripts/build_definition.py index e953cee..76af139 100644 --- a/.github/scripts/build_definition.py +++ b/.github/scripts/build_definition.py @@ -15,52 +15,52 @@ import csv, io, re # 'D' letter means we trigger the docker forkflow to build the docker image. # If a cell contains several letters, all are done. BUILD_DEF = """ - | OS_NAME | COMPILE_CONFIG | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | platform_name | - ===================================================================================================================== + | OS_NAME | COMPILE_CONFIG | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | platform_name | dependency_name | + ============================================================================================================================================== # Bionic is a special case as we need to compile libzim on old arch for python - | bionic | native_mixed | BP | | | | | linux-x86_64-bionic | - | bionic | aarch64_mixed | BP | | | | | linux-aarch64-bionic | - -------------------------------------------------------------------------------------------------------------------- + | bionic | native_mixed | BP | | | | | linux-x86_64-bionic | | + | bionic | aarch64_mixed | BP | | | | | linux-aarch64-bionic | | + ---------------------------------------------------------------------------------------------------------------------------------------------- # Osx builds, build binaries on native_dyn and native_static. On anyother things, build only the libraries - | macos | native_dyn | d | d | dB | B | | | - | macos | native_static | | | BP | BP | | macos-x86_64 | - | macos | native_mixed | BP | BP | | | | macos-x86_64 | - | macos | iOS_arm64 | dB | dB | | | | | - | macos | iOSSimulator_x86_64| dB | dB | | | | | - | macos | iOSSimulator_arm64 | dB | dB | | | | | - | macos | macOS_arm64_static | | | BP | BP | | macos-arm64 | - | macos | macOS_arm64_mixed | BP | BP | | | | macos-arm64 | - | macos | macOS_x86_64 | B | B | | | | | - | macos | apple_all_static | | BP | | | | xcframework | - ---------------------------------------------------------------------------------------------- - | | flatpak | | | | | BP | | - | | native_static | d | d | dBPSD | dBPSD | | linux-x86_64 | - | | native_mixed | BPS | BPS | | | | linux-x86_64 | - | | native_dyn | d | d | dB | dB | dBPS | | + | macos | native_dyn | d | d | dB | B | | | macos-x86_64-dyn | + | macos | native_static | | | BP | BP | | macos-x86_64 | | + | macos | native_mixed | BP | BP | | | | macos-x86_64 | | + | macos | iOS_arm64 | dB | dB | | | | | ios-arm64-dyn | + | macos | iOSSimulator_x86_64| dB | dB | | | | | ios-x86_64-dyn | + | macos | iOSSimulator_arm64 | B | B | | | | | | + | macos | macOS_arm64_static | | | BP | BP | | macos-arm64 | | + | macos | macOS_arm64_mixed | BP | BP | | | | macos-arm64 | | + | macos | macOS_x86_64 | B | B | | | | | | + | macos | apple_all_static | | BP | | | | xcframework | | + ---------------------------------------------------------------------------------------------------------------------------------------------- + | | flatpak | | | | | BP | | | + | | native_static | d | d | dBPSD | dBPSD | | linux-x86_64 | linux-x86_64-static | + | | native_mixed | BPS | BPS | | | | linux-x86_64 | | + | | native_dyn | d | d | dB | dB | dBPS | | linux-x86_64-dyn | # libzim CI is building alpine_dyn but not us - | | android_arm | dBP | dBP | | | | android-arm | - | | android_arm64 | dBP | dBP | | | | android-arm64 | - | | android_x86 | BP | BP | | | | android-x86 | - | | android_x86_64 | BP | BP | | | | android-x86_64 | - | | armv6_static | | | BP | BP | | linux-armv6 | - | | armv6_mixed | BP | | | | | linux-armv6 | - | | armv6_dyn | | | B | B | | | - | | armv8_static | | | BP | BP | | linux-armv8 | - | | armv8_mixed | BP | | | | | linux-armv8 | - | | armv8_dyn | | | B | B | | | - | | aarch64_static | | | BP | BP | | linux-aarch64 | - | | aarch64_mixed | BP | | | | | linux-aarch64 | - | | aarch64_dyn | d | | B | B | | | - | | aarch64_musl_static| | | BP | BP | | linux-aarch64-musl | - | | aarch64_musl_mixed | BP | | | | | linux-aarch64-musl | - | | aarch64_musl_dyn | d | | B | B | | | - | | x86-64_musl_static | | | BP | BP | | linux-x86_64-musl | - | | x86-64_musl_mixed | BP | | | | | linux-x86_64-musl | - | | win32_static | d | dB | dBP | dBP | | win-i686 | - | | win32_dyn | d | dB | dB | dB | | | - | | i586_static | | | BP | BP | | linux-i586 | - | | i586_dyn | | | B | B | | | - | | wasm | dBP | | | | | wasm-emscripten | + | | android_arm | dBP | dBP | | | | android-arm | android-arm | + | | android_arm64 | dBP | dBP | | | | android-arm64 | android-arm64 | + | | android_x86 | BP | BP | | | | android-x86 | | + | | android_x86_64 | BP | BP | | | | android-x86_64 | | + | | armv6_static | | | BP | BP | | linux-armv6 | | + | | armv6_mixed | BP | | | | | linux-armv6 | | + | | armv6_dyn | | | B | B | | | | + | | armv8_static | | | BP | BP | | linux-armv8 | | + | | armv8_mixed | BP | | | | | linux-armv8 | | + | | armv8_dyn | | | B | B | | | | + | | aarch64_static | | | BP | BP | | linux-aarch64 | | + | | aarch64_mixed | BP | | | | | linux-aarch64 | | + | | aarch64_dyn | d | | B | B | | | linux-aarch64-dyn | + | | aarch64_musl_static| | | BP | BP | | linux-aarch64-musl | | + | | aarch64_musl_mixed | BP | | | | | linux-aarch64-musl | | + | | aarch64_musl_dyn | d | | B | B | | | linux-aarch64-musl-dyn | + | | x86-64_musl_static | | | BP | BP | | linux-x86_64-musl | | + | | x86-64_musl_mixed | BP | | | | | linux-x86_64-musl | | + | | win32_static | d | dB | dBP | dBP | | win-i686 |win32-static | + | | win32_dyn | d | dB | dB | dB | | |win32-dyn | + | | i586_static | | | BP | BP | | linux-i586 | | + | | i586_dyn | | | B | B | | | | + | | wasm | dBP | | | | | wasm-emscripten | wasm | """ @@ -151,3 +151,7 @@ def get_column_value(column_name): def get_platform_name(): return get_column_value("platform_name") + + +def get_dependency_archive_name(): + return get_column_value("dependency_name") From 944eedcf565917a09b535c3291357d27ea765ab7 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 25 Mar 2024 15:33:46 +0100 Subject: [PATCH 03/12] Publish deps archive name using declared `dependency_name`. Instead of compose it from `OS_NAME`, `COMPILE_CONFIG`. --- .github/scripts/common.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/common.py b/.github/scripts/common.py index 360f193..85d96bd 100644 --- a/.github/scripts/common.py +++ b/.github/scripts/common.py @@ -10,7 +10,7 @@ import shutil import requests -from build_definition import get_platform_name +from build_definition import get_platform_name, get_dependency_archive_name from kiwixbuild.dependencies.apple_xcframework import AppleXCFramework from kiwixbuild.versions import ( @@ -274,8 +274,8 @@ def filter_install_dir(path): # Full: True if we are creating a full archive to be used as cache by kiwix-build (base_deps2_{os}_{config}_{base_deps_version}.tar.xz) # Full: False if we are creating a archive to be used as pre-cached dependencies for project's CI (deps2_{os}_{config}_{target}.tar.xz) def make_deps_archive(target=None, name=None, full=False): - archive_name = name or "deps2_{}_{}_{}.tar.xz".format( - OS_NAME, COMPILE_CONFIG, target + archive_name = name or "deps_{}_{}.tar.xz".format( + get_dependency_archive_name(), target ) print_message("Create archive {}.", archive_name) files_to_archive = list(filter_install_dir(INSTALL_DIR)) From f42999199a69528154e99ac941bb04dc9a8d0309 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 25 Mar 2024 16:59:00 +0100 Subject: [PATCH 04/12] `pj` now normalize path. --- kiwixbuild/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kiwixbuild/utils.py b/kiwixbuild/utils.py index 1cdc990..322eccc 100644 --- a/kiwixbuild/utils.py +++ b/kiwixbuild/utils.py @@ -13,7 +13,9 @@ from collections import namedtuple, defaultdict from kiwixbuild._global import neutralEnv, option -pj = os.path.join + +def pj(*args): + return os.path.normpath(os.path.join(*args)) COLORS = { From f0b214681c4616c4ca1112eab0e6ccc462344466 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 25 Mar 2024 16:59:21 +0100 Subject: [PATCH 05/12] Properly escape char in `progress_chars` --- kiwixbuild/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kiwixbuild/utils.py b/kiwixbuild/utils.py index 322eccc..8cce351 100644 --- a/kiwixbuild/utils.py +++ b/kiwixbuild/utils.py @@ -79,7 +79,7 @@ def remove_duplicates(iterable, key_function=None): def get_sha256(path): - progress_chars = "/-\|" + progress_chars = "/-\\|" current = 0 batch_size = 1024 * 8 sha256 = hashlib.sha256() @@ -139,7 +139,7 @@ def download_remote(what, where): context = None batch_size = 1024 * 8 extra_args = {"context": context} if sys.version_info >= (3, 4, 3) else {} - progress_chars = "/-\|" + progress_chars = "/-\\|" try: with urllib.request.urlopen(what.url, **extra_args) as resource, open( file_path, "wb" From 734c1ea6fcc41fc38a0e029aa248cbbf3c3337fe Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 26 Mar 2024 11:24:17 +0100 Subject: [PATCH 06/12] Update base_deps_version --- kiwixbuild/versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kiwixbuild/versions.py b/kiwixbuild/versions.py index 1518a87..b61b83a 100644 --- a/kiwixbuild/versions.py +++ b/kiwixbuild/versions.py @@ -39,7 +39,7 @@ release_versions = { # This is the "version" of the whole base_deps_versions dict. # Change this when you change base_deps_versions. -base_deps_meta_version = "98" +base_deps_meta_version = "99" base_deps_versions = { "zlib": "1.2.12", From 0c0eac69c40e4de6867195e28943f302124ade59 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 27 Mar 2024 17:31:40 +0100 Subject: [PATCH 07/12] Introduce arch_name to name build dir base on arch name instead of config. --- .github/scripts/common.py | 1 + kiwixbuild/__init__.py | 12 ++++++++++++ kiwixbuild/buildenv.py | 5 ++++- kiwixbuild/configs/android.py | 4 ++++ kiwixbuild/configs/base.py | 4 ++++ kiwixbuild/configs/flatpak.py | 1 + kiwixbuild/configs/ios.py | 12 ++++++++++++ kiwixbuild/configs/native.py | 9 +++++++++ kiwixbuild/configs/neutral.py | 1 + 9 files changed, 48 insertions(+), 1 deletion(-) diff --git a/.github/scripts/common.py b/.github/scripts/common.py index 85d96bd..7024332 100644 --- a/.github/scripts/common.py +++ b/.github/scripts/common.py @@ -161,6 +161,7 @@ def run_kiwix_build( command.append("--hide-progress") command.append("--fast-clone") command.append("--assume-packages-installed") + command.append("--use-target-arch-name") command.extend(["--config", config]) if build_deps_only: command.append("--build-deps-only") diff --git a/kiwixbuild/__init__.py b/kiwixbuild/__init__.py index 33f2926..5b1b878 100644 --- a/kiwixbuild/__init__.py +++ b/kiwixbuild/__init__.py @@ -125,6 +125,18 @@ def parse_args(): "to develop with the cloned sources." ), ) + subgroup.add_argument( + "--use-target-arch-name", + action="store_true", + help=( + "Name the build directory using the arch name instead of the config name.\n" + "Different configs may create binary for the same arch so this option is " + "not recommended when working with several config on the same computer.\n" + "However, when generating dependencies for other it is better to have a " + "directory named using the target instead of the used config.\n" + "Intended to be used in CI only." + ), + ) options = parser.parse_args() if not options.android_arch: diff --git a/kiwixbuild/buildenv.py b/kiwixbuild/buildenv.py index 37fe7fc..50552b2 100644 --- a/kiwixbuild/buildenv.py +++ b/kiwixbuild/buildenv.py @@ -76,9 +76,12 @@ class NeutralEnv: class BuildEnv: def __init__(self, configInfo): - build_dir = "BUILD_{}".format(configInfo.name) self.configInfo = configInfo self.base_build_dir = pj(option("working_dir"), option("build_dir")) + build_dir = ( + configInfo.arch_name if option("use_target_arch_name") else configInfo.name + ) + build_dir = f"BUILD_{build_dir}" self.build_dir = pj(self.base_build_dir, build_dir) self.install_dir = pj(self.build_dir, "INSTALL") self.toolchain_dir = pj(self.build_dir, "TOOLCHAINS") diff --git a/kiwixbuild/configs/android.py b/kiwixbuild/configs/android.py index 51d6d91..9374944 100644 --- a/kiwixbuild/configs/android.py +++ b/kiwixbuild/configs/android.py @@ -147,6 +147,10 @@ class Android(MetaConfigInfo): name = "android" compatible_hosts = ["fedora", "debian"] + @property + def arch_name(self): + return "multi-linux-android" + @property def subConfigNames(self): return ["android_{}".format(arch) for arch in option("android_arch")] diff --git a/kiwixbuild/configs/base.py b/kiwixbuild/configs/base.py index 43798f2..883f1f7 100644 --- a/kiwixbuild/configs/base.py +++ b/kiwixbuild/configs/base.py @@ -27,6 +27,10 @@ class ConfigInfo(metaclass=_MetaConfig): mixed = False libdir = None + @property + def arch_name(self): + return self.arch_full + @classmethod def get_config(cls, name, targets=None): if name not in cls.all_running_configs: diff --git a/kiwixbuild/configs/flatpak.py b/kiwixbuild/configs/flatpak.py index a49acdd..f481a5d 100644 --- a/kiwixbuild/configs/flatpak.py +++ b/kiwixbuild/configs/flatpak.py @@ -4,6 +4,7 @@ from kiwixbuild._global import option, neutralEnv class FlatpakConfigInfo(ConfigInfo): name = "flatpak" + arch_name = "flatpak" build = "flatpak" static = "" toolchain_names = ["org.kde", "io.qt.qtwebengine"] diff --git a/kiwixbuild/configs/ios.py b/kiwixbuild/configs/ios.py index bce93a3..62a9c1f 100644 --- a/kiwixbuild/configs/ios.py +++ b/kiwixbuild/configs/ios.py @@ -24,6 +24,10 @@ class AppleConfigInfo(ConfigInfo): super().__init__(*args, **kwargs) self._root_path = None + @property + def arch_name(self): + return self.target + @property def root_path(self): if self._root_path is None: @@ -209,6 +213,10 @@ class IOS(MetaConfigInfo): name = "iOS_multi" compatible_hosts = ["Darwin"] + @property + def arch_name(self): + return self.name + @property def subConfigNames(self): return ["iOS_{}".format(arch) for arch in option("ios_arch")] @@ -225,6 +233,10 @@ class AppleStaticAll(MetaConfigInfo): name = "apple_all_static" compatible_hosts = ["Darwin"] + @property + def arch_name(self): + return self.name + @property def subConfigNames(self): return AppleXCFramework.subConfigNames diff --git a/kiwixbuild/configs/native.py b/kiwixbuild/configs/native.py index 6bf3e36..f8ab8ef 100644 --- a/kiwixbuild/configs/native.py +++ b/kiwixbuild/configs/native.py @@ -3,6 +3,9 @@ from .base import ConfigInfo, MixedMixin from kiwixbuild.utils import pj from kiwixbuild._global import option, neutralEnv from kiwixbuild.configs.ios import MIN_MACOS_VERSION +import sysconfig +import platform +import sys class NativeConfigInfo(ConfigInfo): @@ -16,6 +19,12 @@ class NativeConfigInfo(ConfigInfo): env["CFLAGS"] += f"-mmacosx-version-min={MIN_MACOS_VERSION}" return env + @property + def arch_name(self): + if sys.platform == "darwin": + return f"{platform.machine()}-apple-darwin" + return sysconfig.get_platform() + class NativeDyn(NativeConfigInfo): name = "native_dyn" diff --git a/kiwixbuild/configs/neutral.py b/kiwixbuild/configs/neutral.py index f5ae53e..f237c48 100644 --- a/kiwixbuild/configs/neutral.py +++ b/kiwixbuild/configs/neutral.py @@ -3,6 +3,7 @@ from .base import ConfigInfo class NeutralConfigInfo(ConfigInfo): name = "neutral" + arch_name = "neutral" static = "" compatible_hosts = ["fedora", "debian", "Darwin"] From afda1f6673795544246a5327c2ea0218aa9f4a08 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 2 Apr 2024 16:27:19 +0200 Subject: [PATCH 08/12] =?UTF-8?q?Make=20CI=C2=A0common=20script=20use=20th?= =?UTF-8?q?e=20correct=20build=20dir.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add an option to kiwix-build to get it. --- .github/scripts/common.py | 34 ++++++++++++++++++++++------------ kiwixbuild/__init__.py | 8 +++++++- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/.github/scripts/common.py b/.github/scripts/common.py index 7024332..cb668ef 100644 --- a/.github/scripts/common.py +++ b/.github/scripts/common.py @@ -20,11 +20,23 @@ from kiwixbuild.versions import ( ) +def get_build_dir(config) -> Path: + command = ["kiwix-build"] + command.extend(["--config", config]) + command.append("--get-build-dir") + command.append("--use-target-arch-name") + return Path( + subprocess.run(command, cwd=str(HOME), check=True, stdout=subprocess.PIPE) + .stdout[:-1] + .decode("utf8") + ) + + COMPILE_CONFIG = _environ["COMPILE_CONFIG"] OS_NAME = _environ["OS_NAME"] HOME = Path(os.path.expanduser("~")) -BASE_DIR = HOME / "BUILD_{}".format(COMPILE_CONFIG) +BASE_DIR = get_build_dir(COMPILE_CONFIG) SOURCE_DIR = HOME / "SOURCE" ARCHIVE_DIR = HOME / "ARCHIVE" TOOLCHAIN_DIR = BASE_DIR / "TOOLCHAINS" @@ -283,19 +295,17 @@ def make_deps_archive(target=None, name=None, full=False): files_to_archive += HOME.glob("BUILD_*/LOGS") if COMPILE_CONFIG == "apple_all_static": for subconfig in AppleXCFramework.subConfigNames: - base_dir = HOME / "BUILD_{}".format(subconfig) + base_dir = get_build_dir(subconfig) files_to_archive += filter_install_dir(base_dir / "INSTALL") if (base_dir / "meson_cross_file.txt").exists(): files_to_archive.append(base_dir / "meson_cross_file.txt") if COMPILE_CONFIG.endswith("_mixed"): static_config = COMPILE_CONFIG.replace("_mixed", "_static") - files_to_archive += filter_install_dir( - HOME / ("BUILD_" + static_config) / "INSTALL" - ) + files_to_archive += filter_install_dir(get_build_dir(static_config) / "INSTALL") if COMPILE_CONFIG.startswith("android_"): files_to_archive += filter_install_dir(HOME / "BUILD_neutral" / "INSTALL") - base_dir = HOME / "BUILD_{}".format(COMPILE_CONFIG) + base_dir = get_build_dir(COMPILE_CONFIG) if (base_dir / "meson_cross_file.txt").exists(): files_to_archive.append(base_dir / "meson_cross_file.txt") # Copy any toolchain @@ -315,13 +325,13 @@ def make_deps_archive(target=None, name=None, full=False): # Add also static build for mixed target if COMPILE_CONFIG.endswith("_mixed"): static_config = COMPILE_CONFIG.replace("_mixed", "_static") - files_to_archive += (HOME / ("BUILD_" + static_config)).glob("*/.*_ok") + files_to_archive += get_build_dir(static_config).glob("*/.*_ok") # Native dyn and static is needed for potential cross compilation that use native tools (icu) - files_to_archive += (HOME / "BUILD_native_dyn").glob("*/.*_ok") - files_to_archive += (HOME / "BUILD_native_static").glob("*/.*_ok") - files_to_archive += HOME.glob("BUILD_android*/**/.*_ok") - files_to_archive += HOME.glob("BUILD_macOS*/**/.*_ok") - files_to_archive += HOME.glob("BUILD_iOS*/**/.*_ok") + files_to_archive += get_build_dir("native_dyn").glob("*/.*_ok") + files_to_archive += get_build_dir("native_static").glob("*/.*_ok") + files_to_archive += HOME.glob("BUILD_*android*/**/.*_ok") + files_to_archive += HOME.glob("BUILD_*apple-macos*/**/.*_ok") + files_to_archive += HOME.glob("BUILD_*apple-ios*/**/.*_ok") files_to_archive += SOURCE_DIR.glob("*/.*_ok") files_to_archive += SOURCE_DIR.glob("zim-testing-suite-*/*") diff --git a/kiwixbuild/__init__.py b/kiwixbuild/__init__.py index 5b1b878..8e5f645 100644 --- a/kiwixbuild/__init__.py +++ b/kiwixbuild/__init__.py @@ -137,6 +137,9 @@ def parse_args(): "Intended to be used in CI only." ), ) + subgroup.add_argument( + "--get-build-dir", action="store_true", help="Print the output directory." + ) options = parser.parse_args() if not options.android_arch: @@ -157,4 +160,7 @@ def main(): builder = FlatpakBuilder() else: builder = Builder() - builder.run() + if options.get_build_dir: + print(ConfigInfo.get_config(options.config).buildEnv.build_dir) + else: + builder.run() From 6fbe60386e013aff86843d9e2fce15e01eaee1c8 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 3 Apr 2024 11:02:14 +0200 Subject: [PATCH 09/12] Do not pollute stdout with warning and trace. --- kiwixbuild/buildenv.py | 5 +++-- kiwixbuild/configs/base.py | 1 - 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kiwixbuild/buildenv.py b/kiwixbuild/buildenv.py index 50552b2..1f9f98d 100644 --- a/kiwixbuild/buildenv.py +++ b/kiwixbuild/buildenv.py @@ -36,7 +36,8 @@ class NeutralEnv: if _platform == "Windows": print( "ERROR: kiwix-build is not intented to run on Windows platform.\n" - "It should probably not work, but well, you still can have a try." + "It should probably not work, but well, you still can have a try.", + file=sys.stderr, ) cont = input("Do you want to continue ? [y/N]") if cont.lower() != "y": @@ -70,7 +71,7 @@ class NeutralEnv: if required: sys.exit("ERROR: {} command not found".format(name)) else: - print("WARNING: {} command not found".format(name)) + print("WARNING: {} command not found".format(name), file=sys.stderr) return ["{}_NOT_FOUND".format(name.upper())] diff --git a/kiwixbuild/configs/base.py b/kiwixbuild/configs/base.py index 883f1f7..21049d5 100644 --- a/kiwixbuild/configs/base.py +++ b/kiwixbuild/configs/base.py @@ -135,7 +135,6 @@ def MixedMixin(static_name): static = False def add_targets(self, targetName, targets): - print(targetName) if option("target") == targetName: return super().add_targets(targetName, targets) else: From b880da6ca2c92833e46a7d7c507c63b35d16db86 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 3 Apr 2024 15:16:56 +0200 Subject: [PATCH 10/12] native_static is compatible with Darwin --- kiwixbuild/configs/native.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kiwixbuild/configs/native.py b/kiwixbuild/configs/native.py index f8ab8ef..3d72e95 100644 --- a/kiwixbuild/configs/native.py +++ b/kiwixbuild/configs/native.py @@ -35,7 +35,7 @@ class NativeDyn(NativeConfigInfo): class NativeStatic(NativeConfigInfo): name = "native_static" static = True - compatible_hosts = ["fedora", "debian"] + compatible_hosts = ["fedora", "debian", "Darwin"] class NativeMixed(MixedMixin("native_static"), NativeConfigInfo): From 0d27cbd8cf9559aaf79b07da60e24f6ef0025a21 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 8 Apr 2024 14:48:39 +0200 Subject: [PATCH 11/12] Update github's action to download the right deps archive. --- actions/dl_deps_archive/.gitignore | 1 + actions/dl_deps_archive/dist/index.js | 5 ++--- actions/dl_deps_archive/index.js | 5 ++--- 3 files changed, 5 insertions(+), 6 deletions(-) create mode 100644 actions/dl_deps_archive/.gitignore diff --git a/actions/dl_deps_archive/.gitignore b/actions/dl_deps_archive/.gitignore new file mode 100644 index 0000000..3c3629e --- /dev/null +++ b/actions/dl_deps_archive/.gitignore @@ -0,0 +1 @@ +node_modules diff --git a/actions/dl_deps_archive/dist/index.js b/actions/dl_deps_archive/dist/index.js index 7660fb4..63352d6 100644 --- a/actions/dl_deps_archive/dist/index.js +++ b/actions/dl_deps_archive/dist/index.js @@ -30484,7 +30484,6 @@ function addLocalPath(inputPath) { async function run() { try { const base_url = core.getInput("base_url"); - const os_name = getInput("os_name", process.env["OS_NAME"]); const target = core.getInput("target_platform"); const project = getInput( "project", @@ -30501,11 +30500,11 @@ async function run() { let archivePath; try { - const archive_url = `${base_url}/dev_preview/${branch}/deps2_${os_name}_${target}_${project}.tar.xz`; + const archive_url = `${base_url}/dev_preview/${branch}/deps_${target}_${project}.tar.xz`; process.stdout.write("Downloading " + archive_url + "\n"); archivePath = await tc.downloadTool(archive_url); } catch (error) { - const archive_url = `${base_url}/deps2_${os_name}_${target}_${project}.tar.xz`; + const archive_url = `${base_url}/deps_${target}_${project}.tar.xz`; process.stdout.write("Downloading " + archive_url + "\n"); archivePath = await tc.downloadTool(archive_url); } diff --git a/actions/dl_deps_archive/index.js b/actions/dl_deps_archive/index.js index 00b3367..7d0b12c 100644 --- a/actions/dl_deps_archive/index.js +++ b/actions/dl_deps_archive/index.js @@ -18,7 +18,6 @@ function addLocalPath(inputPath) { async function run() { try { const base_url = core.getInput("base_url"); - const os_name = getInput("os_name", process.env["OS_NAME"]); const target = core.getInput("target_platform"); const project = getInput( "project", @@ -35,11 +34,11 @@ async function run() { let archivePath; try { - const archive_url = `${base_url}/dev_preview/${branch}/deps2_${os_name}_${target}_${project}.tar.xz`; + const archive_url = `${base_url}/dev_preview/${branch}/deps_${target}_${project}.tar.xz`; process.stdout.write("Downloading " + archive_url + "\n"); archivePath = await tc.downloadTool(archive_url); } catch (error) { - const archive_url = `${base_url}/deps2_${os_name}_${target}_${project}.tar.xz`; + const archive_url = `${base_url}/deps_${target}_${project}.tar.xz`; process.stdout.write("Downloading " + archive_url + "\n"); archivePath = await tc.downloadTool(archive_url); } From a93b3c889602a79cf07c9a3b779c17c49249512d Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 8 Apr 2024 16:53:17 +0200 Subject: [PATCH 12/12] Remove the `2` in base_deps name --- .github/scripts/common.py | 4 ++-- .github/scripts/ensure_base_deps.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/scripts/common.py b/.github/scripts/common.py index cb668ef..8eff925 100644 --- a/.github/scripts/common.py +++ b/.github/scripts/common.py @@ -284,8 +284,8 @@ def filter_install_dir(path): yield sub_dir -# Full: True if we are creating a full archive to be used as cache by kiwix-build (base_deps2_{os}_{config}_{base_deps_version}.tar.xz) -# Full: False if we are creating a archive to be used as pre-cached dependencies for project's CI (deps2_{os}_{config}_{target}.tar.xz) +# Full: True if we are creating a full archive to be used as cache by kiwix-build (base_deps_{os}_{config}_{base_deps_version}.tar.xz) +# Full: False if we are creating a archive to be used as pre-cached dependencies for project's CI (deps_{config}_{target}.tar.xz) def make_deps_archive(target=None, name=None, full=False): archive_name = name or "deps_{}_{}.tar.xz".format( get_dependency_archive_name(), target diff --git a/.github/scripts/ensure_base_deps.py b/.github/scripts/ensure_base_deps.py index 0995f9a..f900d78 100755 --- a/.github/scripts/ensure_base_deps.py +++ b/.github/scripts/ensure_base_deps.py @@ -32,10 +32,10 @@ def download_base_archive(base_name): return file_path -ARCHIVE_NAME_TEMPLATE = "base_deps2_{os}_{config}_{version}.tar.xz" +ARCHIVE_NAME_TEMPLATE = "base_deps_{os}_{config}_{version}.tar.xz" if COMPILE_CONFIG == "flatpak": - base_dep_archive_name = "base_deps2_flatpak.tar.xz" + base_dep_archive_name = "base_deps_flatpak.tar.xz" else: base_dep_archive_name = ARCHIVE_NAME_TEMPLATE.format( os=OS_NAME,