diff --git a/.github/scripts/build_definition.py b/.github/scripts/build_definition.py index e3ecd80..76af139 100644 --- a/.github/scripts/build_definition.py +++ b/.github/scripts/build_definition.py @@ -15,52 +15,52 @@ import csv, io, re # 'D' letter means we trigger the docker forkflow to build the docker image. # If a cell contains several letters, all are done. BUILD_DEF = """ - | OS_NAME | COMPILE_CONFIG | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | platform_name | - ===================================================================================================================== + | OS_NAME | COMPILE_CONFIG | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | platform_name | dependency_name | + ============================================================================================================================================== # Bionic is a special case as we need to compile libzim on old arch for python - | bionic | native_mixed | BP | | | | | linux-x86_64-bionic | - | bionic | aarch64_mixed | BP | | | | | linux-aarch64-bionic | - -------------------------------------------------------------------------------------------------------------------- + | bionic | native_mixed | BP | | | | | linux-x86_64-bionic | | + | bionic | aarch64_mixed | BP | | | | | linux-aarch64-bionic | | + ---------------------------------------------------------------------------------------------------------------------------------------------- # Osx builds, build binaries on native_dyn and native_static. On anyother things, build only the libraries - | macos | native_dyn | d | d | dB | B | | | - | macos | native_static | | | BP | BP | | macos-x86_64 | - | macos | native_mixed | BP | BP | | | | macos-x86_64 | - | macos | iOS_arm64 | dB | dB | | | | | - | macos | iOSSimulator_x86_64| dB | dB | | | | | - | macos | iOSSimulator_arm64 | dB | dB | | | | | - | macos | macOS_arm64_static | | | BP | BP | | macos-arm64 | - | macos | macOS_arm64_mixed | BP | BP | | | | macos-arm64 | - | macos | macOS_x86_64 | B | B | | | | | - | macos | apple_all_static | | BP | | | | xcframework | - ---------------------------------------------------------------------------------------------- - | | flatpak | | | | | BP | | - | | native_static | d | d | dBPSD | dBPSD | | linux-x86_64 | - | | native_mixed | BPS | BPS | | | | linux-x86_64 | - | | native_dyn | d | d | dB | dB | dBPS | | + | macos | native_dyn | d | d | dB | B | | | macos-x86_64-dyn | + | macos | native_static | | | BP | BP | | macos-x86_64 | | + | macos | native_mixed | BP | BP | | | | macos-x86_64 | | + | macos | iOS_arm64 | dB | dB | | | | | ios-arm64-dyn | + | macos | iOSSimulator_x86_64| dB | dB | | | | | ios-x86_64-dyn | + | macos | iOSSimulator_arm64 | B | B | | | | | | + | macos | macOS_arm64_static | | | BP | BP | | macos-arm64 | | + | macos | macOS_arm64_mixed | BP | BP | | | | macos-arm64 | | + | macos | macOS_x86_64 | B | B | | | | | | + | macos | apple_all_static | | BP | | | | xcframework | | + ---------------------------------------------------------------------------------------------------------------------------------------------- + | | flatpak | | | | | BP | | | + | | native_static | d | d | dBPSD | dBPSD | | linux-x86_64 | linux-x86_64-static | + | | native_mixed | BPS | BPS | | | | linux-x86_64 | | + | | native_dyn | d | d | dB | dB | dBPS | | linux-x86_64-dyn | # libzim CI is building alpine_dyn but not us - | | android_arm | dBP | dBP | | | | android-arm | - | | android_arm64 | dBP | dBP | | | | android-arm64 | - | | android_x86 | BP | BP | | | | android-x86 | - | | android_x86_64 | BP | BP | | | | android-x86_64 | - | | armv6_static | | | BP | BP | | linux-armv6 | - | | armv6_mixed | BP | | | | | linux-armv6 | - | | armv6_dyn | | | B | B | | | - | | armv8_static | | | BP | BP | | linux-armv8 | - | | armv8_mixed | BP | | | | | linux-armv8 | - | | armv8_dyn | | | B | B | | | - | | aarch64_static | | | BP | BP | | linux-aarch64 | - | | aarch64_mixed | BP | | | | | linux-aarch64 | - | | aarch64_dyn | d | | B | B | | | - | | aarch64_musl_static| | | BP | BP | | linux-aarch64-musl | - | | aarch64_musl_mixed | BP | | | | | linux-aarch64-musl | - | | aarch64_musl_dyn | d | | B | B | | | - | | x86-64_musl_static | | | BP | BP | | linux-x86_64-musl | - | | x86-64_musl_mixed | BP | | | | | linux-x86_64-musl | - | | win32_static | d | dB | dBP | dBP | | win-i686 | - | | win32_dyn | d | dB | dB | dB | | | - | | i586_static | | | BP | BP | | linux-i586 | - | | i586_dyn | | | B | B | | | - | | wasm | dBP | | | | | wasm-emscripten | + | | android_arm | dBP | dBP | | | | android-arm | android-arm | + | | android_arm64 | dBP | dBP | | | | android-arm64 | android-arm64 | + | | android_x86 | BP | BP | | | | android-x86 | | + | | android_x86_64 | BP | BP | | | | android-x86_64 | | + | | armv6_static | | | BP | BP | | linux-armv6 | | + | | armv6_mixed | BP | | | | | linux-armv6 | | + | | armv6_dyn | | | B | B | | | | + | | armv8_static | | | BP | BP | | linux-armv8 | | + | | armv8_mixed | BP | | | | | linux-armv8 | | + | | armv8_dyn | | | B | B | | | | + | | aarch64_static | | | BP | BP | | linux-aarch64 | | + | | aarch64_mixed | BP | | | | | linux-aarch64 | | + | | aarch64_dyn | d | | B | B | | | linux-aarch64-dyn | + | | aarch64_musl_static| | | BP | BP | | linux-aarch64-musl | | + | | aarch64_musl_mixed | BP | | | | | linux-aarch64-musl | | + | | aarch64_musl_dyn | d | | B | B | | | linux-aarch64-musl-dyn | + | | x86-64_musl_static | | | BP | BP | | linux-x86_64-musl | | + | | x86-64_musl_mixed | BP | | | | | linux-x86_64-musl | | + | | win32_static | d | dB | dBP | dBP | | win-i686 |win32-static | + | | win32_dyn | d | dB | dB | dB | | |win32-dyn | + | | i586_static | | | BP | BP | | linux-i586 | | + | | i586_dyn | | | B | B | | | | + | | wasm | dBP | | | | | wasm-emscripten | wasm | """ @@ -135,7 +135,7 @@ def select_build_targets(criteria): raise ValueError("No definition match with current context.") -def get_platform_name(): +def get_column_value(column_name): from common import COMPILE_CONFIG, OS_NAME context = Context(COMPILE_CONFIG=COMPILE_CONFIG, OS_NAME=OS_NAME) @@ -143,7 +143,15 @@ def get_platform_name(): reader = csv.DictReader(strip_array(BUILD_DEF), dialect=TableDialect()) for row in reader: if context.match(row): - name = row["platform_name"] + name = row[column_name] return name or None raise ValueError("No definition match with current context.") + + +def get_platform_name(): + return get_column_value("platform_name") + + +def get_dependency_archive_name(): + return get_column_value("dependency_name") diff --git a/.github/scripts/common.py b/.github/scripts/common.py index 360f193..8eff925 100644 --- a/.github/scripts/common.py +++ b/.github/scripts/common.py @@ -10,7 +10,7 @@ import shutil import requests -from build_definition import get_platform_name +from build_definition import get_platform_name, get_dependency_archive_name from kiwixbuild.dependencies.apple_xcframework import AppleXCFramework from kiwixbuild.versions import ( @@ -20,11 +20,23 @@ from kiwixbuild.versions import ( ) +def get_build_dir(config) -> Path: + command = ["kiwix-build"] + command.extend(["--config", config]) + command.append("--get-build-dir") + command.append("--use-target-arch-name") + return Path( + subprocess.run(command, cwd=str(HOME), check=True, stdout=subprocess.PIPE) + .stdout[:-1] + .decode("utf8") + ) + + COMPILE_CONFIG = _environ["COMPILE_CONFIG"] OS_NAME = _environ["OS_NAME"] HOME = Path(os.path.expanduser("~")) -BASE_DIR = HOME / "BUILD_{}".format(COMPILE_CONFIG) +BASE_DIR = get_build_dir(COMPILE_CONFIG) SOURCE_DIR = HOME / "SOURCE" ARCHIVE_DIR = HOME / "ARCHIVE" TOOLCHAIN_DIR = BASE_DIR / "TOOLCHAINS" @@ -161,6 +173,7 @@ def run_kiwix_build( command.append("--hide-progress") command.append("--fast-clone") command.append("--assume-packages-installed") + command.append("--use-target-arch-name") command.extend(["--config", config]) if build_deps_only: command.append("--build-deps-only") @@ -271,30 +284,28 @@ def filter_install_dir(path): yield sub_dir -# Full: True if we are creating a full archive to be used as cache by kiwix-build (base_deps2_{os}_{config}_{base_deps_version}.tar.xz) -# Full: False if we are creating a archive to be used as pre-cached dependencies for project's CI (deps2_{os}_{config}_{target}.tar.xz) +# Full: True if we are creating a full archive to be used as cache by kiwix-build (base_deps_{os}_{config}_{base_deps_version}.tar.xz) +# Full: False if we are creating a archive to be used as pre-cached dependencies for project's CI (deps_{config}_{target}.tar.xz) def make_deps_archive(target=None, name=None, full=False): - archive_name = name or "deps2_{}_{}_{}.tar.xz".format( - OS_NAME, COMPILE_CONFIG, target + archive_name = name or "deps_{}_{}.tar.xz".format( + get_dependency_archive_name(), target ) print_message("Create archive {}.", archive_name) files_to_archive = list(filter_install_dir(INSTALL_DIR)) files_to_archive += HOME.glob("BUILD_*/LOGS") if COMPILE_CONFIG == "apple_all_static": for subconfig in AppleXCFramework.subConfigNames: - base_dir = HOME / "BUILD_{}".format(subconfig) + base_dir = get_build_dir(subconfig) files_to_archive += filter_install_dir(base_dir / "INSTALL") if (base_dir / "meson_cross_file.txt").exists(): files_to_archive.append(base_dir / "meson_cross_file.txt") if COMPILE_CONFIG.endswith("_mixed"): static_config = COMPILE_CONFIG.replace("_mixed", "_static") - files_to_archive += filter_install_dir( - HOME / ("BUILD_" + static_config) / "INSTALL" - ) + files_to_archive += filter_install_dir(get_build_dir(static_config) / "INSTALL") if COMPILE_CONFIG.startswith("android_"): files_to_archive += filter_install_dir(HOME / "BUILD_neutral" / "INSTALL") - base_dir = HOME / "BUILD_{}".format(COMPILE_CONFIG) + base_dir = get_build_dir(COMPILE_CONFIG) if (base_dir / "meson_cross_file.txt").exists(): files_to_archive.append(base_dir / "meson_cross_file.txt") # Copy any toolchain @@ -314,13 +325,13 @@ def make_deps_archive(target=None, name=None, full=False): # Add also static build for mixed target if COMPILE_CONFIG.endswith("_mixed"): static_config = COMPILE_CONFIG.replace("_mixed", "_static") - files_to_archive += (HOME / ("BUILD_" + static_config)).glob("*/.*_ok") + files_to_archive += get_build_dir(static_config).glob("*/.*_ok") # Native dyn and static is needed for potential cross compilation that use native tools (icu) - files_to_archive += (HOME / "BUILD_native_dyn").glob("*/.*_ok") - files_to_archive += (HOME / "BUILD_native_static").glob("*/.*_ok") - files_to_archive += HOME.glob("BUILD_android*/**/.*_ok") - files_to_archive += HOME.glob("BUILD_macOS*/**/.*_ok") - files_to_archive += HOME.glob("BUILD_iOS*/**/.*_ok") + files_to_archive += get_build_dir("native_dyn").glob("*/.*_ok") + files_to_archive += get_build_dir("native_static").glob("*/.*_ok") + files_to_archive += HOME.glob("BUILD_*android*/**/.*_ok") + files_to_archive += HOME.glob("BUILD_*apple-macos*/**/.*_ok") + files_to_archive += HOME.glob("BUILD_*apple-ios*/**/.*_ok") files_to_archive += SOURCE_DIR.glob("*/.*_ok") files_to_archive += SOURCE_DIR.glob("zim-testing-suite-*/*") diff --git a/.github/scripts/ensure_base_deps.py b/.github/scripts/ensure_base_deps.py index 0995f9a..f900d78 100755 --- a/.github/scripts/ensure_base_deps.py +++ b/.github/scripts/ensure_base_deps.py @@ -32,10 +32,10 @@ def download_base_archive(base_name): return file_path -ARCHIVE_NAME_TEMPLATE = "base_deps2_{os}_{config}_{version}.tar.xz" +ARCHIVE_NAME_TEMPLATE = "base_deps_{os}_{config}_{version}.tar.xz" if COMPILE_CONFIG == "flatpak": - base_dep_archive_name = "base_deps2_flatpak.tar.xz" + base_dep_archive_name = "base_deps_flatpak.tar.xz" else: base_dep_archive_name = ARCHIVE_NAME_TEMPLATE.format( os=OS_NAME, diff --git a/actions/dl_deps_archive/.gitignore b/actions/dl_deps_archive/.gitignore new file mode 100644 index 0000000..3c3629e --- /dev/null +++ b/actions/dl_deps_archive/.gitignore @@ -0,0 +1 @@ +node_modules diff --git a/actions/dl_deps_archive/dist/index.js b/actions/dl_deps_archive/dist/index.js index 7660fb4..63352d6 100644 --- a/actions/dl_deps_archive/dist/index.js +++ b/actions/dl_deps_archive/dist/index.js @@ -30484,7 +30484,6 @@ function addLocalPath(inputPath) { async function run() { try { const base_url = core.getInput("base_url"); - const os_name = getInput("os_name", process.env["OS_NAME"]); const target = core.getInput("target_platform"); const project = getInput( "project", @@ -30501,11 +30500,11 @@ async function run() { let archivePath; try { - const archive_url = `${base_url}/dev_preview/${branch}/deps2_${os_name}_${target}_${project}.tar.xz`; + const archive_url = `${base_url}/dev_preview/${branch}/deps_${target}_${project}.tar.xz`; process.stdout.write("Downloading " + archive_url + "\n"); archivePath = await tc.downloadTool(archive_url); } catch (error) { - const archive_url = `${base_url}/deps2_${os_name}_${target}_${project}.tar.xz`; + const archive_url = `${base_url}/deps_${target}_${project}.tar.xz`; process.stdout.write("Downloading " + archive_url + "\n"); archivePath = await tc.downloadTool(archive_url); } diff --git a/actions/dl_deps_archive/index.js b/actions/dl_deps_archive/index.js index 00b3367..7d0b12c 100644 --- a/actions/dl_deps_archive/index.js +++ b/actions/dl_deps_archive/index.js @@ -18,7 +18,6 @@ function addLocalPath(inputPath) { async function run() { try { const base_url = core.getInput("base_url"); - const os_name = getInput("os_name", process.env["OS_NAME"]); const target = core.getInput("target_platform"); const project = getInput( "project", @@ -35,11 +34,11 @@ async function run() { let archivePath; try { - const archive_url = `${base_url}/dev_preview/${branch}/deps2_${os_name}_${target}_${project}.tar.xz`; + const archive_url = `${base_url}/dev_preview/${branch}/deps_${target}_${project}.tar.xz`; process.stdout.write("Downloading " + archive_url + "\n"); archivePath = await tc.downloadTool(archive_url); } catch (error) { - const archive_url = `${base_url}/deps2_${os_name}_${target}_${project}.tar.xz`; + const archive_url = `${base_url}/deps_${target}_${project}.tar.xz`; process.stdout.write("Downloading " + archive_url + "\n"); archivePath = await tc.downloadTool(archive_url); } diff --git a/kiwixbuild/__init__.py b/kiwixbuild/__init__.py index 33f2926..8e5f645 100644 --- a/kiwixbuild/__init__.py +++ b/kiwixbuild/__init__.py @@ -125,6 +125,21 @@ def parse_args(): "to develop with the cloned sources." ), ) + subgroup.add_argument( + "--use-target-arch-name", + action="store_true", + help=( + "Name the build directory using the arch name instead of the config name.\n" + "Different configs may create binary for the same arch so this option is " + "not recommended when working with several config on the same computer.\n" + "However, when generating dependencies for other it is better to have a " + "directory named using the target instead of the used config.\n" + "Intended to be used in CI only." + ), + ) + subgroup.add_argument( + "--get-build-dir", action="store_true", help="Print the output directory." + ) options = parser.parse_args() if not options.android_arch: @@ -145,4 +160,7 @@ def main(): builder = FlatpakBuilder() else: builder = Builder() - builder.run() + if options.get_build_dir: + print(ConfigInfo.get_config(options.config).buildEnv.build_dir) + else: + builder.run() diff --git a/kiwixbuild/buildenv.py b/kiwixbuild/buildenv.py index 37fe7fc..1f9f98d 100644 --- a/kiwixbuild/buildenv.py +++ b/kiwixbuild/buildenv.py @@ -36,7 +36,8 @@ class NeutralEnv: if _platform == "Windows": print( "ERROR: kiwix-build is not intented to run on Windows platform.\n" - "It should probably not work, but well, you still can have a try." + "It should probably not work, but well, you still can have a try.", + file=sys.stderr, ) cont = input("Do you want to continue ? [y/N]") if cont.lower() != "y": @@ -70,15 +71,18 @@ class NeutralEnv: if required: sys.exit("ERROR: {} command not found".format(name)) else: - print("WARNING: {} command not found".format(name)) + print("WARNING: {} command not found".format(name), file=sys.stderr) return ["{}_NOT_FOUND".format(name.upper())] class BuildEnv: def __init__(self, configInfo): - build_dir = "BUILD_{}".format(configInfo.name) self.configInfo = configInfo self.base_build_dir = pj(option("working_dir"), option("build_dir")) + build_dir = ( + configInfo.arch_name if option("use_target_arch_name") else configInfo.name + ) + build_dir = f"BUILD_{build_dir}" self.build_dir = pj(self.base_build_dir, build_dir) self.install_dir = pj(self.build_dir, "INSTALL") self.toolchain_dir = pj(self.build_dir, "TOOLCHAINS") diff --git a/kiwixbuild/configs/android.py b/kiwixbuild/configs/android.py index 51d6d91..9374944 100644 --- a/kiwixbuild/configs/android.py +++ b/kiwixbuild/configs/android.py @@ -147,6 +147,10 @@ class Android(MetaConfigInfo): name = "android" compatible_hosts = ["fedora", "debian"] + @property + def arch_name(self): + return "multi-linux-android" + @property def subConfigNames(self): return ["android_{}".format(arch) for arch in option("android_arch")] diff --git a/kiwixbuild/configs/base.py b/kiwixbuild/configs/base.py index 43798f2..21049d5 100644 --- a/kiwixbuild/configs/base.py +++ b/kiwixbuild/configs/base.py @@ -27,6 +27,10 @@ class ConfigInfo(metaclass=_MetaConfig): mixed = False libdir = None + @property + def arch_name(self): + return self.arch_full + @classmethod def get_config(cls, name, targets=None): if name not in cls.all_running_configs: @@ -131,7 +135,6 @@ def MixedMixin(static_name): static = False def add_targets(self, targetName, targets): - print(targetName) if option("target") == targetName: return super().add_targets(targetName, targets) else: diff --git a/kiwixbuild/configs/flatpak.py b/kiwixbuild/configs/flatpak.py index a49acdd..f481a5d 100644 --- a/kiwixbuild/configs/flatpak.py +++ b/kiwixbuild/configs/flatpak.py @@ -4,6 +4,7 @@ from kiwixbuild._global import option, neutralEnv class FlatpakConfigInfo(ConfigInfo): name = "flatpak" + arch_name = "flatpak" build = "flatpak" static = "" toolchain_names = ["org.kde", "io.qt.qtwebengine"] diff --git a/kiwixbuild/configs/ios.py b/kiwixbuild/configs/ios.py index bce93a3..62a9c1f 100644 --- a/kiwixbuild/configs/ios.py +++ b/kiwixbuild/configs/ios.py @@ -24,6 +24,10 @@ class AppleConfigInfo(ConfigInfo): super().__init__(*args, **kwargs) self._root_path = None + @property + def arch_name(self): + return self.target + @property def root_path(self): if self._root_path is None: @@ -209,6 +213,10 @@ class IOS(MetaConfigInfo): name = "iOS_multi" compatible_hosts = ["Darwin"] + @property + def arch_name(self): + return self.name + @property def subConfigNames(self): return ["iOS_{}".format(arch) for arch in option("ios_arch")] @@ -225,6 +233,10 @@ class AppleStaticAll(MetaConfigInfo): name = "apple_all_static" compatible_hosts = ["Darwin"] + @property + def arch_name(self): + return self.name + @property def subConfigNames(self): return AppleXCFramework.subConfigNames diff --git a/kiwixbuild/configs/native.py b/kiwixbuild/configs/native.py index 6bf3e36..3d72e95 100644 --- a/kiwixbuild/configs/native.py +++ b/kiwixbuild/configs/native.py @@ -3,6 +3,9 @@ from .base import ConfigInfo, MixedMixin from kiwixbuild.utils import pj from kiwixbuild._global import option, neutralEnv from kiwixbuild.configs.ios import MIN_MACOS_VERSION +import sysconfig +import platform +import sys class NativeConfigInfo(ConfigInfo): @@ -16,6 +19,12 @@ class NativeConfigInfo(ConfigInfo): env["CFLAGS"] += f"-mmacosx-version-min={MIN_MACOS_VERSION}" return env + @property + def arch_name(self): + if sys.platform == "darwin": + return f"{platform.machine()}-apple-darwin" + return sysconfig.get_platform() + class NativeDyn(NativeConfigInfo): name = "native_dyn" @@ -26,7 +35,7 @@ class NativeDyn(NativeConfigInfo): class NativeStatic(NativeConfigInfo): name = "native_static" static = True - compatible_hosts = ["fedora", "debian"] + compatible_hosts = ["fedora", "debian", "Darwin"] class NativeMixed(MixedMixin("native_static"), NativeConfigInfo): diff --git a/kiwixbuild/configs/neutral.py b/kiwixbuild/configs/neutral.py index f5ae53e..f237c48 100644 --- a/kiwixbuild/configs/neutral.py +++ b/kiwixbuild/configs/neutral.py @@ -3,6 +3,7 @@ from .base import ConfigInfo class NeutralConfigInfo(ConfigInfo): name = "neutral" + arch_name = "neutral" static = "" compatible_hosts = ["fedora", "debian", "Darwin"] diff --git a/kiwixbuild/utils.py b/kiwixbuild/utils.py index 1cdc990..8cce351 100644 --- a/kiwixbuild/utils.py +++ b/kiwixbuild/utils.py @@ -13,7 +13,9 @@ from collections import namedtuple, defaultdict from kiwixbuild._global import neutralEnv, option -pj = os.path.join + +def pj(*args): + return os.path.normpath(os.path.join(*args)) COLORS = { @@ -77,7 +79,7 @@ def remove_duplicates(iterable, key_function=None): def get_sha256(path): - progress_chars = "/-\|" + progress_chars = "/-\\|" current = 0 batch_size = 1024 * 8 sha256 = hashlib.sha256() @@ -137,7 +139,7 @@ def download_remote(what, where): context = None batch_size = 1024 * 8 extra_args = {"context": context} if sys.version_info >= (3, 4, 3) else {} - progress_chars = "/-\|" + progress_chars = "/-\\|" try: with urllib.request.urlopen(what.url, **extra_args) as resource, open( file_path, "wb" diff --git a/kiwixbuild/versions.py b/kiwixbuild/versions.py index 1518a87..b61b83a 100644 --- a/kiwixbuild/versions.py +++ b/kiwixbuild/versions.py @@ -39,7 +39,7 @@ release_versions = { # This is the "version" of the whole base_deps_versions dict. # Change this when you change base_deps_versions. -base_deps_meta_version = "98" +base_deps_meta_version = "99" base_deps_versions = { "zlib": "1.2.12",