Merge pull request #819 from kiwix/release-zim-tools-3.6.0

Relase zim-tools 3.6.0

.github/configs/aqt.ini

@@ -0,0 +1,46 @@
+[DEFAULTS]
+
+[aqt]
+concurrency: 4
+baseurl: https://qt.mirror.constant.com/
+7zcmd: 7z
+print_stacktrace_on_error: False
+always_keep_archives: False
+archive_download_location: .
+min_module_size: 41
+
+[requests]
+connection_timeout: 3.5
+response_timeout: 30
+max_retries_on_connection_error: 5
+retry_backoff: 0.1
+max_retries_on_checksum_error: 5
+max_retries_to_retrieve_hash: 5
+hash_algorithm: sha256
+INSECURE_NOT_FOR_PRODUCTION_ignore_hash: False
+
+[mirrors]
+trusted_mirrors:
+    https://download.qt.io
+    https://qt.mirror.constant.com/
+    https://ftp.fau.de/qtproject/
+blacklist:
+    http://mirrors.ocf.berkeley.edu
+    http://mirrors.tuna.tsinghua.edu.cn
+    http://mirrors.geekpie.club
+fallbacks:
+    https://qtproject.mirror.liquidtelecom.com/
+    https://mirrors.aliyun.com/qt/
+    https://mirrors.ustc.edu.cn/qtproject/
+    https://ftp.jaist.ac.jp/pub/qtproject/
+    https://ftp.yz.yamagata-u.ac.jp/pub/qtproject/
+    https://qt-mirror.dannhauer.de/
+    https://ftp.fau.de/qtproject/
+    https://mirror.netcologne.de/qtproject/
+    https://mirrors.dotsrc.org/qtproject/
+    https://www.nic.funet.fi/pub/mirrors/download.qt-project.org/
+    https://master.qt.io/
+    https://mirrors.ukfast.co.uk/sites/qt.io/
+    https://ftp2.nluug.nl/languages/qt/
+    https://ftp1.nluug.nl/languages/qt/
+    https://qt.mirror.constant.com/

.github/scripts/build_definition.py

@@ -0,0 +1,161 @@
+from typing import NamedTuple
+import csv, io, re
+
+# Definition of what to build.
+# Array is read line by line.
+# Empty cells under (OS_NAME, COMPILE_CONFIG) mean "always match" (catch all, or `.*` regex)
+# Once a cell doesn't match, skip to the next line.
+# Once a line matches, other lines are not read, so put more specific combination first.
+# Lines composed of `-` , or `=`, or starting by `#` are ignored.
+# 'B' letter means that the project is build in the CI
+# 'd' letter means that the project's dependencies are build and published to be used by the project's CI.
+# 'P' letter means that (build) project must be publish when we do a release.
+#     (This is used to avoid two publication of the same archive)
+# 'S' letter means that source code must be publish (almost by definition, S should be put only with a P)
+# 'D' letter means we trigger the docker forkflow to build the docker image.
+# If a cell contains several letters, all are done.
+BUILD_DEF = """
+    | OS_NAME   | COMPILE_CONFIG     | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | platform_name           | dependency_name        |
+    ==============================================================================================================================================
+# manylinux is a special case as we need to compile libzim on old arch for python
+    | manylinux | native_mixed       | BP     |          |           |             |               | linux-x86_64-manylinux  |                        |
+    | manylinux | aarch64_mixed      | BP     |          |           |             |               | linux-aarch64-manylinux |                        |
+    ----------------------------------------------------------------------------------------------------------------------------------------------
+# On Windows, we build only libzim for now. And only native_mixed as xapian doesn't compile as dll
+    | windows   | native_static      | Bd     | d        | BPd       | BPd         |               | win-x86_64              | win-x86_64-static      |
+    | windows   | native_dyn         | Bd     |          |           |             |               | win-x86_64              | win-x86_64-dyn         |
+    | windows   | native_mixed       | BPd    | d        |           |             | BPd           | win-x86_64              | win-x86_64-mixed       |
+    ----------------------------------------------------------------------------------------------------------------------------------------------
+# Osx builds, build binaries on native_dyn and native_static. On anyother things, build only the libraries
+    | macos     | native_dyn         | d      | d        | dB        | B           |               |                         | macos-x86_64-dyn       |
+    | macos     | native_static      |        |          | BP        | BP          |               | macos-x86_64            |                        |
+    | macos     | native_mixed       | BP     | BP       |           |             |               | macos-x86_64            |                        |
+    | macos     | iOS_arm64          | dB     | dB       |           |             |               |                         | ios-arm64-dyn          |
+    | macos     | iOSSimulator_x86_64| dB     | dB       |           |             |               |                         | ios-x86_64-dyn         |
+    | macos     | iOSSimulator_arm64 | B      | B        |           |             |               |                         |                        |
+    | macos     | macOS_arm64_static |        |          | BP        | BP          |               | macos-arm64             |                        |
+    | macos     | macOS_arm64_mixed  | dBP    | dBP      | d         |             |               | macos-arm64             | macos-aarch64-dyn      |
+    | macos     | macOS_x86_64       | B      | B        |           |             |               |                         |                        |
+    | macos     | apple_all_static   |        | BP       |           |             |               | xcframework             |                        |
+    ----------------------------------------------------------------------------------------------------------------------------------------------
+    | focal     | flatpak            |        |          |           |             | BP            |                         |                        |
+    | focal     | native_static      | d      | d        | dBPSD     | dBPSD       |               | linux-x86_64            | linux-x86_64-static    |
+    | focal     | native_mixed       | BPS    | BPS      |           |             |               | linux-x86_64            |                        |
+    | focal     | native_dyn         | d      | d        | dB        | dB          |               |                         | linux-x86_64-dyn       |
+    | jammy     | native_dyn         |        |          |           |             | dBPS          |                         | linux-x86_64-dyn       |
+    # libzim CI is building alpine_dyn but not us
+    | focal     | android_arm        | dBP    | dBP      |           |             |               | android-arm             | android-arm            |
+    | focal     | android_arm64      | dBP    | dBP      |           |             |               | android-arm64           | android-arm64          |
+    | focal     | android_x86        | BP     | BP       |           |             |               | android-x86             |                        |
+    | focal     | android_x86_64     | BP     | BP       |           |             |               | android-x86_64          |                        |
+    | focal     | armv6_static       |        |          | BP        | BP          |               | linux-armv6             |                        |
+    | focal     | armv6_mixed        | BP     |          |           |             |               | linux-armv6             |                        |
+    | focal     | armv6_dyn          |        |          | B         | B           |               |                         |                        |
+    | focal     | armv8_static       |        |          | BP        | BP          |               | linux-armv8             |                        |
+    | focal     | armv8_mixed        | BP     |          |           |             |               | linux-armv8             |                        |
+    | focal     | armv8_dyn          |        |          | B         | B           |               |                         |                        |
+    | focal     | aarch64_static     |        |          | BP        | BP          |               | linux-aarch64           |                        |
+    | focal     | aarch64_mixed      | BP     |          |           |             |               | linux-aarch64           |                        |
+    | focal     | aarch64_dyn        | d      |          | B         | B           |               |                         | linux-aarch64-dyn      |
+    | focal     | aarch64_musl_static|        |          | BP        | BP          |               | linux-aarch64-musl      |                        |
+    | focal     | aarch64_musl_mixed | BP     |          |           |             |               | linux-aarch64-musl      |                        |
+    | focal     | aarch64_musl_dyn   | d      |          | B         | B           |               |                         | linux-aarch64-musl-dyn |
+    | focal     | x86-64_musl_static |        |          | BP        | BP          |               | linux-x86_64-musl       |                        |
+    | focal     | x86-64_musl_mixed  | BP     |          |           |             |               | linux-x86_64-musl       |                        |
+    | focal     | i586_static        |        |          | BP        | BP          |               | linux-i586              |                        |
+    | focal     | i586_dyn           |        |          | B         | B           |               |                         |                        |
+    | focal     | wasm               | dBP    |          |           |             |               | wasm-emscripten         | wasm                   |
+"""
+
+
+class TableDialect(csv.Dialect):
+    delimiter = "|"
+    quoting = csv.QUOTE_NONE
+    lineterminator = "\n"
+
+
+def strip_array(array_str):
+    """Return a iterable of lines, skiping "decorative lines" and with all values in the line's cells stripped"""
+    for line in array_str.splitlines():
+        line = line.strip()
+        line_set = set(line)
+        if (
+            not line
+            or line.startswith("#")
+            or (len(line_set) == 1 and line_set.pop() in "-=")
+        ):
+            continue
+        yield "|".join(c.strip() for c in line.split("|"))
+
+
+def selector_match(selector, value):
+    if not selector:
+        return True
+    return re.fullmatch(selector, value) is not None
+
+
+class Context(NamedTuple):
+    OS_NAME: str
+    COMPILE_CONFIG: str
+
+    def match(self, row):
+        for key in ["OS_NAME", "COMPILE_CONFIG"]:
+            context_value = getattr(self, key)
+            selector = row[key]
+            if not selector_match(selector, context_value):
+                return False
+        return True
+
+
+BUILD = "B"
+PUBLISH = "P"
+SOURCE_PUBLISH = "S"
+DEPS = "d"
+DOCKER = "D"
+
+
+def select_build_targets(criteria):
+    from common import COMPILE_CONFIG, OS_NAME
+
+    context = Context(COMPILE_CONFIG=COMPILE_CONFIG, OS_NAME=OS_NAME)
+
+    reader = csv.DictReader(strip_array(BUILD_DEF), dialect=TableDialect())
+    for row in reader:
+        if context.match(row):
+            build_order = [
+                k
+                for k in (
+                    "libzim",
+                    "libkiwix",
+                    "zim-tools",
+                    "kiwix-tools",
+                    "kiwix-desktop",
+                )
+                if criteria in row[k]
+            ]
+            print(build_order)
+            return build_order
+
+    raise ValueError("No definition match with current context.")
+
+
+def get_column_value(column_name):
+    from common import COMPILE_CONFIG, OS_NAME
+
+    context = Context(COMPILE_CONFIG=COMPILE_CONFIG, OS_NAME=OS_NAME)
+
+    reader = csv.DictReader(strip_array(BUILD_DEF), dialect=TableDialect())
+    for row in reader:
+        if context.match(row):
+            name = row[column_name]
+            return name or None
+
+    raise ValueError("No definition match with current context.")
+
+
+def get_platform_name():
+    return get_column_value("platform_name")
+
+
+def get_dependency_archive_name():
+    return get_column_value("dependency_name")

.github/scripts/build_projects.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+
+from build_definition import select_build_targets, BUILD
+from common import (
+    run_kiwix_build,
+    make_archive,
+    create_desktop_image,
+    fix_macos_rpath,
+    upload_archive,
+    OS_NAME,
+    COMPILE_CONFIG,
+    DEV_BRANCH,
+)
+
+for target in select_build_targets(BUILD):
+    run_kiwix_build(target, config=COMPILE_CONFIG)
+    if target == "kiwix-desktop":
+        archive = create_desktop_image(make_release=False)
+    else:
+        if COMPILE_CONFIG == "native_mixed" and OS_NAME == "macos":
+            fix_macos_rpath(target)
+        archive = make_archive(target, make_release=False)
+    if archive and DEV_BRANCH:
+        upload_archive(archive, target, make_release=False, dev_branch=DEV_BRANCH)

.github/scripts/build_release_nightly.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+
+import os
+
+from common import (
+    run_kiwix_build,
+    main_project_versions,
+    release_versions,
+    make_archive,
+    create_desktop_image,
+    update_flathub_git,
+    upload_archive,
+    fix_macos_rpath,
+    BASE_DIR,
+    OS_NAME,
+    COMPILE_CONFIG,
+    MAKE_RELEASE,
+    notarize_macos_build,
+)
+
+from build_definition import select_build_targets, BUILD, PUBLISH, SOURCE_PUBLISH
+
+
+def release_filter(project):
+    return release_versions.get(project) is not None
+
+
+# Filter what to build if we are doing a release.
+TARGETS = select_build_targets(PUBLISH)
+
+if MAKE_RELEASE:
+    TARGETS = tuple(filter(release_filter, TARGETS))
+
+for target in TARGETS:
+    run_kiwix_build(target, config=COMPILE_CONFIG, make_release=MAKE_RELEASE)
+    if target == "kiwix-desktop":
+        archive = create_desktop_image(make_release=MAKE_RELEASE)
+    else:
+        if OS_NAME == "macos" and COMPILE_CONFIG.endswith("_mixed"):
+            fix_macos_rpath(target)
+            notarize_macos_build(target)
+        archive = make_archive(target, make_release=MAKE_RELEASE)
+    if archive:
+        upload_archive(archive, target, make_release=MAKE_RELEASE)
+
+# We have few more things to do for release:
+if MAKE_RELEASE:
+    # Publish source archives
+    source_published_targets = select_build_targets(SOURCE_PUBLISH)
+    for target in TARGETS:
+        # Looping on TARGETS instead of source_published_targets ensures we are
+        # publishing sources only for target we are building.
+        # (source_published_targets must be a subset of TARGETS)
+        if release_versions.get(target) != 0:
+            continue
+        if target not in source_published_targets:
+            continue
+        run_kiwix_build(
+            target, config=COMPILE_CONFIG, make_release=MAKE_RELEASE, make_dist=True
+        )
+        full_target_name = "{}-{}".format(target, main_project_versions[target])
+        if target == "kiwix-desktop":
+            archive = BASE_DIR / full_target_name / "{}.tar.gz".format(full_target_name)
+        else:
+            archive = (
+                BASE_DIR
+                / full_target_name
+                / "meson-dist"
+                / "{}.tar.xz".format(full_target_name)
+            )
+        upload_archive(archive, target, make_release=MAKE_RELEASE)
+
+    # Publish flathub
+    if COMPILE_CONFIG == "flatpak" and "kiwix-desktop" in TARGETS:
+        update_flathub_git()

.github/scripts/common.py

@@ -0,0 +1,771 @@
+import os
+from os import environ as _environ
+from pathlib import Path, PurePosixPath
+from datetime import date
+import tarfile
+import zipfile
+import subprocess
+import re
+import shutil
+import platform
+
+import requests
+
+from build_definition import get_platform_name, get_dependency_archive_name
+
+from kiwixbuild.dependencies.apple_xcframework import AppleXCFramework
+from kiwixbuild.versions import (
+    main_project_versions,
+    release_versions,
+    base_deps_versions,
+)
+
+
+def get_build_dir(config) -> Path:
+    command = ["kiwix-build"]
+    command.extend(["--config", config])
+    command.append("--get-build-dir")
+    command.append("--use-target-arch-name")
+    return Path(
+        subprocess.run(command, cwd=str(HOME), check=True, stdout=subprocess.PIPE)
+        .stdout.strip()
+        .decode("utf8")
+    )
+
+
+COMPILE_CONFIG = _environ["COMPILE_CONFIG"]
+OS_NAME = _environ["OS_NAME"]
+HOME = Path(os.path.expanduser("~"))
+
+BASE_DIR = get_build_dir(COMPILE_CONFIG)
+SOURCE_DIR = HOME / "SOURCE"
+ARCHIVE_DIR = HOME / "ARCHIVE"
+TOOLCHAIN_DIR = BASE_DIR / "TOOLCHAINS"
+INSTALL_DIR = BASE_DIR / "INSTALL"
+default_tmp_dir = os.getenv("TEMP") if platform.system() == "Windows" else "/tmp"
+TMP_DIR = Path(os.getenv("TMP_DIR", default_tmp_dir))
+if platform.system() == "Windows":
+    KBUILD_SOURCE_DIR = Path(_environ["GITHUB_WORKSPACE"])
+    BIN_EXT = ".exe"
+else:
+    KBUILD_SOURCE_DIR = HOME / "kiwix-build"
+    BIN_EXT = ""
+
+
+_ref = _environ.get("GITHUB_REF", "").split("/")[-1]
+MAKE_RELEASE = re.fullmatch(r"r_[0-9]+", _ref) is not None
+MAKE_RELEASE = MAKE_RELEASE and (_environ.get("GITHUB_EVENT_NAME") != "schedule")
+
+if not MAKE_RELEASE and _ref != "main":
+    DEV_BRANCH = _ref
+else:
+    DEV_BRANCH = None
+
+FLATPAK_HTTP_GIT_REMOTE = "https://github.com/flathub/org.kiwix.desktop.git"
+FLATPAK_GIT_REMOTE = "git@github.com:flathub/org.kiwix.desktop.git"
+
+
+def major_version(version: str) -> str:
+    return version.split(".")[0]
+
+
+# Depending of base distribution, libraries are in "lib64" (redhat base) or "lib/<arch>" (debian base).
+# On top of that, when cross-compiling, libraries are always put in `lib/<arch>`.
+# As we use this as glob regex to select which files to add to archive, this is not a problem to have both.
+def lib_prefix(file):
+    yield "lib64/" + file
+    yield "lib/*/" + file
+
+
+# We have build everything. Now create archives for public deployement.
+EXPORT_FILES = {
+    "kiwix-tools": (
+        INSTALL_DIR / "bin",
+        [f + BIN_EXT for f in ("kiwix-manage", "kiwix-search", "kiwix-serve")]
+        + ["icu*.dll"],
+    ),
+    "zim-tools": (
+        INSTALL_DIR / "bin",
+        [
+            f + BIN_EXT
+            for f in (
+                "zimbench",
+                "zimcheck",
+                "zimdump",
+                "zimsearch",
+                "zimdiff",
+                "zimpatch",
+                "zimsplit",
+                "zimwriterfs",
+                "zimrecreate",
+            )
+        ]
+        + ["icu*.dll"],
+    ),
+    "libzim": (
+        INSTALL_DIR,
+        (
+            ## Linux
+            # We need to package all dependencies (`*.a`) on wasm
+            *lib_prefix("libzim.a" if COMPILE_CONFIG != "wasm" else "*.a"),
+            *lib_prefix("libzim.so"),
+            *lib_prefix(
+                "libzim.so.{version}".format(version=main_project_versions["libzim"])
+            ),
+            *lib_prefix(
+                "libzim.so.{version}".format(
+                    version=major_version(main_project_versions["libzim"])
+                )
+            ),
+            ## MacOS
+            "lib/libzim.{}.dylib".format(
+                major_version(main_project_versions["libzim"])
+            ),
+            "lib/libzim.dylib",
+            "lib/*/libzim.pc",
+            ## Windows
+            "bin/zim-{version}.dll".format(
+                version=major_version(main_project_versions["libzim"])
+            ),
+            "bin/icu*.dll",
+            "bin/zim-{version}.pdb".format(
+                version=major_version(main_project_versions["libzim"])
+            ),
+            "lib/zim.lib",
+            ## Includes and others
+            "include/zim/**/*.h",
+            "share/icu/{}/icudt{}l.dat".format(
+                base_deps_versions["icu4c"], major_version(base_deps_versions["icu4c"])
+            ),
+        ),
+    ),
+    "libkiwix": (
+        INSTALL_DIR,
+        (
+            "lib/CoreKiwix.xcframework/",
+            "lib/*/libkiwix.so",
+            "lib/*/libkiwix.so.{version}".format(
+                version=main_project_versions["libkiwix"]
+            ),
+            "lib/*/libkiwix.so.{version}".format(
+                version=major_version(main_project_versions["libkiwix"])
+            ),
+            "lib/libkiwix.{}.dylib".format(
+                major_version(main_project_versions["libkiwix"])
+            ),
+            "bin/kiwix-{version}.dll".format(
+                version=major_version(main_project_versions["libkiwix"])
+            ),
+            "bin/icu*.dll",
+            "bin/kiwix-{version}.pdb".format(
+                version=major_version(main_project_versions["libkiwix"])
+            ),
+            "lib/kiwix.lib",
+            "lib/libkiwix.dylib",
+            "lib/*/libkiwix.pc",
+            "include/kiwix/**/*.h",
+            "share/icu/{}/icudt{}l.dat".format(
+                base_deps_versions["icu4c"], major_version(base_deps_versions["icu4c"])
+            ),
+        ),
+    ),
+}
+
+DATE = date.today().isoformat()
+
+
+def print_message(message, *args, **kwargs):
+    message = message.format(*args, **kwargs)
+    message = "{0} {1} {0}".format("-" * 3, message)
+    print(message, flush=True)
+
+
+MANIFEST_TEMPLATE = """{archive_name}
+***************************
+
+Dependencies archive for {target} using config {config}
+Generated at {date}
+"""
+
+
+def write_manifest(manifest_file, archive_name, target, config):
+    with manifest_file.open(mode="w") as f:
+        f.write(
+            MANIFEST_TEMPLATE.format(
+                archive_name=archive_name,
+                target=target,
+                config=config,
+                date=DATE,
+            )
+        )
+
+
+def run_kiwix_build(
+    target,
+    config,
+    build_deps_only=False,
+    target_only=False,
+    make_release=False,
+    make_dist=False,
+):
+    command = ["kiwix-build"]
+    command.append(target)
+    command.append("--hide-progress")
+    command.append("--fast-clone")
+    command.append("--assume-packages-installed")
+    command.append("--use-target-arch-name")
+    command.extend(["--config", config])
+    if build_deps_only:
+        command.append("--build-deps-only")
+    if target_only:
+        command.append("--build-nodeps")
+    if make_release:
+        command.append("--make-release")
+    if make_dist:
+        command.append("--make-dist")
+    print_message(
+        "Build {} (deps={}, release={}, dist={})",
+        target,
+        build_deps_only,
+        make_release,
+        make_dist,
+    )
+    env = dict(_environ)
+    env["SKIP_BIG_MEMORY_TEST"] = "1"
+    subprocess.check_call(command, cwd=str(HOME), env=env)
+    print_message("Build ended")
+
+
+try:
+    import paramiko
+
+    def upload(file_to_upload, host, dest_path):
+        if not file_to_upload.exists():
+            print_message("No {} to upload!", file_to_upload)
+            return
+
+        if ":" in host:
+            host, port = host.split(":", 1)
+        else:
+            port = "22"
+        if "@" in host:
+            user, host = host.split("@", 1)
+        else:
+            user = None
+
+        from contextlib import contextmanager
+
+        @contextmanager
+        def get_client():
+            client = paramiko.client.SSHClient()
+            client.set_missing_host_key_policy(paramiko.client.WarningPolicy)
+            print_message(f"Connect to {host}:{port}")
+            client.connect(
+                host,
+                port=port,
+                username=user,
+                key_filename=_environ.get("SSH_KEY"),
+                look_for_keys=False,
+                compress=True,
+            )
+            try:
+                yield client
+            finally:
+                client.close()
+
+        @contextmanager
+        def get_sftp():
+            with get_client() as client:
+                sftp = client.open_sftp()
+                try:
+                    yield sftp
+                finally:
+                    sftp.close()
+
+        dest_path = PurePosixPath(dest_path)
+        remote_file = dest_path.joinpath(file_to_upload.name)
+
+        with get_sftp() as sftp:
+            for part in list(reversed(dest_path.parents)) + [dest_path]:
+                part = str(part)
+                try:
+                    sftp.stat(part)
+                except FileNotFoundError:
+                    sftp.mkdir(part)
+
+            print_message(f"Sending archive {file_to_upload} to {remote_file}")
+            sftp.put(str(file_to_upload), str(remote_file), confirm=True)
+
+except ModuleNotFoundError:
+    # On old system (bionic) paramiko is really complex to install
+    # Keep the old implementaion on sush system.
+
+    def upload(file_to_upload, host, dest_path):
+        if not file_to_upload.exists():
+            print_message("No {} to upload!", file_to_upload)
+            return
+
+        if ":" in host:
+            host, port = host.split(":", 1)
+        else:
+            port = "22"
+
+        # Using SFTP to create the directory hierarchy because we can not
+        # use SSH (no shell for this user); and then scp to upload the file.
+        #
+        # Sending SFTP mkdir command to the SFTP interactive mode and not batch (-b) mode
+        # as the latter would exit on any mkdir error while it is most likely
+        # the first parts of the destination is already present and thus can't be created
+        sftp_commands = "\n".join(
+            [
+                f"mkdir {part}"
+                for part in list(reversed(Path(dest_path).parents)) + [dest_path]
+            ]
+        )
+        command = [
+            "sftp",
+            "-c",
+            "aes128-ctr",
+            "-i",
+            _environ.get("SSH_KEY"),
+            "-P",
+            port,
+            "-o",
+            "StrictHostKeyChecking=no",
+            host,
+        ]
+        print_message("Creating dest path {}", dest_path)
+        subprocess.run(command, input=sftp_commands.encode("utf-8"), check=True)
+
+        command = [
+            "scp",
+            "-c",
+            "aes128-ctr",
+            "-rp",
+            "-P",
+            port,
+            "-i",
+            _environ.get("SSH_KEY"),
+            "-o",
+            "StrictHostKeyChecking=no",
+            str(file_to_upload),
+            "{}:{}".format(host, dest_path),
+        ]
+        print_message("Sending archive with command {}", command)
+        subprocess.check_call(command)
+
+
+def upload_archive(archive, project, make_release, dev_branch=None):
+    if not archive.exists():
+        print_message("No archive {} to upload!", archive)
+        return
+
+    if project.startswith("kiwix-") or project in ["libkiwix"]:
+        host = "ci@master.download.kiwix.org:30022"
+        dest_path = "/data/download/"
+    else:
+        host = "ci@download.openzim.org:30022"
+        dest_path = "/data/openzim/"
+
+    if make_release:
+        dest_path = dest_path + "release/" + project
+    else:
+        dest_path = dest_path + "nightly/" + DATE
+
+    if dev_branch:
+        dest_path = "/data/tmp/ci/dev_preview/" + dev_branch
+    else:
+        # Make the archive read only. This way, scp will preserve rights.
+        # If somehow we try to upload twice the same archive, scp will fails.
+        archive.chmod(0o444)
+
+    upload(archive, host, dest_path)
+
+
+# This remove "share/doc" and "share/man" from the thing to copy in the deps archive
+def filter_install_dir(path):
+    for dir in path.glob("*"):
+        if dir.name not in ["share"]:
+            yield dir
+        else:
+            for sub_dir in dir.glob("*"):
+                if sub_dir.name not in ["doc", "man"]:
+                    yield sub_dir
+
+
+# Full: True if we are creating a full archive to be used as cache by kiwix-build (base_deps_{os}_{config}_{base_deps_version}.tar.gz)
+# Full: False if we are creating a archive to be used as pre-cached dependencies for project's CI (deps_{config}_{target}.tar.gz)
+def make_deps_archive(target=None, name=None, full=False):
+    archive_name = name or "deps_{}_{}.tar.gz".format(
+        get_dependency_archive_name(), target
+    )
+    print_message("Create archive {}.", archive_name)
+    files_to_archive = list(filter_install_dir(INSTALL_DIR))
+    files_to_archive += HOME.glob("BUILD_*/LOGS")
+    if COMPILE_CONFIG == "apple_all_static":
+        for subconfig in AppleXCFramework.subConfigNames:
+            base_dir = get_build_dir(subconfig)
+            files_to_archive += filter_install_dir(base_dir / "INSTALL")
+            if (base_dir / "meson_cross_file.txt").exists():
+                files_to_archive.append(base_dir / "meson_cross_file.txt")
+
+    if COMPILE_CONFIG.endswith("_mixed"):
+        static_config = COMPILE_CONFIG.replace("_mixed", "_static")
+        files_to_archive += filter_install_dir(get_build_dir(static_config) / "INSTALL")
+    if COMPILE_CONFIG.startswith("android_"):
+        files_to_archive += filter_install_dir(HOME / "BUILD_neutral" / "INSTALL")
+        base_dir = get_build_dir(COMPILE_CONFIG)
+        if (base_dir / "meson_cross_file.txt").exists():
+            files_to_archive.append(base_dir / "meson_cross_file.txt")
+    # Copy any toolchain
+    files_to_archive += [TOOLCHAIN_DIR]
+    files_to_archive += HOME.glob("BUILD_neutral/TOOLCHAINS/*")
+    if (BASE_DIR / "meson_cross_file.txt").exists():
+        files_to_archive.append(BASE_DIR / "meson_cross_file.txt")
+
+    manifest_file = BASE_DIR / "manifest.txt"
+    write_manifest(manifest_file, archive_name, target, COMPILE_CONFIG)
+    files_to_archive.append(manifest_file)
+
+    relative_path = HOME
+    if full:
+        files_to_archive += ARCHIVE_DIR.glob(".*_ok")
+        files_to_archive += BASE_DIR.glob("*/.*_ok")
+        # Add also static build for mixed target
+        if COMPILE_CONFIG.endswith("_mixed"):
+            static_config = COMPILE_CONFIG.replace("_mixed", "_static")
+            files_to_archive += get_build_dir(static_config).glob("*/.*_ok")
+        # Native dyn and static is needed for potential cross compilation that use native tools (icu)
+        files_to_archive += get_build_dir("native_dyn").glob("*/.*_ok")
+        files_to_archive += get_build_dir("native_static").glob("*/.*_ok")
+        files_to_archive += HOME.glob("BUILD_*android*/**/.*_ok")
+        files_to_archive += HOME.glob("BUILD_*apple-macos*/**/.*_ok")
+        files_to_archive += HOME.glob("BUILD_*apple-ios*/**/.*_ok")
+        files_to_archive += SOURCE_DIR.glob("*/.*_ok")
+        files_to_archive += SOURCE_DIR.glob("zim-testing-suite-*/*")
+
+    archive_file = TMP_DIR / archive_name
+    with tarfile.open(str(archive_file), "w:gz") as tar:
+        for name in set(files_to_archive):
+            print(".{}".format(name), flush=True)
+            tar.add(str(name), arcname=str(name.relative_to(relative_path)))
+
+    return archive_file
+
+
+def get_postfix(project):
+    postfix = main_project_versions[project]
+    extra = release_versions.get(project)
+    if extra:
+        postfix = "{}-{}".format(postfix, extra)
+    return postfix
+
+
+def sign_binary(path):
+    # We assume here that signtool and certificate are properly configured.
+    # Env var `SIGNTOOL_THUMBPRINT` must contain thumbprint of the certificate to use.
+    command = [
+        os.getenv("SIGNTOOL_PATH", "signtool.exe"),
+        "sign",
+        "/fd",
+        "sha256",
+        "/tr",
+        "http://ts.ssl.com",
+        "/td",
+        "sha256",
+        "/sha1",
+        os.environ["SIGNTOOL_THUMBPRINT"],
+        str(path),
+    ]
+    subprocess.run(command, check=True)
+
+
+def make_archive(project, make_release):
+    platform_name = get_platform_name()
+    if not platform_name:
+        return None
+
+    try:
+        base_dir, export_files = EXPORT_FILES[project]
+    except KeyError:
+        # No binary files to export
+        return None
+
+    if make_release:
+        postfix = get_postfix(project)
+    else:
+        postfix = DATE
+
+    archive_name = "{}_{}-{}".format(project, platform_name, postfix)
+
+    files_to_archive = []
+    for export_file in export_files:
+        files_to_archive.extend(base_dir.glob(export_file))
+
+    if make_release and platform.system() == "Windows":
+        for file in files_to_archive:
+            if str(file).endswith(".exe"):
+                sign_binary(file)
+
+    if platform_name == "win-i686" or platform.system() == "Windows":
+        open_archive = lambda a: zipfile.ZipFile(
+            str(a), "w", compression=zipfile.ZIP_DEFLATED
+        )
+        archive_add = lambda a, f: a.write(str(f), arcname=str(f.relative_to(base_dir)))
+        archive_ext = ".zip"
+    else:
+        open_archive = lambda a: tarfile.open(str(a), "w:gz")
+        archive_add = lambda a, f: a.add(
+            str(f), arcname="{}/{}".format(archive_name, str(f.relative_to(base_dir)))
+        )
+        archive_ext = ".tar.gz"
+
+    archive = TMP_DIR / "{}{}".format(archive_name, archive_ext)
+    print_message("create archive {} with {}", archive, files_to_archive)
+    with open_archive(archive) as arch:
+        for f in files_to_archive:
+            archive_add(arch, f)
+    return archive
+
+
+def create_desktop_image(make_release):
+    print_message("creating desktop image")
+    if make_release:
+        postfix = get_postfix("kiwix-desktop")
+        src_dir = SOURCE_DIR / "kiwix-desktop_release"
+    else:
+        postfix = DATE
+        src_dir = SOURCE_DIR / "kiwix-desktop"
+
+    if COMPILE_CONFIG == "flatpak":
+        build_path = BASE_DIR / "org.kiwix.desktop.flatpak"
+        app_name = "org.kiwix.desktop.{}.flatpak".format(postfix)
+        print_message("archive is {}", build_path)
+    elif platform.system() == "Windows":
+        archive_basename = "kiwix-desktop_windows_x64_{}".format(postfix)
+        working_dir = INSTALL_DIR / archive_basename
+        build_path = Path(str(working_dir) + ".zip")
+        app_name = build_path.name
+        command = [
+            "python",
+            KBUILD_SOURCE_DIR / "scripts" / "package_kiwix-desktop_windows.py",
+            str(INSTALL_DIR),
+            str(working_dir),
+            str(build_path),
+        ]
+        if make_release:
+            command += ["-s"]
+        print_message("Package archive of kiwix-desktop")
+        subprocess.check_call(command, cwd=str(HOME))
+    else:
+        build_path = HOME / "Kiwix-{}-x86_64.AppImage".format(postfix)
+        app_name = "kiwix-desktop_x86_64_{}.appimage".format(postfix)
+        command = [
+            KBUILD_SOURCE_DIR / "scripts" / "create_kiwix-desktop_appImage.sh",
+            str(INSTALL_DIR),
+            str(src_dir),
+            str(HOME / "AppDir"),
+        ]
+        env = dict(os.environ)
+        env["VERSION"] = postfix
+        print_message("Build AppImage of kiwix-desktop")
+        subprocess.check_call(command, cwd=str(HOME), env=env)
+
+    print_message("Copy Build to {}".format(TMP_DIR / app_name))
+    shutil.copy(str(build_path), str(TMP_DIR / app_name))
+    return TMP_DIR / app_name
+
+
+def update_flathub_git():
+    git_repo_dir = TMP_DIR / "org.kiwix.desktop"
+    env = dict(os.environ)
+    env["GIT_AUTHOR_NAME"] = env["GIT_COMMITTER_NAME"] = "KiwixBot"
+    env["GIT_AUTHOR_EMAIL"] = env["GIT_COMMITTER_EMAIL"] = "release@kiwix.org"
+
+    def call(command, cwd=None):
+        cwd = cwd or git_repo_dir
+        print_message("call {}", command)
+        subprocess.check_call(command, env=env, cwd=str(cwd))
+
+    command = ["git", "clone", FLATPAK_HTTP_GIT_REMOTE]
+    call(command, cwd=TMP_DIR)
+
+    branch_name = "v_{}".format(get_postfix("kiwix-desktop"))
+    command = ["git", "checkout", "-b", branch_name]
+    call(command)
+    shutil.copy(str(BASE_DIR / "org.kiwix.desktop.json"), str(git_repo_dir))
+    patch_dir = KBUILD_SOURCE_DIR / "kiwixbuild" / "patches"
+    for dep in ["pugixml", "libmicrohttpd"]:
+        for f in patch_dir.glob("{}_*.patch".format(dep)):
+            shutil.copy(str(f), str(git_repo_dir / "patches"))
+    command = ["git", "add", "-A", "."]
+    call(command)
+    command = [
+        "git",
+        "commit",
+        "-m",
+        "Update to version {}".format(get_postfix("kiwix-desktop")),
+    ]
+    try:
+        call(command)
+    except subprocess.CalledProcessError:
+        # This may fail if there is nothing to commit (a rebuild of the CI for exemple)
+        return
+    command = ["git", "config", "remote.origin.pushurl", FLATPAK_GIT_REMOTE]
+    call(command)
+    command = ["git", "push", "origin", branch_name]
+    env["GIT_SSH_COMMAND"] = "ssh -o StrictHostKeyChecking=no -i " + _environ.get(
+        "SSH_KEY"
+    )
+    call(command)
+
+
+def fix_macos_rpath(project):
+    base_dir, export_files = EXPORT_FILES[project]
+    for file in filter(lambda f: f.endswith(".dylib"), export_files):
+        lib = base_dir / file
+        if lib.is_symlink():
+            continue
+        command = ["install_name_tool", "-id", lib.name, str(lib)]
+        print_message("call {}", " ".join(command))
+        subprocess.check_call(command, env=os.environ)
+
+
+def trigger_workflow(repo, workflow="docker.yml", ref="main", inputs=None):
+    """triggers a `workflow_dispatch` event to the specified workflow on its repo
+
+    repo: {user}/{repo} format
+    workflow: workflow ID or workflow file name
+    ref: branch or tag name
+    inputs: dict of inputs to pass to the workflow"""
+    print_message(
+        "triggering workflow `{workflow}` on {repo}@{ref} " "with inputs={inputs}",
+        workflow=workflow,
+        repo=repo,
+        ref=ref,
+        inputs=inputs,
+    )
+
+    url = "{base_url}/repos/{repo}/actions/workflows/{workflow}/dispatches".format(
+        base_url=os.getenv("GITHUB_API_URL", "https://api.github.com"),
+        repo=repo,
+        workflow=workflow,
+    )
+
+    resp = requests.post(
+        url,
+        headers={
+            "Content-Type": "application/json",
+            "Authorization": "token {token}".format(token=os.getenv("GITHUB_PAT", "")),
+            "Accept": "application/vnd.github.v3+json",
+        },
+        json={"ref": ref, "inputs": inputs},
+        timeout=5,
+    )
+    if resp.status_code != 204:
+        raise ValueError(
+            "Unexpected HTTP {code}: {reason}".format(
+                code=resp.status_code, reason=resp.reason
+            )
+        )
+
+
+def trigger_docker_publish(target):
+    if target not in ("zim-tools", "kiwix-tools"):
+        return
+
+    version = get_postfix(target)
+    repo = {"zim-tools": "openzim/zim-tools", "kiwix-tools": "kiwix/kiwix-tools"}.get(
+        target
+    )
+
+    try:
+        trigger_workflow(
+            repo, workflow="docker.yml", ref="main", inputs={"version": version}
+        )
+        print_message("triggered docker workflow on {repo}", repo=repo)
+    except Exception as exc:
+        print_message("Error triggering workflow: {exc}", exc=exc)
+        raise exc
+
+
+def notarize_macos_build(project):
+    """sign and notarize files for macOS
+
+    Expects the following environment:
+    - `SIGNING_IDENTITY` environ with Certificate name/identity
+    - `KEYCHAIN` environ with path to the keychain storing credentials
+    - `KEYCHAIN_PROFILE` environ with name of the profile in that keychain
+    - `KEYCHAIN_PASSWORD` environ with password to unlock the keychain
+    """
+    if project != "libzim":
+        return
+
+    # currently only supports libzim use case: sign every dylib
+    base_dir, export_files = EXPORT_FILES[project]
+    filepaths = [
+        base_dir.joinpath(file)
+        for file in filter(lambda f: f.endswith(".dylib"), export_files)
+        if not base_dir.joinpath(file).is_symlink()
+    ]
+
+    if not filepaths:
+        return
+
+    for filepath in filepaths:
+        subprocess.check_call(
+            [
+                "/usr/bin/codesign",
+                "--force",
+                "--sign",
+                os.getenv("SIGNING_IDENTITY", "no-signing-ident"),
+                "--keychain",
+                os.getenv("KEYCHAIN", "no-keychain-path"),
+                str(filepath),
+                "--deep",
+                "--timestamp",
+            ],
+            env=os.environ,
+        )
+
+    # create a zip of the dylibs and upload for notarization
+    zip_name = "{}.zip".format(project)
+    subprocess.check_call(
+        ["/usr/bin/ditto", "-c", "-k", "--keepParent"]
+        + [str(f) for f in filepaths]
+        + [zip_name],
+        env=os.environ,
+    )
+
+    # make sure keychain is unlocked
+    subprocess.check_call(
+        [
+            "/usr/bin/security",
+            "unlock-keychain",
+            "-p",
+            os.getenv("KEYCHAIN_PASSWORD", "no-keychain-password"),
+            os.getenv("KEYCHAIN", "no-keychain-path"),
+        ],
+        env=os.environ,
+    )
+
+    subprocess.check_call(
+        [
+            "/usr/bin/xcrun",
+            "notarytool",
+            "submit",
+            "--keychain",
+            os.getenv("KEYCHAIN", "no-keychain-path"),
+            "--keychain-profile",
+            os.getenv("KEYCHAIN_PROFILE", "no-keychain-profile"),
+            "--wait",
+            str(zip_name),
+        ],
+        env=os.environ,
+    )
+
+    # check notarization of a file (should be in-progress atm and this != 0)
+    subprocess.call(
+        ["/usr/sbin/spctl", "--assess", "-vv", "--type", "install", filepaths[-1]],
+        env=os.environ,
+    )

.github/scripts/compile_all_deps.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+
+import os
+
+from common import (
+    run_kiwix_build,
+    make_deps_archive,
+    upload,
+    COMPILE_CONFIG,
+    DEV_BRANCH,
+)
+from build_definition import select_build_targets, DEPS
+
+for target in select_build_targets(DEPS):
+    run_kiwix_build(target, config=COMPILE_CONFIG, build_deps_only=True)
+    archive_file = make_deps_archive(target=target)
+    if DEV_BRANCH:
+        destination = "/data/tmp/ci/dev_preview/" + DEV_BRANCH
+    else:
+        destination = "/data/tmp/ci"
+    upload(archive_file, "ci@tmp.kiwix.org:30022", destination)
+    os.remove(str(archive_file))

.github/scripts/ensure_base_deps.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+
+import os
+import tarfile
+from urllib.request import urlopen
+from urllib.error import URLError
+
+from kiwixbuild.versions import base_deps_meta_version
+
+from common import (
+    print_message,
+    run_kiwix_build,
+    upload,
+    make_deps_archive,
+    HOME,
+    COMPILE_CONFIG,
+    OS_NAME,
+    MAKE_RELEASE,
+)
+
+
+def download_base_archive(base_name):
+    url = "http://tmp.kiwix.org/ci/{}".format(base_name)
+    file_path = str(HOME / base_name)
+    batch_size = 1024 * 1024 * 8
+    with urlopen(url) as resource, open(file_path, "wb") as file:
+        while True:
+            batch = resource.read(batch_size)
+            if not batch:
+                break
+            print(".", end="", flush=True)
+            file.write(batch)
+    return file_path
+
+
+def get_archive_name():
+    ARCHIVE_NAME_TEMPLATE = "base_deps_{os}_{config}_{version}.tar.gz"
+
+    if COMPILE_CONFIG == "flatpak":
+        return "base_deps_flatpak.tar.gz"
+
+    return ARCHIVE_NAME_TEMPLATE.format(
+        os=OS_NAME,
+        config=COMPILE_CONFIG,
+        version=base_deps_meta_version,
+    )
+
+
+def main():
+    base_dep_archive_name = get_archive_name()
+    print_message("Getting archive {}", base_dep_archive_name)
+    try:
+        local_filename = download_base_archive(base_dep_archive_name)
+        with tarfile.open(local_filename) as f:
+            f.extractall(str(HOME))
+        os.remove(str(local_filename))
+    except URLError:
+        if COMPILE_CONFIG == "flatpak":
+            print_message("Cannot get archive. Move on")
+        else:
+            print_message("Cannot get archive. Build dependencies")
+            run_kiwix_build("alldependencies", config=COMPILE_CONFIG)
+            archive_file = make_deps_archive(name=base_dep_archive_name, full=True)
+            upload(archive_file, "ci@tmp.kiwix.org:30022", "/data/tmp/ci")
+            os.remove(str(archive_file))
+
+
+if __name__ == "__main__":
+    if MAKE_RELEASE:
+        print_message("We are building release. Don't download deps archive.")
+    else:
+        main()

.github/scripts/trigger_docker_workflow.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+
+from common import (
+    trigger_docker_publish,
+    MAKE_RELEASE,
+)
+
+from build_definition import select_build_targets, DOCKER
+
+# Filter what to build if we are doing a release.
+if MAKE_RELEASE:
+    docker_trigger = select_build_targets(DOCKER)
+else:
+    docker_trigger = []
+
+for target in docker_trigger:
+    trigger_docker_publish(target)

.github/scripts/upload_failure_logs.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+
+import tarfile
+from pathlib import Path
+from common import upload, OS_NAME, COMPILE_CONFIG, HOME
+
+ARCHIVE_NAME = Path(f"fail_log_{OS_NAME}_{COMPILE_CONFIG}.tar.gz")
+
+
+files_to_archive = []
+files_to_archive += HOME.glob("BUILD_*")
+files_to_archive += [HOME / "SOURCE", HOME / "LOGS", HOME / "TOOLCHAINS"]
+
+with tarfile.open(ARCHIVE_NAME, "w:xz") as tar:
+    for name in set(files_to_archive):
+        tar.add(str(name))
+
+upload(ARCHIVE_NAME, "ci@tmp.kiwix.org:30022", "/data/tmp/ci")

.github/workflows/cd.yml

@@ -0,0 +1,326 @@
+name: CD
+
+on:
+ push:
+   tags:
+     - r_[0-9]+
+ schedule:
+   - cron: '0 3 * * *'
+
+jobs:
+  Windows:
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - native_mixed
+          - native_dyn
+          - native_static
+    runs-on: windows-2022
+    env:
+      OS_NAME: windows
+      COMPILE_CONFIG: ${{matrix.config}}
+      HOME: 'C:\\Users\\runneradmin'
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+    - name: Setup python 3.12
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+    - name: Install packages
+      run: |
+        choco.exe install pkgconfiglite ninja
+    - name: Install python modules
+      shell: bash
+      run: |
+        pip3 install meson pytest requests distro paramiko
+        pip3 install --no-deps $GITHUB_WORKSPACE
+    - name: Install QT
+      if: ${{ matrix.config == 'native_mixed' }}
+      uses: jurplel/install-qt-action@v4
+      with:
+        version: 5.15.2
+        modules: "qtwebengine"
+        setup-python: false
+      env:
+        AQT_CONFIG: ${{ github.workspace }}/.github/configs/aqt.ini
+    - name: Setup MSVC compiler
+      uses: bus1/cabuild/action/msdevshell@v1
+      with:
+        architecture: x64
+    - name: secret
+      shell: bash
+      run: |
+        echo "${{secrets.ssh_key}}" > $SSH_KEY
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+    - name: Install and configure eSigner CKA and Windows SDK
+      if: github.event_name == 'push'
+      env:
+        ESIGNER_URL: https://github.com/SSLcom/eSignerCKA/releases/download/v1.0.7/SSL.COM-eSigner-CKA_1.0.7.zip
+      run: |
+          Set-StrictMode -Version 'Latest'
+
+          # Download and Unzip eSignerCKA Setup
+          Invoke-WebRequest -OutFile eSigner_CKA_Setup.zip "$env:ESIGNER_URL"
+          Expand-Archive -Force eSigner_CKA_Setup.zip
+          Remove-Item eSigner_CKA_Setup.zip
+          Move-Item -Destination “eSigner_CKA_Installer.exe” -Path “eSigner_CKA_*\*.exe”
+
+          # Install eSignerCKA
+          New-Item -ItemType Directory -Force -Path "C:\esigner"
+          ./eSigner_CKA_Installer.exe /CURRENTUSER /VERYSILENT /SUPPRESSMSGBOXES /DIR=”C:\esigner” /TYPE=automatic | Out-Null
+          Remove-Item "eSigner_CKA_Installer.exe"
+
+          # Configure the CKA with SSL.com credentials
+          C:\esigner\eSignerCKATool.exe config -mode product -user "${{ secrets.ESIGNER_USERNAME }}" -pass "${{ secrets.ESIGNER_PASSWORD }}" -totp "${{ secrets.ESIGNER_TOTP_SECRET }}" -key "C:\esigner\master.key" -r
+          C:\esigner\eSignerCKATool.exe unload
+          C:\esigner\eSignerCKATool.exe load
+
+          # Find certificate
+          $CodeSigningCert = Get-ChildItem Cert:\CurrentUser\My -CodeSigningCert | Select-Object -First 1
+          echo Certificate: $CodeSigningCert
+
+          # Extract thumbprint and subject name
+          $Thumbprint = $CodeSigningCert.Thumbprint
+          echo "SIGNTOOL_THUMBPRINT=$Thumbprint" >> $env:GITHUB_ENV
+
+    - name: Ensure base deps
+      run: |
+        python .github\\scripts\\ensure_base_deps.py
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+    - name: Build Release
+      run: |
+        python .github\\scripts\\build_release_nightly.py
+      env:
+        SIGNTOOL_PATH: "C:/Program Files (x86)/Windows Kits/10/bin/10.0.22621.0/x86/signtool.exe"
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+    - name: Upload failure logs
+      if: failure()
+      run: |
+        python .github\\scripts\\upload_failure_logs.py
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+
+  Linux:
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - native_static
+          - native_mixed
+          - native_dyn
+          - wasm
+          - armv6_static
+          - armv6_mixed
+          - armv8_static
+          - armv8_mixed
+          - aarch64_static
+          - aarch64_mixed
+          - aarch64_musl_static
+          - aarch64_musl_mixed
+          - x86-64_musl_static
+          - x86-64_musl_mixed
+          - i586_static
+          - android_arm
+          - android_arm64
+          - android_x86
+          - android_x86_64
+        image_variant: ['focal']
+        include:
+          - config: native_mixed
+            image_variant: manylinux
+          - config: aarch64_mixed
+            image_variant: manylinux
+          - config: native_dyn
+            image_variant: jammy
+    env:
+      HOME: /home/runner
+      SSH_KEY: /tmp/id_rsa
+    runs-on: ubuntu-22.04
+    container:
+      image: "ghcr.io/kiwix/kiwix-build_ci_${{matrix.image_variant}}:2024-11-30"
+      options: "--device /dev/fuse --privileged"
+    steps:
+    - name: Checkout code
+      shell: bash
+      run: |
+        cd $HOME
+        git clone https://github.com/${REP}
+        cd ./${REP##*/}
+        git checkout --force ${GITHUB_SHA}
+        pip3 install --user --no-deps .
+      env:
+        REP: ${{github.repository}}
+    - name: secret
+      shell: bash
+      run: |
+        echo "${{secrets.ssh_key}}" > $SSH_KEY
+        chmod 600 $SSH_KEY
+    - name: Ensure base deps
+      shell: bash
+      run: |
+        cd $HOME
+        kiwix-build/.github/scripts/ensure_base_deps.py
+      env:
+        COMPILE_CONFIG: ${{matrix.config}}
+    - name: Build release
+      shell: bash
+      run: |
+        cd $HOME
+        kiwix-build/.github/scripts/build_release_nightly.py
+      env:
+        COMPILE_CONFIG: ${{matrix.config}}
+    - name: Upload failure logs
+      if: failure()
+      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py
+      env:
+        COMPILE_CONFIG: ${{matrix.config}}
+
+  Flatpak:
+    strategy:
+      fail-fast: false
+    env:
+      HOME: /home/runner
+      SSH_KEY: /tmp/id_rsa
+      COMPILE_CONFIG: flatpak
+      OS_NAME: focal
+    runs-on: ubuntu-22.04
+    steps:
+    - name: Checkout code
+      shell: bash
+      run: |
+        cd $HOME
+        git clone https://github.com/${REP}
+        cd ./${REP##*/}
+        git checkout --force ${GITHUB_SHA}
+        pip3 install --user --no-deps .
+      env:
+        REP: ${{github.repository}}
+    - name: Install flatpak tools
+      run: |
+        sudo apt-get update
+        sudo apt-get install flatpak-builder ninja-build meson
+    - name: secret
+      shell: bash
+      run: |
+        echo "${{secrets.ssh_key}}" > $SSH_KEY
+        chmod 600 $SSH_KEY
+    - name: Ensure base deps
+      shell: bash
+      run: |
+        cd $HOME
+        kiwix-build/.github/scripts/ensure_base_deps.py
+    - name: Build release
+      shell: bash
+      run: |
+        cd $HOME
+        kiwix-build/.github/scripts/build_release_nightly.py
+    - name: Upload failure logs
+      if: failure()
+      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py
+
+  Macos:
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - native_dyn
+          - native_static
+          - native_mixed
+          - macOS_arm64_static
+          - macOS_arm64_mixed
+          - apple_all_static
+    runs-on: macos-13
+    env:
+      SSH_KEY: /tmp/id_rsa
+      OS_NAME: macos
+      CERTIFICATE: /tmp/wmch-devid.p12
+      SIGNING_IDENTITY: ${{ secrets.APPLE_SIGNING_IDENTITY }}
+      KEYCHAIN: /Users/runner/build.keychain-db
+      KEYCHAIN_PASSWORD: mysecretpassword
+      KEYCHAIN_PROFILE: build-profile
+    steps:
+    - name: Set Xcode version (15.0.1)
+      # https://github.com/actions/runner-images/blob/main/images/macos/macos-13-Readme.md#xcode
+      run: sudo xcode-select -s /Applications/Xcode_15.0.1.app
+    - name: Checkout code
+      uses: actions/checkout@v4
+    - name: Setup python 3.10
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.10'
+    - name: Install packages
+      run:
+        brew install automake # ninja
+    - name: Install python modules
+      run: |
+        pip3 install meson pytest requests distro
+        pip3 install --no-deps $GITHUB_WORKSPACE
+    - name: secret
+      shell: bash
+      run: |
+        echo "${{secrets.ssh_key}}" > $SSH_KEY
+        chmod 600 $SSH_KEY
+    - name: install Apple certificate
+      shell: bash
+      run: |
+        echo "${{ secrets.APPLE_SIGNING_CERTIFICATE }}" | base64 --decode -o $CERTIFICATE
+        security create-keychain -p $KEYCHAIN_PASSWORD $KEYCHAIN
+        security default-keychain -s $KEYCHAIN
+        security set-keychain-settings $KEYCHAIN
+        security unlock-keychain -p $KEYCHAIN_PASSWORD $KEYCHAIN
+        security import $CERTIFICATE -k $KEYCHAIN -P "${{ secrets.APPLE_SIGNING_P12_PASSWORD }}" -A -T "/usr/bin/codesign"
+        rm $CERTIFICATE
+        security set-key-partition-list -S apple-tool:,apple: -s -k $KEYCHAIN_PASSWORD $KEYCHAIN
+        security find-identity -v $KEYCHAIN
+        xcrun notarytool store-credentials \
+            --apple-id "${{ secrets.APPLE_SIGNING_ALTOOL_USERNAME }}" \
+            --password "${{ secrets.APPLE_SIGNING_ALTOOL_PASSWORD }}" \
+            --team-id "${{ secrets.APPLE_SIGNING_TEAM }}" \
+            --validate \
+            --keychain $KEYCHAIN \
+            $KEYCHAIN_PROFILE
+    - name: Ensure base deps
+      shell: bash
+      run: |
+        cd $HOME
+        $GITHUB_WORKSPACE/.github/scripts/ensure_base_deps.py
+      env:
+        COMPILE_CONFIG: ${{matrix.config}}
+    - name: Build release
+      shell: bash
+      run: |
+        cd $HOME
+        $GITHUB_WORKSPACE/.github/scripts/build_release_nightly.py
+      env:
+        COMPILE_CONFIG: ${{matrix.config}}
+    - name: Upload failure logs
+      if: failure()
+      run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.py
+      env:
+        COMPILE_CONFIG: ${{matrix.config}}
+
+
+  Trigger_Docker:
+    needs: [Linux]
+    runs-on: ubuntu-22.04
+    env:
+      COMPILE_CONFIG: native_static
+      OS_NAME: focal
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+    - name: Install python modules
+      shell: bash
+      run: |
+        pip3 install --user --no-deps $GITHUB_WORKSPACE
+    - name: Trigger docker workflow
+      shell: bash
+      run: |
+        cd $HOME
+        $GITHUB_WORKSPACE/.github/scripts/trigger_docker_workflow.py
+      env:
+        GITHUB_PAT: ${{secrets.DOCKER_TRIGGER_GITHUB_PAT}}

.github/workflows/ci.yml

@@ -0,0 +1,280 @@
+name: CI
+
+on:
+ push:
+ schedule:
+   - cron: '0 1 * * *'
+
+jobs:
+  Windows:
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - native_mixed
+          - native_dyn
+          - native_static
+    runs-on: windows-2022
+    env:
+      OS_NAME: windows
+      COMPILE_CONFIG: ${{matrix.config}}
+      HOME: 'C:\\Users\\runneradmin'
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+    - name: Setup python 3.12
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+    - name: Install packages
+      run: |
+        choco.exe install pkgconfiglite ninja
+    - name: Install python modules
+      shell: bash
+      run: |
+        pip3 install meson pytest requests distro paramiko
+        pip3 install --no-deps $GITHUB_WORKSPACE
+    - name: Install QT
+      if: ${{ matrix.config == 'native_mixed' }}
+      uses: jurplel/install-qt-action@v4
+      with:
+        version: 6.4.3
+        modules: "qtwebengine qtwebchannel qtpositioning"
+        setup-python: false
+      env:
+        AQT_CONFIG: ${{ github.workspace }}/.github/configs/aqt.ini
+    - name: Setup MSVC compiler
+      uses: bus1/cabuild/action/msdevshell@v1
+      with:
+        architecture: x64
+    - name: secret
+      shell: bash
+      run: |
+        echo "${{secrets.ssh_key}}" > $SSH_KEY
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+    - name: Ensure base deps
+      run: |
+        python .github\\scripts\\ensure_base_deps.py
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+    - name: Compile all deps
+      run: |
+        python .github\\scripts\\compile_all_deps.py
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+    - name: Build projects
+      run: |
+        python .github\\scripts\\build_projects.py
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+    - name: Upload failure logs
+      if: failure()
+      run: |
+        python .github\\scripts\\upload_failure_logs.py
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+
+  Linux:
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - native_static
+          - native_dyn
+          - native_mixed
+          - wasm
+          - armv6_static
+          - armv6_dyn
+          - armv6_mixed
+          - armv8_static
+          - armv8_dyn
+          - armv8_mixed
+          - aarch64_static
+          - aarch64_dyn
+          - aarch64_mixed
+          - aarch64_musl_static
+          - aarch64_musl_dyn
+          - aarch64_musl_mixed
+          - x86-64_musl_static
+          - x86-64_musl_mixed
+          - i586_static
+          - i586_dyn
+          - android_arm
+          - android_arm64
+          - android_x86
+          - android_x86_64
+        image_variant: ['focal']
+        include:
+          - config: native_mixed
+            image_variant: manylinux
+          - config: aarch64_mixed
+            image_variant: manylinux
+          - config: native_dyn
+            image_variant: jammy
+    env:
+      HOME: /home/runner
+      SSH_KEY: /tmp/id_rsa
+      OS_NAME: ${{matrix.image_variant}}
+    runs-on: ubuntu-22.04
+    container:
+      image: "ghcr.io/kiwix/kiwix-build_ci_${{matrix.image_variant}}:2024-11-30"
+      options: "--device /dev/fuse --privileged"
+    steps:
+    - name: Checkout code
+      shell: bash
+      run: |
+        cd $HOME
+        git clone https://github.com/${REP}
+        cd ./${REP##*/}
+        git checkout --force ${GITHUB_SHA}
+        pip3 install --user --no-deps .
+      env:
+        REP: ${{github.repository}}
+    - name: Install paramiko
+      if: ${{matrix.image_variant != 'bionic' }}
+      shell: bash
+      run: pip3 install --user paramiko
+    - name: secret
+      shell: bash
+      run: |
+        echo "${{secrets.ssh_key}}" > $SSH_KEY
+        chmod 600 $SSH_KEY
+    - name: Ensure base deps
+      shell: bash
+      run: |
+        cd $HOME
+        kiwix-build/.github/scripts/ensure_base_deps.py
+      env:
+        COMPILE_CONFIG: ${{matrix.config}}
+    - name: Compile all deps
+      shell: bash
+      run: |
+        cd $HOME
+        kiwix-build/.github/scripts/compile_all_deps.py
+      env:
+        COMPILE_CONFIG: ${{matrix.config}}
+    - name: Build projects
+      shell: bash
+      run: |
+        cd $HOME
+        kiwix-build/.github/scripts/build_projects.py
+      env:
+        COMPILE_CONFIG: ${{matrix.config}}
+    - name: Upload failure logs
+      if: failure()
+      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py
+      env:
+        COMPILE_CONFIG: ${{matrix.config}}
+
+  Flatpak:
+    strategy:
+      fail-fast: false
+    env:
+      HOME: /home/runner
+      SSH_KEY: /tmp/id_rsa
+      COMPILE_CONFIG: flatpak
+      OS_NAME: focal
+    runs-on: ubuntu-22.04
+    steps:
+    - name: Checkout code
+      shell: bash
+      run: |
+        cd $HOME
+        git clone https://github.com/${REP}
+        cd ./${REP##*/}
+        git checkout --force ${GITHUB_SHA}
+        pip3 install --user paramiko
+        pip3 install --user --no-deps .
+      env:
+        REP: ${{github.repository}}
+    - name: Install flatpak tools
+      run: |
+        sudo apt-get update
+        sudo apt-get install flatpak-builder ninja-build meson
+    - name: secret
+      shell: bash
+      run: |
+        echo "${{secrets.ssh_key}}" > $SSH_KEY
+        chmod 600 $SSH_KEY
+    - name: Ensure base deps
+      shell: bash
+      run: |
+        cd $HOME
+        kiwix-build/.github/scripts/ensure_base_deps.py
+    - name: Build projects
+      shell: bash
+      run: |
+        cd $HOME
+        kiwix-build/.github/scripts/build_projects.py
+    - name: Upload failure logs
+      if: failure()
+      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py
+
+  Macos:
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - native_dyn
+          - native_static
+          - native_mixed
+          - iOS_arm64
+          - iOSSimulator_x86_64
+          - iOSSimulator_arm64
+          - macOS_arm64_static
+          - macOS_arm64_mixed
+          - macOS_x86_64
+          - apple_all_static
+    runs-on: macos-13
+    env:
+      SSH_KEY: /tmp/id_rsa
+      OS_NAME: macos
+    steps:
+    - name: Set Xcode version (15.0.1)
+      # https://github.com/actions/runner-images/blob/main/images/macos/macos-13-Readme.md#xcode
+      run: sudo xcode-select -s /Applications/Xcode_15.0.1.app
+    - name: Checkout code
+      uses: actions/checkout@v4
+    - name: Setup Python 3.10
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.10'
+    - name: Install packages
+      run:
+        brew install automake # ninja
+    - name: Install python modules
+      run: |
+        pip3 install meson pytest requests distro paramiko
+        pip3 install --no-deps $GITHUB_WORKSPACE
+    - name: secret
+      shell: bash
+      run: |
+        echo "${{secrets.ssh_key}}" > $SSH_KEY
+        chmod 600 $SSH_KEY
+    - name: Ensure base deps
+      shell: bash
+      run: |
+        cd $HOME
+        $GITHUB_WORKSPACE/.github/scripts/ensure_base_deps.py
+      env:
+        COMPILE_CONFIG: ${{matrix.config}}
+    - name: Compile all deps
+      shell: bash
+      run: |
+        cd $HOME
+        $GITHUB_WORKSPACE/.github/scripts/compile_all_deps.py
+      env:
+        COMPILE_CONFIG: ${{matrix.config}}
+    - name: Build projects
+      shell: bash
+      run: |
+        cd $HOME
+        $GITHUB_WORKSPACE/.github/scripts/build_projects.py
+      env:
+        COMPILE_CONFIG: ${{matrix.config}}
+    - name: Upload failure logs
+      if: failure()
+      run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.py
+      env:
+        COMPILE_CONFIG: ${{matrix.config}}

.gitignore

@@ -109,3 +109,4 @@ ENV/
 # mypy
 .mypy_cache/
 
+.idea

.travis.yml

@@ -1,91 +0,0 @@
-language: cpp
-dist: trusty
-sudo: required
-branches:
-  only:
-  - master
-  - /\d+\.\d+\.\d+$/
-before_install:
-- eval "${MATRIX_EVAL}"
-- ${CXX} --version
-- openssl aes-256-cbc -K $encrypted_eba2f7543984_key -iv $encrypted_eba2f7543984_iv
-  -in travis/travisci_builder_id_key.enc -out travis/travisci_builder_id_key -d
-- chmod 600 travis/travisci_builder_id_key
-before_cache:
-  - rm -f $HOME/.gradle/caches/modules-2/modules-2.lock
-  - rm -fr $HOME/.gradle/caches/*/plugin-resolution/
-cache:
-  ccache: true
-  directories:
-    - $HOME/.cache/pip
-    - $HOME/.gradle/caches/
-    - $HOME/.gradle/wrapper/
-    - $HOME/.android/build-cache
-install: travis/install_extra_deps.sh
-script: travis/compile_all.py
-after_failure: travis/upload_all_log.sh
-deploy:
-  - provider: script
-    skip_cleanup: true
-    script: travis/deploy.sh
-    on:
-      tags: true
-  - provider: script
-    skip_cleanup: true
-    script: travis/deploy.sh
-    on:
-      branch: master
-      condition: $TRAVIS_EVENT_TYPE = cron
-env:
-  global:
-    - MATRIX_EVAL="CC=gcc-5 && CXX=g++-5"
-  matrix:
-    - PLATFORM="native_dyn"
-    - PLATFORM="native_static"
-    - PLATFORM="win32_dyn"
-    - PLATFORM="win32_static"
-    - PLATFORM="armhf_dyn"
-    - PLATFORM="armhf_static"
-    - PLATFORM="android_arm"
-    - PLATFORM="android_arm64"
-    - PLATFORM="android_mips"
-    - PLATFORM="android_mips64"
-    - PLATFORM="android_x86"
-    - PLATFORM="android_x86_64"
-matrix:
-  allow_failures:
-    - env: PLATFORM="android_arm"
-    - env: PLATFORM="android_arm64"
-    - env: PLATFORM="android_mips"
-    - env: PLATFORM="android_mips64"
-    - env: PLATFORM="android_x86"
-    - env: PLATFORM="android_x86_64"
-
-notifications:
-  irc:
-    channels:
-     - "chat.freenode.net#kiwix"
-    on_success: change
-    on_failure: always
-addons:
-  ssh_known_hosts:
-  - download.kiwix.org
-  - download.openzim.org
-  apt:
-    sources:
-      - ubuntu-toolchain-r-test
-    packages:
-      - g++-5
-      - cmake
-      - python3-pip
-      - zlib1g-dev
-      - libjpeg-dev
-      - libmagic-dev
-      - ctpp2-utils
-      - libctpp2-dev
-      - libmicrohttpd-dev
-      - g++-mingw-w64-i686
-      - gcc-mingw-w64-i686
-      - gcc-mingw-w64-base
-      - mingw-w64-tools
-      - default-jdk

MANIFEST.in

@@ -0,0 +1,3 @@
+include kiwixbuild/templates/*_cross_file.txt
+include kiwixbuild/patches/*.patch
+include kiwixbuild/dependencies/icu4c_data_filter.json

README.md

@@ -1,104 +1,181 @@
-Build status: [![Build Status](https://travis-ci.org/kiwix/kiwix-build.svg?branch=master)](https://travis-ci.org/kiwix/kiwix-build)
+# Kiwix Build
 
-Kiwix is an offline reader for web content. It's especially thought to
+Kiwix Build provides advanced tools to (cross-)compile easily
-make Wikipedia available offline. This is done by reading the content
+[Kiwix](https://kiwix.org) & [openZIM](https://openzim.org) softwares
-of the project stored in a file format ZIM, a high compressed open
+and libraries and deploy them. They have been tested on
-format with additional meta-data.
+[Fedora](https://getfedora.org) 35+ & [Ubuntu](https://ubuntu.com)
+20.04+.
 
-This repository contains advanced tools to (cross-)compile easily
+Kiwix Build audience is:
-Kiwix softwares and library and deploy them. They have been tested on
+* Advanced users who don't want/can handle all the dependencies
-Fedora 23 and Ubuntu 16.10.
+  compilations manually
+* Kiwix developer team for its own CI/CD
 
-# Prerequesites
+[![CI Build Status](https://github.com/kiwix/kiwix-build/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/kiwix/kiwix-build/actions/workflows/ci.yml?query=branch%3Amain)
+[![CD Build Status](https://github.com/kiwix/kiwix-build/actions/workflows/cd.yml/badge.svg?branch=main)](https://github.com/kiwix/kiwix-build/actions/workflows/cd.yml?query=branch%3Amain)
+[![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
 
-You will need a recent version of `meson` (0.34) and `ninja` (1.6) If
+Prerequisites
-your distribution provides a recent enough versions for them, just
+-------------
-install them with your package manager. Continue to read the
+
-instructions otherwise.
+You will need a recent version of [Meson](https://mesonbuild.com/) (>=
+0.34) and [Ninja](https://ninja-build.org) (>= 1.6) If your
+distribution provides a recent enough versions for them, just install
+them with your package manager. Continue to read the instructions
+otherwise.
 
 Before anything else you need to install Python3 related tools. On Debian
 based systems:
-
+```bash
-```
 sudo apt-get install python3-pip virtualenv
 ```
 
-Then install Meson:
+Create a virtual environment to install python module in it instead
+of modifying the system.
+```bash
+virtualenv -p python3 ./ # Create virtualenv
+source bin/activate      # Activate the virtualenv
 ```
+
+Then, download and install kiwix-build and its dependencies:
+```bash
+git clone https://github.com/kiwix/kiwix-build.git
+cd kiwix-build
+pip3 install .
+hash -r # Refresh bash paths
+```
+
+Compilation
+-----------
+
+The compilation is handled by the `kiwix-build` command. It will compile
+everything. If you are using a supported platform (Redhat or Debian
+based) it will install missing packages using `sudo`. You can get
+`kiwix-build` usage like this:
+```bash
+kiwix-build --help
+```
+
+#### Target
+
+You may want to compile a specific target so you will have to specify it on the
+
+command line:
+```bash
+kiwix-build libkiwix # will build kiwix-build and its dependencies
+kiwix-build kiwix-desktop # will build kiwix-desktop and its dependencies
+kiwix-build zim-tools # will build zim-tools and its dependencies
+```
+
+By default, `kiwix-build` will build `kiwix-tools` .
+
+To see the whole list of available targets run with non existing target, ex:
+
+```bash
+kiwix-build not-existing-target
+...
+invalid choice: 'not-existing-target' (choose from 'alldependencies', 'android-ndk',
+...
+```
+
+#### Config
+
+If no config is specified, the default will be `native_dyn`.
+
+You can select another config using the option
+`--config`. For now, there is ten different supported
+platforms:
+
+- native_dyn
+- native_mixed
+- native_static
+- android
+- android_arm
+- android_arm64
+- android_x86
+- android_x86_64
+- flatpak
+
+All `native_*` config means using the native compiler without any cross-compilation option.
+Other may simply use cross-compilation or may download a specific toolchain to use.
+
+Android
+-------
+
+`kiwix-android` (https://github.com/kiwix/kiwix-android) depends of
+the `libkiwix` project.
+
+When building `libkiwix`, you should directly use the
+target-platform `android_<arch>`:
+```bash
+kiwix-build libkiwix --config android_arm
+```
+
+You may directly use the special config `android` which will build different android architectures
+```bash
+kiwix-build --config android libkiwix
+```
+
+By default, it will build for all android architecture,
+you can limit this with option `--android-arch`:
+```bash
+kiwix-build libkiwix --config android --android-arch arm # aar with arm architecture
+kiwix-build libkiwix --config android --android-arch arm --android-arch arm64 # aan with arm and arm64 architectures
+```
+
+To build `kiwix-android` itself, you should see the documentation of `kiwix-android`.
+
+iOS
+---
+
+When building for ios, we may want to compile a "fat library", a library
+for several architectures.
+
+To do so, you should directly use the target-platfrom `ios_multi`.
+As for `android`, `kiwix-build` will build the library several times
+(once for each platform) and then create the fat library.
+```bash
+kiwix-build --config iOS_multi libkiwix
+```
+
+You can specify the supported architectures with the option `--ios-arch`:
+```bash
+kiwix-build --config iOS_multi libkiwix # all architetures
+kiwix-build --config iOS_multi --ios-arch arm --ios-arch arm64 # arm and arm64 arch only
+```
+
+Outputs
+-------
+
+Kiwix-build.py will create several directories:
+- `ARCHIVES`: All the downloaded archives go there.
+- `SOURCES`: All the sources (extracted from archives and patched) go there.
+- `BUILD_<config>`: All the build files go there.
+- `BUILD_<config>/INSTALL`: The installed files go there.
+- `BUILD_<config>/LOGS`: The logs files of the build.
+
+If you want to install all those directories elsewhere, you can pass the
+`--working-dir` option to `kiwix-build`:
+
+Troubleshooting
+---------------
+
+If you need to install [Meson](https://mesonbuild.com/) "manually":
+```bash
 virtualenv -p python3 ./ # Create virtualenv
 source bin/activate      # Activate the virtualenv
 pip3 install meson       # Install Meson
 hash -r                  # Refresh bash paths
 ```
 
-Finally we need the Ninja tool (available in the $PATH). Here is a
+If you need to install [Ninja](https://ninja-build.org) "manually":
-solution to download, build and install it locally:
+```bash
-
+wget https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip
-```
+unzip ninja-linux.zip ninja -d $HOME/bin
-git clone git://github.com/ninja-build/ninja.git
-cd ninja
-git checkout release
-./configure.py --bootstrap
-mkdir ../bin
-cp ninja ../bin
-cd ..
 ```
 
-# Compilation
+License
+-------
 
-The compilation is handled by `kiwix-build.py`. It will compile
+[GPLv3](https://www.gnu.org/licenses/gpl-3.0) or later, see
-everything. If you are using a supported platform (Redhat or Debian
+[LICENSE](LICENSE) for more details.
-based) it will install missing packages using `sudo`. You can get
-`kiwix-build.py` usage like this:
-
-```
-./kiwix-build.py -h
-```
-
-## Target
-
-By default, `kiwix-build.py` will build `kiwix-tools`. If you want to
-compile another target only (let's said kiwixlib), you can specify it:
-
-```
-./kiwix-build Kiwixlib
-```
-
-## Target platform
-
-By default, `kiwix-build.py` will build everything for the current (native)
-platform using dynamic linkage (hence the `native_dyn` of the
-BUILD_native_dyn directory).
-
-But you can select another target platform using the option
-`target-platform`. For now, there is ten different supported
-platforms :
-
-- native_dyn
-- native_static
-- win32_dyn
-- win32_static
-- android_arm
-- android_arm64
-- android_mips
-- android_mips64
-- android_x86
-- android_x86_64
-
-So, if you want to compile for win32 using static linkage:
-
-```
-./kiwix-build.py --target-platform win32_dyn
-```
-
-# Outputs
-
-Kiwix-build.py will create several directories:
-- `ARCHIVES`: All the downloaded archives go there.
-- `SOURCES`: All the sources (extracted from archives and patched) go there.
-- `BUILD_native_dyn`: All the build files go there.
-- `BUILD_native_dyn/INSTALL`: The installed files go there.
-- `BUILD_native_dyn/LOGS`: The logs files of the build.
-
-If you want to install all those directories elsewhere, you can pass the
-`--working-dir` option to `kiwix-build.py`:

actions/dl_deps_archive/.gitignore

@@ -0,0 +1 @@
+node_modules

actions/dl_deps_archive/README.md

@@ -0,0 +1,55 @@
+# Download dependencies archive action
+
+This action download dependencies archive made by kiwix-build.
+It is intended to be used in projects using theses dependencies
+
+## Inputs
+
+### `base_url`
+
+The base url where we can download the archive.
+This input is provided for greater customization but you probably shouldn't set it.
+
+### `os_name`
+
+The os "name" on which the compilation is done.
+By default this use the `OS_NAME` env var, which is set in the docker file.
+
+
+### `target_platform`
+
+**Required** The targeted platform. Must be provided. Values are kind of :
+- native_dyn
+- android_arm
+- ...
+
+### `project`
+
+The name of the project being compiled.
+By default, the name of the repository.
+
+
+### `branch`
+
+The name of the "branch" to try to download (`/dev_preview/<branch>`).
+By default, the current branch on which the action is run.
+
+### `extract_dir`
+
+Where to extract the dependencies archive. By default it is `$HOME`
+
+
+## Example usage
+
+```yaml
+uses: kiwix/kiwix-build/actions/dl_deps_archive@main
+with:
+  target_platform: ${{ matrix.target_platform }}
+```
+
+```
+uses: kiwix/kiwix-build/actions/dl_deps_archive@main
+with:
+  target_platform: native_mixed
+  os_name: windows
+```

actions/dl_deps_archive/action.yml

@@ -0,0 +1,25 @@
+name: 'DL deps archive'
+description: 'Download the dependencies archive of the project'
+inputs:
+  base_url:  # id of input
+    description: 'What is the base url to download the archive'
+    required: false
+    default: 'https://tmp.kiwix.org/ci'
+  os_name:
+    description: "On which os are we running. [Default to env var `OS_NAME`]"
+    required: false
+  target_platform:
+    description: "What platform are we targetting"
+    required: true
+  project:
+    description: "What project are we building"
+    required: false
+  branch:
+    description: "On which branch are we ?"
+    required: false
+  extract_dir:
+    description: "Where to extract our dependencies. [Default to env var `HOME`]"
+    required: false
+runs:
+  using: 'node20'
+  main: 'dist/index.js'

actions/dl_deps_archive/index.js

@@ -0,0 +1,59 @@
+const tc = require("@actions/tool-cache");
+const core = require("@actions/core");
+const path = require("path");
+const os = require("os");
+
+function getInput(name, dflt) {
+  const val = process.env[`INPUT_${name.replace(/ /g, "_").toUpperCase()}`];
+  if (!val) {
+    return dflt;
+  }
+  return val;
+}
+
+function addLocalPath(inputPath) {
+  process.env["PATH"] = `${inputPath}${path.delimiter}${process.env["PATH"]}`;
+}
+
+async function run() {
+  try {
+    const base_url = core.getInput("base_url");
+    const target = core.getInput("target_platform");
+    const project = getInput(
+      "project",
+      process.env["GITHUB_REPOSITORY"].split("/")[1],
+    );
+    const branch = getInput(
+      "branch",
+      process.env["GITHUB_HEAD_REF"] || process.env["GITHUB_REF_NAME"],
+    );
+    const extract_dir = getInput(
+      "extract_dir",
+      process.env["HOME"] || process.env["GITHUB_WORKSPACE"],
+    );
+
+    let archivePath;
+    try {
+      const archive_url = `${base_url}/dev_preview/${branch}/deps_${target}_${project}.tar.gz`;
+      process.stdout.write("Downloading " + archive_url + "\n");
+      archivePath = await tc.downloadTool(archive_url);
+    } catch (error) {
+      const archive_url = `${base_url}/deps_${target}_${project}.tar.gz`;
+      process.stdout.write("Downloading " + archive_url + "\n");
+      archivePath = await tc.downloadTool(archive_url);
+    }
+
+    process.stdout.write("Extracting " + archivePath + " to " + extract_dir);
+    const archive_dir = await tc.extractTar(archivePath, extract_dir);
+    process.stdout.write("Extracted to " + archive_dir);
+  } catch (error) {
+    core.setFailed(error.message);
+  }
+}
+
+if (os.platform() === "win32") {
+  addLocalPath("C:\\Program Files\\Git\\usr\\bin");
+}
+core.setCommandEcho(true);
+
+run();

actions/dl_deps_archive/package-lock.json

@@ -0,0 +1,144 @@
+{
+  "name": "dl_deps_archive",
+  "version": "1.0.0",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "dl_deps_archive",
+      "version": "1.0.0",
+      "license": "ISC",
+      "dependencies": {
+        "@actions/core": "^1.10.1",
+        "@actions/tool-cache": "^2.0.1",
+        "path": "^0.12.7"
+      }
+    },
+    "node_modules/@actions/core": {
+      "version": "1.10.1",
+      "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.10.1.tgz",
+      "integrity": "sha512-3lBR9EDAY+iYIpTnTIXmWcNbX3T2kCkAEQGIQx4NVQ0575nk2k3GRZDTPQG+vVtS2izSLmINlxXf0uLtnrTP+g==",
+      "dependencies": {
+        "@actions/http-client": "^2.0.1",
+        "uuid": "^8.3.2"
+      }
+    },
+    "node_modules/@actions/exec": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@actions/exec/-/exec-1.1.1.tgz",
+      "integrity": "sha512-+sCcHHbVdk93a0XT19ECtO/gIXoxvdsgQLzb2fE2/5sIZmWQuluYyjPQtrtTHdU1YzTZ7bAPN4sITq2xi1679w==",
+      "dependencies": {
+        "@actions/io": "^1.0.1"
+      }
+    },
+    "node_modules/@actions/http-client": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.2.0.tgz",
+      "integrity": "sha512-q+epW0trjVUUHboliPb4UF9g2msf+w61b32tAkFEwL/IwP0DQWgbCMM0Hbe3e3WXSKz5VcUXbzJQgy8Hkra/Lg==",
+      "dependencies": {
+        "tunnel": "^0.0.6",
+        "undici": "^5.25.4"
+      }
+    },
+    "node_modules/@actions/io": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/@actions/io/-/io-1.1.3.tgz",
+      "integrity": "sha512-wi9JjgKLYS7U/z8PPbco+PvTb/nRWjeoFlJ1Qer83k/3C5PHQi28hiVdeE2kHXmIL99mQFawx8qt/JPjZilJ8Q=="
+    },
+    "node_modules/@actions/tool-cache": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/@actions/tool-cache/-/tool-cache-2.0.1.tgz",
+      "integrity": "sha512-iPU+mNwrbA8jodY8eyo/0S/QqCKDajiR8OxWTnSk/SnYg0sj8Hp4QcUEVC1YFpHWXtrfbQrE13Jz4k4HXJQKcA==",
+      "dependencies": {
+        "@actions/core": "^1.2.6",
+        "@actions/exec": "^1.0.0",
+        "@actions/http-client": "^2.0.1",
+        "@actions/io": "^1.1.1",
+        "semver": "^6.1.0",
+        "uuid": "^3.3.2"
+      }
+    },
+    "node_modules/@actions/tool-cache/node_modules/uuid": {
+      "version": "3.4.0",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
+      "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==",
+      "deprecated": "Please upgrade  to version 7 or higher.  Older versions may use Math.random() in certain circumstances, which is known to be problematic.  See https://v8.dev/blog/math-random for details.",
+      "bin": {
+        "uuid": "bin/uuid"
+      }
+    },
+    "node_modules/@fastify/busboy": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.0.tgz",
+      "integrity": "sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==",
+      "engines": {
+        "node": ">=14"
+      }
+    },
+    "node_modules/inherits": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
+      "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw=="
+    },
+    "node_modules/path": {
+      "version": "0.12.7",
+      "resolved": "https://registry.npmjs.org/path/-/path-0.12.7.tgz",
+      "integrity": "sha512-aXXC6s+1w7otVF9UletFkFcDsJeO7lSZBPUQhtb5O0xJe8LtYhj/GxldoL09bBj9+ZmE2hNoHqQSFMN5fikh4Q==",
+      "dependencies": {
+        "process": "^0.11.1",
+        "util": "^0.10.3"
+      }
+    },
+    "node_modules/process": {
+      "version": "0.11.10",
+      "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz",
+      "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==",
+      "engines": {
+        "node": ">= 0.6.0"
+      }
+    },
+    "node_modules/semver": {
+      "version": "6.3.1",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
+      "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
+      "bin": {
+        "semver": "bin/semver.js"
+      }
+    },
+    "node_modules/tunnel": {
+      "version": "0.0.6",
+      "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz",
+      "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==",
+      "engines": {
+        "node": ">=0.6.11 <=0.7.0 || >=0.7.3"
+      }
+    },
+    "node_modules/undici": {
+      "version": "5.28.4",
+      "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.4.tgz",
+      "integrity": "sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g==",
+      "dependencies": {
+        "@fastify/busboy": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=14.0"
+      }
+    },
+    "node_modules/util": {
+      "version": "0.10.4",
+      "resolved": "https://registry.npmjs.org/util/-/util-0.10.4.tgz",
+      "integrity": "sha512-0Pm9hTQ3se5ll1XihRic3FDIku70C+iHUdT/W926rSgHV5QgXsYbKZN8MSC3tJtSkhuROzvsQjAaFENRXr+19A==",
+      "dependencies": {
+        "inherits": "2.0.3"
+      }
+    },
+    "node_modules/uuid": {
+      "version": "8.3.2",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
+      "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
+      "bin": {
+        "uuid": "dist/bin/uuid"
+      }
+    }
+  }
+}

actions/dl_deps_archive/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "dl_deps_archive",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@actions/core": "^1.10.1",
+    "@actions/tool-cache": "^2.0.1",
+    "path": "^0.12.7"
+  }
+}

build_custom_app.pl

@@ -1,97 +0,0 @@
-#!/usr/bin/perl
-
-use utf8;
-use strict;
-use warnings;
-use Getopt::Long;
-
-# get the params
-my $zim_url;
-my $custom_app;
-my $keystore;
-my $api_key;
-my $version = "0";
-my $cmd;
-
-# Get console line arguments
-GetOptions('zim_url=s' => \$zim_url,
-	   'custom_app=s' => \$custom_app,
-	   'keystore=s' => \$keystore,
-	   'api_key=s' => \$api_key,
-	   'version=s' => \$version
-	   );
-
-# Print usage() if necessary
-if (!$zim_url || !$custom_app || !$keystore || !$api_key) {
-    print "usage: ./build_custom_app.pl --keystore=kiwix-android.keystore --api_key=google.json --zim_url=\"https://download.kiwix.org/zim/wikipedia_en_medicine_novid.zim\" --custom_app=wikimed [--version=1]\n";
-    exit;
-}
-
-# Clean signed ap
-$cmd = "rm ./signed_apks/*apk"; `$cmd`;
-
-# Download ZIM file
-$cmd = "wget \"$zim_url\" -O content.zim"; `$cmd`;
-
-# Get ZIM file size
-$cmd = "stat -c %s content.zim";
-my $zim_size = `$cmd` =~ s/\n//gr ;
-$ENV{ZIM_SIZE} = $zim_size;
-
-# Compute version code base
-$cmd = "date +%y%j";
-my $version_code_base = `$cmd` =~ s/\n//gr . $version;
-
-# Compute content version code
-my $content_version_code = $version_code_base;
-$ENV{CONTENT_VERSION_CODE} = $content_version_code;
-
-# Compute custom app date
-$cmd = "date +%Y-%m";
-my $date = `$cmd` =~ s/\n//gr;
-$ENV{VERSION_NAME} = $date;
-
-# Compile apps
-$ENV{VERSION_CODE} = $version_code_base;
-$cmd = "./kiwix-build.py --target-platform android_arm --android-custom-app $custom_app --zim-file-size $zim_size kiwix-android-custom"; system $cmd;
-
-$ENV{VERSION_CODE} = "1" . $version_code_base;
-$cmd = "./kiwix-build.py --target-platform android_arm64 --android-custom-app $custom_app --zim-file-size $zim_size kiwix-android-custom"; system $cmd;
-
-$ENV{VERSION_CODE} = "2" . $version_code_base;
-$cmd = "./kiwix-build.py --target-platform android_x86 --android-custom-app $custom_app --zim-file-size $zim_size kiwix-android-custom"; system $cmd;
-
-$ENV{VERSION_CODE} = "3" . $version_code_base;
-$cmd = "./kiwix-build.py --target-platform android_x86_64 --android-custom-app $custom_app --zim-file-size $zim_size kiwix-android-custom"; system $cmd;
-
-$ENV{VERSION_CODE} = "4" . $version_code_base;
-$cmd = "./kiwix-build.py --target-platform android_mips --android-custom-app $custom_app --zim-file-size $zim_size kiwix-android-custom"; system $cmd;
-
-$ENV{VERSION_CODE} = "5" . $version_code_base;
-$cmd = "./kiwix-build.py --target-platform android_mips64 --android-custom-app $custom_app --zim-file-size $zim_size kiwix-android-custom"; system $cmd;
-
-# Sign apps
-$cmd = "./TOOLCHAINS/android-sdk-r25.2.3/build-tools/25.0.2/apksigner sign -ks \"${keystore}\" --out signed_apks/app-${version_code_base}-release-signed.apk BUILD_android_arm/kiwix-android-custom_${custom_app}/app/build/outputs/apk/${custom_app}/release/app-${custom_app}-release-unsigned.apk";
-system $cmd;
-
-$cmd = "./TOOLCHAINS/android-sdk-r25.2.3/build-tools/25.0.2/apksigner sign -ks \"${keystore}\" --out signed_apks/app-1${version_code_base}-release-signed.apk BUILD_android_arm64/kiwix-android-custom_${custom_app}/app/build/outputs/apk/${custom_app}/release/app-${custom_app}-release-unsigned.apk";
-system $cmd;
-
-$cmd = "./TOOLCHAINS/android-sdk-r25.2.3/build-tools/25.0.2/apksigner sign -ks \"${keystore}\" --out signed_apks/app-2${version_code_base}-release-signed.apk BUILD_android_x86/kiwix-android-custom_${custom_app}/app/build/outputs/apk/${custom_app}/release/app-${custom_app}-release-unsigned.apk";
-system $cmd;
-
-$cmd = "./TOOLCHAINS/android-sdk-r25.2.3/build-tools/25.0.2/apksigner sign -ks \"${keystore}\" --out signed_apks/app-3${version_code_base}-release-signed.apk BUILD_android_x86_64/kiwix-android-custom_${custom_app}/app/build/outputs/apk/${custom_app}/release/app-${custom_app}-release-unsigned.apk";
-system $cmd;
-
-$cmd = "./TOOLCHAINS/android-sdk-r25.2.3/build-tools/25.0.2/apksigner sign -ks \"${keystore}\" --out signed_apks/app-4${version_code_base}-release-signed.apk BUILD_android_mips/kiwix-android-custom_${custom_app}/app/build/outputs/apk/${custom_app}/release/app-${custom_app}-release-unsigned.apk";
-system $cmd;
-
-$cmd = "./TOOLCHAINS/android-sdk-r25.2.3/build-tools/25.0.2/apksigner sign -ks \"${keystore}\" --out signed_apks/app-5${version_code_base}-release-signed.apk BUILD_android_mips64/kiwix-android-custom_${custom_app}/app/build/outputs/apk/${custom_app}/release/app-${custom_app}-release-unsigned.apk";
-system $cmd;
-
-# Upload
-
-$cmd = "./build_custom_app.py --step publish --custom-app ${custom_app} --google-api-key ${api_key} --zim-path content.zim --apks-dir signed_apks --content-version-code ${content_version_code}";
-system $cmd;
-
-exit;

build_custom_app.py

@@ -1,430 +0,0 @@
-#!/usr/bin/env python3
-
-import \
-    argparse, \
-    datetime, \
-    glob, \
-    json, \
-    os, \
-    ssl, \
-    sys, \
-    tempfile, \
-    time, \
-    urllib
-from uuid import uuid4
-from contextlib import contextmanager
-from urllib.parse import urlparse
-
-from utils import (
-    Remotefile,
-    download_remote
-)
-
-import requests
-import httplib2
-from apiclient.discovery import build
-from apiclient.errors import HttpError
-from oauth2client import client
-from oauth2client.service_account import ServiceAccountCredentials
-
-tmpl_request_url = "https://api.travis-ci.org/repo/{organisation}%2F{repository}/{endpoint}"
-tmpl_message = """Build of custom app {app} with zim file {zim}.
-
-UUID:#{uuid}#"""
-
-description = """Launch a custom application build.
-This command will launch a custom application build on Travis-CI.
-Travis-CI jobs will compile the application and upload the build apks on
-google play, using tha 'alpha' track of the application.
-
-You will need to have a valid TRAVIS_TOKEN associated to your personnal account
-and the kiwix-build repository on travis.
-Use the 'travis' command line tool (https://github.com/travis-ci/travis.rb)
-to generate a token.
-"""
-
-def parse_args():
-    parser = argparse.ArgumentParser(description=description,
-        formatter_class=argparse.RawDescriptionHelpFormatter)
-    parser.add_argument('--custom-app')
-    parser.add_argument('--travis-token')
-
-    advance = parser.add_argument_group('advance', "Some advanced options.")
-    advance.add_argument('--extra-code', type=int, default=0)
-    advance.add_argument('--version-name', default=None,
-                         help="The version of the application (seen by user). Get from json info file by default.")
-    advance.add_argument('--check-certificate', default=True)
-    advance.add_argument('--zim-url', default=None, help="Get from json info file by default.")
-    advance.add_argument('--no-android-upload', action='store_false', dest='android_upload')
-
-    # Hidden options
-    parser.add_argument('--step', default='launch', choices=['launch', 'publish'], help=argparse.SUPPRESS)
-    parser.add_argument('--apks-dir', help=argparse.SUPPRESS)
-    parser.add_argument('--zim-path', default=None, help=argparse.SUPPRESS)
-    parser.add_argument('--content-version-code', type=int, help=argparse.SUPPRESS)
-    parser.add_argument('--package-name', default=None, help=argparse.SUPPRESS)
-    parser.add_argument('--google-api-key', help=argparse.SUPPRESS)
-
-    options = parser.parse_args()
-
-    if (not options.package_name
-     or not (options.zim_url or options.zim_path)
-     or not options.version_name):
-        if not options.package_name:
-            print("Try to get package name from info.json file")
-        if not options.zim_url:
-            print("Try to get zim url from info.json file")
-        if not options.version_name:
-            print("Try to get version_name form info.json file")
-        request_url = ('https://raw.githubusercontent.com/kiwix/kiwix-android-custom/master/{}/info.json'
-                      .format(options.custom_app))
-        json_request = requests.get(request_url)
-        if json_request.status_code != 200:
-            print("Error while getting json file.")
-            print("Reason is '{}'".format(json_request.reason))
-            sys.exit(-1)
-        json_data = json.loads(json_request.text)
-        if not options.package_name:
-            print("Found package_name '{}'".format(json_data['package']))
-            options.package_name = json_data['package']
-        if not options.zim_url:
-            print("Found zim_url '{}'".format(json_data['zim_url']))
-            options.zim_url = json_data['zim_url']
-        if not options.version_name:
-            print("Found version_name '{}'".format(json_data['version_name']))
-            options.version_name = json_data['version_name']
-
-    options.base_version = "{}{}".format(
-        datetime.date.today().strftime('%y%j'),
-        options.extra_code)
-
-    return options
-
-
-def download_zim_file(zim_url, dest_dir=None):
-    if dest_dir is None:
-        dest_dir = os.getcwd()
-    out_filename = urlparse(zim_url).path
-    out_filename = os.path.basename(out_filename)
-    zim_file = Remotefile(out_filename, '', zim_url)
-    download_remote(zim_file, dest_dir)
-    return os.path.join(dest_dir, out_filename)
-
-
-def get_zim_size(*, zim_url=None, zim_path=None, check_certificate=True):
-    print("Try to get zim size")
-    if not zim_path:
-        if not check_certificate:
-            context = ssl.create_default_context()
-            context.check_hostname = False
-            context.verify_mode = ssl.CERT_NONE
-        else:
-            context = None
-        extra_args = {'context':context} if sys.version_info >= (3, 4, 3) else {}
-        with urllib.request.urlopen(zim_url, **extra_args) as resource:
-            size = resource.getheader('Content-Length', None)
-        if size is not None:
-            size = int(size)
-            print("Zim size is {}".format(size))
-            return size
-        else:
-            print("No 'Content-Length' header in http answer from the server.\n"
-                  "We need to download the zim file to get its size.")
-            zim_path = download_zim_file(zim_url, tempfile.gettempdir())
-
-    size = os.path.getsize(zim_path)
-    print("Zim size is {}".format(size))
-    return size
-
-
-def do_launch(options):
-    if options.zim_path:
-        zim_size = get_zim_size(zim_path=options.zim_path)
-    else:
-        zim_size = get_zim_size(zim_url=options.zim_url,
-                                check_certificate=options.check_certificate)
-    travis_launch_build('kiwix', 'kiwix-build', options, zim_size)
-    print("Travis build has been launch.")
-
-
-def do_publish(options):
-    zim_path = options.zim_path or download_zim_file(options.zim_url)
-    googleService = Google(options)
-    with googleService.new_request():
-        versionCodes = []
-        for apk in glob.iglob(options.apks_dir+"/*.apk"):
-            result = googleService.upload_apk(apk)
-            versionCodes.append(result['versionCode'])
-        googleService.upload_expansionfile(
-            zim_path, options.content_version_code, versionCodes)
-        googleService.publish_release(options, versionCodes)
-
-
-def travis_launch_build(organisation, repository, options, zim_size):
-    request_url = tmpl_request_url.format(
-        organisation=organisation,
-        repository=repository,
-        endpoint='requests')
-    headers = {
-        "Travis-API-Version": "3",
-        "Content-Type": "application/json",
-        "Authorization": "token {}".format(options.travis_token),
-        "User-Agent": "kiwix-build"
-    }
-    uuid = uuid4()
-    envs = []
-    for platform_index, platform in enumerate(['arm', 'arm64', 'x86', 'x86_64', 'mips', 'mips64']):
-        d = {
-            'PLATFORM': "android_{}".format(platform),
-            'VERSION_CODE': gen_version_code(platform_index, options.base_version)
-        }
-        envs.append(d)
-
-    global_env = [
-        { 'CUSTOM_APP': options.custom_app},
-        { 'ZIM_FILE_SIZE': zim_size},
-        { 'PACKAGE_NAME': options.package_name},
-        { 'ZIM_URL': options.zim_url},
-        { 'EXTRA_CODE': options.extra_code},
-        { 'CONTENT_VERSION_CODE': gen_version_code(0, options.base_version)},
-        { 'VERSION_NAME': options.version_name},
-        # google_key
-        { 'secure': ('VAgKBMx0KEIyJlSnpM4YrHKLALIbaibkhlsgiv19ITa6dODoEIqeYHz'
-                     'wFTiL3mRHU6HwtXtdNb/JeMle9NfHJVFSV56ZgFzX7ev9zr0YG0qZQv'
-                     'tl8vHQlFPBErARder/L2tblOTM194/TiJk/q89a0XWDanKswXExwjcW'
-                     'Z0tnDYQXTHSAKEt+xW8hjbnhqqB/v16lX6dUjZI+sVlsw+qAM4VT/qf'
-                     'FCyDO5eJCzWIEL2LDUWI7jKSETNih5hl5fMMvCCNRPnkgGnytw5kF/t'
-                     'Lw8YAbLRxkGsO4FCx5mB7HF5pNHyWOCCalMTKheyg/qUV/VcXW9Unlr'
-                     'puMu0+d3hpLZESplS/NkvDxSrx16ank7EORS8OxLOufu56TW2hDuBzz'
-                     'w1CBAj1p6s+Z6Kc4RMYYdxRgR1TjXg/ZVUn3T69d9igdS/5lAPFx2Ww'
-                     '8x82FWCLSaiXymxXRNsNcKx5ifuvtv307r4yh31QjlKFYwadOCaCHHZ'
-                     'zGE1mXcOu3j6W9WIaZfYRTpxmOrcfDIHxZSdLf11hOSZEUUFpj9hQlV'
-                     'Op0RHkDEJUMNs6vkXUhZq9yPuqgrcb6GaN+UhOT8iHlijKmlG8NJEPk'
-                     'Hp8RnL1hsr44N57ZzLqmSUZtvC83u/5e+YUb7beUDGsMyJV/fcMiGMM'
-                     'LVtRnuPCFyNVNQUf2CphtG0=') },
-        # pass
-        { 'secure': ('AtbgKUukES2uJPpEWNEDHLg0WcghLlCGL171Ah3+4CckBI8y1Fn+VpH'
-                     'U2vEXzsV8tKoxX1IyB2tFivzuyo6CQXHSuWGJYQexwkBeGCgOfzKJLj'
-                     'MAy75ATYA6JnFrikV+UcqdEz/9Dow3J1K7Slp3jpsQhERHbNeqkr4I+'
-                     'XCL1LLnpewfOZo9OIEu93p6b6YlqvIPXJHyQe5xnMd8jFWg3/uIYqFn'
-                     'XPvigeZqC2lhNp48mj4JdwwF2tmiArgyXOmgxiuHJNVVI7okbhc7kmI'
-                     'Y3MmCSFgG0XPUEBU3Kdr4o/2hy8DDP6Gff+rUZW8nPI/2UWXRLWtOxv'
-                     'XGGRyjHHTxGWzI4JyZbli9dls5M32MMjsXVKtciSFVwsMM8qn7wFnRi'
-                     'q248a1Sg5fDNX/WYowmsHlWjffHZ7+UqUqJxAKtZ0vpQL+4SPIALPnK'
-                     'V6j6CoorQp+VhMF01EFlZ0c8bkNmk4YW7R7RyNLIcaHKfd1ud8QF9PD'
-                     'AnQ7Jr1GRBxzkjHvHfFrE14WPUu+FjVvDO7UPVMNQX7RS1IVACpKSRu'
-                     '7N8KnIK3vSnLpn5GXKsbx0JB2vtyoTaFZvC9c3qyAw1nlpn7Lp3sPs3'
-                     'bgIBU4tWOzg5g46eHbc4ad5nyB9Soz715lbMdECvKs2HHJUG3tubLKj'
-                     'L0S/LRGRQ+IDgC7xrjQj8aA=') },
-    ]
-
-    env = {
-        'matrix': envs,
-        'global': global_env
-    }
-
-    data = {
-        'request': {
-            'message' : tmpl_message.format(app=options.custom_app, zim=options.zim_url, uuid=uuid),
-            'branch'  : "custom_app",
-            'config'  : {
-                'before_install' : [
-                    ( 'pip3 install pyOpenSSl google-api-python-client'
-                      ' httplib2 apiclient requests'),
-                    ( 'openssl aes-256-cbc -k $google_key'
-                      ' -in travis/googleplay_android_developer-5a411156212c.json.enc'
-                      ' -out travis/googleplay_android_developer-5a411156212c.json'
-                      ' -d'),
-                    ( 'openssl aes-256-cbc -k $google_key'
-                      ' -in travis/kiwix-android.keystore.enc'
-                      ' -out travis/kiwix-android.keystore -d'),
-                    ( 'openssl aes-256-cbc -K $encrypted_eba2f7543984_key'
-                      ' -iv $encrypted_eba2f7543984_iv'
-                      ' -in travis/travisci_builder_id_key.enc'
-                      ' -out travis/travisci_builder_id_key -d'),
-                    'chmod 600 travis/travisci_builder_id_key'
-                ],
-                'env' : env,
-                'script' : 'travis_wait 30 travis/compile_custom_app.sh',
-                'deploy' : {
-                     'provider': 'script',
-                     'skip_cleanup': True,
-                     'script': 'travis/deploy_apk.sh',
-                     'on': {
-                         'branch': 'custom_app'
-                     }
-                }
-            }
-        }
-    }
-
-    if options.android_upload:
-        data['request']['config']['jobs'] = {
-            'include': [
-                {
-                    'stage' : 'make_release',
-                    'install': 'pip3 install -r requirements_build_custom_app.txt',
-                    'script': True,
-                    'env': global_env,
-                    'deploy' : {
-                        'provider': 'script',
-                        'skip_cleanup': True,
-                        'script': 'travis/make_release.sh',
-                        'on': {
-                            'branch': 'custom_app'
-                        }
-                    }
-                }
-            ]
-        }
-        global_env.append({
-            'DEPLOY_DIR' : '/home/nightlybot/apks/{}_{}'.format(
-                options.custom_app, options.base_version)
-        })
-    else:
-        global_env.append({
-            'DEPLOY_DIR' : '/var/www/tmp.kiwix.org/custom_apps/{}_{}'.format(
-                options.custom_app, options.base_version)
-        })
-
-
-    r = requests.post(request_url, headers=headers, json=data)
-    if r.status_code != 202:
-        print("Error while requesting build:")
-        print(r.reason)
-        print("Have you forget to give the travis token ?")
-        sys.exit(-1)
-    else:
-        request_id = r.json()['request']['id']
-        print("Request {} has been schedule.".format(request_id))
-        request_left = 10
-        found = False
-        request_url = tmpl_request_url.format(
-            organisation=organisation,
-            repository=repository,
-            endpoint='builds')
-        while request_left:
-            time.sleep(1)
-            print("Try to get associated build.")
-            r = requests.get(request_url, headers=headers)
-            json_data = json.loads(r.text)
-            for build in json_data['builds']:
-                if build['event_type'] != 'api':
-                    continue
-                message = build['commit']['message']
-                if str(uuid) in message:
-                    found = True
-                    break
-            if found:
-                break
-            print("Cannot find build. Wait 1 second and try again")
-            print("{} tries left".format(request_left))
-            request_left -= 1
-        if found:
-            print("Associated build found:  {}.".format(build['number']))
-            print("https://travis-ci.org/kiwix/kiwix-build/builds/{}".format(build['id']))
-        else:
-            print("Request has been accepted by travis-ci but I cannot found "
-                  "the associated build. Have a look here "
-                  "https://travis-ci.org/kiwix/kiwix-build/builds"
-                  "if you found it.")
-        if not options.android_upload:
-            print(("Automatic upload to android play store has been deactivated.\n"
-                   "You will find the apks at this address once they have been compiled :"
-                   " http://tmp.kiwix.org/custom_apps/{}_{}").format(
-                       options.custom_app, options.base_version))
-
-
-ERROR_MSG_EDIT_CHANGE = "A change was made to the application outside of this Edit, please create a new edit."
-
-class Google:
-    def __init__(self, options):
-        scope = 'https://www.googleapis.com/auth/androidpublisher'
-        key = options.google_api_key
-        credentials = ServiceAccountCredentials.from_json_keyfile_name(
-            key,
-            scopes=[scope])
-
-        http = httplib2.Http()
-        http = credentials.authorize(http)
-
-        self.service = build('androidpublisher', 'v2', http=http)
-        self.packageName = options.package_name
-        self.edit_id = None
-
-    @contextmanager
-    def new_request(self):
-        edit_request = self.service.edits().insert(
-            packageName=self.packageName,
-            body={})
-        result = edit_request.execute()
-        print("create", result)
-        self.edit_id = result['id']
-
-        yield
-
-        commit_request = self.service.edits().commit(
-            editId=self.edit_id,
-            packageName=self.packageName)
-        result = commit_request.execute()
-        print("commit", result)
-
-        self.edit_id = None
-
-    def make_request(self, section, method, **kwargs):
-        request_content = self._build_request_content(kwargs)
-        _section = getattr(self._edits, section)()
-        _method = getattr(_section, method)
-        print(">", request_content)
-        request = _method(**request_content)
-        result = request.execute()
-        print("<", result)
-        return result
-
-    def _build_request_content(self, kwargs):
-        d = kwargs.copy()
-        d['editId'] = self.edit_id
-        d['packageName'] = self.packageName
-        return d
-
-    @property
-    def _edits(self):
-        return self.service.edits()
-
-    def upload_expansionfile(self, comp_file, contentVersionCode, versionCodes):
-        versionCodes = [int(v) for v in versionCodes]
-        self.make_request('expansionfiles', 'upload',
-            expansionFileType='main',
-            apkVersionCode=contentVersionCode,
-            media_body=comp_file,
-            media_mime_type='application/octet-stream')
-        for versionCode in versionCodes:
-            if versionCode == contentVersionCode:
-                continue
-            self.make_request('expansionfiles', 'update',
-                expansionFileType='main',
-                apkVersionCode=versionCode,
-                body={'referencesVersion': contentVersionCode}
-            )
-
-    def upload_apk(self, apk_file):
-        return self.make_request('apks', 'upload',
-            media_body=apk_file)
-
-    def publish_release(self, options, versionCodes):
-        return self.make_request('tracks', 'update',
-            track="alpha",
-            body={'versionCodes': versionCodes})
-
-
-def gen_version_code(platform_index, base_version):
-    str_version = "{platform}{base_version}".format(
-        platform=platform_index,
-        base_version=base_version
-    )
-    return int(str_version)
-
-if __name__ == "__main__":
-    options = parse_args()
-    func = globals()['do_{}'.format(options.step)]
-    func(options)

dependencies.py

@@ -1,547 +0,0 @@
-import shutil, os, json
-from urllib.parse import urlparse
-
-from dependency_utils import (
-    Dependency,
-    ReleaseDownload,
-    GitClone,
-    SvnClone,
-    NoopSource,
-    MakeBuilder,
-    CMakeBuilder,
-    MesonBuilder,
-    GradleBuilder,
-    NoopBuilder,
-    Builder as BaseBuilder)
-
-from utils import Remotefile, pj, SkipCommand, copy_tree, add_execution_right
-
-# *************************************
-# Missing dependencies
-# Is this ok to assume that those libs
-# exist in your "distri" (linux/mac) ?
-# If not, we need to compile them here
-# *************************************
-# aria2
-# Argtable
-# MSVirtual
-# Android
-# libiconv
-# gettext
-# *************************************
-
-
-class zlib(Dependency):
-    name = 'zlib'
-
-    class Source(ReleaseDownload):
-        archive = Remotefile('zlib-1.2.8.tar.gz',
-                             '36658cb768a54c1d4dec43c3116c27ed893e88b02ecfcb44f2166f9c0b7f2a0d')
-        patches = ['zlib_std_libname.patch']
-
-    class Builder(MakeBuilder):
-        dynamic_configure_option = "--shared"
-        static_configure_option = "--static"
-
-        def _pre_build_script(self, context):
-            context.try_skip(self.build_path)
-            shutil.copytree(self.source_path, self.build_path)
-
-        @property
-        def all_configure_option(self):
-            return '--static' if self.buildEnv.platform_info.static else '--shared'
-
-        @property
-        def configure_option(self):
-            options = "-DINSTALL_PKGCONFIG_DIR={}".format(pj(self.buildEnv.install_dir, self.buildEnv.libprefix, 'pkgconfig'))
-            if self.buildEnv.platform_info.static:
-                options += " -DBUILD_SHARED_LIBS=false"
-            else:
-                options += " -DBUILD_SHARED_LIBS=true"
-            return options
-
-        def _configure(self, context):
-            if self.buildEnv.platform_info.build == 'win32':
-                raise SkipCommand()
-            return super()._configure(context)
-
-        @property
-        def make_option(self):
-            if self.buildEnv.platform_info.build == 'win32':
-                return "--makefile win32/Makefile.gcc PREFIX={host}- SHARED_MODE={static} INCLUDE_PATH={include_path} LIBRARY_PATH={library_path} BINARY_PATH={binary_path}".format(
-                    host='i686-w64-mingw32',
-                    static="0" if self.buildEnv.platform_info.static else "1",
-                    include_path=pj(self.buildEnv.install_dir, 'include'),
-                    library_path=pj(self.buildEnv.install_dir, self.buildEnv.libprefix),
-                    binary_path=pj(self.buildEnv.install_dir, 'bin'),
-                    )
-            return ""
-
-class lzma(Dependency):
-    name = 'lzma'
-
-    class Source(ReleaseDownload):
-        archive = Remotefile('xz-5.0.4.tar.bz2',
-                             '5cd9b060d3a1ad396b3be52c9b9311046a1c369e6062aea752658c435629ce92')
-
-    class Builder(MakeBuilder):
-        @property
-        def configure_option(self):
-            return "--disable-assembler"
-
-class UUID(Dependency):
-    name = 'uuid'
-
-    class Source(ReleaseDownload):
-        archive = Remotefile('e2fsprogs-libs-1.43.4.tar.gz',
-                             'eed4516325768255c9745e7b82c9d7d0393abce302520a5b2cde693204b0e419',
-                             'https://www.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v1.43.4/e2fsprogs-libs-1.43.4.tar.gz')
-        extract_dir = 'e2fsprogs-libs-1.43.4'
-
-    class Builder(MakeBuilder):
-        configure_option = ("--enable-libuuid --disable-debugfs --disable-imager --disable-resizer --disable-defrag --enable-fsck"
-                            " --disable-uuidd")
-        configure_env = {'_format_CFLAGS': "{env.CFLAGS} -fPIC"}
-        static_configure_option = dynamic_configure_option = ""
-        make_target = 'libs'
-        make_install_target = 'install-libs'
-
-
-class Xapian(Dependency):
-    name = "xapian-core"
-
-    class Source(ReleaseDownload):
-        archive = Remotefile('xapian-core-1.4.5.tar.xz',
-                             '85b5f952de9df925fd13e00f6e82484162fd506d38745613a50b0a2064c6b02b')
-
-    class Builder(MakeBuilder):
-        configure_option = "--disable-sse --disable-backend-inmemory --disable-documentation"
-        configure_env = {'_format_LDFLAGS': "-L{buildEnv.install_dir}/{buildEnv.libprefix}",
-                         '_format_CXXFLAGS': "-I{buildEnv.install_dir}/include"}
-
-    @property
-    def dependencies(self):
-        deps = ['zlib', 'lzma']
-        if (self.buildEnv.platform_info.build == 'win32'
-         or self.buildEnv.distname == 'Darwin'):
-            return deps
-        return deps + ['uuid']
-
-
-class CTPP2(Dependency):
-    name = "ctpp2"
-
-    class Source(ReleaseDownload):
-        name = "ctpp2"
-        source_dir = "ctpp2-2.8.3"
-        archive = Remotefile('ctpp2-2.8.3.tar.gz',
-                             'a83ffd07817adb575295ef40fbf759892512e5a63059c520f9062d9ab8fb42fc')
-        patches = ["ctpp2_include.patch",
-                   "ctpp2_no_src_modification.patch",
-                   "ctpp2_fix-static-libname.patch",
-                   "ctpp2_mingw32.patch",
-                   "ctpp2_dll_export_VMExecutable.patch",
-                   "ctpp2_win_install_lib_in_lib_dir.patch",
-                   "ctpp2_iconv_support.patch",
-                   "ctpp2_compile_ctpp2c_static.patch",
-                  ]
-
-    class Builder(CMakeBuilder):
-        configure_option = "-DMD5_SUPPORT=OFF"
-
-
-class CTPP2C(CTPP2):
-    name = "ctpp2c"
-    force_native_build = True
-
-    class Builder(CTPP2.Builder):
-        make_target = "ctpp2c"
-
-        @property
-        def build_path(self):
-            return super().build_path+"_native"
-
-        def _install(self, context):
-            context.try_skip(self.build_path)
-            command = "cp {ctpp2c}* {install_dir}".format(
-                ctpp2c=pj(self.build_path, 'ctpp2c'),
-                install_dir=pj(self.buildEnv.install_dir, 'bin')
-            )
-            self.buildEnv.run_command(command, self.build_path, context)
-
-
-class Pugixml(Dependency):
-    name = "pugixml"
-
-    class Source(ReleaseDownload):
-        archive = Remotefile('pugixml-1.2.tar.gz',
-                             '0f422dad86da0a2e56a37fb2a88376aae6e931f22cc8b956978460c9db06136b')
-        patches = ["pugixml_meson.patch"]
-
-    Builder = MesonBuilder
-
-
-class MicroHttpd(Dependency):
-    name = "libmicrohttpd"
-
-    class Source(ReleaseDownload):
-        archive = Remotefile('libmicrohttpd-0.9.46.tar.gz',
-                             '06dbd2654f390fa1e8196fe063fc1449a6c2ed65a38199a49bf29ad8a93b8979',
-                             'http://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.46.tar.gz')
-
-    class Builder(MakeBuilder):
-        configure_option = "--disable-https --without-libgcrypt --without-libcurl"
-
-
-class Gumbo(Dependency):
-    name = "gumbo"
-
-    class Source(ReleaseDownload):
-        archive = Remotefile('gumbo-0.10.1.tar.gz',
-                             '28463053d44a5dfbc4b77bcf49c8cee119338ffa636cc17fc3378421d714efad',
-                             'https://github.com/google/gumbo-parser/archive/v0.10.1.tar.gz')
-
-        def _post_prepare_script(self, context):
-            context.try_skip(self.extract_path)
-            command = "./autogen.sh"
-            self.buildEnv.run_command(command, self.extract_path, context)
-
-    Builder = MakeBuilder
-
-
-class Icu(Dependency):
-    name = "icu4c"
-
-    class Source(SvnClone):
-        name = "icu4c"
-        svn_remote = "http://source.icu-project.org/repos/icu/tags/release-58-2/icu4c"
-        svn_dir = "icu4c"
-
-        patches = ["icu4c_fix_static_lib_name_mingw.patch",
-                   "icu4c_android_elf64_st_info.patch",
-                   "icu4c_custom_data.patch",
-                   "icu4c_noxlocale.patch"]
-
-
-    class Builder(MakeBuilder):
-        subsource_dir = "source"
-
-        @property
-        def configure_option(self):
-            options = "--disable-samples --disable-tests --disable-extras --disable-dyload"
-            if self.buildEnv.platform_info.build == 'android':
-                options += " --with-data-packaging=archive"
-            return options
-
-
-class Icu_native(Icu):
-    name = "icu4c_native"
-    force_native_build = True
-
-    class Builder(Icu.Builder):
-        @property
-        def build_path(self):
-            return super().build_path+"_native"
-
-        def _install(self, context):
-            raise SkipCommand()
-
-
-class Icu_cross_compile(Icu):
-    name = "icu4c_cross-compile"
-    dependencies = ['icu4c_native']
-
-    class Builder(Icu.Builder):
-        @property
-        def configure_option(self):
-            icu_native_dep = self.buildEnv.targetsDict['icu4c_native']
-            return super().configure_option + " --with-cross-build=" + icu_native_dep.builder.build_path
-
-
-class Libzim(Dependency):
-    name = "libzim"
-
-    @property
-    def dependencies(self):
-        base_dependencies = ['zlib', 'lzma', 'xapian-core']
-        if self.buildEnv.platform_info.build != 'native':
-            return base_dependencies + ["icu4c_cross-compile"]
-        else:
-            return base_dependencies + ["icu4c"]
-
-    class Source(GitClone):
-        git_remote = "https://github.com/openzim/libzim.git"
-        git_dir = "libzim"
-
-    class Builder(MesonBuilder):
-        test_option = "-t 8"
-
-
-class ZimTools(Dependency):
-    name = "zim-tools"
-    dependencies = ['libzim']
-
-    class Source(GitClone):
-        git_remote = "https://github.com/openzim/zim-tools.git"
-        git_dir = "zim-tools"
-
-    class Builder(MesonBuilder):
-        @property
-        def configure_option(self):
-            if self.buildEnv.platform_info.static:
-                return "-Dstatic-linkage=true"
-            return ""
-
-
-class Zimwriterfs(Dependency):
-    name = "zimwriterfs"
-    extra_packages = ['file']
-
-    @property
-    def dependencies(self):
-        base_dependencies = ['libzim', 'zlib', 'lzma', 'xapian-core', 'gumbo']
-        if self.buildEnv.platform_info.build != 'native':
-            return base_dependencies + ["icu4c_cross-compile"]
-        else:
-            return base_dependencies + ["icu4c"]
-
-    class Source(GitClone):
-        git_remote = "https://github.com/openzim/zimwriterfs.git"
-        git_dir = "zimwriterfs"
-        release_git_ref = "1.1"
-
-        def _post_prepare_script(self, context):
-            context.try_skip(self.git_path)
-            command = "./autogen.sh"
-            self.buildEnv.run_command(command, self.git_path, context)
-
-    Builder = MakeBuilder
-
-
-class Kiwixlib(Dependency):
-    name = "kiwix-lib"
-
-    @property
-    def dependencies(self):
-        base_dependencies = ["pugixml", "libzim", "zlib", "lzma"]
-        if ( self.buildEnv.platform_info.build != 'android'
-          and self.buildEnv.distname != 'Darwin'):
-            base_dependencies += ['ctpp2c', 'ctpp2']
-        if self.buildEnv.platform_info.build != 'native':
-            return base_dependencies + ["icu4c_cross-compile"]
-        else:
-            return base_dependencies + ["icu4c"]
-
-    class Source(GitClone):
-        git_remote = "https://github.com/kiwix/kiwix-lib.git"
-        git_dir = "kiwix-lib"
-
-    class Builder(MesonBuilder):
-        @property
-        def configure_option(self):
-            base_option = "-Dctpp2-install-prefix={buildEnv.install_dir}"
-            if self.buildEnv.platform_info.build == 'android':
-                base_option += ' -Dandroid=true'
-            return base_option
-
-        @property
-        def library_type(self):
-            if self.buildEnv.platform_info.build == 'android':
-                return 'shared'
-            return super().library_type
-
-
-class KiwixTools(Dependency):
-    name = "kiwix-tools"
-    dependencies = ["kiwix-lib", "libmicrohttpd", "zlib"]
-
-    class Source(GitClone):
-        git_remote = "https://github.com/kiwix/kiwix-tools.git"
-        git_dir = "kiwix-tools"
-
-    class Builder(MesonBuilder):
-        @property
-        def configure_option(self):
-            if self.buildEnv.platform_info.static:
-                return "-Dstatic-linkage=true"
-            return ""
-
-
-class Gradle(Dependency):
-    name = "Gradle"
-
-    class Source(ReleaseDownload):
-        archive = Remotefile('gradle-4.1-bin.zip',
-                             'd55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43',
-                             'https://services.gradle.org/distributions/gradle-4.1-bin.zip')
-
-    class Builder(BaseBuilder):
-        def build(self):
-            self.command('install', self._install)
-
-        def _install(self, context):
-            copy_tree(
-                pj(self.source_path, "bin"),
-                pj(self.buildEnv.install_dir, "bin"),
-                post_copy_function = add_execution_right)
-            copy_tree(
-                pj(self.source_path, "lib"),
-                pj(self.buildEnv.install_dir, "lib"))
-
-
-class AllBaseDependencies(Dependency):
-    name = "alldependencies"
-
-    @property
-    def dependencies(self):
-        base_deps = ['zlib', 'lzma', 'xapian-core', 'gumbo', 'pugixml', 'libmicrohttpd']
-        if self.buildEnv.platform_info.build != 'native':
-            base_deps += ["icu4c_cross-compile"]
-        else:
-            base_deps += ["icu4c"]
-        if ( self.buildEnv.platform_info.build != 'android'
-           and self.buildEnv.distname != 'Darwin'):
-            base_deps += ['ctpp2c', 'ctpp2']
-        if self.buildEnv.platform_info.build == 'android':
-            base_deps += ['Gradle']
-
-        return base_deps
-
-
-    Source = NoopSource
-    Builder = NoopBuilder
-
-
-class KiwixAndroid(Dependency):
-    name = "kiwix-android"
-    dependencies = ["Gradle", "kiwix-lib"]
-
-    class Source(GitClone):
-        git_remote = "https://github.com/kiwix/kiwix-android"
-        git_dir = "kiwix-android"
-
-    class Builder(GradleBuilder):
-        def build(self):
-            if self.buildEnv.options.targets == 'kiwix-android-custom':
-                print("SKIP")
-            else:
-                super().build()
-
-        def _configure(self, context):
-            if not os.path.exists(self.build_path):
-                shutil.copytree(self.source_path, self.build_path, symlinks=True)
-            try:
-                shutil.rmtree(pj(self.build_path, 'kiwixlib', 'src', 'main'))
-            except FileNotFoundError:
-                pass
-            shutil.copytree(pj(self.buildEnv.install_dir, 'kiwix-lib'),
-                            pj(self.build_path, 'kiwixlib', 'src', 'main'))
-            os.makedirs(
-                pj(self.build_path, 'app', 'src', 'main', 'assets', 'icu'),
-                exist_ok=True)
-            shutil.copy2(pj(self.buildEnv.install_dir, 'share', 'icu', '58.2',
-                            'icudt58l.dat'),
-                         pj(self.build_path, 'app', 'src', 'main', 'assets',
-                            'icu', 'icudt58l.dat'))
-
-
-class KiwixCustomApp(Dependency):
-    name = "kiwix-android-custom"
-    dependencies = ["kiwix-android", "kiwix-lib"]
-
-    def __init__(self, buildEnv):
-        super().__init__(buildEnv)
-        self.custom_name = buildEnv.options.android_custom_app
-
-    class Source(GitClone):
-        git_remote = "https://github.com/kiwix/kiwix-android-custom"
-        git_dir = "kiwix-android-custom"
-
-    class Builder(GradleBuilder):
-        @property
-        def gradle_target(self):
-            return "assemble{}".format(self.target.custom_name)
-
-        @property
-        def gradle_option(self):
-            template = ("-i -P customDir={customDir}"
-                        " -P zim_file_size={zim_size}"
-                        " -P version_code={version_code}"
-                        " -P version_name={version_name}"
-                        " -P content_version_code={content_version_code}")
-            return template.format(
-                customDir=pj(self.build_path, 'custom'),
-                zim_size=self._get_zim_size(),
-                version_code=os.environ['VERSION_CODE'],
-                version_name=os.environ['VERSION_NAME'],
-                content_version_code=os.environ['CONTENT_VERSION_CODE'])
-
-        @property
-        def build_path(self):
-            return pj(self.buildEnv.build_dir, "{}_{}".format(self.target.full_name, self.target.custom_name))
-
-        @property
-        def custom_build_path(self):
-            return pj(self.build_path, 'custom', self.target.custom_name)
-
-        def _get_zim_size(self):
-            try:
-                zim_size = self.buildEnv.options.zim_file_size
-            except AttributeError:
-                with open(pj(self.source_path, self.target.custom_name, 'info.json')) as f:
-                    app_info = json.load(f)
-                zim_size = os.path.getsize(pj(self.custom_build_path, app_info['zim_file']))
-            return zim_size
-
-        def build(self):
-            self.command('configure', self._configure)
-            self.command('download_zim', self._download_zim)
-            self.command('compile', self._compile)
-
-        def _download_zim(self, context):
-            zim_url = self.buildEnv.options.zim_file_url
-            if zim_url is None:
-                raise SkipCommand()
-            with open(pj(self.source_path, self.target.custom_name, 'info.json')) as f:
-                app_info = json.load(f)
-            zim_url = app_info.get('zim_url', zim_url)
-            out_filename = urlparse(zim_url).path
-            out_filename = os.path.basename(out_filename)
-            zim_file = Remotefile(out_filename, '', zim_url)
-            self.buildEnv.download(zim_file)
-            shutil.copy(pj(self.buildEnv.archive_dir, out_filename),
-                        pj(self.custom_build_path, app_info['zim_file']))
-
-        def _configure(self, context):
-            # Copy kiwix-android in build dir.
-            kiwix_android_dep = self.buildEnv.targetsDict['kiwix-android']
-            if not os.path.exists(self.build_path):
-                shutil.copytree(kiwix_android_dep.source_path, self.build_path)
-
-            # Copy kiwix-lib application in build dir
-            try:
-                shutil.rmtree(pj(self.build_path, 'kiwixlib', 'src', 'main'))
-            except FileNotFoundError:
-                pass
-            shutil.copytree(pj(self.buildEnv.install_dir, 'kiwix-lib'),
-                            pj(self.build_path, 'kiwixlib', 'src', 'main'))
-            os.makedirs(
-                pj(self.build_path, 'app', 'src', 'main', 'assets', 'icu'),
-                exist_ok=True)
-            shutil.copy2(pj(self.buildEnv.install_dir, 'share', 'icu', '58.2',
-                            'icudt58l.dat'),
-                         pj(self.build_path, 'app', 'src', 'main', 'assets',
-                            'icu', 'icudt58l.dat'))
-
-            # Generate custom directory
-            try:
-                shutil.rmtree(pj(self.build_path, 'custom'))
-            except FileNotFoundError:
-                pass
-            os.makedirs(pj(self.build_path, 'custom'))
-            command = "./gen-custom-android-directory.py {custom_name} --output-dir {custom_dir}"
-            command = command.format(
-                custom_name=self.target.custom_name,
-                custom_dir=pj(self.build_path, 'custom', self.target.custom_name)
-            )
-            self.buildEnv.run_command(command, self.source_path, context)

dependency_utils.py

@@ -1,431 +0,0 @@
-import subprocess
-import os
-import shutil
-
-from utils import pj, Context, SkipCommand, extract_archive, Defaultdict, StopBuild
-from dependency_versions import main_project_versions, base_deps_versions
-
-SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
-
-
-class _MetaDependency(type):
-    def __new__(cls, name, bases, dct):
-        _class = type.__new__(cls, name, bases, dct)
-        if name != 'Dependency':
-            dep_name = dct['name']
-            Dependency.all_deps[dep_name] = _class
-        return _class
-
-
-class Dependency(metaclass=_MetaDependency):
-    all_deps = {}
-    dependencies = []
-    force_native_build = False
-
-    def __init__(self, buildEnv):
-        self.buildEnv = buildEnv
-        self.source = self.Source(self)
-        self.builder = self.Builder(self)
-        self.skip = False
-
-    @property
-    def version(self):
-        return base_deps_versions.get(self.name, None)
-
-    @property
-    def full_name(self):
-        if self.version:
-            return "{}-{}".format(self.name, self.version)
-        return self.name
-
-    @property
-    def source_path(self):
-        return pj(self.buildEnv.source_dir, self.source.source_dir)
-
-    @property
-    def _log_dir(self):
-        return self.buildEnv.log_dir
-
-    def command(self, name, function, *args):
-        print("  {} {} : ".format(name, self.name), end="", flush=True)
-        log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
-        context = Context(name, log, self.force_native_build)
-        try:
-            ret = function(*args, context=context)
-            context._finalise()
-            print("OK")
-            return ret
-        except SkipCommand:
-            print("SKIP")
-        except subprocess.CalledProcessError:
-            print("ERROR")
-            try:
-                with open(log, 'r') as f:
-                    print(f.read())
-            except:
-                pass
-            raise StopBuild()
-        except:
-            print("ERROR")
-            raise
-
-
-class Source:
-    """Base Class to the real preparator
-       A source preparator must install source in the self.source_dir attribute
-       inside the buildEnv.source_dir."""
-    def __init__(self, target):
-        self.target = target
-        self.buildEnv = target.buildEnv
-
-    @property
-    def name(self):
-        return self.target.name
-
-    @property
-    def source_dir(self):
-        return self.target.full_name
-
-    def _patch(self, context):
-        source_path = pj(self.buildEnv.source_dir, self.source_dir)
-        context.try_skip(source_path)
-        context.force_native_build = True
-        for p in self.patches:
-            with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input:
-                self.buildEnv.run_command("patch -p1", source_path, context, input=patch_input.read())
-
-    def command(self, *args, **kwargs):
-        return self.target.command(*args, **kwargs)
-
-
-class NoopSource(Source):
-    def prepare(self):
-        pass
-
-
-class ReleaseDownload(Source):
-    archive_top_dir = None
-
-    @property
-    def extract_path(self):
-        return pj(self.buildEnv.source_dir, self.source_dir)
-
-    def _download(self, context):
-        context.try_skip(self.buildEnv.archive_dir, self.name)
-        self.buildEnv.download(self.archive)
-
-    def _extract(self, context):
-        context.try_skip(self.extract_path)
-        if os.path.exists(self.extract_path):
-            shutil.rmtree(self.extract_path)
-        extract_archive(pj(self.buildEnv.archive_dir, self.archive.name),
-                        self.buildEnv.source_dir,
-                        topdir=self.archive_top_dir,
-                        name=self.source_dir)
-
-    def prepare(self):
-        self.command('download', self._download)
-        self.command('extract', self._extract)
-        if hasattr(self, 'patches'):
-            self.command('patch', self._patch)
-        if hasattr(self, '_post_prepare_script'):
-            self.command('post_prepare_script', self._post_prepare_script)
-
-
-class GitClone(Source):
-    base_git_ref = "master"
-
-    @property
-    def release_git_ref(self):
-        return main_project_versions.get(self.name, "master")
-
-    @property
-    def source_dir(self):
-        if self.buildEnv.make_release:
-            return "{}_release".format(self.git_dir)
-        else:
-            return self.git_dir
-
-    @property
-    def git_path(self):
-        return pj(self.buildEnv.source_dir, self.source_dir)
-
-    @property
-    def git_ref(self):
-        if self.buildEnv.make_release:
-            return self.release_git_ref
-        else:
-            return self.base_git_ref
-
-    def _git_clone(self, context):
-        context.force_native_build = True
-        if os.path.exists(self.git_path):
-            raise SkipCommand()
-        command = "git clone --depth=1 --branch {} {} {}".format(
-            self.git_ref, self.git_remote, self.source_dir)
-        self.buildEnv.run_command(command, self.buildEnv.source_dir, context)
-
-    def _git_update(self, context):
-        context.force_native_build = True
-        command = "git fetch origin {}".format(
-            self.git_ref)
-        self.buildEnv.run_command(command, self.git_path, context)
-        self.buildEnv.run_command("git checkout "+self.git_ref, self.git_path, context)
-
-    def prepare(self):
-        self.command('gitclone', self._git_clone)
-        self.command('gitupdate', self._git_update)
-        if hasattr(self, '_post_prepare_script'):
-            self.command('post_prepare_script', self._post_prepare_script)
-
-
-class SvnClone(Source):
-    @property
-    def source_dir(self):
-        return self.svn_dir
-
-    @property
-    def svn_path(self):
-        return pj(self.buildEnv.source_dir, self.svn_dir)
-
-    def _svn_checkout(self, context):
-        context.force_native_build = True
-        if os.path.exists(self.svn_path):
-            raise SkipCommand()
-        command = "svn checkout {} {}".format(self.svn_remote, self.svn_dir)
-        self.buildEnv.run_command(command, self.buildEnv.source_dir, context)
-
-    def _svn_update(self, context):
-        context.force_native_build = True
-        self.buildEnv.run_command("svn update", self.svn_path, context)
-
-    def prepare(self):
-        self.command('svncheckout', self._svn_checkout)
-        self.command('svnupdate', self._svn_update)
-        if hasattr(self, 'patches'):
-            self.command('patch', self._patch)
-
-
-class Builder:
-    subsource_dir = None
-
-    def __init__(self, target):
-        self.target = target
-        self.buildEnv = target.buildEnv
-
-    @property
-    def name(self):
-        return self.target.name
-
-    @property
-    def source_path(self):
-        base_source_path = self.target.source_path
-        if self.subsource_dir:
-            return pj(base_source_path, self.subsource_dir)
-        return base_source_path
-
-    @property
-    def build_path(self):
-        return pj(self.buildEnv.build_dir, self.target.full_name)
-
-    def command(self, *args, **kwargs):
-        return self.target.command(*args, **kwargs)
-
-    def build(self):
-        if hasattr(self, '_pre_build_script'):
-            self.command('pre_build_script', self._pre_build_script)
-        self.command('configure', self._configure)
-        self.command('compile', self._compile)
-        if hasattr(self, '_test'):
-            self.command('test', self._test)
-        self.command('install', self._install)
-        if hasattr(self, '_post_build_script'):
-            self.command('post_build_script', self._post_build_script)
-
-    def make_dist(self):
-        if hasattr(self, '_pre_build_script'):
-            self.command('pre_build_script', self._pre_build_script)
-        self.command('configure', self._configure)
-        self.command('make_dist', self._make_dist)
-
-
-class NoopBuilder(Builder):
-    def build(self):
-        pass
-
-    def make_dist(self):
-        pass
-
-
-class MakeBuilder(Builder):
-    configure_option = ""
-    dynamic_configure_option = "--enable-shared --disable-static"
-    static_configure_option = "--enable-static --disable-shared"
-    make_option = ""
-    install_option = ""
-    configure_script = "configure"
-    configure_env = None
-    make_target = ""
-    make_install_target = "install"
-
-    @property
-    def all_configure_option(self):
-        return "{} {} {}".format(
-            self.configure_option,
-            self.static_configure_option if self.buildEnv.platform_info.static else self.dynamic_configure_option,
-            self.buildEnv.configure_option if not self.target.force_native_build else "")
-
-    def _configure(self, context):
-        context.try_skip(self.build_path)
-        command = "{configure_script} {configure_option} --prefix {install_dir} --libdir {libdir}"
-        command = command.format(
-            configure_script=pj(self.source_path, self.configure_script),
-            configure_option=self.all_configure_option,
-            install_dir=self.buildEnv.install_dir,
-            libdir=pj(self.buildEnv.install_dir, self.buildEnv.libprefix)
-        )
-        env = Defaultdict(str, os.environ)
-        if self.buildEnv.platform_info.static:
-            env['CFLAGS'] = env['CFLAGS'] + ' -fPIC'
-        if self.configure_env:
-            for k in self.configure_env:
-                if k.startswith('_format_'):
-                    v = self.configure_env.pop(k)
-                    v = v.format(buildEnv=self.buildEnv, env=env)
-                    self.configure_env[k[8:]] = v
-            env.update(self.configure_env)
-        self.buildEnv.run_command(command, self.build_path, context, env=env)
-
-    def _compile(self, context):
-        context.try_skip(self.build_path)
-        command = "make -j4 {make_target} {make_option}".format(
-            make_target=self.make_target,
-            make_option=self.make_option
-        )
-        self.buildEnv.run_command(command, self.build_path, context)
-
-    def _install(self, context):
-        context.try_skip(self.build_path)
-        command = "make {make_install_target} {make_option}".format(
-            make_install_target=self.make_install_target,
-            make_option=self.make_option
-        )
-        self.buildEnv.run_command(command, self.build_path, context)
-
-    def _make_dist(self, context):
-        context.try_skip(self.build_path)
-        command = "make dist"
-        self.buildEnv.run_command(command, self.build_path, context)
-
-
-class CMakeBuilder(MakeBuilder):
-    def _configure(self, context):
-        context.try_skip(self.build_path)
-        cross_option = ""
-        if not self.target.force_native_build and self.buildEnv.cmake_crossfile:
-            cross_option = "-DCMAKE_TOOLCHAIN_FILE={}".format(self.buildEnv.cmake_crossfile)
-        command = ("cmake {configure_option}"
-                   " -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON"
-                   " -DCMAKE_INSTALL_PREFIX={install_dir}"
-                   " -DCMAKE_INSTALL_LIBDIR={libdir}"
-                   " {source_path}"
-                   " {cross_option}")
-        command = command.format(
-            configure_option="{} {}".format(self.buildEnv.cmake_option, self.configure_option),
-            install_dir=self.buildEnv.install_dir,
-            libdir=self.buildEnv.libprefix,
-            source_path=self.source_path,
-            cross_option=cross_option
-        )
-        env = Defaultdict(str, os.environ)
-        if self.buildEnv.platform_info.static:
-            env['CFLAGS'] = env['CFLAGS'] + ' -fPIC'
-        if self.configure_env:
-            for k in self.configure_env:
-                if k.startswith('_format_'):
-                    v = self.configure_env.pop(k)
-                    v = v.format(buildEnv=self.buildEnv, env=env)
-                    self.configure_env[k[8:]] = v
-            env.update(self.configure_env)
-        self.buildEnv.run_command(command, self.build_path, context, env=env, cross_env_only=True)
-
-
-class MesonBuilder(Builder):
-    configure_option = ""
-    test_option = ""
-
-    @property
-    def library_type(self):
-        return 'static' if self.buildEnv.platform_info.static else 'shared'
-
-    def _configure(self, context):
-        context.try_skip(self.build_path)
-        if os.path.exists(self.build_path):
-            shutil.rmtree(self.build_path)
-        os.makedirs(self.build_path)
-        configure_option = self.configure_option.format(buildEnv=self.buildEnv)
-        cross_option = ""
-        if not self.target.force_native_build and self.buildEnv.meson_crossfile:
-            cross_option = "--cross-file {}".format(
-                self.buildEnv.meson_crossfile)
-        command = ("{command} . {build_path}"
-                   " --default-library={library_type}"
-                   " {configure_option}"
-                   " --prefix={buildEnv.install_dir}"
-                   " --libdir={buildEnv.libprefix}"
-                   " {cross_option}")
-        command = command.format(
-            command=self.buildEnv.meson_command,
-            library_type=self.library_type,
-            configure_option=configure_option,
-            build_path=self.build_path,
-            buildEnv=self.buildEnv,
-            cross_option=cross_option
-        )
-        self.buildEnv.run_command(command, self.source_path, context, cross_env_only=True)
-
-    def _compile(self, context):
-        command = "{} -v".format(self.buildEnv.ninja_command)
-        self.buildEnv.run_command(command, self.build_path, context)
-
-    def _test(self, context):
-        if ( self.buildEnv.platform_info.build == 'android'
-             or (self.buildEnv.platform_info.build != 'native'
-                 and not self.buildEnv.platform_info.static)
-           ):
-            raise SkipCommand()
-        command = "{} --verbose {}".format(self.buildEnv.mesontest_command, self.test_option)
-        self.buildEnv.run_command(command, self.build_path, context)
-
-    def _install(self, context):
-        command = "{} -v install".format(self.buildEnv.ninja_command)
-        self.buildEnv.run_command(command, self.build_path, context)
-
-    def _make_dist(self, context):
-        command = "{} -v dist".format(self.buildEnv.ninja_command)
-        self.buildEnv.run_command(command, self.build_path, context)
-
-
-class GradleBuilder(Builder):
-    gradle_target = "build"
-    gradle_option = "-i"
-
-    def build(self):
-        self.command('configure', self._configure)
-        if hasattr(self, '_pre_compile_script'):
-            self.command('pre_compile_script', self._pre_compile_script)
-        self.command('compile', self._compile)
-
-    def _configure(self, context):
-        # We don't have a lot to configure by itself
-        context.try_skip(self.build_path)
-        if os.path.exists(self.build_path):
-            shutil.rmtree(self.build_path)
-        shutil.copytree(self.source_path, self.build_path)
-
-    def _compile(self, context):
-        command = "gradle {gradle_target} {gradle_option}"
-        command = command.format(
-            gradle_target=self.gradle_target,
-            gradle_option=self.gradle_option)
-        self.buildEnv.run_command(command, self.build_path, context)

dependency_versions.py

@@ -1,27 +0,0 @@
-
-main_project_versions = {
-    'kiwix-lib': '1.0.2',
-    'kiwix-tools': '0.3.0',
-    'libzim': '3.0.0',
-    'zim-tools': '0.0.1',
-    'zimwriterfs': '1.1',
-}
-
-
-# This is the "version" of the whole base_deps_versions dict.
-# Change this when you change base_deps_versions.
-base_deps_meta_version = '0'
-
-
-base_deps_versions = {
-  'zlib' : '1.2.8',
-  'lzma' : '5.0.4',
-  'uuid' : '1.43.4',
-  'xapian-core' : '1.4.5',
-  'ctpp2' : '2.8.3',
-  'pugixml' : '1.2',
-  'libmicrohttpd' : '0.9.46',
-  'gumbo' : '0.10.1',
-  'icu4c' : '58.2',
-  'Gradle' : '3.4',
-}

kiwix-deploy.py

@@ -1,98 +0,0 @@
-#!/usr/bin/env python3
-
-import os, sys, stat
-import argparse
-import datetime
-import subprocess
-import tarfile
-import zipfile
-
-pj = os.path.join
-
-FILES_TO_UPLOAD = [
-    'kiwix-index',
-    'kiwix-install',
-    'kiwix-manage',
-    'kiwix-read',
-    'kiwix-search',
-    'kiwix-serve'
-]
-
-class Archiver:
-    def __init__(self, options):
-        self.options = options
-        self.working_directory = "{:%Y-%m-%d}".format(datetime.date.today())
-        os.makedirs(self.working_directory, exist_ok=True)
-        self.archive_basename = "kiwix-tools_{:%Y-%m-%d}".format(datetime.date.today())
-        self.files_to_upload = list(self._gen_file_list())
-
-    def _gen_file_list(self):
-        bin_dir = pj(self.options.install_dir, 'bin')
-        for filename in os.listdir(bin_dir):
-            basename, _ = os.path.splitext(filename)
-            if basename in FILES_TO_UPLOAD:
-                yield pj(bin_dir, filename), pj(self.archive_basename, filename)
-
-    def build_tar(self):
-        archive_name = "{}.tar.gz".format(self.archive_basename)
-        archive_name = pj(self.working_directory, archive_name)
-        with tarfile.open(archive_name, "w:gz") as archive:
-            for filename, arcname in self.files_to_upload:
-                archive.add(filename, arcname=arcname)
-        return archive_name
-
-    def build_zip(self):
-        archive_name = "{}.zip".format(self.archive_basename)
-        archive_name = pj(self.working_directory, archive_name)
-        with zipfile.ZipFile(archive_name, "w") as archive:
-            for filename, arcname in self.files_to_upload:
-                archive.write(filename, arcname)
-        return archive_name
-
-
-class Deployer:
-    def __init__(self, options):
-        self.options = options
-
-    def deploy(self, directory):
-        if not os.path.isdir(directory):
-            return
-        command = "scp -v -r -p -i {id_file} {directory} {host_addr}".format(
-            id_file=self.options.ssh_private_key,
-            directory=directory,
-            host_addr="{}:{}".format(self.options.server, self.options.base_path)
-        )
-        return subprocess.check_call(command, shell=True)
-
-
-def parse_args():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('install_dir')
-    parser.add_argument('--deploy', action="store_true")
-    group = parser.add_argument_group('deploy options')
-    group.add_argument('--ssh_private_key')
-    group.add_argument('--server')
-    group.add_argument('--base_path')
-    parser.add_argument('--tar', action="store_true")
-    parser.add_argument('--zip', action="store_true")
-    parser.add_argument('--verbose', '-v', action="store_true",
-                        help=("Print all logs on stdout instead of in specific"
-                              " log files per commands"))
-    return parser.parse_args()
-
-if __name__ == "__main__":
-    options = parse_args()
-    options.install_dir = os.path.abspath(options.install_dir)
-
-    archiver = Archiver(options)
-    archive_list = []
-    if options.tar:
-        print("Generating tar archive")
-        archive_list.append(archiver.build_tar())
-    if options.zip:
-        print("Generating zip archive")
-        archive_list.append(archiver.build_zip())
-
-    if options.deploy:
-       deployer = Deployer(options)
-       deployer.deploy(archiver.working_directory)

kiwixbuild/__init__.py

@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+
+import os, sys
+import argparse
+
+from .dependencies import Dependency
+from .configs import ConfigInfo
+from .builder import Builder
+from .flatpak_builder import FlatpakBuilder
+from . import _global
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "target",
+        default="kiwix-tools",
+        nargs="?",
+        metavar="TARGET",
+        choices=Dependency.all_deps.keys(),
+    )
+    parser.add_argument(
+        "--working-dir",
+        default=".",
+        help=(
+            "Directory where kiwix-build puts all its files "
+            "(source, archive and build)\n"
+            "working-dir can be absolute path or a relative (to cwd) one."
+        ),
+    )
+    parser.add_argument(
+        "--build-dir",
+        default=".",
+        help=(
+            "Directory where kiwix-build puts all build files.\n"
+            "build-dir can be absolute path or a relative (to working-dir) one."
+        ),
+    )
+    parser.add_argument("--libprefix", default=None)
+    parser.add_argument(
+        "--config", choices=ConfigInfo.all_configs, default="native_dyn"
+    )
+    parser.add_argument(
+        "--verbose",
+        "-v",
+        action="store_true",
+        help=(
+            "Print all logs on stdout instead of in specific" " log files per commands"
+        ),
+    )
+    parser.add_argument(
+        "--hide-progress",
+        action="store_false",
+        dest="show_progress",
+        help="Hide intermediate progress information.",
+    )
+    parser.add_argument(
+        "--skip-source-prepare",
+        action="store_true",
+        help="Skip the source download part",
+    )
+    parser.add_argument(
+        "--build-deps-only",
+        action="store_true",
+        help="Build only the dependencies of the specified target.",
+    )
+    parser.add_argument(
+        "--build-nodeps",
+        action="store_true",
+        help="Build only the target, not its dependencies.",
+    )
+    parser.add_argument(
+        "--make-dist",
+        action="store_true",
+        help="Build distrubution (dist) source archive",
+    )
+    parser.add_argument(
+        "--make-release", action="store_true", help="Build a release version"
+    )
+    subgroup = parser.add_argument_group("advanced")
+    subgroup.add_argument(
+        "--no-cert-check",
+        action="store_true",
+        help="Skip SSL certificate verification during download",
+    )
+    subgroup.add_argument(
+        "--clean-at-end",
+        action="store_true",
+        help="Clean all intermediate files after the (successfull) build",
+    )
+    subgroup.add_argument(
+        "--dont-install-packages",
+        action="store_true",
+        help="Do not try to install packages before compiling",
+    )
+    subgroup.add_argument(
+        "--assume-packages-installed",
+        action="store_true",
+        help="Assume the package to install to be aleady installed",
+    )
+    subgroup.add_argument(
+        "--android-arch",
+        action="append",
+        help=(
+            "Specify the architecture to build for android application/libraries.\n"
+            "Can be specified several times to build for several architectures.\n"
+            "If not specified, all architectures will be build."
+        ),
+    )
+    subgroup.add_argument(
+        "--ios-arch",
+        action="append",
+        help=(
+            "Specify the architecture to build for ios application/libraries.\n"
+            "Can be specified several times to build for several architectures.\n"
+            "If not specified, all architectures will be build."
+        ),
+    )
+    subgroup.add_argument(
+        "--fast-clone",
+        action="store_true",
+        help=(
+            "Do not clone the whole repository.\n"
+            "This is useful for one shot build but it is not recommended if you want "
+            "to develop with the cloned sources."
+        ),
+    )
+    subgroup.add_argument(
+        "--use-target-arch-name",
+        action="store_true",
+        help=(
+            "Name the build directory using the arch name instead of the config name.\n"
+            "Different configs may create binary for the same arch so this option is "
+            "not recommended when working with several config on the same computer.\n"
+            "However, when generating dependencies for other it is better to have a "
+            "directory named using the target instead of the used config.\n"
+            "Intended to be used in CI only."
+        ),
+    )
+    subgroup.add_argument(
+        "--get-build-dir", action="store_true", help="Print the output directory."
+    )
+    options = parser.parse_args()
+
+    if not options.android_arch:
+        options.android_arch = ["arm", "arm64", "x86", "x86_64"]
+    if not options.ios_arch:
+        options.ios_arch = ["arm64", "x86_64"]
+
+    return options
+
+
+def main():
+    options = parse_args()
+    options.working_dir = os.path.abspath(options.working_dir)
+    _global.set_options(options)
+    neutralEnv = buildenv.NeutralEnv(options.get_build_dir)
+    _global.set_neutralEnv(neutralEnv)
+    if options.config == "flatpak":
+        builder = FlatpakBuilder()
+    else:
+        builder = Builder()
+    if options.get_build_dir:
+        print(ConfigInfo.get_config(options.config).buildEnv.build_dir)
+    else:
+        builder.run()

kiwixbuild/__main__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python3
+
+from . import main
+
+
+if __name__ == "__main__":
+    main()

kiwixbuild/_global.py

@@ -0,0 +1,40 @@
+from collections import OrderedDict as _OrderedDict
+import platform
+
+_neutralEnv = None
+_options = None
+_target_steps = _OrderedDict()
+
+
+def set_neutralEnv(env):
+    global _neutralEnv
+    _neutralEnv = env
+
+
+def neutralEnv(what):
+    return getattr(_neutralEnv, what)
+
+
+def set_options(options):
+    global _options
+    _options = options
+
+
+def option(what):
+    return getattr(_options, what)
+
+
+def add_target_step(key, what):
+    _target_steps[key] = what
+
+
+def get_target_step(key, default_context=None):
+    if isinstance(key, tuple):
+        context, target = key
+    else:
+        context, target = default_context, key
+    return _target_steps[(context, target)]
+
+
+def target_steps():
+    return _target_steps

kiwixbuild/buildenv.py

@@ -0,0 +1,183 @@
+import os, sys, shutil
+import subprocess
+import platform
+import distro
+
+from .utils import pj, download_remote, escape_path
+from ._global import neutralEnv, option
+
+
+class NeutralEnv:
+    def __init__(self, dummy_run):
+        self.working_dir = option("working_dir")
+        self.source_dir = pj(self.working_dir, "SOURCE")
+        self.archive_dir = pj(self.working_dir, "ARCHIVE")
+        self.toolchain_dir = pj(self.working_dir, "TOOLCHAINS")
+        self.log_dir = pj(self.working_dir, "LOGS")
+        for d in (self.source_dir, self.archive_dir, self.toolchain_dir, self.log_dir):
+            os.makedirs(d, exist_ok=True)
+        self.detect_platform()
+        if dummy_run:
+            # If this is for a dummy run, we will not run anything.
+            # To check for command (and so, don't enforce their presence)
+            return
+        self.ninja_command = self._detect_command(
+            "ninja", default=[["ninja"], ["ninja-build"]]
+        )
+        self.meson_command = self._detect_command(
+            "meson", default=[["meson.py"], ["meson"]]
+        )
+        self.mesontest_command = [*self.meson_command, "test"]
+        self.patch_command = self._detect_command("patch")
+        self.git_command = self._detect_command("git")
+        if platform.system() == "Windows":
+            self.make_command = self._detect_command("nmake", options=["/?", "/NOLOGO"])
+        else:
+            self.make_command = self._detect_command("make")
+        self.cmake_command = self._detect_command("cmake")
+        self.qmake_command = self._detect_command(
+            "qmake", required=False, default=[["qmake"], ["qmake-qt5"]]
+        )
+
+    def detect_platform(self):
+        _platform = platform.system()
+        self.distname = _platform
+        if _platform == "Linux":
+            self.distname = distro.id()
+            if self.distname == "ubuntu":
+                self.distname = "debian"
+
+    def download(self, what, where=None):
+        where = where or self.archive_dir
+        download_remote(what, where)
+
+    def _detect_command(self, name, default=None, options=["--version"], required=True):
+        if default is None:
+            default = [[name]]
+        env_key = "KBUILD_{}_COMMAND".format(name.upper())
+        if env_key in os.environ:
+            default = [os.environ[env_key].split()] + default
+        for command in default:
+            try:
+                retcode = subprocess.check_call(
+                    command + options, stdout=subprocess.DEVNULL
+                )
+            except (FileNotFoundError, PermissionError, OSError):
+                # Doesn't exist in PATH or isn't executable
+                continue
+            if retcode == 0:
+                return command
+        else:
+            if required:
+                sys.exit("ERROR: {} command not found".format(name))
+            else:
+                print("WARNING: {} command not found".format(name), file=sys.stderr)
+                return ["{}_NOT_FOUND".format(name.upper())]
+
+
+class BuildEnv:
+    def __init__(self, configInfo):
+        self.configInfo = configInfo
+        self.base_build_dir = pj(option("working_dir"), option("build_dir"))
+        build_dir = (
+            configInfo.arch_name if option("use_target_arch_name") else configInfo.name
+        )
+        build_dir = f"BUILD_{build_dir}"
+        self.build_dir = pj(self.base_build_dir, build_dir)
+        self.install_dir = pj(self.build_dir, "INSTALL")
+        self.toolchain_dir = pj(self.build_dir, "TOOLCHAINS")
+        self.log_dir = pj(self.build_dir, "LOGS")
+        for d in (self.build_dir, self.install_dir, self.toolchain_dir, self.log_dir):
+            os.makedirs(d, exist_ok=True)
+
+        self.libprefix = option("libprefix") or self._detect_libdir()
+
+    def clean_intermediate_directories(self):
+        for subdir in os.listdir(self.build_dir):
+            subpath = pj(self.build_dir, subdir)
+            if subpath == self.install_dir:
+                continue
+            if os.path.isdir(subpath):
+                shutil.rmtree(subpath)
+            else:
+                os.remove(subpath)
+
+    def _is_debianlike(self):
+        return os.path.isfile("/etc/debian_version")
+
+    def _detect_libdir(self):
+        if self.configInfo.libdir is not None:
+            return self.configInfo.libdir
+        if self._is_debianlike():
+            try:
+                pc = subprocess.Popen(
+                    ["dpkg-architecture", "-qDEB_HOST_MULTIARCH"],
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.DEVNULL,
+                )
+                (stdo, _) = pc.communicate()
+                if pc.returncode == 0:
+                    archpath = stdo.decode().strip()
+                    return "lib/" + archpath
+            except Exception:
+                pass
+        if os.path.isdir("/usr/lib64") and not os.path.islink("/usr/lib64"):
+            return "lib64"
+        return "lib"
+
+    def get_env(self, *, cross_comp_flags, cross_compilers, cross_path):
+        env = self.configInfo.get_env()
+        pkgconfig_path = pj(self.install_dir, self.libprefix, "pkgconfig")
+        env["PKG_CONFIG_PATH"].append(pkgconfig_path)
+
+        env["PATH"].insert(0, pj(self.install_dir, "bin"))
+
+        env["LD_LIBRARY_PATH"].extend(
+            [
+                pj(self.install_dir, "lib"),
+                pj(self.install_dir, self.libprefix),
+            ]
+        )
+
+        env["QMAKE_CXXFLAGS"] = " ".join(
+            [escape_path("-I" + pj(self.install_dir, "include")), env["QMAKE_CXXFLAGS"]]
+        )
+        env["CPPFLAGS"] = " ".join(
+            [escape_path("-I" + pj(self.install_dir, "include")), env["CPPFLAGS"]]
+        )
+        env["QMAKE_LFLAGS"] = " ".join(
+            [
+                escape_path("-L" + pj(self.install_dir, "lib")),
+                escape_path("-L" + pj(self.install_dir, self.libprefix)),
+                env["QMAKE_LFLAGS"],
+            ]
+        )
+        env["LDFLAGS"] = " ".join(
+            [
+                escape_path("-L" + pj(self.install_dir, "lib")),
+                escape_path("-L" + pj(self.install_dir, self.libprefix)),
+                env["LDFLAGS"],
+            ]
+        )
+
+        if cross_comp_flags:
+            self.configInfo.set_comp_flags(env)
+        if cross_compilers:
+            self.configInfo.set_compiler(env)
+        if cross_path:
+            env["PATH"][0:0] = self.configInfo.get_bin_dir()
+        return env
+
+    @property
+    def configure_wrapper(self):
+        try:
+            yield self.configInfo.configure_wrapper
+        except AttributeError:
+            pass
+
+    @property
+    def make_wrapper(self):
+        try:
+            yield self.configInfo.make_wrapper
+        except AttributeError:
+            pass

kiwixbuild/builder.py

@@ -0,0 +1,213 @@
+import sys
+from collections import OrderedDict
+from .buildenv import *
+
+from .configs import ConfigInfo
+from .utils import remove_duplicates, StopBuild, colorize
+from .dependencies import Dependency
+from .packages import PACKAGE_NAME_MAPPERS
+from ._global import (
+    neutralEnv,
+    option,
+    add_target_step,
+    get_target_step,
+    target_steps,
+)
+from . import _global
+
+
+class Builder:
+    def __init__(self):
+        self._targets = {}
+        ConfigInfo.get_config("neutral", self._targets)
+
+        config_name = option("config")
+        config = ConfigInfo.get_config(config_name, self._targets)
+        if neutralEnv("distname") not in config.compatible_hosts:
+            print(
+                (
+                    colorize("ERROR") + ": The config {} cannot be build on host {}.\n"
+                    "Select another config or change your host system."
+                ).format(config.name, neutralEnv("distname"))
+            )
+        self.targetDefs = config.add_targets(option("target"), self._targets)
+
+    def finalize_target_steps(self):
+        steps = []
+        for targetDef in self.targetDefs:
+            steps += self.order_steps(targetDef)
+        steps = list(remove_duplicates(steps))
+
+        if option("build_nodeps"):
+            # add all config steps
+            for dep in steps:
+                stepClass = Dependency.all_deps[dep[1]]
+                if stepClass.dont_skip:
+                    add_target_step(dep, self._targets[dep])
+
+            src_targetDef = ("source", targetDef[1])
+            add_target_step(src_targetDef, self._targets[src_targetDef])
+            add_target_step(targetDef, self._targets[targetDef])
+        else:
+            for dep in steps:
+                if option("build_deps_only") and dep[1] == targetDef[1]:
+                    continue
+                add_target_step(dep, self._targets[dep])
+        self.instanciate_steps()
+
+    def order_steps(self, targetDef):
+        for cfgName in ConfigInfo.all_running_configs:
+            cfg = ConfigInfo.all_configs[cfgName]
+            for tlcName in cfg.toolchain_names:
+                tlc = Dependency.all_deps[tlcName]
+                yield ("source", tlcName)
+                yield ("neutral" if tlc.neutral else cfgName, tlcName)
+        _targets = dict(self._targets)
+        yield from self.order_dependencies(targetDef, _targets)
+
+    def order_dependencies(self, targetDef, targets):
+        targetConfigName, targetName = targetDef
+        if targetConfigName == "source":
+            # Do not try to order sources, they will be added as dep by the
+            # build step two lines later.
+            return
+        try:
+            target = targets.pop(targetDef)
+        except KeyError:
+            return
+
+        targetConfig = ConfigInfo.get_config(targetConfigName)
+        for dep in target.get_dependencies(targetConfig, True):
+            depConfig, depName = targetConfig.get_fully_qualified_dep(dep)
+            if (depConfig, depName) in targets:
+                yield from self.order_dependencies((depConfig, depName), targets)
+        yield ("source", targetName)
+        yield targetDef
+
+    def instanciate_steps(self):
+        for stepDef in list(target_steps()):
+            stepConfig, stepName = stepDef
+            stepClass = Dependency.all_deps[stepName]
+            if stepConfig == "source":
+                source = get_target_step(stepDef)(stepClass)
+                add_target_step(stepDef, source)
+            else:
+                source = get_target_step(stepName, "source")
+                env = ConfigInfo.get_config(stepConfig).buildEnv
+                builder = get_target_step(stepDef)(stepClass, source, env)
+                add_target_step(stepDef, builder)
+
+    def prepare_sources(self):
+        if option("skip_source_prepare"):
+            print(colorize("SKIP"))
+            return
+
+        sourceDefs = remove_duplicates(
+            tDef for tDef in target_steps() if tDef[0] == "source"
+        )
+        for sourceDef in sourceDefs:
+            print("prepare sources {} :".format(sourceDef[1]))
+            source = get_target_step(sourceDef)
+            source.prepare()
+
+    def build(self):
+        builderDefs = (tDef for tDef in target_steps() if tDef[0] != "source")
+        for builderDef in builderDefs:
+            builder = get_target_step(builderDef)
+            if option("make_dist") and builderDef[1] == option("target"):
+                print("make dist {} ({}):".format(builder.name, builderDef[0]))
+                builder.make_dist()
+                continue
+            print("build {} ({}):".format(builder.name, builderDef[0]))
+            add_target_step(builderDef, builder)
+            builder.build()
+
+    def _get_packages(self):
+        packages_list = []
+        for config in ConfigInfo.all_running_configs.values():
+            mapper_name = "{host}_{config}".format(
+                host=neutralEnv("distname"), config=config
+            )
+            package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {})
+            packages_list += package_name_mapper.get("COMMON", [])
+
+        to_drop = []
+        for builderDef in self._targets:
+            configName, builderName = builderDef
+            mapper_name = "{host}_{config}".format(
+                host=neutralEnv("distname"), config=configName
+            )
+            package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {})
+            packages = package_name_mapper.get(builderName)
+            if packages:
+                to_drop.append(builderDef)
+                if packages is not True:
+                    # True means "assume the dependency is install but do not try to install anything for it"
+                    packages_list += packages
+        for dep in to_drop:
+            del self._targets[dep]
+        return packages_list
+
+    def install_packages(self):
+        packages_to_have = self._get_packages()
+        packages_to_have = remove_duplicates(packages_to_have)
+
+        if option("assume_packages_installed"):
+            print(colorize("SKIP") + ", Assume package installed")
+            return
+
+        distname = neutralEnv("distname")
+        if distname in ("fedora", "redhat", "centos"):
+            package_installer = "sudo dnf install {}"
+            package_checker = "rpm -q --quiet {}"
+        elif distname in ("debian", "Ubuntu"):
+            package_installer = "sudo apt-get install {}"
+            package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1'
+        elif distname == "Darwin":
+            package_installer = "brew install {}"
+            package_checker = "brew ls --version {} > /dev/null"
+
+        packages_to_install = []
+        for package in packages_to_have:
+            print(" - {} : ".format(package), end="")
+            command = package_checker.format(package)
+            try:
+                subprocess.check_call(command, shell=True)
+            except subprocess.CalledProcessError:
+                print(colorize("NEEDED"))
+                packages_to_install.append(package)
+            else:
+                print(colorize("SKIP"))
+
+        if packages_to_install:
+            command = package_installer.format(" ".join(packages_to_install))
+            print(command)
+            subprocess.check_call(command, shell=True)
+        else:
+            print(colorize("SKIP") + ", No package to install.")
+
+    def run(self):
+        try:
+            print("[INSTALL PACKAGES]")
+            if option("dont_install_packages"):
+                print(colorize("SKIP"))
+            else:
+                self.install_packages()
+            self.finalize_target_steps()
+            print("[SETUP TOOLCHAINS]")
+            for config in ConfigInfo.all_running_configs.values():
+                config.finalize_setup()
+            print("[PREPARE]")
+            self.prepare_sources()
+            print("[BUILD]")
+            self.build()
+            # No error, clean intermediate file at end of build if needed.
+            print("[CLEAN]")
+            if option("clean_at_end"):
+                for config in ConfigInfo.all_running_configs.values():
+                    config.clean_intermediate_directories()
+            else:
+                print(colorize("SKIP"))
+        except StopBuild as e:
+            print(e)
+            sys.exit("Stopping build due to errors")

kiwixbuild/configs/__init__.py

@@ -0,0 +1,3 @@
+from .base import *
+
+from . import android, armhf, musl, flatpak, i586, ios, native, neutral, wasm

kiwixbuild/configs/android.py

@@ -0,0 +1,169 @@
+from .base import ConfigInfo, MetaConfigInfo
+from kiwixbuild.utils import pj
+from kiwixbuild._global import get_target_step, option
+
+
+class AndroidConfigInfo(ConfigInfo):
+    build = "android"
+    static = True
+    toolchain_names = ["android-ndk"]
+    compatible_hosts = ["fedora", "debian"]
+
+    def __str__(self):
+        return "android"
+
+    @property
+    def libdir(self):
+        return "lib/{}".format(self.arch_full)
+
+    @property
+    def binaries_name(self):
+        arch_full = self.arch_full
+        return {
+            "CC": "{}-{}".format(arch_full, "clang"),
+            "CXX": "{}-{}".format(arch_full, "clang++"),
+            "AR": "{}-{}".format(arch_full, "ar"),
+            "STRIP": "{}-{}".format(arch_full, "strip"),
+            "RANLIB": "{}-{}".format(arch_full, "ranlib"),
+            "LD": "{}-{}".format(arch_full, "ld"),
+        }
+
+    def binaries(self):
+        install_path = self.install_path
+        binaries = {
+            k: pj(install_path, "bin", v) for k, v in self.binaries_name.items()
+        }
+        binaries["PKGCONFIG"] = "pkg-config"
+        return binaries
+
+    @property
+    def ndk_builder(self):
+        return get_target_step("android-ndk", self.name)
+
+    @property
+    def install_path(self):
+        return self.ndk_builder.install_path
+
+    def get_cross_config(self):
+        extra_libs = [
+            "-llog",
+            "-Wl,--exclude-libs,libgcc.a",
+            "-Wl,--exclude-libs,libunwind.a",
+        ]
+        extra_cflags = [
+            "-I{}".format(include_dir) for include_dir in self.get_include_dirs()
+        ]
+        if hasattr(self, "march"):
+            extra_libs.append("-march={}".format(self.march))
+            extra_cflags.append("-march={}".format(self.march))
+        return {
+            "exe_wrapper_def": "",
+            "install_path": self.install_path,
+            "binaries": self.binaries(),
+            "root_path": pj(self.install_path, "sysroot"),
+            "extra_libs": extra_libs,
+            "extra_cflags": extra_cflags,
+            "host_machine": {
+                "system": "Android",
+                "lsystem": "android",
+                "cpu_family": self.arch,
+                "cpu": self.cpu,
+                "endian": "little",
+                "abi": self.abi,
+            },
+        }
+
+    def get_env(self):
+        env = super().get_env()
+        root_path = pj(self.install_path, "sysroot")
+        env["PKG_CONFIG_LIBDIR"] = pj(root_path, "lib", "pkgconfig")
+        env["NDK_DEBUG"] = "0"
+        return env
+
+    def get_bin_dir(self):
+        return [pj(self.install_path, "bin")]
+
+    def set_comp_flags(self, env):
+        super().set_comp_flags(env)
+        root_path = pj(self.install_path, "sysroot")
+        march = "-march={}".format(self.march) if hasattr(self, "march") else ""
+        env["CFLAGS"] = (
+            "-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} {} ".format(
+                root_path, march
+            )
+            + env["CFLAGS"]
+        )
+        env["CXXFLAGS"] = (
+            "-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} {} ".format(
+                root_path, march
+            )
+            + env["CXXFLAGS"]
+        )
+        env["LDFLAGS"] = "--sysroot={} {} ".format(root_path, march) + env["LDFLAGS"]
+
+    def set_compiler(self, env):
+        binaries = self.binaries()
+        for k, v in binaries.items():
+            env[k] = v
+
+    @property
+    def configure_options(self):
+        yield "--host={}".format(self.arch_full)
+
+    def finalize_setup(self):
+        super().finalize_setup()
+        self.buildEnv.cmake_crossfile = self._gen_crossfile(
+            "cmake_android_cross_file.txt", "cmake_cross_file.txt"
+        )
+        self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
+
+
+class AndroidArm(AndroidConfigInfo):
+    name = "android_arm"
+    arch = cpu = "arm"
+    arch_full = "arm-linux-androideabi"
+    abi = "armeabi-v7a"
+    march = "armv7-a"
+
+
+class AndroidArm64(AndroidConfigInfo):
+    name = "android_arm64"
+    arch = "arm64"
+    arch_full = "aarch64-linux-android"
+    cpu = "aarch64"
+    abi = "arm64-v8a"
+
+
+class AndroidX86(AndroidConfigInfo):
+    name = "android_x86"
+    arch = abi = "x86"
+    arch_full = "i686-linux-android"
+    cpu = "i686"
+
+
+class AndroidX8664(AndroidConfigInfo):
+    name = "android_x86_64"
+    arch = cpu = abi = "x86_64"
+    arch_full = "x86_64-linux-android"
+
+
+class Android(MetaConfigInfo):
+    name = "android"
+    compatible_hosts = ["fedora", "debian"]
+
+    @property
+    def arch_name(self):
+        return "multi-linux-android"
+
+    @property
+    def subConfigNames(self):
+        return ["android_{}".format(arch) for arch in option("android_arch")]
+
+    def add_targets(self, targetName, targets):
+        return super().add_targets(targetName, targets)
+
+    def __str__(self):
+        return self.name
+
+    def set_comp_flags(self, env):
+        pass

kiwixbuild/configs/armhf.py

@@ -0,0 +1,183 @@
+from .base import ConfigInfo, MixedMixin
+
+from kiwixbuild.utils import pj
+from kiwixbuild._global import get_target_step
+
+
+# Base config for arm
+class ArmConfigInfo(ConfigInfo):
+    compatible_hosts = ["fedora", "debian", "almalinux"]
+
+    def get_cross_config(self):
+        return {
+            "binaries": self.binaries,
+            "exe_wrapper_def": "",
+            "root_path": self.root_path,
+            "extra_libs": [],
+            "extra_cflags": [
+                "-I{}".format(include_dir) for include_dir in self.get_include_dirs()
+            ],
+            "host_machine": {
+                "system": "linux",
+                "lsystem": "linux",
+                "cpu_family": self.cpu_family,
+                "cpu": self.cpu,
+                "endian": "little",
+                "abi": "",
+            },
+        }
+
+    @property
+    def libdir(self):
+        return "lib/{}".format(self.arch_full)
+
+    @property
+    def toolchain(self):
+        return get_target_step(self.build, "neutral")
+
+    @property
+    def root_path(self):
+        return self.toolchain.build_path
+
+    @property
+    def binaries(self):
+        binaries = (
+            (k, "{}-{}".format(self.arch_full, v))
+            for k, v in (
+                ("CC", "gcc"),
+                ("CXX", "g++"),
+                ("AR", "ar"),
+                ("STRIP", "strip"),
+                ("WINDRES", "windres"),
+                ("RANLIB", "ranlib"),
+                ("LD", "ld"),
+                ("LDSHARED", "g++ -shared"),
+            )
+        )
+        binaries = {k: pj(self.root_path, "bin", v) for k, v in binaries}
+        binaries["PKGCONFIG"] = "pkg-config"
+        return binaries
+
+    @property
+    def exe_wrapper_def(self):
+        try:
+            which("qemu-arm")
+        except subprocess.CalledProcessError:
+            return ""
+        else:
+            return "exe_wrapper = 'qemu-arm'"
+
+    @property
+    def configure_options(self):
+        yield "--host={}".format(self.arch_full)
+
+    def get_bin_dir(self):
+        return [pj(self.root_path, "bin")]
+
+    def get_env(self):
+        env = super().get_env()
+        env["LD_LIBRARY_PATH"][0:0] = [
+            pj(self.root_path, self.arch_full, "lib64"),
+            pj(self.root_path, "lib"),
+        ]
+        env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig")
+        env["QEMU_LD_PREFIX"] = pj(self.root_path, self.arch_full, "libc")
+        env["QEMU_SET_ENV"] = "LD_LIBRARY_PATH={}".format(
+            ":".join(
+                [pj(self.root_path, self.arch_full, "lib"), str(env["LD_LIBRARY_PATH"])]
+            )
+        )
+        return env
+
+    def set_comp_flags(self, env):
+        super().set_comp_flags(env)
+        env["CFLAGS"] = (
+            " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
+            + env["CFLAGS"]
+        )
+        env["CXXFLAGS"] = (
+            " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
+            + env["CXXFLAGS"]
+        )
+
+    def set_compiler(self, env):
+        for k, v in self.binaries.items():
+            env[k] = v
+
+    def finalize_setup(self):
+        super().finalize_setup()
+        self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt")
+        self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
+
+
+class Armv6(ArmConfigInfo):
+    build = "armv6"
+    arch_full = "armv6-rpi-linux-gnueabihf"
+    toolchain_names = ["armv6"]
+    cpu_family = "arm"
+    cpu = "armv6"
+
+
+class Armv6Dyn(Armv6):
+    name = "armv6_dyn"
+    static = False
+
+
+class Armv6Static(Armv6):
+    name = "armv6_static"
+    static = True
+
+
+class Armv6Mixed(MixedMixin("armv6_static"), Armv6):
+    name = "armv6_mixed"
+    static = False
+
+
+class Armv8(ArmConfigInfo):
+    build = "armv8"
+    arch_full = "armv8-rpi3-linux-gnueabihf"
+    toolchain_names = ["armv8"]
+    cpu_family = "arm"
+    cpu = "armv8"
+
+
+class Armv8Dyn(Armv8):
+    name = "armv8_dyn"
+    static = False
+
+
+class Armv8Static(Armv8):
+    name = "armv8_static"
+    static = True
+
+
+class Armv8Mixed(MixedMixin("armv8_static"), Armv8):
+    name = "armv8_mixed"
+    static = False
+
+
+class Aarch64(ArmConfigInfo):
+    build = "aarch64"
+    arch_full = "aarch64-linux-gnu"
+    toolchain_names = ["aarch64"]
+    cpu_family = "aarch64"
+    cpu = "aarch64"
+
+    @property
+    def root_path(self):
+        return self.toolchain.build_path
+
+
+class Aarch64Dyn(Aarch64):
+    name = "aarch64_dyn"
+    static = False
+
+
+class Aarch64Static(Aarch64):
+    name = "aarch64_static"
+    static = True
+
+
+class Aarch64Mixed(MixedMixin("aarch64_static"), Aarch64):
+    name = "aarch64_mixed"
+    static = False

kiwixbuild/configs/base.py

@@ -0,0 +1,179 @@
+import os, sys
+import subprocess
+
+from kiwixbuild.dependencies import Dependency
+from kiwixbuild.utils import pj, remove_duplicates, DefaultEnv
+from kiwixbuild.buildenv import BuildEnv
+from kiwixbuild._global import neutralEnv, option, target_steps
+
+_SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
+TEMPLATES_DIR = pj(os.path.dirname(_SCRIPT_DIR), "templates")
+
+
+class _MetaConfig(type):
+    def __new__(cls, name, bases, dct):
+        _class = type.__new__(cls, name, bases, dct)
+        if name not in ("ConfigInfo", "MetaConfigInfo") and "name" in dct:
+            dep_name = dct["name"]
+            ConfigInfo.all_configs[dep_name] = _class
+        return _class
+
+
+class ConfigInfo(metaclass=_MetaConfig):
+    all_configs = {}
+    all_running_configs = {}
+    toolchain_names = []
+    configure_options = []
+    mixed = False
+    libdir = None
+
+    @property
+    def arch_name(self):
+        return self.arch_full
+
+    @classmethod
+    def get_config(cls, name, targets=None):
+        if name not in cls.all_running_configs:
+            if targets is None:
+                print("Should not got there.")
+                print(cls.all_running_configs)
+                raise KeyError(name)
+            cls.all_running_configs[name] = cls.all_configs[name](targets)
+        return cls.all_running_configs[name]
+
+    def __init__(self, targets):
+        self.all_running_configs[self.name] = self
+        self.buildEnv = BuildEnv(self)
+        self.setup_toolchains(targets)
+
+    def __str__(self):
+        return "{}_{}".format(self.build, "static" if self.static else "dyn")
+
+    def setup_toolchains(self, targets):
+        for tlc_name in self.toolchain_names:
+            ToolchainClass = Dependency.all_deps[tlc_name]
+            targets[("source", tlc_name)] = ToolchainClass.Source
+            cfg_name = "neutral" if ToolchainClass.neutral else self.name
+            targets[(cfg_name, tlc_name)] = ToolchainClass.Builder
+
+    def add_targets(self, targetName, targets):
+        if (self.name, targetName) in targets:
+            return []
+        targetClass = Dependency.all_deps[targetName]
+        targets[("source", targetName)] = targetClass.Source
+        targets[(self.name, targetName)] = targetClass.Builder
+        for dep in targetClass.Builder.get_dependencies(self, False):
+            if isinstance(dep, tuple):
+                depConfigName, depName = dep
+            else:
+                depConfigName, depName = self.name, dep
+            depConfig = self.get_config(depConfigName, targets)
+            depConfig.add_targets(depName, targets)
+        return [(self.name, targetName)]
+
+    def get_fully_qualified_dep(self, dep):
+        if isinstance(dep, tuple):
+            return dep
+        else:
+            return self.name, dep
+
+    def get_cross_config(self):
+        return {}
+
+    def get_include_dirs(self):
+        return [pj(self.buildEnv.install_dir, "include")]
+
+    def get_env(self):
+        return DefaultEnv()
+
+    def get_bin_dir(self):
+        return []
+
+    def set_compiler(self, env):
+        pass
+
+    def set_comp_flags(self, env):
+        if self.static:
+            env["CFLAGS"] = env["CFLAGS"] + " -fPIC"
+            env["CXXFLAGS"] = env["CXXFLAGS"] + " -fPIC"
+
+    def _gen_crossfile(self, name, outname=None):
+        if outname is None:
+            outname = name
+        crossfile = pj(self.buildEnv.build_dir, outname)
+        template_file = pj(TEMPLATES_DIR, name)
+        with open(template_file, "r") as f:
+            template = f.read()
+        content = template.format(**self.get_cross_config())
+        with open(crossfile, "w") as outfile:
+            outfile.write(content)
+        return crossfile
+
+    def finalize_setup(self):
+        self.buildEnv.cross_config = self.get_cross_config()
+        self.buildEnv.meson_crossfile = None
+        self.buildEnv.cmake_crossfile = None
+
+    def clean_intermediate_directories(self):
+        self.buildEnv.clean_intermediate_directories()
+
+
+class MetaConfigInfo(ConfigInfo):
+    subConfigNames = []
+
+    def add_targets(self, targetName, targets):
+        targetDefs = []
+        for configName in self.subConfigNames:
+            config = self.get_config(configName, targets)
+            targetDefs += config.add_targets(targetName, targets)
+        return targetDefs
+
+
+def MixedMixin(static_name):
+    class MixedMixinClass:
+        mixed = True
+        static = False
+
+        def add_targets(self, targetName, targets):
+            if option("target") == targetName:
+                return super().add_targets(targetName, targets)
+            else:
+                static_config = self.get_config(static_name, targets)
+                return static_config.add_targets(targetName, targets)
+
+        def get_fully_qualified_dep(self, dep):
+            if isinstance(dep, tuple):
+                return dep
+            if option("target") == dep:
+                return self.name, dep
+            return static_name, dep
+
+        @property
+        def static_buildEnv(self):
+            static_config = self.get_config(static_name)
+            return static_config.buildEnv
+
+        def get_include_dirs(self):
+            return [
+                pj(self.buildEnv.install_dir, "include"),
+                pj(self.static_buildEnv.install_dir, "include"),
+            ]
+
+        def get_env(self):
+            env = super().get_env()
+            env["PATH"].insert(0, pj(self.static_buildEnv.install_dir, "bin"))
+            pkgconfig_path = pj(
+                self.static_buildEnv.install_dir,
+                self.static_buildEnv.libprefix,
+                "pkgconfig",
+            )
+            env["PKG_CONFIG_PATH"].append(pkgconfig_path)
+            env["CPPFLAGS"] = " ".join(
+                [
+                   "-I" + pj(self.static_buildEnv.install_dir, "include"),
+                    env["CPPFLAGS"],
+                ]
+            )
+            return env
+
+    return MixedMixinClass

kiwixbuild/configs/flatpak.py

@@ -0,0 +1,19 @@
+from .base import ConfigInfo
+from kiwixbuild._global import option, neutralEnv
+
+
+class FlatpakConfigInfo(ConfigInfo):
+    name = "flatpak"
+    arch_name = "flatpak"
+    build = "flatpak"
+    static = ""
+    toolchain_names = ["org.kde", "io.qt.qtwebengine"]
+    compatible_hosts = ["debian", "fedora"]
+
+    def __str__(self):
+        return "flatpak"
+
+    def get_env(self):
+        env = super().get_env()
+        env["FLATPAK_USER_DIR"] = self.buildEnv.build_dir
+        return env

kiwixbuild/configs/i586.py

@@ -0,0 +1,79 @@
+import os
+
+from .base import ConfigInfo
+from kiwixbuild.utils import which, pj
+
+
+class I586ConfigInfo(ConfigInfo):
+    build = "i586"
+    arch_full = "i586-linux-gnu"
+    compatible_hosts = ["fedora", "debian"]
+
+    def get_cross_config(self):
+        return {
+            "binaries": self.binaries,
+            "exe_wrapper_def": "",
+            "extra_libs": ["-m32", "-march=i586", "-mno-sse"],
+            "extra_cflags": [
+                "-m32",
+                "-march=i586",
+                "-mno-sse",
+                *(
+                    "-I{}".format(include_dir)
+                    for include_dir in self.get_include_dirs()
+                ),
+            ],
+            "host_machine": {
+                "system": "linux",
+                "lsystem": "linux",
+                "cpu_family": "x86",
+                "cpu": "i586",
+                "endian": "little",
+                "abi": "",
+            },
+        }
+
+    @property
+    def configure_options(self):
+        yield f"--host={self.arch_full}"
+
+    @property
+    def binaries(self):
+        return {
+            k: which(v)
+            for k, v in (
+                ("CC", os.environ.get("CC", "gcc")),
+                ("CXX", os.environ.get("CXX", "g++")),
+                ("AR", "ar"),
+                ("STRIP", "strip"),
+                ("RANLIB", "ranlib"),
+                ("LD", "ld"),
+                ("PKGCONFIG", "pkg-config"),
+            )
+        }
+
+    def set_comp_flags(self, env):
+        super().set_comp_flags(env)
+        env["CFLAGS"] = "-m32 -march=i586 -mno-sse " + env["CFLAGS"]
+        env["CXXFLAGS"] = "-m32 -march=i586 -mno-sse " + env["CXXFLAGS"]
+        env["LDFLAGS"] = "-m32 -march=i586 -mno-sse " + env["LDFLAGS"]
+
+    def get_bin_dir(self):
+        return []
+
+    def finalize_setup(self):
+        super().finalize_setup()
+        self.buildEnv.cmake_crossfile = self._gen_crossfile(
+            "cmake_i586_cross_file.txt", "cmake_cross_file.txt"
+        )
+        self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
+
+
+class I586Dyn(I586ConfigInfo):
+    name = "i586_dyn"
+    static = False
+
+
+class I586Static(I586ConfigInfo):
+    name = "i586_static"
+    static = True

kiwixbuild/configs/ios.py

@@ -0,0 +1,249 @@
+import subprocess
+
+from kiwixbuild._global import option
+from kiwixbuild.utils import pj, xrun_find
+from .base import ConfigInfo, MetaConfigInfo, MixedMixin
+from kiwixbuild.dependencies.apple_xcframework import AppleXCFramework
+
+
+MIN_MACOS_VERSION = "12.0"
+
+
+class AppleConfigInfo(ConfigInfo):
+    build = "iOS"
+    static = True
+    compatible_hosts = ["Darwin"]
+    arch = None
+    host = None
+    target = None
+    sdk_name = None
+    min_iphoneos_version = None
+    min_macos_version = None
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._root_path = None
+
+    @property
+    def arch_name(self):
+        return self.target
+
+    @property
+    def root_path(self):
+        if self._root_path is None:
+            command = "xcrun --sdk {} --show-sdk-path".format(self.sdk_name)
+            self._root_path = subprocess.check_output(command, shell=True)[:-1].decode()
+        return self._root_path
+
+    def __str__(self):
+        return "iOS"
+
+    def finalize_setup(self):
+        super().finalize_setup()
+        self.buildEnv.cmake_crossfile = self._gen_crossfile(
+            "cmake_ios_cross_file.txt", "cmake_cross_file.txt"
+        )
+        self.buildEnv.meson_crossfile = self._gen_crossfile(
+            "meson_ios_cross_file.txt", "meson_cross_file.txt"
+        )
+
+    def get_cross_config(self):
+        config = {
+            "root_path": self.root_path,
+            "binaries": self.binaries,
+            "exe_wrapper_def": "",
+            "extra_libs": [
+                "-isysroot",
+                self.root_path,
+                "-arch",
+                self.arch,
+                "-target",
+                self.target,
+            ],
+            "extra_cflags": [
+                "-isysroot",
+                self.root_path,
+                "-arch",
+                self.arch,
+                "-target",
+                self.target,
+                *(
+                    "-I{}".format(include_dir)
+                    for include_dir in self.get_include_dirs()
+                ),
+            ],
+            "host_machine": {
+                "system": "Darwin",
+                "lsystem": "darwin",
+                "cpu_family": self.arch,
+                "cpu": self.cpu,
+                "endian": "",
+                "abi": "",
+            },
+        }
+        if self.min_iphoneos_version:
+            config["extra_libs"].append(
+                "-miphoneos-version-min={}".format(self.min_iphoneos_version)
+            )
+            config["extra_cflags"].append(
+                "-miphoneos-version-min={}".format(self.min_iphoneos_version)
+            )
+        if self.min_macos_version:
+            config["extra_libs"].append(
+                "-mmacosx-version-min={}".format(self.min_macos_version)
+            )
+            config["extra_cflags"].append(
+                "-mmacosx-version-min={}".format(self.min_macos_version)
+            )
+        return config
+
+    def get_env(self):
+        env = super().get_env()
+        cflags = [env["CFLAGS"]]
+        if self.min_iphoneos_version:
+            cflags.append("-miphoneos-version-min={}".format(self.min_iphoneos_version))
+        if self.min_macos_version:
+            cflags.append("-mmacosx-version-min={}".format(self.min_macos_version))
+        env["CFLAGS"] = " ".join(cflags)
+        return env
+
+    def set_comp_flags(self, env):
+        super().set_comp_flags(env)
+        cflags = [
+            "-isysroot {}".format(self.root_path),
+            "-arch {}".format(self.arch),
+            "-target {}".format(self.target),
+            env["CFLAGS"],
+        ]
+        if self.min_iphoneos_version:
+            cflags.append("-miphoneos-version-min={}".format(self.min_iphoneos_version))
+        env["CFLAGS"] = " ".join(cflags)
+        env["CXXFLAGS"] = " ".join(
+            [
+                env["CFLAGS"],
+                "-std=c++11",
+                env["CXXFLAGS"],
+            ]
+        )
+        env["LDFLAGS"] = " ".join(
+            [
+                " -arch {}".format(self.arch),
+                "-isysroot {}".format(self.root_path),
+            ]
+        )
+
+    def get_bin_dir(self):
+        return [pj(self.root_path, "bin")]
+
+    @property
+    def binaries(self):
+        return {
+            "CC": xrun_find("clang"),
+            "CXX": xrun_find("clang++"),
+            "AR": xrun_find("ar"),
+            "STRIP": xrun_find("strip"),
+            "RANLIB": xrun_find("ranlib"),
+            "LD": xrun_find("ld"),
+            "PKGCONFIG": "pkg-config",
+        }
+
+    @property
+    def configure_options(self):
+        yield f"--host={self.host}"
+
+
+class iOSArm64(AppleConfigInfo):
+    name = "iOS_arm64"
+    arch = cpu = "arm64"
+    host = "arm-apple-darwin"
+    target = "aarch64-apple-ios"
+    sdk_name = "iphoneos"
+    min_iphoneos_version = "15.0"
+
+
+class iOSx64Simulator(AppleConfigInfo):
+    name = "iOSSimulator_x86_64"
+    arch = cpu = "x86_64"
+    host = "x86_64-apple-darwin"
+    target = "x86-apple-ios-simulator"
+    sdk_name = "iphonesimulator"
+    min_iphoneos_version = "15.0"
+
+
+class iOSArm64Simulator(AppleConfigInfo):
+    name = "iOSSimulator_arm64"
+    arch = cpu = "arm64"
+    host = "arm-apple-darwin"
+    target = "aarch64-apple-ios-simulator"
+    sdk_name = "iphonesimulator"
+    min_iphoneos_version = "15.0"
+
+
+class macOSArm64(AppleConfigInfo):
+    name = "macOS_arm64_static"
+    arch = cpu = "arm64"
+    host = "aarch64-apple-darwin"
+    target = "arm64-apple-macos"
+    sdk_name = "macosx"
+    min_iphoneos_version = None
+    min_macos_version = MIN_MACOS_VERSION
+
+
+class macOSArm64Mixed(MixedMixin("macOS_arm64_static"), AppleConfigInfo):
+    name = "macOS_arm64_mixed"
+    arch = cpu = "arm64"
+    host = "aarch64-apple-darwin"
+    target = "arm64-apple-macos"
+    sdk_name = "macosx"
+    min_iphoneos_version = None
+    min_macos_version = MIN_MACOS_VERSION
+
+
+class macOSx64(AppleConfigInfo):
+    name = "macOS_x86_64"
+    arch = cpu = "x86_64"
+    host = "x86_64-apple-darwin"
+    target = "x86_64-apple-macos"
+    sdk_name = "macosx"
+    min_iphoneos_version = None
+    min_macos_version = MIN_MACOS_VERSION
+
+
+class IOS(MetaConfigInfo):
+    name = "iOS_multi"
+    compatible_hosts = ["Darwin"]
+
+    @property
+    def arch_name(self):
+        return self.name
+
+    @property
+    def subConfigNames(self):
+        return ["iOS_{}".format(arch) for arch in option("ios_arch")]
+
+    def add_targets(self, targetName, targets):
+        super().add_targets(targetName, targets)
+        return ConfigInfo.add_targets(self, "_ios_fat_lib", targets)
+
+    def __str__(self):
+        return self.name
+
+
+class AppleStaticAll(MetaConfigInfo):
+    name = "apple_all_static"
+    compatible_hosts = ["Darwin"]
+
+    @property
+    def arch_name(self):
+        return self.name
+
+    @property
+    def subConfigNames(self):
+        return AppleXCFramework.subConfigNames
+
+    def add_targets(self, targetName, targets):
+        super().add_targets(targetName, targets)
+        return ConfigInfo.add_targets(self, "apple_xcframework", targets)
+
+    def __str__(self):
+        return self.name

kiwixbuild/configs/musl.py

@@ -0,0 +1,157 @@
+from .base import ConfigInfo, MixedMixin
+
+from kiwixbuild.utils import pj
+from kiwixbuild._global import get_target_step
+
+
+class MuslConfigInfo(ConfigInfo):
+    compatible_hosts = ["fedora", "debian"]
+
+    def get_cross_config(self):
+        return {
+            "binaries": self.binaries,
+            "exe_wrapper_def": "",
+            "root_path": self.root_path,
+            "extra_libs": [],
+            "extra_cflags": [
+                "-I{}".format(include_dir) for include_dir in self.get_include_dirs()
+            ],
+            "host_machine": {
+                "system": "linux",
+                "lsystem": "linux",
+                "cpu_family": self.cpu_family,
+                "cpu": self.cpu,
+                "endian": "little",
+                "abi": "",
+            },
+        }
+
+    @property
+    def toolchain(self):
+        return get_target_step(self.build, "neutral")
+
+    @property
+    def root_path(self):
+        return self.toolchain.build_path
+
+    @property
+    def binaries(self):
+        binaries = (
+            (k, "{}-{}".format(self.arch_full, v))
+            for k, v in (
+                ("CC", "gcc"),
+                ("CXX", "g++"),
+                ("AR", "ar"),
+                ("STRIP", "strip"),
+                ("WINDRES", "windres"),
+                ("RANLIB", "ranlib"),
+                ("LD", "ld"),
+                ("LDSHARED", "g++ -shared"),
+            )
+        )
+        binaries = {k: pj(self.root_path, "bin", v) for k, v in binaries}
+        binaries["PKGCONFIG"] = "pkg-config"
+        return binaries
+
+    @property
+    def exe_wrapper_def(self):
+        try:
+            which(self.qemu)
+        except subprocess.CalledProcessError:
+            return ""
+        except AttributeError:
+            return ""
+        else:
+            return f"exe_wrapper = '{self.qemu}'"
+
+    @property
+    def configure_options(self):
+        return [f"--host={self.arch_full}"]
+
+    def get_bin_dir(self):
+        return [pj(self.root_path, "bin")]
+
+    def get_env(self):
+        env = super().get_env()
+        env["LD_LIBRARY_PATH"][0:0] = [
+            pj(self.root_path, self.arch_full, "lib64"),
+            pj(self.root_path, "lib"),
+        ]
+        env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig")
+        env["QEMU_LD_PREFIX"] = pj(self.root_path, self.arch_full, "libc")
+        env["QEMU_SET_ENV"] = "LD_LIBRARY_PATH={}".format(
+            ":".join(
+                [pj(self.root_path, self.arch_full, "lib"), str(env["LD_LIBRARY_PATH"])]
+            )
+        )
+        return env
+
+    def set_comp_flags(self, env):
+        super().set_comp_flags(env)
+        env["LD_LIBRARY_PATH"].insert(0, pj(self.root_path, self.arch_full, "lib"))
+        env["CFLAGS"] = (
+            " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
+            + env["CFLAGS"]
+        )
+        env["CXXFLAGS"] = (
+            " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
+            + env["CXXFLAGS"]
+        )
+
+    def set_compiler(self, env):
+        for k, v in self.binaries.items():
+            env[k] = v
+
+    def finalize_setup(self):
+        super().finalize_setup()
+        self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt")
+        self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
+
+
+class Aarch64MuslConfigInfo(MuslConfigInfo):
+    build = "aarch64_musl"
+    arch_full = "aarch64-linux-musl"
+    toolchain_names = ["aarch64_musl"]
+    libdir = "lib/aarch64-linux-musl"
+    cpu_family = "arm"
+    cpu = "armhf"
+    qemu = "qemu-arm"
+
+
+class Aarch64MuslDyn(Aarch64MuslConfigInfo):
+    name = "aarch64_musl_dyn"
+    static = False
+
+
+class Aarch64MuslStatic(Aarch64MuslConfigInfo):
+    name = "aarch64_musl_static"
+    static = True
+
+
+class Aarch64MuslMixed(MixedMixin("aarch64_musl_static"), Aarch64MuslConfigInfo):
+    name = "aarch64_musl_mixed"
+    static = False
+
+
+class X86_64MuslConfigInfo(MuslConfigInfo):
+    build = "x86-64_musl"
+    arch_full = "x86_64-linux-musl"
+    toolchain_names = ["x86-64_musl"]
+    libdir = "lib/x86_64-linux-musl"
+    cpu_family = "x86_64"
+    cpu = "x86_64"
+
+
+class X86_64MuslDyn(X86_64MuslConfigInfo):
+    name = "x86-64_musl_dyn"
+    static = False
+
+
+class X86_64MuslStatic(X86_64MuslConfigInfo):
+    name = "x86-64_musl_static"
+    static = True
+
+
+class x86_64MuslMixed(MixedMixin("x86-64_musl_static"), X86_64MuslConfigInfo):
+    name = "x86-64_musl_mixed"
+    static = False

kiwixbuild/configs/native.py

@@ -0,0 +1,44 @@
+from .base import ConfigInfo, MixedMixin
+
+from kiwixbuild.utils import pj
+from kiwixbuild._global import option, neutralEnv
+from kiwixbuild.configs.ios import MIN_MACOS_VERSION
+import sysconfig
+import platform
+import sys
+
+
+class NativeConfigInfo(ConfigInfo):
+    build = "native"
+
+    def get_env(self):
+        env = super().get_env()
+        if neutralEnv("distname") == "fedora":
+            env["QT_SELECT"] = "5-64"
+        if neutralEnv("distname") == "Darwin":
+            env["CFLAGS"] += f"-mmacosx-version-min={MIN_MACOS_VERSION}"
+        return env
+
+    @property
+    def arch_name(self):
+        if sys.platform == "darwin":
+            return f"{platform.machine()}-apple-darwin"
+        return sysconfig.get_platform()
+
+
+class NativeDyn(NativeConfigInfo):
+    name = "native_dyn"
+    static = False
+    compatible_hosts = ["fedora", "debian", "Darwin", "almalinux", "Windows"]
+
+
+class NativeStatic(NativeConfigInfo):
+    name = "native_static"
+    static = True
+    compatible_hosts = ["fedora", "debian", "Darwin", "almalinux", "Windows"]
+
+
+class NativeMixed(MixedMixin("native_static"), NativeConfigInfo):
+    name = "native_mixed"
+    static = False
+    compatible_hosts = ["fedora", "debian", "Darwin", "almalinux", "Windows"]

kiwixbuild/configs/neutral.py

@@ -0,0 +1,11 @@
+from .base import ConfigInfo
+
+
+class NeutralConfigInfo(ConfigInfo):
+    name = "neutral"
+    arch_name = "neutral"
+    static = ""
+    compatible_hosts = ["fedora", "debian", "Darwin"]
+
+    def __str__(self):
+        return "neutral"

kiwixbuild/configs/wasm.py

@@ -0,0 +1,113 @@
+from .base import ConfigInfo
+
+from kiwixbuild.utils import pj
+from kiwixbuild._global import get_target_step
+
+
+class WasmConfigInfo(ConfigInfo):
+    name = "wasm"
+    static = True
+    build = "wasm"
+    arch_full = "wasm64-emscripten"
+    libdir = "lib"
+    # arch_full = 'wasm64-linux'
+    toolchain_names = ["emsdk"]
+    compatible_hosts = ["fedora", "debian"]
+    exe_wrapper_def = ""
+
+    def get_cross_config(self):
+        return {
+            "binaries": self.binaries,
+            "exe_wrapper_def": "",
+            "root_path": self.root_path,
+            "extra_libs": [],
+            "extra_cflags": [],
+            "host_machine": {
+                "system": "emscripten",
+                "lsystem": "emscripten",
+                "cpu_family": "wasm64",
+                "cpu": "wasm64",
+                "endian": "little",
+                "abi": "",
+            },
+        }
+
+    @property
+    def wasm_sdk(self):
+        return get_target_step("emsdk", self.name)
+
+    @property
+    def install_path(self):
+        return self.wasm_sdk.install_path
+
+    @property
+    def root_path(self):
+        return self.install_path
+
+    @property
+    def binaries(self):
+        binaries = (
+            ("CC", "emcc"),
+            ("CXX", "em++"),
+            ("AR", "emar"),
+            ("STRIP", "emstrip"),
+            ("WINDRES", "windres"),
+            ("RANLIB", "emranlib"),
+            ("LD", "wasm-ld"),
+        )
+        binaries = {
+            k: pj(self.install_path, "upstream", "emscripten", v) for k, v in binaries
+        }
+        binaries["PKGCONFIG"] = "pkg-config"
+        return binaries
+
+    @property
+    def configure_options(self):
+        # return ""
+        return [f"--host={self.arch_full}"]
+
+    @property
+    def configure_wrapper(self):
+        return "emconfigure"
+
+    @property
+    def make_wrapper(self):
+        return "emmake"
+
+    def get_bin_dir(self):
+        return [pj(self.install_path, "bin")]
+
+    def get_env(self):
+        env = super().get_env()
+        env["PATH"].extend(
+            [
+                self.install_path,
+                pj(self.install_path, "upstream", "emscripten"),
+                pj(self.install_path, "node", "14.18.2_64bit", "bin"),
+            ]
+        )
+        env["EMSDK"] = self.install_path
+        env["EMSDK_NODE"] = pj(
+            self.install_path, "node", "14.18.2_64bit", "bin", "node"
+        )
+        return env
+
+    def set_comp_flags(self, env):
+        super().set_comp_flags(env)
+        env["CFLAGS"] = (
+            " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
+            + env["CFLAGS"]
+        )
+        env["CXXFLAGS"] = (
+            " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
+            + env["CXXFLAGS"]
+        )
+
+    def set_compiler(self, env):
+        for k, v in self.binaries.items():
+            env[k] = v
+
+    def finalize_setup(self):
+        super().finalize_setup()
+        self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt")
+        self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")

kiwixbuild/configs/win64.py

@@ -0,0 +1,103 @@
+import subprocess
+
+from .base import ConfigInfo
+from kiwixbuild.utils import which, pj
+from kiwixbuild._global import neutralEnv
+
+
+class Win64ConfigInfo(ConfigInfo):
+    extra_libs = [
+        "-lmingw32",
+        "-lwinmm",
+        "-lws2_32",
+        "-lshlwapi",
+        "-lrpcrt4",
+        "-lmsvcr100",
+        "-liphlpapi",
+        "-lshell32",
+        "-lkernel32",
+    ]
+    build = "win64"
+    compatible_hosts = ["fedora", "debian"]
+    arch_full = "x86_64-w64-mingw32"
+
+    def get_cross_config(self):
+        return {
+            "exe_wrapper_def": self.exe_wrapper_def,
+            "binaries": self.binaries,
+            "root_path": self.root_path,
+            "extra_libs": self.extra_libs,
+            "extra_cflags": ["-DWIN32"],
+            "host_machine": {
+                "system": "Windows",
+                "lsystem": "windows",
+                "cpu_family": "x86_64",
+                "cpu": "x86_64",
+                "endian": "little",
+                "abi": "",
+            },
+        }
+
+    def finalize_setup(self):
+        super().finalize_setup()
+        self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt")
+        self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
+
+    @property
+    def root_path(self):
+        root_paths = {
+            "fedora": "/usr/x86_64-w64-mingw32/sys-root/mingw",
+            "debian": "/usr/x86_64-w64-mingw32",
+        }
+        return root_paths[neutralEnv("distname")]
+
+    @property
+    def binaries(self):
+        return {
+            k: which("{}-{}".format(self.arch_full, v))
+            for k, v in (
+                ("CC", "gcc"),
+                ("CXX", "g++"),
+                ("AR", "ar"),
+                ("STRIP", "strip"),
+                ("WINDRES", "windres"),
+                ("RANLIB", "ranlib"),
+                ("PKGCONFIG", "pkg-config"),
+            )
+        }
+
+    @property
+    def exe_wrapper_def(self):
+        try:
+            which("wine")
+        except subprocess.CalledProcessError:
+            return ""
+        else:
+            return "exe_wrapper = 'wine'"
+
+    @property
+    def configure_options(self):
+        return [f"--host={self.arch_full}"]
+
+    def set_compiler(self, env):
+        for k, v in self.binaries.items():
+            env[k] = v
+
+    def get_bin_dir(self):
+        return [pj(self.root_path, "bin")]
+
+    def get_env(self):
+        env = super().get_env()
+        env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig")
+        env["LIBS"] = " ".join(self.extra_libs) + " " + env["LIBS"]
+        return env
+
+
+class Win64Dyn(Win64ConfigInfo):
+    name = "win64_dyn"
+    static = False
+
+
+class Win64Static(Win64ConfigInfo):
+    name = "win64_static"
+    static = True

kiwixbuild/dependencies/__init__.py

@@ -0,0 +1,32 @@
+from .base import *
+from . import (
+    all_dependencies,
+    boostregex,
+    tc_android_ndk,
+    aria2,
+    tc_armhf,
+    tc_musl,
+    docoptcpp,
+    tc_emsdk,
+    tc_flatpak,
+    gumbo,
+    icu4c,
+    ios_fat_lib,
+    mustache,
+    kiwix_desktop,
+    kiwix_tools,
+    libcurl,
+    libkiwix,
+    libmagic,
+    libmicrohttpd,
+    libzim,
+    lzma,
+    qt,
+    pugixml,
+    uuid,
+    xapian,
+    zim_tools,
+    zim_testing_suite,
+    zlib,
+    zstd,
+)

kiwixbuild/dependencies/all_dependencies.py

@@ -0,0 +1,61 @@
+from os import environ
+
+from .base import Dependency, NoopSource, NoopBuilder
+
+from kiwixbuild._global import neutralEnv
+
+
+class AllBaseDependencies(Dependency):
+    name = "alldependencies"
+
+    Source = NoopSource
+
+    class Builder(NoopBuilder):
+        @classmethod
+        def get_dependencies(cls, configInfo, allDeps):
+            if configInfo.build == "wasm" or environ.get("OS_NAME") == "manylinux":
+                return ["zlib", "lzma", "zstd", "icu4c", "xapian-core"]
+
+            if neutralEnv("distname") == "Windows":
+                base_deps = [
+                    "zlib",
+                    "zstd",
+                    "xapian-core",
+                    "zim-testing-suite",
+                    "icu4c",
+                    "boostregex",
+                    "docoptcpp",
+                ]
+
+                if not configInfo.name.endswith("_dyn"):
+                    base_deps += [
+                        "pugixml",
+                        "libcurl",
+                        "mustache",
+                        "libmicrohttpd",
+                    ]
+            else:
+                base_deps = [
+                    "zlib",
+                    "lzma",
+                    "zstd",
+                    "xapian-core",
+                    "pugixml",
+                    "libcurl",
+                    "icu4c",
+                    "mustache",
+                    "libmicrohttpd",
+                    "zim-testing-suite",
+                ]
+                # Add specific dependencies depending of the config
+                if configInfo.build not in ("android", "iOS"):
+                    # For zimtools
+                    base_deps += ["docoptcpp", "libmagic", "gumbo"]
+                    if (
+                        configInfo.build == "native"
+                        and neutralEnv("distname") != "Darwin"
+                    ):
+                        # We compile kiwix-desktop only on native and not on `Darwin`
+                        # So we need aria2 only there
+                        base_deps += ["aria2"]
+            return base_deps

kiwixbuild/dependencies/apple_xcframework.py

@@ -0,0 +1,126 @@
+import os
+import shutil
+from pathlib import Path
+
+from kiwixbuild.configs import ConfigInfo
+from kiwixbuild.utils import pj, run_command
+from .base import Dependency, NoopSource, Builder as BaseBuilder
+
+
+class AppleXCFramework(Dependency):
+    name = "apple_xcframework"
+    subConfigNames = [
+        "macOS_x86_64",
+        "macOS_arm64_static",
+        "iOS_arm64",
+        "iOSSimulator_x86_64",
+        "iOSSimulator_arm64",
+    ]
+    Source = NoopSource
+
+    class Builder(BaseBuilder):
+        @property
+        def all_subconfigs(self):
+            return self.buildEnv.configInfo.subConfigNames
+
+        @property
+        def macos_subconfigs(self):
+            return [
+                target for target in self.all_subconfigs if target.startswith("macOS")
+            ]
+
+        @property
+        def iossimulator_subconfigs(self):
+            return [
+                target
+                for target in self.all_subconfigs
+                if target.startswith("iOSSimulator")
+            ]
+
+        @property
+        def ios_subconfigs(self):
+            return [
+                target for target in self.all_subconfigs if target.startswith("iOS_")
+            ]
+
+        @classmethod
+        def get_dependencies(cls, configInfo, alldeps):
+            return [(target, "libkiwix") for target in AppleXCFramework.subConfigNames]
+
+        @property
+        def final_path(self):
+            return pj(self.buildEnv.install_dir, "lib", "CoreKiwix.xcframework")
+
+        def _remove_if_exists(self, context):
+            if not os.path.exists(self.final_path):
+                return
+
+            shutil.rmtree(self.final_path)
+
+        def _merge_libs(self, context):
+            """create merged.a in all targets to bundle all static archives"""
+            xcf_libs = []
+            for target in self.all_subconfigs:
+                static_ars = []
+
+                cfg = ConfigInfo.get_config(target)
+                lib_dir = pj(cfg.buildEnv.install_dir, "lib")
+                static_ars = [str(f) for f in Path(lib_dir).glob("*.a")]
+
+                # create merged.a from all *.a in install_dir/lib
+                command = ["libtool", "-static", "-o", "merged.a", *static_ars]
+                run_command(command, lib_dir, context)
+
+                # will be included in xcframework
+                if target in self.ios_subconfigs:
+                    xcf_libs.append(pj(lib_dir, "merged.a"))
+
+            return xcf_libs
+
+        def make_fat_with(self, configs, folder_name, context):
+            """create fat merged.a in {folder_name} install/lib with {configs}"""
+            libs = []
+            for target in configs:
+                cfg = ConfigInfo.get_config(target)
+                libs.append(pj(cfg.buildEnv.install_dir, "lib", "merged.a"))
+
+            fat_dir = pj(self.buildEnv.build_dir, folder_name)
+            os.makedirs(fat_dir, exist_ok=True)
+
+            output_merged = pj(fat_dir, "merged.a")
+            command = ["lipo", "-create", "-output", output_merged, *libs]
+            run_command(command, self.buildEnv.build_dir, context)
+
+            return [output_merged]
+
+        def _build_xcframework(self, xcf_libs, context):
+            # create xcframework
+            ref_conf = ConfigInfo.get_config(self.macos_subconfigs[0])
+            command = ["xcodebuild", "-create-xcframework"]
+            for lib in xcf_libs:
+                command += [
+                    "-library",
+                    lib,
+                    "-headers",
+                    pj(ref_conf.buildEnv.install_dir, "include"),
+                ]
+            command += ["-output", self.final_path]
+            run_command(command, self.buildEnv.build_dir, context)
+
+        def build(self):
+            xcf_libs = []
+            self.command("remove_if_exists", self._remove_if_exists)
+            xcf_libs += self.command("merge_libs", self._merge_libs)
+            xcf_libs += self.command(
+                "make_macos_fat",
+                self.make_fat_with,
+                self.macos_subconfigs,
+                "macOS_fat",
+            )
+            xcf_libs += self.command(
+                "make_simulator_fat",
+                self.make_fat_with,
+                self.iossimulator_subconfigs,
+                "iOS-simulator_fat",
+            )
+            self.command("build_xcframework", self._build_xcframework, xcf_libs)

kiwixbuild/dependencies/aria2.py

@@ -0,0 +1,53 @@
+from .base import Dependency, ReleaseDownload, MakeBuilder, NoopBuilder
+
+from kiwixbuild.utils import Remotefile, run_command, pj
+import platform
+from shutil import copy2
+
+# Important: in case of aria2c update,
+# 'scripts/create_kiwix-desktop_appImage.sh' should not be forgotten!
+
+class Aria2(Dependency):
+    name = "aria2"
+
+    if platform.system() == "Windows":
+
+        class Source(ReleaseDownload):
+            archive = Remotefile(
+                "aria2-1.37.0-win-64bit-build1.zip",
+                "67d015301eef0b612191212d564c5bb0a14b5b9c4796b76454276a4d28d9b288",
+                "https://dev.kiwix.org/kiwix-desktop/aria2-1.37.0-win-64bit-build1.zip",
+            )
+
+        class Builder(NoopBuilder):
+            def build(self):
+                self.command("copy_binary", self._copy_binary)
+
+            def _copy_binary(self, context):
+                context.try_skip(self.build_path)
+                copy2(
+                    pj(self.source_path, "aria2c.exe"),
+                    pj(self.buildEnv.install_dir, "bin"),
+                )
+
+    else:
+
+        class Source(ReleaseDownload):
+            archive = Remotefile(
+                "aria2-1.37.0.tar.xz",
+                "60a420ad7085eb616cb6e2bdf0a7206d68ff3d37fb5a956dc44242eb2f79b66b",
+                "https://dev.kiwix.org/kiwix-desktop/aria2-1.37.0.tar.xz",
+            )
+
+            def _post_prepare_script(self, context):
+                context.try_skip(self.extract_path)
+                command = ["autoreconf", "-i"]
+                run_command(command, self.extract_path, context)
+
+        class Builder(MakeBuilder):
+            dependencies = ["zlib"]
+            configure_options = [
+                "--disable-libaria2",
+                "--disable-websocket",
+                "--without-sqlite3",
+            ]

kiwixbuild/dependencies/base.py

@@ -0,0 +1,615 @@
+import subprocess
+import os
+import shutil
+import time
+import platform
+
+from kiwixbuild.utils import (
+    pj,
+    Context,
+    SkipCommand,
+    WarningMessage,
+    extract_archive,
+    StopBuild,
+    run_command,
+    colorize,
+    copy_tree,
+)
+from kiwixbuild.versions import main_project_versions, base_deps_versions
+from kiwixbuild._global import neutralEnv, option, get_target_step
+
+SCRIPT_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+
+
+class _MetaDependency(type):
+    def __new__(cls, name, bases, dct):
+        _class = type.__new__(cls, name, bases, dct)
+        if name != "Dependency":
+            dep_name = dct["name"]
+            Dependency.all_deps[dep_name] = _class
+        return _class
+
+
+class Dependency(metaclass=_MetaDependency):
+    all_deps = {}
+    force_build = False
+    force_native_build = False
+    dont_skip = False
+
+    @classmethod
+    def version(cls):
+        if cls.name in base_deps_versions:
+            return base_deps_versions[cls.name]
+        elif option("make_release"):
+            return main_project_versions.get(cls.name, None)
+        return None
+
+    @classmethod
+    def full_name(cls):
+        if cls.version():
+            return "{}-{}".format(cls.name, cls.version())
+        return cls.name
+
+
+class Source:
+    """Base Class to the real preparator
+    A source preparator must install source in the self.source_dir attribute
+    inside the neutralEnv.source_dir."""
+
+    def __init__(self, target):
+        self.target = target
+
+    @property
+    def name(self):
+        return self.target.name
+
+    @property
+    def full_name(self):
+        return self.target.full_name()
+
+    @property
+    def source_dir(self):
+        return self.target.full_name()
+
+    @property
+    def source_path(self):
+        return pj(neutralEnv("source_dir"), self.source_dir)
+
+    @property
+    def _log_dir(self):
+        return neutralEnv("log_dir")
+
+    def _patch(self, context):
+        context.try_skip(self.source_path)
+        for p in self.patches:
+            patch_file_path = pj(SCRIPT_DIR, "patches", p)
+            patch_command = [*neutralEnv("patch_command"), "-p1", "-i", patch_file_path]
+            run_command(patch_command, self.source_path, context)
+
+    def command(self, name, function, *args):
+        print("  {} {} : ".format(name, self.name), end="", flush=True)
+        log = pj(self._log_dir, "cmd_{}_{}.log".format(name, self.name))
+        context = Context(name, log, True)
+        try:
+            start_time = time.time()
+            ret = function(*args, context=context)
+            context._finalise()
+            duration = time.time() - start_time
+            print(colorize("OK"), "({:.1f}s)".format(duration))
+            return ret
+        except WarningMessage as e:
+            print(e)
+        except SkipCommand as e:
+            print(e)
+        except subprocess.CalledProcessError:
+            print(colorize("ERROR"))
+            try:
+                with open(log, "r") as f:
+                    print(f.read())
+            except:
+                pass
+            raise StopBuild()
+        except:
+            print(colorize("ERROR"))
+            raise
+
+
+class NoopSource(Source):
+    def prepare(self):
+        pass
+
+
+class ReleaseDownload(Source):
+    archive_top_dir = None
+
+    @property
+    def archives(self):
+        return (self.archive,)
+
+    @property
+    def extract_path(self):
+        return pj(neutralEnv("source_dir"), self.source_dir)
+
+    def _download(self, context):
+        context.try_skip(neutralEnv("archive_dir"), self.full_name)
+        archive_iter = iter(self.archives)
+        archive = next(archive_iter, None)
+        while archive:
+            try:
+                neutralEnv("download")(archive)
+            except SkipCommand as e:
+                archive = next(archive_iter, None)
+                if not archive:
+                    raise e
+                continue
+            archive = next(archive_iter, None)
+
+    def _extract(self, context):
+        context.try_skip(self.extract_path)
+        if os.path.exists(self.extract_path):
+            shutil.rmtree(self.extract_path)
+        for archive in self.archives:
+            extract_archive(
+                pj(neutralEnv("archive_dir"), archive.name),
+                neutralEnv("source_dir"),
+                topdir=self.archive_top_dir,
+                name=self.source_dir,
+            )
+
+    def prepare(self):
+        self.command("download", self._download)
+        self.command("extract", self._extract)
+        if hasattr(self, "patches"):
+            self.command("patch", self._patch)
+        if hasattr(self, "_post_prepare_script"):
+            self.command("post_prepare_script", self._post_prepare_script)
+
+
+class GitClone(Source):
+    base_git_ref = "main"
+    force_full_clone = False
+
+    @property
+    def release_git_ref(self):
+        return main_project_versions.get(self.name, self.base_git_ref)
+
+    @property
+    def source_dir(self):
+        if option("make_release"):
+            return "{}_release".format(self.git_dir)
+        else:
+            return self.git_dir
+
+    @property
+    def git_path(self):
+        return pj(neutralEnv("source_dir"), self.source_dir)
+
+    @property
+    def git_ref(self):
+        if option("make_release"):
+            return self.release_git_ref
+        else:
+            return self.base_git_ref
+
+    def _git_init(self, context):
+        if option("fast_clone") and self.force_full_clone == False:
+            command = [
+                *neutralEnv("git_command"),
+                "clone",
+                "--depth=1",
+                "--branch",
+                self.git_ref,
+                self.git_remote,
+                self.source_dir,
+            ]
+            run_command(command, neutralEnv("source_dir"), context)
+        else:
+            command = [
+                *neutralEnv("git_command"),
+                "clone",
+                self.git_remote,
+                self.source_dir,
+            ]
+            run_command(command, neutralEnv("source_dir"), context)
+            command = [*neutralEnv("git_command"), "checkout", self.git_ref]
+            run_command(command, self.git_path, context)
+
+    def _git_update(self, context):
+        command = [*neutralEnv("git_command"), "fetch", "origin", self.git_ref]
+        run_command(command, self.git_path, context)
+        try:
+            command = [
+                *neutralEnv("git_command"),
+                "merge",
+                "--ff-only",
+                f"origin/{self.git_ref}",
+            ]
+            run_command(command, self.git_path, context)
+        except subprocess.CalledProcessError:
+            raise WarningMessage("Cannot update, please check log for information")
+
+    def prepare(self):
+        if not os.path.exists(self.git_path):
+            self.command("gitinit", self._git_init)
+        else:
+            self.command("gitupdate", self._git_update)
+        if hasattr(self, "_post_prepare_script"):
+            self.command("post_prepare_script", self._post_prepare_script)
+
+
+class Builder:
+    subsource_dir = None
+    dependencies = []
+
+    def __init__(self, target, source, buildEnv):
+        self.target = target
+        self.source = source
+        self.buildEnv = buildEnv
+
+    @classmethod
+    def get_dependencies(cls, configInfo, allDeps):
+        return cls.dependencies
+
+    @property
+    def name(self):
+        return self.target.name
+
+    @property
+    def source_path(self):
+        base_source_path = self.source.source_path
+        if self.subsource_dir:
+            return pj(base_source_path, self.subsource_dir)
+        return base_source_path
+
+    @property
+    def build_path(self):
+        return pj(self.buildEnv.build_dir, self.target.full_name())
+
+    @property
+    def _log_dir(self):
+        return self.buildEnv.log_dir
+
+    def command(self, name, function, *args):
+        print("  {} {} : ".format(name, self.name), end="", flush=True)
+        log = pj(self._log_dir, "cmd_{}_{}.log".format(name, self.name))
+        context = Context(name, log, self.target.force_native_build)
+        if self.target.force_build:
+            context.no_skip = True
+        try:
+            start_time = time.time()
+            ret = function(*args, context=context)
+            context._finalise()
+            duration = time.time() - start_time
+            print(colorize("OK"), "({:.1f}s)".format(duration))
+            return ret
+        except SkipCommand as e:
+            print(e)
+        except WarningMessage as e:
+            print(e)
+        except subprocess.CalledProcessError:
+            print(colorize("ERROR"))
+            try:
+                with open(log, "r") as f:
+                    print(f.read())
+            except:
+                pass
+            raise StopBuild()
+        except:
+            print(colorize("ERROR"))
+            raise
+
+    def build(self):
+        if hasattr(self, "_pre_build_script"):
+            self.command("pre_build_script", self._pre_build_script)
+        self.command("configure", self._configure)
+        if hasattr(self, "_post_configure_script"):
+            self.command("post_configure_script", self._post_configure_script)
+        self.command("compile", self._compile)
+        if hasattr(self, "_test"):
+            self.command("test", self._test)
+        self.command("install", self._install)
+        if hasattr(self, "_post_build_script"):
+            self.command("post_build_script", self._post_build_script)
+
+    def make_dist(self):
+        if hasattr(self, "_pre_build_script"):
+            self.command("pre_build_script", self._pre_build_script)
+        self.command("configure", self._configure)
+        self.command("make_dist", self._make_dist)
+
+    def set_flatpak_buildsystem(self, module):
+        if getattr(self, "flatpak_buildsystem", None):
+            module["buildsystem"] = self.flatpak_buildsystem
+        if getattr(self, "subsource_dir", None):
+            module["subdir"] = self.subsource_dir
+        if getattr(self, "flatpack_build_options", None):
+            module["build-options"] = self.flatpack_build_options
+        if getattr(self, "configure_options", ""):
+            module["config-opts"] = list(self.configure_options)
+
+    def get_env(self, *, cross_comp_flags, cross_compilers, cross_path):
+        env = self.buildEnv.get_env(
+            cross_comp_flags=cross_comp_flags,
+            cross_compilers=cross_compilers,
+            cross_path=cross_path,
+        )
+        for dep in self.get_dependencies(self.buildEnv.configInfo, False):
+            try:
+                builder = get_target_step(dep, self.buildEnv.configInfo.name)
+                builder.set_env(env)
+            except KeyError:
+                # Some target may be missing (installed by a package, ...)
+                pass
+        return env
+
+    def set_env(self, env):
+        pass
+
+
+class NoopBuilder(Builder):
+    def build(self):
+        pass
+
+    def make_dist(self):
+        pass
+
+
+class TcCopyBuilder(Builder):
+    src_subdir = None
+
+    @property
+    def build_path(self):
+        return pj(self.buildEnv.toolchain_dir, self.target.full_name())
+
+    def build(self):
+        self.command("copy", self._copy)
+
+    def _copy(self, context):
+        context.try_skip(self.build_path)
+        if self.src_subdir:
+            source_path = pj(self.source_path, self.src_subdir)
+        else:
+            source_path = self.source_path
+        copy_tree(source_path, self.build_path)
+
+    def make_dist(self):
+        pass
+
+
+class MakeBuilder(Builder):
+    configure_options = []
+    dynamic_configure_options = ["--enable-shared", "--disable-static"]
+    static_configure_options = ["--enable-static", "--disable-shared"]
+    make_options = ["-j4"]
+    install_options = []
+    configure_script = "configure"
+    configure_env = {
+        "_format_CFLAGS": "{env[CFLAGS]} -O3",
+        "_format_CXXFLAGS": "{env[CXXFLAGS]} -O3",
+    }
+    make_targets = []
+    flatpak_buildsystem = None
+
+    @property
+    def make_install_targets(self):
+        if self.buildEnv.configInfo.build in ("iOS", "wasm"):
+            yield "install"
+        else:
+            yield "install-strip"
+
+    @property
+    def all_configure_options(self):
+        yield from self.configure_options
+        if self.buildEnv.configInfo.static:
+            yield from self.static_configure_options
+        else:
+            yield from self.dynamic_configure_options
+        if not self.target.force_native_build:
+            yield from self.buildEnv.configInfo.configure_options
+        yield from ("--prefix", self.buildEnv.install_dir)
+        yield from ("--libdir", pj(self.buildEnv.install_dir, self.buildEnv.libprefix))
+
+    def set_configure_env(self, env):
+        dep_conf_env = self.configure_env
+        if not dep_conf_env:
+            return
+        for k, v in dep_conf_env.items():
+            if k.startswith("_format_"):
+                v = v.format(buildEnv=self.buildEnv, env=env)
+                env[k[8:]] = v
+            else:
+                env[k] = v
+
+    def _configure(self, context):
+        context.try_skip(self.build_path)
+        command = [
+            *self.buildEnv.configure_wrapper,
+            pj(self.source_path, self.configure_script),
+            *self.all_configure_options,
+        ]
+        env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
+        self.set_configure_env(env)
+        run_command(command, self.build_path, context, env=env)
+
+    def _compile(self, context):
+        context.try_skip(self.build_path)
+        command = [
+            *self.buildEnv.make_wrapper,
+            *neutralEnv("make_command"),
+            *self.make_targets,
+            *self.make_options,
+        ]
+        env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
+        run_command(command, self.build_path, context, env=env)
+
+    def _install(self, context):
+        context.try_skip(self.build_path)
+        command = [
+            *self.buildEnv.make_wrapper,
+            *neutralEnv("make_command"),
+            *self.make_install_targets,
+            *self.make_options,
+        ]
+        env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
+        run_command(command, self.build_path, context, env=env)
+
+    def _make_dist(self, context):
+        context.try_skip(self.build_path)
+        command = [*self.buildEnv.make_wrapper, *neutralEnv("make_command"), "dist"]
+        env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
+        run_command(command, self.build_path, context, env=env)
+
+
+class CMakeBuilder(MakeBuilder):
+    flatpak_buildsystem = "cmake"
+
+    def _configure(self, context):
+        context.try_skip(self.build_path)
+        cross_options = []
+        if not self.target.force_native_build and self.buildEnv.cmake_crossfile:
+            cross_options += [f"-DCMAKE_TOOLCHAIN_FILE={self.buildEnv.cmake_crossfile}"]
+        command = [
+            *neutralEnv("cmake_command"),
+            *self.configure_options,
+            "-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON",
+            f"-DCMAKE_INSTALL_PREFIX={self.buildEnv.install_dir}",
+            f"-DCMAKE_INSTALL_LIBDIR={self.buildEnv.libprefix}",
+            self.source_path,
+            *cross_options,
+        ]
+        env = self.get_env(
+            cross_comp_flags=True, cross_compilers=False, cross_path=True
+        )
+        self.set_configure_env(env)
+        run_command(command, self.build_path, context, env=env)
+
+    def set_flatpak_buildsystem(self, module):
+        super().set_flatpak_buildsystem(module)
+        module["buildir"] = True
+
+
+class QMakeBuilder(MakeBuilder):
+    qmake_targets = []
+    flatpak_buildsystem = "qmake"
+
+    @property
+    def make_options(self):
+        if platform.system() == "Windows":
+            return []
+        return super().make_options
+
+    @property
+    def env_options(self):
+        if "QMAKE_CC" in os.environ:
+            yield f"QMAKE_CC={os.environ['QMAKE_CC']}"
+        if "QMAKE_CXX" in os.environ:
+            yield f"QMAKE_CXX={os.environ['QMAKE_CXX']}"
+
+    def _configure(self, context):
+        context.try_skip(self.build_path)
+        command = [
+            *neutralEnv("qmake_command"),
+            *self.configure_options,
+            *self.env_options,
+            self.source_path,
+        ]
+        env = self.get_env(
+            cross_comp_flags=True, cross_compilers=False, cross_path=True
+        )
+        self.set_configure_env(env)
+        run_command(command, self.build_path, context, env=env)
+
+    def _make_dist(self, context):
+        command = [
+            *neutralEnv("git_command"),
+            "archive",
+            "-o",
+            f"{self.build_path}/{self.target.full_name()}.tar.gz",
+            f"--prefix={self.target.full_name()}/",
+            "HEAD",
+        ]
+        run_command(command, self.source_path, context)
+
+
+class MesonBuilder(Builder):
+    configure_options = []
+    test_options = []
+    flatpak_buildsystem = "meson"
+
+    @property
+    def build_type(self):
+        if platform.system() == "Windows":
+            return "release"
+
+        return "release" if option("make_release") else "debug"
+
+    @property
+    def strip_options(self):
+        if option("make_release"):
+            yield "--strip"
+
+    @property
+    def library_type(self):
+        return "static" if self.buildEnv.configInfo.static else "shared"
+
+    def _configure(self, context):
+        context.try_skip(self.build_path)
+        if os.path.exists(self.build_path):
+            shutil.rmtree(self.build_path)
+        os.makedirs(self.build_path)
+        cross_options = []
+        if not self.target.force_native_build and self.buildEnv.meson_crossfile:
+            cross_options += ["--cross-file", self.buildEnv.meson_crossfile]
+        command = [
+            *neutralEnv("meson_command"),
+            "setup",
+            ".",
+            self.build_path,
+            f"--buildtype={self.build_type}",
+            *self.strip_options,
+            f"--default-library={self.library_type}",
+            *self.configure_options,
+            f"--prefix={self.buildEnv.install_dir}",
+            f"--libdir={self.buildEnv.libprefix}",
+            *cross_options,
+        ]
+        env = self.get_env(
+            cross_comp_flags=False, cross_compilers=False, cross_path=True
+        )
+        run_command(command, self.source_path, context, env=env)
+
+    def _compile(self, context):
+        context.try_skip(self.build_path)
+        command = [*neutralEnv("ninja_command"), "-v"]
+        env = self.get_env(
+            cross_comp_flags=False, cross_compilers=False, cross_path=True
+        )
+        run_command(command, self.build_path, context, env=env)
+
+    def _test(self, context):
+        context.try_skip(self.build_path)
+        if self.buildEnv.configInfo.build == "android" or (
+            self.buildEnv.configInfo.build != "native"
+            and not self.buildEnv.configInfo.static
+        ):
+            raise SkipCommand()
+        command = [*neutralEnv("mesontest_command"), "--verbose", *self.test_options]
+        env = self.get_env(
+            cross_comp_flags=False, cross_compilers=False, cross_path=True
+        )
+        run_command(command, self.build_path, context, env=env)
+
+    def _install(self, context):
+        context.try_skip(self.build_path)
+        command = [*neutralEnv("ninja_command"), "-v", "install"]
+        env = self.get_env(
+            cross_comp_flags=False, cross_compilers=False, cross_path=True
+        )
+        run_command(command, self.build_path, context, env=env)
+
+    def _make_dist(self, context):
+        command = [*neutralEnv("ninja_command"), "-v", "dist"]
+        env = self.get_env(
+            cross_comp_flags=False, cross_compilers=False, cross_path=True
+        )
+        run_command(command, self.build_path, context, env=env)

kiwixbuild/dependencies/boostregex.py

@@ -0,0 +1,27 @@
+from .base import Dependency, ReleaseDownload, Builder as BaseBuilder
+
+from kiwixbuild.utils import Remotefile, pj
+from shutil import copytree
+
+
+class BoostRegex(Dependency):
+    name = "boostregex"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile(
+            "regex-boost-1.86.0.zip",
+            "",
+            "https://codeload.github.com/boostorg/regex/zip/refs/tags/boost-1.86.0",
+        )
+
+    class Builder(BaseBuilder):
+        def build(self):
+            self.command("copy_headers", self._copy_headers)
+
+        def _copy_headers(self, context):
+            context.try_skip(self.build_path)
+            copytree(
+                pj(self.source_path, "include", "boost"),
+                pj(self.buildEnv.install_dir, "include", "boost"),
+                dirs_exist_ok=True,
+            )

kiwixbuild/dependencies/docoptcpp.py

@@ -0,0 +1,34 @@
+from .base import Dependency, ReleaseDownload, MesonBuilder
+
+from kiwixbuild.utils import Remotefile
+from kiwixbuild._global import neutralEnv
+
+class docoptcpp(Dependency):
+    name = "docoptcpp"
+
+    class Source(ReleaseDownload):
+        name = "docoptcpp"
+        src_archive = Remotefile(
+            "v0.6.3.tar.gz",
+            "28af5a0c482c6d508d22b14d588a3b0bd9ff97135f99c2814a5aa3cbff1d6632",
+            "https://github.com/docopt/docopt.cpp/archive/v0.6.3.tar.gz",
+        )
+
+        meson_archive = Remotefile(
+            "docopt_0.6.3-3_patch.zip",
+            "1f641187f9d3f35b0a5ebd2011876ef8e9b04b69b7b163095dd7dfa16219ad01",
+            "https://wrapdb.mesonbuild.com/v2/docopt_0.6.3-3/get_patch",
+        )
+
+        archives = [src_archive, meson_archive]
+        patches = [
+            "docopt_meson_install_pkgconfig.patch",
+            "docopt_meson_use_boostregex.patch",
+        ]
+
+    class Builder(MesonBuilder):
+        @classmethod
+        def get_dependencies(cls, configInfo, allDeps):
+            if neutralEnv("distname") == "Windows":
+                return ["boostregex"]
+            return []

kiwixbuild/dependencies/gumbo.py

@@ -0,0 +1,21 @@
+from .base import Dependency, ReleaseDownload, MakeBuilder
+
+from kiwixbuild.utils import Remotefile, run_command
+
+
+class Gumbo(Dependency):
+    name = "gumbo"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile(
+            "gumbo-0.10.1.tar.gz",
+            "28463053d44a5dfbc4b77bcf49c8cee119338ffa636cc17fc3378421d714efad",
+            "https://github.com/google/gumbo-parser/archive/v0.10.1.tar.gz",
+        )
+
+        def _post_prepare_script(self, context):
+            context.try_skip(self.extract_path)
+            command = ["./autogen.sh"]
+            run_command(command, self.extract_path, context)
+
+    Builder = MakeBuilder

kiwixbuild/dependencies/icu4c.py

@@ -0,0 +1,191 @@
+from .base import (
+    Dependency,
+    ReleaseDownload,
+    MakeBuilder,
+    Builder as BaseBuilder,
+)
+
+from kiwixbuild.utils import pj, SkipCommand, Remotefile, extract_archive
+from kiwixbuild._global import get_target_step, neutralEnv
+import os, shutil
+import fileinput
+import platform
+
+if platform.system() == "Windows":
+
+    class Icu(Dependency):
+        name = "icu4c"
+
+        class Source(ReleaseDownload):
+            archive = Remotefile(
+                "icu4c-74_1-Win64-MSVC2022.zip",
+                "",
+                "https://github.com/unicode-org/icu/releases/download/release-74-1/icu4c-74_1-Win64-MSVC2022.zip",
+            )
+
+        class Builder(BaseBuilder):
+            def build(self):
+                self.command("copy_headers", self._copy_headers)
+                self.command("copy_bins", self._copy_bin)
+                self.command("generate_pkg_config", self._generate_pkg_config)
+
+            def _copy_headers(self, context):
+                context.try_skip(self.build_path)
+                shutil.copytree(
+                    pj(self.source_path, "include", "unicode"),
+                    pj(self.buildEnv.install_dir, "include", "unicode"),
+                )
+
+            def _copy_bin(self, context):
+                context.try_skip(self.build_path)
+                shutil.copytree(
+                    pj(self.source_path, "lib64"),
+                    pj(self.buildEnv.install_dir, "lib"),
+                    dirs_exist_ok=True,
+                )
+
+                def ignore_non_dll(path, names):
+                    return [n for n in names if not n.endswith(".dll")]
+
+                shutil.copytree(
+                    pj(self.source_path, "bin64"),
+                    pj(self.buildEnv.install_dir, "bin"),
+                    ignore=ignore_non_dll,
+                    dirs_exist_ok=True,
+                )
+
+            def _generate_pkg_config(self, context):
+                context.try_skip(self.build_path)
+
+                pkg_config_template = """ Copyright (C) 2016 and later: Unicode, Inc. and others.
+# License & terms of use: http://www.unicode.org/copyright.html
+# Copyright (C) 2010-2013, International Business Machines Corporation. All Rights Reserved.
+
+# CFLAGS contains only anything end users should set
+CFLAGS = 
+# CXXFLAGS contains only anything end users should set
+CXXFLAGS =  -std=c++11
+# DEFS only contains those UCONFIG_CPPFLAGS which are not auto-set by platform.h
+DEFS = 
+prefix = {prefix}
+exec_prefix = ${{prefix}}
+libdir = ${{exec_prefix}}/lib
+includedir = ${{prefix}}/include
+baselibs = -lpthread -lm 
+UNICODE_VERSION=15.0
+ICUPREFIX=icu
+ICULIBSUFFIX=
+LIBICU=lib${{ICUPREFIX}}
+pkglibdir=${{libdir}}/icu${{ICULIBSUFFIX}}/73.1
+ICUDATA_NAME = icudt73l
+ICUDESC=International Components for Unicode
+
+Version: 73.1
+Cflags: -I${{includedir}}
+Description: International Components for Unicode: Internationalization library
+Name: icu-i18n
+Libs: -L${{libdir}} -licuin -licuuc -licudt"""
+
+                pkg_config_content = pkg_config_template.format(
+                    prefix=self.buildEnv.install_dir
+                )
+
+                os.makedirs(
+                    pj(self.buildEnv.install_dir, "lib", "pkgconfig"), exist_ok=True
+                )
+                with open(
+                    pj(self.buildEnv.install_dir, "lib", "pkgconfig", "icu-i18n.pc"),
+                    mode="w",
+                ) as f:
+                    f.write(pkg_config_content)
+
+else:
+
+    class Icu(Dependency):
+        name = "icu4c"
+
+        class Source(ReleaseDownload):
+            archive_src = Remotefile(
+                "icu4c-73_2-src.tgz",
+                "818a80712ed3caacd9b652305e01afc7fa167e6f2e94996da44b90c2ab604ce1",
+                "https://github.com/unicode-org/icu/releases/download/release-73-2/icu4c-73_2-src.tgz",
+            )
+            archive_data = Remotefile(
+                "icu4c-73_2-data.zip",
+                "ca1ee076163b438461e484421a7679fc33a64cd0a54f9d4b401893fa1eb42701",
+                "https://github.com/unicode-org/icu/releases/download/release-73-2/icu4c-73_2-data.zip",
+            )
+
+            archives = [archive_src, archive_data]
+
+            def _extract(self, context):
+                context.try_skip(self.extract_path)
+                if os.path.exists(self.extract_path):
+                    shutil.rmtree(self.extract_path)
+                extract_archive(
+                    pj(neutralEnv("archive_dir"), self.archive_src.name),
+                    neutralEnv("source_dir"),
+                    topdir=None,
+                    name=self.source_dir,
+                )
+                shutil.rmtree(
+                    pj(neutralEnv("source_dir"), self.source_dir, "source", "data")
+                )
+                extract_archive(
+                    pj(neutralEnv("archive_dir"), self.archive_data.name),
+                    pj(neutralEnv("source_dir"), self.source_dir, "source"),
+                    topdir="data",
+                    name="data",
+                )
+
+            patches = [
+                "icu4c_fix_static_lib_name_mingw.patch",
+                "icu4c_rpath.patch",
+                "icu4c_wasm.patch",
+            ]
+
+        class Builder(MakeBuilder):
+            subsource_dir = "source"
+            make_install_targets = ["install"]
+
+            @classmethod
+            def get_dependencies(cls, configInfo, allDeps):
+                plt = "native_static" if configInfo.static else "native_dyn"
+                return [(plt, "icu4c")]
+
+            @property
+            def configure_options(self):
+                yield "--disable-samples"
+                yield "--disable-tests"
+                yield "--disable-extras"
+                yield "--disable-dyload"
+                yield "--enable-rpath"
+                yield "--disable-icuio"
+                yield "--disable-layoutex"
+                configInfo = self.buildEnv.configInfo
+                if configInfo.build != "native":
+                    icu_native_builder = get_target_step(
+                        "icu4c", "native_static" if configInfo.static else "native_dyn"
+                    )
+                    yield f"--with-cross-build={icu_native_builder.build_path}"
+                    yield "--disable-tools"
+                if configInfo.build in ("android", "wasm"):
+                    yield "--with-data-packaging=archive"
+
+            def set_env(self, env):
+                env["ICU_DATA_FILTER_FILE"] = pj(
+                    os.path.dirname(os.path.realpath(__file__)),
+                    "icu4c_data_filter.json",
+                )
+
+            def _post_configure_script(self, context):
+                if self.buildEnv.configInfo.build != "wasm":
+                    context.skip()
+                context.try_skip(self.build_path)
+                for line in fileinput.input(
+                    pj(self.build_path, "Makefile"), inplace=True
+                ):
+                    if line == "#DATASUBDIR = data\n":
+                        print("DATASUBDIR = data")
+                    else:
+                        print(line, end="")

kiwixbuild/dependencies/icu4c_data_filter.json

@@ -0,0 +1,14 @@
+{
+    "strategy": "additive",
+    "featureFilters": {
+        "lang_tree": "include",
+        "locales_tree": "include",
+        "translit": "include",
+        "misc": {
+            "includelist": [
+               "likelySubtags",
+               "metadata"
+            ]
+        }
+    }
+}

kiwixbuild/dependencies/ios_fat_lib.py

@@ -0,0 +1,50 @@
+import os
+
+from kiwixbuild.configs import ConfigInfo
+from kiwixbuild.utils import pj, copy_tree, run_command
+from kiwixbuild._global import option
+from .base import Dependency, NoopSource, Builder as BaseBuilder
+
+
+class IOSFatLib(Dependency):
+    name = "_ios_fat_lib"
+
+    Source = NoopSource
+
+    class Builder(BaseBuilder):
+        @classmethod
+        def get_dependencies(self, platfomInfo, alldeps):
+            base_target = option("target")
+            return [("iOS_{}".format(arch), base_target) for arch in option("ios_arch")]
+
+        def _copy_headers(self, context):
+            plt = ConfigInfo.get_config("iOS_{}".format(option("ios_arch")[0]))
+            include_src = pj(plt.buildEnv.install_dir, "include")
+            include_dst = pj(self.buildEnv.install_dir, "include")
+            copy_tree(include_src, include_dst)
+
+        def _merge_libs(self, context):
+            lib_dirs = []
+            for arch in option("ios_arch"):
+                plt = ConfigInfo.get_config("iOS_{}".format(arch))
+                lib_dirs.append(pj(plt.buildEnv.install_dir, "lib"))
+            libs = []
+            for f in os.listdir(lib_dirs[0]):
+                if os.path.islink(pj(lib_dirs[0], f)):
+                    continue
+                if f.endswith(".a") or f.endswith(".dylib"):
+                    libs.append(f)
+            os.makedirs(pj(self.buildEnv.install_dir, "lib"), exist_ok=True)
+            for l in libs:
+                command = [
+                    "lipo",
+                    "-create",
+                    *[pj(d, l) for d in lib_dirs],
+                    "-output",
+                    pj(self.buildEnv.install_dir, "lib", l),
+                ]
+                run_command(command, self.buildEnv.install_dir, context)
+
+        def build(self):
+            self.command("copy_headers", self._copy_headers)
+            self.command("merge_libs", self._merge_libs)

kiwixbuild/dependencies/kiwix_desktop.py

@@ -0,0 +1,39 @@
+from kiwixbuild._global import option
+from .base import Dependency, GitClone, QMakeBuilder
+import platform
+
+
+class KiwixDesktop(Dependency):
+    name = "kiwix-desktop"
+    force_build = True
+
+    class Source(GitClone):
+        git_remote = "https://github.com/kiwix/kiwix-desktop.git"
+        git_dir = "kiwix-desktop"
+
+    class Builder(QMakeBuilder):
+        dependencies = ["qt", "qtwebengine", "libkiwix", "aria2"]
+        configure_env = None
+
+        flatpack_build_options = {"env": {"QMAKEPATH": "/app"}}
+
+        @property
+        def make_targets(self):
+            if platform.system() == "Windows":
+                yield "release-all"
+            else:
+                yield from super().make_targets
+
+        @property
+        def make_install_targets(self):
+            if platform.system() == "Windows":
+                yield "release-install"
+            else:
+                yield "install"
+
+        @property
+        def configure_options(self):
+            if self.buildEnv.configInfo.name != "flatpak":
+                yield f"PREFIX={self.buildEnv.install_dir}"
+            if self.buildEnv.configInfo.static:
+                yield "CONFIG+=static"

kiwixbuild/dependencies/kiwix_tools.py

@@ -0,0 +1,18 @@
+from .base import Dependency, GitClone, MesonBuilder
+
+
+class KiwixTools(Dependency):
+    name = "kiwix-tools"
+    force_build = True
+
+    class Source(GitClone):
+        git_remote = "https://github.com/kiwix/kiwix-tools.git"
+        git_dir = "kiwix-tools"
+
+    class Builder(MesonBuilder):
+        dependencies = ["libkiwix", "docoptcpp"]
+
+        @property
+        def configure_options(self):
+            if self.buildEnv.configInfo.static:
+                yield "-Dstatic-linkage=true"

kiwixbuild/dependencies/libcurl.py

@@ -0,0 +1,61 @@
+import os
+
+from .base import (
+    Dependency,
+    ReleaseDownload,
+    MesonBuilder,
+)
+
+from kiwixbuild.utils import Remotefile, pj, Defaultdict, SkipCommand, run_command
+from kiwixbuild._global import get_target_step
+
+
+class LibCurl(Dependency):
+    name = "libcurl"
+
+    class Source(ReleaseDownload):
+        name = "libcurl"
+        src_archive = Remotefile(
+            "curl-8.4.0.tar.xz",
+            "16c62a9c4af0f703d28bda6d7bbf37ba47055ad3414d70dec63e2e6336f2a82d",
+            "https://curl.se/download/curl-8.4.0.tar.xz",
+        )
+        meson_archive = Remotefile(
+            "curl_8.4.0-2_patch.zip",
+            "bbb6ae75225c36ef9bb336cface729794c7c070c623a003fff40bd416042ff6e",
+            "https://dev.kiwix.org/libkiwix/curl_8.4.0-2_patch.zip",
+        )
+        archives = [src_archive, meson_archive]
+
+    class Builder(MesonBuilder):
+        dependencies = ["zlib"]
+        configure_options = [
+            f"-D{p}=disabled"
+            for p in (
+                "psl",
+                "kerberos-auth",
+                "gss-api",
+                "ssh",
+                "rtmp",
+                "http2",
+                "idn",
+                "brotli",
+                "ftp",
+                "file",
+                "ldap",
+                "ldaps",
+                "rtsp",
+                "dict",
+                "telnet",
+                "tftp",
+                "pop3",
+                "imap",
+                "smb",
+                "smtp",
+                "gopher",
+                "tool",
+            )
+        ]
+
+        def _test(self, context):
+            context.skip("No Test")

kiwixbuild/dependencies/libkiwix.py

@@ -0,0 +1,52 @@
+import shutil, os
+
+from .base import Dependency, GitClone, MesonBuilder
+from kiwixbuild.utils import pj, copy_tree
+from kiwixbuild._global import option, get_target_step, neutralEnv
+
+
+class Libkiwix(Dependency):
+    name = "libkiwix"
+    force_build = True
+
+    class Source(GitClone):
+        git_remote = "https://github.com/kiwix/libkiwix.git"
+        git_dir = "libkiwix"
+
+    class Builder(MesonBuilder):
+        dependencies = [
+            "pugixml",
+            "libzim",
+            "zlib",
+            "libcurl",
+            "libmicrohttpd",
+            "icu4c",
+            "mustache",
+            "xapian-core",
+        ]
+        strip_options = []
+
+        @property
+        def build_type(self):
+            if self.buildEnv.configInfo.build == "android":
+                return "debug"
+            return super().build_type
+
+        @property
+        def configure_options(self):
+            configInfo = self.buildEnv.configInfo
+            if configInfo.build == "android":
+                yield "-Dstatic-linkage=true"
+                yield "-Dwerror=false"
+            if configInfo.build == "iOS":
+                yield "-Db_bitcode=true"
+            if configInfo.name == "flatpak":
+                yield "--wrap-mode=nodownload"
+            if configInfo.mixed and option("target") == "libkiwix":
+                yield "-Dstatic-linkage=true"
+
+        @property
+        def library_type(self):
+            if self.buildEnv.configInfo.build == "android":
+                return "shared"
+            return super().library_type

kiwixbuild/dependencies/libmagic.py

@@ -0,0 +1,50 @@
+import os
+
+from .base import (
+    Dependency,
+    ReleaseDownload,
+    MakeBuilder,
+)
+
+from kiwixbuild.utils import Remotefile, pj, SkipCommand, run_command
+from kiwixbuild._global import get_target_step
+
+
+class LibMagic(Dependency):
+    name = "libmagic"
+
+    class Source(ReleaseDownload):
+        name = "libmagic"
+        source_dir = "libmagic"
+        archive_top_dir = "file-5.44"
+        archive = Remotefile(
+            "file-5.44.tar.gz",
+            "3751c7fba8dbc831cb8d7cc8aff21035459b8ce5155ef8b0880a27d028475f3b",
+        )
+
+    class Builder(MakeBuilder):
+        @property
+        def configure_options(self):
+            yield "--disable-bzlib"
+            yield "--disable-xzlib"
+            yield "--disable-zstdlib"
+            yield "--disable-lzlib"
+
+        @classmethod
+        def get_dependencies(cls, configInfo, allDeps):
+            if configInfo.build != "native":
+                return [("native_static", "libmagic")]
+            return []
+
+        def _compile(self, context):
+            configInfo = self.buildEnv.configInfo
+            if configInfo.build == "native":
+                return super()._compile(context)
+            context.try_skip(self.build_path)
+            command = ["make", "-j4", *self.make_targets, *self.make_options]
+            env = self.buildEnv.get_env(
+                cross_comp_flags=True, cross_compilers=True, cross_path=True
+            )
+            libmagic_native_builder = get_target_step("libmagic", "native_static")
+            env["PATH"].insert(0, pj(libmagic_native_builder.build_path, "src"))
+            run_command(command, self.build_path, context, env=env)

kiwixbuild/dependencies/libmicrohttpd.py

@@ -0,0 +1,33 @@
+from .base import Dependency, ReleaseDownload, MesonBuilder
+
+from kiwixbuild.utils import Remotefile
+
+
+class MicroHttpd(Dependency):
+    name = "libmicrohttpd"
+
+    class Source(ReleaseDownload):
+        src_archive = Remotefile(
+            "libmicrohttpd-0.9.76.tar.gz",
+            "f0b1547b5a42a6c0f724e8e1c1cb5ce9c4c35fb495e7d780b9930d35011ceb4c",
+            "https://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.76.tar.gz",
+        )
+        meson_archive = Remotefile(
+            "libmicrohttpd_0.9.16-3_patch.zip",
+            "0954c094a0d4cfe0dd799d8df8a04face6669f7b4d51a7386a9c3e2d37b9c3b3",
+            "https://wrapdb.mesonbuild.com/v2/libmicrohttpd_0.9.76-3/get_patch",
+        )
+        archives = [src_archive, meson_archive]
+        patches = [
+            "libmicrohttpd_meson_pkgconfig.patch",
+            "libmicrohttpd_meson_timeval_tvsec_size.patch",
+            "libmicrohttpd_meson_winet6.patch",
+        ]
+
+    class Builder(MesonBuilder):
+        configure_options = [
+            "-Dgnutls=disabled",
+            "-Dgcrypt=disabled",
+            "-Dcurl=disabled",
+            "-Dexpat=disabled",
+        ]

kiwixbuild/dependencies/libzim.py

@@ -0,0 +1,57 @@
+from .base import Dependency, GitClone, MesonBuilder
+from kiwixbuild._global import option, get_target_step, neutralEnv
+
+
+class Libzim(Dependency):
+    name = "libzim"
+    force_build = True
+
+    class Source(GitClone):
+        git_remote = "https://github.com/openzim/libzim.git"
+        git_dir = "libzim"
+
+    class Builder(MesonBuilder):
+        test_options = ["-t", "8"]
+        strip_options = []
+
+        @property
+        def build_type(self):
+            if self.buildEnv.configInfo.build == "android":
+                return "debug"
+            return super().build_type
+
+        @classmethod
+        def get_dependencies(cls, configInfo, allDeps):
+            if neutralEnv("distname") == "Windows":
+                return ["zstd", "xapian-core", "icu4c", "zim-testing-suite"]
+            deps = ["lzma", "zstd", "xapian-core", "icu4c"]
+            if configInfo.name not in ("flatpak", "wasm"):
+                deps.append("zim-testing-suite")
+            return deps
+
+        @property
+        def configure_options(self):
+            configInfo = self.buildEnv.configInfo
+            if neutralEnv("distname") == "Windows":
+                yield "-Dwith_xapian_fuller=false"
+                yield "-Dwerror=false"
+            if configInfo.build == "android":
+                yield "-DUSE_BUFFER_HEADER=false"
+                yield "-Dstatic-linkage=true"
+            if configInfo.mixed and option("target") == "libzim":
+                yield "-Dstatic-linkage=true"
+            if configInfo.name == "flatpak":
+                yield "--wrap-mode=nodownload"
+                yield "-Dtest_data_dir=none"
+            if configInfo.name == "wasm":
+                yield "-Dexamples=false"
+                yield "-DUSE_MMAP=false"
+            if configInfo.name not in ("flatpak", "wasm"):
+                zim_testing_suite = get_target_step("zim-testing-suite", "source")
+                yield "-Dtest_data_dir={}".format(zim_testing_suite.source_path)
+
+        @property
+        def library_type(self):
+            if self.buildEnv.configInfo.build == "android":
+                return "shared"
+            return super().library_type

kiwixbuild/dependencies/lzma.py

@@ -0,0 +1,26 @@
+from .base import (
+    Dependency,
+    ReleaseDownload,
+    MesonBuilder)
+
+from kiwixbuild.utils import Remotefile
+
+
+class lzma(Dependency):
+    name = 'lzma'
+
+    class Source(ReleaseDownload):
+        src_archive = Remotefile(
+            "xz-5.2.6.tar.gz",
+            "a2105abee17bcd2ebd15ced31b4f5eda6e17efd6b10f921a01cda4a44c91b3a0",
+            "https://altushost-swe.dl.sourceforge.net/project/lzmautils/xz-5.2.6.tar.gz",
+        )
+        meson_patch = Remotefile(
+            "liblzma_5.2.6-3_patch.zip",
+            "1c71536d364e1a3ce6bea61266576f89cc5cce4d3b9e11f3494417dafa29780b",
+            "https://wrapdb.mesonbuild.com/v2/liblzma_5.2.6-3/get_patch",
+        )
+        archives = [src_archive, meson_patch]
+        patches = ['lzma_meson_install.patch']
+
+    Builder = MesonBuilder

kiwixbuild/dependencies/mustache.py

@@ -0,0 +1,30 @@
+from .base import Dependency, ReleaseDownload, Builder as BaseBuilder
+
+from kiwixbuild.utils import Remotefile, pj
+from shutil import copy2
+
+
+class Mustache(Dependency):
+    name = "mustache"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile(
+            "Mustache-4.1.tar.gz",
+            "acd66359feb4318b421f9574cfc5a511133a77d916d0b13c7caa3783c0bfe167",
+            "https://github.com/kainjow/Mustache/archive/v4.1.tar.gz",
+        )
+
+    class Builder(BaseBuilder):
+        def build(self):
+            self.command("copy_header", self._copy_header)
+
+        def _copy_header(self, context):
+            context.try_skip(self.build_path)
+            copy2(
+                pj(self.source_path, "mustache.hpp"),
+                pj(self.buildEnv.install_dir, "include"),
+            )
+
+        def set_flatpak_buildsystem(self, module):
+            module["buildsystem"] = "simple"
+            module["build-commands"] = ["cp mustache.hpp /app/include"]

kiwixbuild/dependencies/pugixml.py

@@ -0,0 +1,18 @@
+from .base import Dependency, ReleaseDownload, MesonBuilder
+
+from kiwixbuild.utils import Remotefile
+
+
+class Pugixml(Dependency):
+    name = "pugixml"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile(
+            "pugixml-1.2.tar.gz",
+            "0f422dad86da0a2e56a37fb2a88376aae6e931f22cc8b956978460c9db06136b",
+        )
+        patches = ["pugixml_meson.patch"]
+        flatpak_dest = "src"
+
+    class Builder(MesonBuilder):
+        strip_options = []

kiwixbuild/dependencies/qt.py

@@ -0,0 +1,25 @@
+import shutil
+
+from .base import Dependency, NoopBuilder, NoopSource
+
+from kiwixbuild.utils import SkipCommand, colorize
+
+
+class Qt(Dependency):
+    name = "qt"
+
+    Source = NoopSource
+
+    class Builder(NoopBuilder):
+        def build(self):
+            error_msg = f"""WARNING: kiwix-build cannot build {self.name} for you.
+You must install it yourself using official Qt installer or your distribution system."""
+            print(colorize(error_msg, "WARNING"))
+
+
+class QtWebEngine(Dependency):
+    name = "qtwebengine"
+
+    Source = NoopSource
+
+    Builder = Qt.Builder

kiwixbuild/dependencies/tc_android_ndk.py

@@ -0,0 +1,93 @@
+import os
+
+from .base import Dependency, ReleaseDownload, Builder
+from kiwixbuild.utils import Remotefile, add_execution_right, run_command
+
+pj = os.path.join
+
+
+class android_ndk(Dependency):
+    dont_skip = True
+    neutral = False
+    name = "android-ndk"
+    gccver = "4.9.x"
+    api = "24"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile(
+            "android-ndk-r21e-linux-x86_64.zip",
+            "ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e",
+            "https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip",
+        )
+
+        @property
+        def source_dir(self):
+            return self.target.full_name()
+
+        patches = [
+            "android-ndk-r21e-linux-x86_64-python3.12+.patch",
+        ]
+
+    class Builder(Builder):
+        @property
+        def install_path(self):
+            return pj(self.buildEnv.toolchain_dir, self.target.full_name())
+
+        @property
+        def api(self):
+            return self.target.api
+
+        @property
+        def config(self):
+            return "android-" + self.api
+
+        @property
+        def arch(self):
+            return self.buildEnv.configInfo.arch
+
+        @property
+        def arch_full(self):
+            return self.buildEnv.configInfo.arch_full
+
+        def _build_toolchain(self, context):
+            context.try_skip(self.build_path)
+            script = pj(self.source_path, "build/tools/make_standalone_toolchain.py")
+            add_execution_right(script)
+            command = [
+                script,
+                f"--arch={self.arch}",
+                f"--api={self.api}",
+                f"--install-dir={self.install_path}",
+                "--force",
+            ]
+            env = self.buildEnv.get_env(
+                cross_comp_flags=False, cross_compilers=False, cross_path=False
+            )
+            run_command(command, self.build_path, context, env=env)
+
+        def _fix_permission_right(self, context):
+            context.try_skip(self.build_path)
+            bin_dirs = [
+                pj(self.install_path, "bin"),
+                pj(self.install_path, self.arch_full, "bin"),
+                pj(
+                    self.install_path,
+                    "libexec",
+                    "gcc",
+                    self.arch_full,
+                    self.target.gccver,
+                ),
+            ]
+            for root, dirs, files in os.walk(self.install_path):
+                if not root in bin_dirs:
+                    continue
+
+                for file_ in files:
+                    file_path = pj(root, file_)
+                    if os.path.islink(file_path):
+                        continue
+                    add_execution_right(file_path)
+
+        def build(self):
+            self.command("build_toolchain", self._build_toolchain)
+            self.command("fix_permission_right", self._fix_permission_right)

kiwixbuild/dependencies/tc_armhf.py

@@ -0,0 +1,55 @@
+from .base import Dependency, ReleaseDownload, TcCopyBuilder
+from kiwixbuild.utils import Remotefile
+
+# The arm toolchains
+# This is based on toolchains published here : https://github.com/tttapa/docker-arm-cross-toolchain
+
+base_url = (
+    "https://github.com/tttapa/docker-arm-cross-toolchain/releases/download/0.1.0/"
+)
+
+
+class armv6_toolchain(Dependency):
+    dont_skip = True
+    neutral = True
+    name = "armv6"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile(
+            "x-tools-armv6-rpi-linux-gnueabihf.tar.xz",
+            "4c371c4c5b55ebd1f3d7dd26b14703632d9ba47423f901bcd9303d83ad444434",
+            base_url + "x-tools-armv6-rpi-linux-gnueabihf.tar.xz",
+        )
+
+    class Builder(TcCopyBuilder):
+        src_subdir = "armv6-rpi-linux-gnueabihf"
+
+
+class armv8_toolchain(Dependency):
+    dont_skip = True
+    neutral = True
+    name = "armv8"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile(
+            "x-tools-armv8-rpi-linux-gnueabihf.tar.xz",
+            "cc28f5c3f6a3e7d9985f98779c4e72224b4eb5a7e4dc2bcdefd90cb241fb94a5",
+            base_url + "x-tools-armv8-rpi3-linux-gnueabihf.tar.xz",
+        )
+
+    class Builder(TcCopyBuilder):
+        src_subdir = "armv8-rpi3-linux-gnueabihf"
+
+
+class aarch64_toolchain(Dependency):
+    dont_skip = True
+    neutral = True
+    name = "aarch64"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile(
+            "cross-gcc-6.3.0-pi_64.tar.gz",
+            "1b048bb8886ad63d21797cd9129fc37b9ea0dfaac7e3c36f888aa16fbec1d320",
+        )
+
+    Builder = TcCopyBuilder

kiwixbuild/dependencies/tc_emsdk.py

@@ -0,0 +1,47 @@
+import os
+
+from .base import Dependency, ReleaseDownload, Builder
+from kiwixbuild.utils import Remotefile, run_command, copy_tree
+
+pj = os.path.join
+
+
+class emsdk(Dependency):
+    dont_skip = True
+    neutral = False
+    name = "emsdk"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile(
+            "emsdk-3.1.41.tar.gz",
+            "147a2d72df34227bdb4ffedc587a8cb674a42269c40458f3f69ae37e8966cdc6",
+            "https://codeload.github.com/emscripten-core/emsdk/tar.gz/refs/tags/3.1.41",
+        )
+
+        @property
+        def source_dir(self):
+            return self.target.full_name()
+
+    class Builder(Builder):
+        @property
+        def install_path(self):
+            return pj(self.buildEnv.toolchain_dir, self.target.full_name())
+
+        def _copy_source(self, context):
+            context.try_skip(self.build_path)
+            copy_tree(self.source_path, self.install_path)
+
+        def _install(self, context):
+            context.try_skip(self.build_path)
+            command = ["./emsdk", "install", "3.1.24"]
+            run_command(command, self.install_path, context)
+
+        def _activate(self, context):
+            context.try_skip(self.build_path)
+            command = ["./emsdk", "activate", "3.1.24"]
+            run_command(command, self.install_path, context)
+
+        def build(self):
+            self.command("copy_source", self._copy_source)
+            self.command("install", self._install)
+            self.command("activate", self._activate)

kiwixbuild/dependencies/tc_flatpak.py

@@ -0,0 +1,89 @@
+import os
+
+from .base import Dependency, NoopSource, Builder
+from kiwixbuild.utils import Remotefile, add_execution_right, run_command
+
+pj = os.path.join
+
+
+class org_kde(Dependency):
+    neutral = False
+    name = "org.kde"
+
+    Source = NoopSource
+
+    class Builder(Builder):
+        def _setup_remote(self, context):
+            command = [
+                "flatpak",
+                "--user",
+                "remote-add",
+                "--if-not-exists",
+                "flathub",
+                "https://flathub.org/repo/flathub.flatpakrepo",
+            ]
+            env = self.buildEnv.get_env(
+                cross_comp_flags=False, cross_compilers=False, cross_path=False
+            )
+            run_command(command, self.buildEnv.build_dir, context, env=env)
+
+        def _install_sdk(self, context):
+            command = [
+                "flatpak",
+                "--user",
+                "install",
+                "--noninteractive",
+                "--verbose",
+                "-y",
+                "flathub",
+                f"{self.target.name}.Sdk//{self.target.version()}",
+                f"{self.target.name}.Platform//{self.target.version()}",
+            ]
+            env = self.buildEnv.get_env(
+                cross_comp_flags=False, cross_compilers=False, cross_path=False
+            )
+            run_command(command, self.buildEnv.build_dir, context, env=env)
+
+        def build(self):
+            self.command("setup_remote", self._setup_remote)
+            self.command("install_sdk", self._install_sdk)
+
+
+class io_qt_qtwebengine(Dependency):
+    neutral = False
+    name = "io.qt.qtwebengine"
+
+    Source = NoopSource
+
+    class Builder(Builder):
+        def _setup_remote(self, context):
+            command = [
+                "flatpak",
+                "--user",
+                "remote-add",
+                "--if-not-exists",
+                "flathub",
+                "https://flathub.org/repo/flathub.flatpakrepo",
+            ]
+            env = self.buildEnv.get_env(
+                cross_comp_flags=False, cross_compilers=False, cross_path=False
+            )
+            run_command(command, self.buildEnv.build_dir, context, env=env)
+
+        def _install_sdk(self, context):
+            command = [
+                "flatpak",
+                "--user",
+                "install",
+                "-y",
+                "flathub",
+                f"{self.target.name}.BaseApp//{self.target.version()}",
+            ]
+            env = self.buildEnv.get_env(
+                cross_comp_flags=False, cross_compilers=False, cross_path=False
+            )
+            run_command(command, self.buildEnv.build_dir, context, env=env)
+
+        def build(self):
+            self.command("setup_remote", self._setup_remote)
+            self.command("install_sdk", self._install_sdk)

kiwixbuild/dependencies/tc_musl.py

@@ -0,0 +1,32 @@
+from .base import Dependency, ReleaseDownload, TcCopyBuilder
+from kiwixbuild.utils import Remotefile
+
+
+class aarch64_musl_toolchain(Dependency):
+    dont_skip = True
+    neutral = True
+    name = "aarch64_musl"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile(
+            "aarch64-linux-musl-cross.tgz",
+            "0f18a885b161815520bbb5757a4b4ab40d0898c29bebee58d0cddd6112e59cc6",
+            "https://more.musl.cc/10/x86_64-linux-musl/aarch64-linux-musl-cross.tgz",
+        )
+
+    Builder = TcCopyBuilder
+
+
+class x86_64_musl_toolchain(Dependency):
+    dont_skip = True
+    neutral = True
+    name = "x86-64_musl"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile(
+            "x86_64-linux-musl-cross.tgz",
+            "a3d55de8105739fcfb8b10eaa72cdb5d779319726bacff24149388d7608d1ed8",
+            "https://more.musl.cc/10/x86_64-linux-musl/x86_64-linux-musl-cross.tgz",
+        )
+
+    Builder = TcCopyBuilder

kiwixbuild/dependencies/uuid.py

@@ -0,0 +1,30 @@
+from .base import Dependency, ReleaseDownload, MakeBuilder
+
+from kiwixbuild.utils import Remotefile
+
+
+class UUID(Dependency):
+    name = "uuid"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile(
+            "e2fsprogs-libs-1.43.4.tar.gz",
+            "eed4516325768255c9745e7b82c9d7d0393abce302520a5b2cde693204b0e419",
+            "https://www.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v1.43.4/e2fsprogs-libs-1.43.4.tar.gz",
+        )
+        extract_dir = "e2fsprogs-libs-1.43.4"
+
+    class Builder(MakeBuilder):
+        configure_options = [
+            "--enable-libuuid",
+            "--disable-debugfs",
+            "--disable-imager",
+            "--disable-resizer",
+            "--disable-defrag",
+            "--enable-fsck",
+            "--disable-uuidd",
+        ]
+        configure_env = {"_format_CFLAGS": "{env.CFLAGS} -O3 -fPIC"}
+        static_configure_options = dynamic_configure_options = []
+        make_targets = ["libs"]
+        make_install_targets = ["install-libs"]

kiwixbuild/dependencies/xapian.py

@@ -0,0 +1,55 @@
+from .base import Dependency, GitClone, ReleaseDownload, MakeBuilder, MesonBuilder
+
+from kiwixbuild.utils import Remotefile
+from kiwixbuild._global import neutralEnv
+
+import platform
+
+
+class Xapian(Dependency):
+    name = "xapian-core"
+
+    if platform.system() == "Windows":
+
+        class Source(GitClone):
+            git_remote = "https://github.com/openzim/xapian-meson.git"
+            git_dir = "xapian-core"
+
+        class Builder(MesonBuilder):
+            configure_options = [
+                "-Denable-sse=false",
+                "-Denable-backend-chert=false",
+                "-Denable-backend-remote=false",
+            ]
+            subsource_dir = "xapian-core"
+
+            @classmethod
+            def get_dependencies(cls, configInfo, allDeps):
+                return ["zlib"]
+
+    else:
+
+        class Source(ReleaseDownload):
+            archive = Remotefile(
+                "xapian-core-1.4.23.tar.xz",
+                "30d3518172084f310dab86d262b512718a7f9a13635aaa1a188e61dc26b2288c",
+            )
+
+        class Builder(MakeBuilder):
+            configure_options = [
+                "--disable-sse",
+                "--disable-backend-chert",
+                "--disable-backend-remote",
+                "--disable-documentation",
+            ]
+            configure_env = {
+                "_format_LDFLAGS": "{env.LDFLAGS} -L{buildEnv.install_dir}/{buildEnv.libprefix}",
+                "_format_CXXFLAGS": "{env.CXXFLAGS} -I{buildEnv.install_dir}/include",
+            }
+
+            @classmethod
+            def get_dependencies(cls, configInfo, allDeps):
+                deps = ["zlib", "lzma"]
+                if configInfo.build == "wasm" or neutralEnv("distname") == "Darwin":
+                    return deps
+                return deps + ["uuid"]

kiwixbuild/dependencies/zim_testing_suite.py

@@ -0,0 +1,17 @@
+from .base import Dependency, ReleaseDownload, NoopBuilder
+
+from kiwixbuild.utils import Remotefile
+
+
+class ZimTestingSuite(Dependency):
+    name = "zim-testing-suite"
+    dont_skip = True
+
+    class Source(ReleaseDownload):
+        archive = Remotefile(
+            "zim-testing-suite-0.8.0.tar.gz",
+            "28f51449a3f9aea02652ca21f32c5598fd610d6cec3810fa552bd0c0f7a2d5fc",
+            "https://github.com/openzim/zim-testing-suite/releases/download/0.8.0/zim-testing-suite-0.8.0.tar.gz",
+        )
+
+    Builder = NoopBuilder

kiwixbuild/dependencies/zim_tools.py

@@ -0,0 +1,27 @@
+from .base import Dependency, GitClone, MesonBuilder
+from kiwixbuild._global import neutralEnv
+
+
+class ZimTools(Dependency):
+    name = "zim-tools"
+    force_build = True
+
+    class Source(GitClone):
+        git_remote = "https://github.com/openzim/zim-tools.git"
+        git_dir = "zim-tools"
+
+    class Builder(MesonBuilder):
+        @classmethod
+        def get_dependencies(cls, configInfo, allDeps):
+            base_deps = ["libzim", "docoptcpp", "mustache"]
+            if neutralEnv("distname") != "Windows":
+                base_deps += ["libmagic", "gumbo"]
+            return base_deps
+
+        @property
+        def configure_options(self):
+            # We don't build zimwriterfs on Windows, and so we don't have magic
+            if neutralEnv("distname") != "Windows":
+                yield f"-Dmagic-install-prefix={self.buildEnv.install_dir}"
+            if self.buildEnv.configInfo.static:
+                yield "-Dstatic-linkage=true"

kiwixbuild/dependencies/zlib.py

@@ -0,0 +1,27 @@
+import shutil
+
+from .base import (
+    Dependency,
+    ReleaseDownload,
+    MesonBuilder)
+
+from kiwixbuild.utils import Remotefile, pj, SkipCommand
+from kiwixbuild._global import neutralEnv
+
+
+class zlib(Dependency):
+    name = "zlib"
+
+    class Source(ReleaseDownload):
+        src_archive = Remotefile(
+            "zlib-1.2.12.tar.gz",
+            "91844808532e5ce316b3c010929493c0244f3d37593afd6de04f71821d5136d9"
+        )
+        meson_patch = Remotefile(
+            "zlib_1.2.12-1_patch.zip",
+            "8ec8344f3fe7b06ad4be768fd416694bc56cb4545ce78b0f1c18b3e72b3ec936",
+            "https://wrapdb.mesonbuild.com/v2/zlib_1.2.12-1/get_patch")
+        archives = [src_archive, meson_patch]
+        #patches = ['zlib_std_libname.patch']
+
+    Builder = MesonBuilder

kiwixbuild/dependencies/zstd.py

@@ -0,0 +1,20 @@
+from .base import Dependency, ReleaseDownload, MesonBuilder
+
+from kiwixbuild.utils import Remotefile
+
+
+class zstd(Dependency):
+    name = "zstd"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile(
+            "zstd-1.5.5.tar.gz",
+            "98e9c3d949d1b924e28e01eccb7deed865eefebf25c2f21c702e5cd5b63b85e1",
+            "https://github.com/facebook/zstd/archive/refs/tags/v1.5.5.tar.gz",
+        )
+
+    class Builder(MesonBuilder):
+        subsource_dir = "build/meson"
+        build_type = "release"
+        strip_options = []
+        configure_options = ["-Dbin_programs=false", "-Dbin_contrib=false"]

kiwixbuild/flathub.flatpakrepo

@@ -0,0 +1,8 @@
+[Flatpak Repo]
+Title=Flathub
+Url=https://dl.flathub.org/repo/
+Homepage=https://flathub.org/
+Comment=Central repository of Flatpak applications
+Description=Central repository of Flatpak applications
+Icon=https://dl.flathub.org/repo/logo.svg
+GPGKey=mQINBFlD2sABEADsiUZUOYBg1UdDaWkEdJYkTSZD68214m8Q1fbrP5AptaUfCl8KYKFMNoAJRBXn9FbE6q6VBzghHXj/rSnA8WPnkbaEWR7xltOqzB1yHpCQ1l8xSfH5N02DMUBSRtD/rOYsBKbaJcOgW0K21sX+BecMY/AI2yADvCJEjhVKrjR9yfRX+NQEhDcbXUFRGt9ZT+TI5yT4xcwbvvTu7aFUR/dH7+wjrQ7lzoGlZGFFrQXSs2WI0WaYHWDeCwymtohXryF8lcWQkhH8UhfNJVBJFgCY8Q6UHkZG0FxMu8xnIDBMjBmSZKwKQn0nwzwM2afskZEnmNPYDI8nuNsSZBZSAw+ThhkdCZHZZRwzmjzyRuLLVFpOj3XryXwZcSefNMPDkZAuWWzPYjxS80cm2hG1WfqrG0Gl8+iX69cbQchb7gbEb0RtqNskTo9DDmO0bNKNnMbzmIJ3/rTbSahKSwtewklqSP/01o0WKZiy+n/RAkUKOFBprjJtWOZkc8SPXV/rnoS2dWsJWQZhuPPtv3tefdDiEyp7ePrfgfKxuHpZES0IZRiFI4J/nAUP5bix+srcIxOVqAam68CbAlPvWTivRUMRVbKjJiGXIOJ78wAMjqPg3QIC0GQ0EPAWwAOzzpdgbnG7TCQetaVV8rSYCuirlPYN+bJIwBtkOC9SWLoPMVZTwQARAQABtC5GbGF0aHViIFJlcG8gU2lnbmluZyBLZXkgPGZsYXRodWJAZmxhdGh1Yi5vcmc+iQJUBBMBCAA+FiEEblwF2XnHba+TwIE1QYTdTZB6fK4FAllD2sACGwMFCRLMAwAFCwkIBwIGFQgJCgsCBBYCAwECHgECF4AACgkQQYTdTZB6fK5RJQ/+Ptd4sWxaiAW91FFk7+wmYOkEe1NY2UDNJjEEz34PNP/1RoxveHDt43kYJQ23OWaPJuZAbu+fWtjRYcMBzOsMCaFcRSHFiDIC9aTp4ux/mo+IEeyarYt/oyKb5t5lta6xaAqg7rwt65jW5/aQjnS4h7eFZ+dAKta7Y/fljNrOznUp81/SMcx4QA5G2Pw0hs4Xrxg59oONOTFGBgA6FF8WQghrpR7SnEe0FSEOVsAjwQ13Cfkfa7b70omXSWp7GWfUzgBKyoWxKTqzMN3RQHjjhPJcsQnrqH5enUu4Pcb2LcMFpzimHnUgb9ft72DP5wxfzHGAWOUiUXHbAekfq5iFks8cha/RST6wkxG3Rf44Zn09aOxh1btMcGL+5xb1G0BuCQnA0fP/kDYIPwh9z22EqwRQOspIcvGeLVkFeIfubxpcMdOfQqQnZtHMCabV5Q/Rk9K1ZGc8M2hlg8gHbXMFch2xJ0Wu72eXbA/UY5MskEeBgawTQnQOK/vNm7t0AJMpWK26Qg6178UmRghmeZDj9uNRc3EI1nSbgvmGlpDmCxaAGqaGL1zW4KPW5yN25/qeqXcgCvUjZLI9PNq3Kvizp1lUrbx7heRiSoazCucvHQ1VHUzcPVLUKKTkoTP8okThnRRRsBcZ1+jI4yMWIDLOCT7IW3FePr+3xyuy5eEo9a25Ag0EWUPa7AEQALT/CmSyZ8LWlRYQZKYw417p7Z2hxqd6TjwkwM3IQ1irumkWcTZBZIbBgrSOg6CcXD2oWydCQHWi9qaxhuhEl2bJL5LskmBcMxVdQeD0LLHd8QUnbnnIby8ocvWN1alPfvJFjCUTrmD22U1ycOzRw2lIe4kiQONbOZtdWrVImQQSndjFlisitbmlWHvHm2lOOYy8+GJB7YffVV193hmnBSJffCy4bvkuLxsI+n1DhOzc7MPV3z6HGk4HiEcF0yyt9tCYhpsxHFdBoq2h771HfAcS0s98EVAqYMFnf9em+4cnYpdI6mhIfS1FQiKl6DBAYA8tT3ggla00DurPo0JwX/zN+PaO5h/6O9aCZwV7G6rbkgMuqMergXaf8oP38gr0z+MqWnkfM63Bodq68GP4l4hd02BoFBbDf38TMuGQB14+twJMdfbAxo2MbgluvQgfwHfZ2ca6gyEY+9s/YD1gugLjV+S6CB51WkFNe1z4tAPgJZNxUcKCbeaHNbthl8Hks/pY9RCEseX/EdfzF18epbSjJMPh4DPQXbUoFwmyuYcoBOPmvZHNl9hK7B/1RP8w1ZrXk8qdupC0SNbafX7270B7lMMVImzZetGsM9ypXJ6llhp3FwW09iseNyGJGPsr/dvTMGDXqOPfU/9SAS1LSTY4K9PbRtdrBE318YX8mIk5ABEBAAGJBHIEGAEIACYWIQRuXAXZecdtr5PAgTVBhN1NkHp8rgUCWUPa7AIbAgUJEswDAAJACRBBhN1NkHp8rsF0IAQZAQgAHRYhBFSmzd2JGfsgQgDYrFYnAunj7X7oBQJZQ9rsAAoJEFYnAunj7X7oR6AP/0KYmiAFeqx14Z43/6s2gt3VhxlSd8bmcVV7oJFbMhdHBIeWBp2BvsUf00I0Zl14ZkwCKfLwbbORC2eIxvzJ+QWjGfPhDmS4XUSmhlXxWnYEveSek5Tde+fmu6lqKM8CHg5BNx4GWIX/vdLi1wWJZyhrUwwICAxkuhKxuP2Z1An48930eslTD2GGcjByc27+9cIZjHKa07I/aLffo04V+oMT9/tgzoquzgpVV4jwekADo2MJjhkkPveSNI420bgT+Q7Fi1l0X1aFUniBvQMsaBa27PngWm6xE2ZYvh7nWCdd5g0c0eLIHxWwzV1lZ4Ryx4ITO/VL25ItECcjhTRdYa64sA62MYSaB0x3eR+SihpgP3wSNPFu3MJo6FKTFdi4CBAEmpWHFW7FcRmd+cQXeFrHLN3iNVWryy0HK/CUEJmiZEmpNiXecl4vPIIuyF0zgSCztQtKoMr+injpmQGC/rF/ELBVZTUSLNB350S0Ztvw0FKWDAJSxFmoxt3xycqvvt47rxTrhi78nkk6jATKGyvP55sO+K7Q7Wh0DXA69hvPrYW2eu8jGCdVGxi6HX7L1qcfEd0378S71dZ3g9o6KKl1OsDWWQ6MJ6FGBZedl/ibRfs8p5+sbCX3lQSjEFy3rx6n0rUrXx8U2qb+RCLzJlmC5MNBOTDJwHPcX6gKsUcXZrEQALmRHoo3SrewO41RCr+5nUlqiqV3AohBMhnQbGzyHf2+drutIaoh7Rj80XRh2bkkuPLwlNPf+bTXwNVGse4bej7B3oV6Ae1N7lTNVF4Qh+1OowtGjmfJPWo0z1s6HFJVxoIof9z58Msvgao0zrKGqaMWaNQ6LUeC9g9Aj/9Uqjbo8X54aLiYs8Z1WNc06jKP+gv8AWLtv6CR+l2kLez1YMDucjm7v6iuCMVAmZdmxhg5I/X2+OM3vBsqPDdQpr2TPDLX3rCrSBiS0gOQ6DwN5N5QeTkxmY/7QO8bgLo/Wzu1iilH4vMKW6LBKCaRx5UEJxKpL4wkgITsYKneIt3NTHo5EOuaYk+y2+Dvt6EQFiuMsdbfUjs3seIHsghX/cbPJa4YUqZAL8C4OtVHaijwGo0ymt9MWvS9yNKMyT0JhN2/BdeOVWrHk7wXXJn/ZjpXilicXKPx4udCF76meE+6N2u/T+RYZ7fP1QMEtNZNmYDOfA6sViuPDfQSHLNbauJBo/n1sRYAsL5mcG22UDchJrlKvmK3EOADCQg+myrm8006LltubNB4wWNzHDJ0Ls2JGzQZCd/xGyVmUiidCBUrD537WdknOYE4FD7P0cHaM9brKJ/M8LkEH0zUlo73bY4XagbnCqve6PvQb5G2Z55qhWphd6f4B6DGed86zJEa/RhS

kiwixbuild/flatpak_builder.py

@@ -0,0 +1,321 @@
+import sys
+from collections import OrderedDict
+from .buildenv import *
+
+from .configs import ConfigInfo
+from .utils import remove_duplicates, run_command, StopBuild, Context
+from .dependencies import Dependency
+from .packages import PACKAGE_NAME_MAPPERS
+from .versions import base_deps_versions
+from ._global import (
+    neutralEnv,
+    option,
+    add_target_step,
+    get_target_step,
+    target_steps,
+)
+from . import _global
+from .dependencies.base import (
+    Source,
+    Builder,
+    ReleaseDownload,
+    GitClone,
+    MesonBuilder,
+    CMakeBuilder,
+    QMakeBuilder,
+    MakeBuilder,
+    SCRIPT_DIR,
+)
+import json
+from shutil import copyfile
+from urllib.parse import urlparse
+from urllib.request import urlopen
+import json
+
+MANIFEST = {
+    "app-id": "org.kiwix.desktop",
+    "runtime": "org.kde.Platform",
+    "runtime-version": base_deps_versions["org.kde"],
+    "base": "io.qt.qtwebengine.BaseApp",
+    "base-version": base_deps_versions["io.qt.qtwebengine"],
+    "sdk": "org.kde.Sdk",
+    "command": "kiwix-desktop",
+    "rename-icon": "kiwix-desktop",
+    "finish-args": [
+        "--device=dri",
+        "--env=QTWEBENGINEPROCESS_PATH=/app/bin/QtWebEngineProcess",
+        "--socket=wayland",
+        "--socket=fallback-x11",
+        "--socket=pulseaudio",
+        "--share=network",
+        "--share=ipc",
+        "--filesystem=host:ro",
+    ],
+    "cleanup": [
+        "/include",
+        "/lib/pkgconfig",
+        "/lib/cmake",
+        "/lib/*.la",
+        "/bin/curl",
+        "/bin/copydatabase",
+        "/bin/kiwix-compile-resources",
+        "/bin/kiwix-manage",
+        "/bin/kiwix-read",
+        "/bin/kiwix-search",
+        "/bin/quest",
+        "/bin/simple*",
+        "/bin/xapian-*",
+        "/share/aclocal",
+        "/share/doc",
+        "/share/man",
+    ],
+    "cleanup-commands": ["/app/cleanup-BaseApp.sh"],
+}
+
+GET_REF_URL_API_TEMPLATE = "https://api.github.com/repos{repo}/git/refs/tags/{ref}"
+
+
+class FlatpakBuilder:
+    def __init__(self):
+        self._targets = {}
+        ConfigInfo.get_config("neutral", self._targets)
+        self.config = ConfigInfo.get_config("flatpak", self._targets)
+        if neutralEnv("distname") not in self.config.compatible_hosts:
+            print(
+                (
+                    "ERROR: The config {} cannot be build on host {}.\n"
+                    "Select another config or change your host system."
+                ).format(self.config.name, neutralEnv("distname"))
+            )
+        self.targetDefs = self.config.add_targets(option("target"), self._targets)
+
+    def finalize_target_steps(self):
+        steps = []
+        for targetDef in self.targetDefs:
+            steps += self.order_steps(targetDef)
+        steps = list(remove_duplicates(steps))
+
+        for cfgName in ConfigInfo.all_running_configs:
+            cfg = ConfigInfo.all_configs[cfgName]
+            for tlcName in cfg.toolchain_names:
+                tlc = Dependency.all_deps[tlcName]
+                src_cfg_step = ("source", tlcName)
+                add_target_step(src_cfg_step, self._targets[src_cfg_step])
+                blt_cfg_step = ("neutral" if tlc.neutral else cfgName, tlcName)
+                add_target_step(blt_cfg_step, self._targets[blt_cfg_step])
+
+        for dep in steps:
+            add_target_step(dep, self._targets[dep])
+        self.instanciate_steps()
+
+    def order_steps(self, targetDef):
+        _targets = dict(self._targets)
+        yield from self.order_dependencies(targetDef, _targets)
+
+    def order_dependencies(self, targetDef, targets):
+        targetConfigName, targetName = targetDef
+        if targetConfigName == "source":
+            # Do not try to order sources, they will be added as dep by the
+            # build step two lines later.
+            return
+        try:
+            target = targets.pop(targetDef)
+        except KeyError:
+            return
+
+        targetConfig = ConfigInfo.get_config(targetConfigName)
+        for dep in target.get_dependencies(targetConfig, True):
+            if isinstance(dep, tuple):
+                depConfig, depName = dep
+            else:
+                depConfig, depName = targetConfigName, dep
+            if (depConfig, depName) in targets:
+                yield from self.order_dependencies((depConfig, depName), targets)
+        yield ("source", targetName)
+        yield targetDef
+
+    def instanciate_steps(self):
+        for stepDef in list(target_steps()):
+            stepConfig, stepName = stepDef
+            stepClass = Dependency.all_deps[stepName]
+            if stepConfig == "source":
+                source = get_target_step(stepDef)(stepClass)
+                add_target_step(stepDef, source)
+            else:
+                source = get_target_step(stepName, "source")
+                env = ConfigInfo.get_config(stepConfig).buildEnv
+                builder = get_target_step(stepDef)(stepClass, source, env)
+                add_target_step(stepDef, builder)
+
+    def configure(self):
+        steps = remove_duplicates(target_steps())
+        modules = OrderedDict()
+        for stepDef in steps:
+            module = modules.setdefault(stepDef[1], {})
+            module["name"] = stepDef[1]
+            if stepDef[0] == "source":
+                source = get_target_step(stepDef)
+                if getattr(source, "flatpak_no_autogen", False):
+                    module["no-autogen"] = True
+                module_sources = module.setdefault("sources", [])
+                if isinstance(source, ReleaseDownload):
+                    for archive in source.archives:
+                        src = {
+                            "type": "archive",
+                            "dest-filename": archive.name,
+                            "sha256": archive.sha256,
+                            "url": archive.url,
+                        }
+                        if hasattr(source, "flatpak_dest"):
+                            src["dest"] = source.flatpak_dest
+                        module_sources.append(src)
+                elif isinstance(source, GitClone):
+                    src = {
+                        "type": "git",
+                        "url": source.git_remote,
+                        "tag": source.git_ref,
+                    }
+                    try:
+                        parsed = urlparse(source.git_remote)
+                        if parsed.hostname == "github.com":
+                            repo = parsed.path[:-4]
+                            api_url = GET_REF_URL_API_TEMPLATE.format(
+                                repo=repo, ref=source.git_ref
+                            )
+                            with urlopen(api_url) as r:
+                                ret = json.loads(r.read())
+                            src["commit"] = ret["object"]["sha"]
+                    except:
+                        pass
+                    module_sources.append(src)
+                for p in getattr(source, "patches", []):
+                    patch = {"type": "patch", "path": "patches/" + p}
+                    module_sources.append(patch)
+
+                if hasattr(source, "flatpak_command"):
+                    command = {"type": "shell", "commands": [source.flatpak_command]}
+                    module_sources.append(command)
+
+            else:
+                builder = get_target_step(stepDef)
+                builder.set_flatpak_buildsystem(module)
+                print(module["name"])
+
+        manifest = MANIFEST.copy()
+        modules = [m for m in modules.values() if m.get("sources")]
+        for m in modules:
+            temp = m["sources"]
+            del m["sources"]
+            m["sources"] = temp
+        manifest["modules"] = modules
+        manifest_name = "{}.json".format(MANIFEST["app-id"])
+        manifest_path = pj(self.config.buildEnv.build_dir, manifest_name)
+        with open(manifest_path, "w") as f:
+            f.write(json.dumps(manifest, indent=4))
+
+    def copy_patches(self):
+        sourceDefs = (tDef for tDef in target_steps() if tDef[0] == "source")
+        for sourceDef in sourceDefs:
+            source = get_target_step(sourceDef)
+            if not hasattr(source, "patches"):
+                continue
+            for p in source.patches:
+                path = pj(SCRIPT_DIR, "patches", p)
+                os.makedirs(
+                    pj(self.config.buildEnv.build_dir, "patches"), exist_ok=True
+                )
+                dest = pj(self.config.buildEnv.build_dir, "patches", p)
+                copyfile(path, dest)
+
+    def build(self):
+        log = pj(self.config.buildEnv.log_dir, "cmd_build_flatpak.log")
+        context = Context("build", log, False)
+        command = [
+            "flatpak-builder",
+            "--user",
+            "--ccache",
+            "--force-clean",
+            "--keep-build-dirs",
+            "--disable-rofiles-fuse",
+            "--repo=repo",
+            "builddir",
+            f"{MANIFEST['app-id']}.json",
+        ]
+        try:
+            run_command(
+                command,
+                self.config.buildEnv.build_dir,
+                context,
+                env=self.config.get_env(),
+            )
+            context._finalise()
+        except subprocess.CalledProcessError:
+            with open(log, "r") as f:
+                print(f.read())
+            raise StopBuild()
+
+    def bundle(self):
+        log = pj(self.config.buildEnv.log_dir, "cmd_bundle_flatpak.log")
+        context = Context("bundle", log, False)
+        app_id = MANIFEST["app-id"]
+        command = ["flatpak", "build-bundle", "repo", f"{app_id}.flatpak", app_id]
+        try:
+            run_command(
+                command,
+                self.config.buildEnv.build_dir,
+                context,
+                env=self.config.get_env(),
+            )
+            context._finalise()
+        except subprocess.CalledProcessError:
+            with open(log, "r") as f:
+                print(f.read())
+            raise StopBuild()
+
+    def _get_packages(self):
+        package_name_mapper = PACKAGE_NAME_MAPPERS.get("flatpak", {})
+
+        to_drop = []
+        for builderDef in self._targets:
+            configName, builderName = builderDef
+            packages = package_name_mapper.get(builderName)
+            if packages:
+                to_drop.append(builderDef)
+        for dep in to_drop:
+            del self._targets[dep]
+
+    def run(self):
+        try:
+            # This is a small hack, we don't need the list of packages to
+            # install in a flatpak sdk, but _get_packages() will drop the
+            # dependencies we already have in the sdk.
+            self._get_packages()
+            self.finalize_target_steps()
+            print("[SETUP TOOLCHAINS]")
+            for config in ConfigInfo.all_running_configs.values():
+                config.finalize_setup()
+            for cfgName in ConfigInfo.all_running_configs:
+                cfg = ConfigInfo.all_configs[cfgName]
+                for tlcName in cfg.toolchain_names:
+                    tlc = Dependency.all_deps[tlcName]
+                    builderDef = (cfgName, tlcName)
+                    builder = get_target_step(builderDef)
+                    print("build {} ({}):".format(builder.name, cfgName))
+                    add_target_step(builderDef, builder)
+                    builder.build()
+            print("[GENERATE FLATPAK MANIFEST]")
+            self.configure()
+            self.copy_patches()
+            print("[BUILD FLATBACK]")
+            self.build()
+            print("[BUNDLE]")
+            self.bundle()
+            # No error, clean intermediate file at end of build if needed.
+            print("[CLEAN]")
+            if option("clean_at_end"):
+                for config in ConfigInfo.all_running_configs.values():
+                    config.clean_intermediate_directories()
+            else:
+                print("SKIP")
+        except StopBuild:
+            sys.exit("Stopping build due to errors")

kiwixbuild/packages.py

@@ -0,0 +1,102 @@
+_fedora_common = [
+    "automake",
+    "libtool",
+    "cmake",
+    "git",
+    "subversion",
+    "ccache",
+    "pkgconf-pkg-config",
+    "gcc-c++",
+    "gettext-devel",
+]
+_debian_common = [
+    "automake",
+    "libtool",
+    "cmake",
+    "git",
+    "subversion",
+    "ccache",
+    "pkg-config",
+    "gcc",
+    "autopoint",
+]
+PACKAGE_NAME_MAPPERS = {
+    "flatpak": {
+        "zlib": True,
+        "lzma": True,
+        "icu4c": True,
+        "qt": True,
+        "qtwebengine": True,
+        "uuid": True,
+        "libxml2": True,
+        "libssl": True,
+        "libcurl": True,
+    },
+    "fedora_native_dyn": {
+        "COMMON": _fedora_common,
+        "uuid": ["libuuid-devel"],
+        "xapian-core": None,  # Not the right version on fedora 25
+        "pugixml": None,  # ['pugixml-devel'] but package doesn't provide pkg-config file
+        "libmicrohttpd": ["libmicrohttpd-devel"],
+        "zlib": ["zlib-devel"],
+        "lzma": ["xz-devel"],
+        "icu4c": None,
+        "zimlib": None,
+        "file": ["file-devel"],
+        "gumbo": ["gumbo-parser-devel"],
+        "aria2": ["aria2"],
+        "qt": ["qt5-qtbase-devel", "qt5-qtsvg"],
+        "qtwebengine": ["qt5-qtwebengine-devel"],
+    },
+    "fedora_native_static": {
+        "COMMON": _fedora_common + ["glibc-static", "libstdc++-static"],
+        "lzma": ["xz-devel", "xz-static"],
+        # Either there is no packages, or no static or too old
+    },
+    "fedora_i586_dyn": {
+        "COMMON": _fedora_common + ["glibc-devel.i686", "libstdc++-devel.i686"],
+    },
+    "fedora_i586_static": {
+        "COMMON": _fedora_common + ["glibc-devel.i686"],
+    },
+    "fedora_armhf_static": {"COMMON": _fedora_common},
+    "fedora_armhf_dyn": {"COMMON": _fedora_common},
+    "fedora_android": {"COMMON": _fedora_common},
+    "debian_native_dyn": {
+        "COMMON": _debian_common + ["libbz2-dev", "libmagic-dev"],
+        "zlib": ["zlib1g-dev"],
+        "uuid": ["uuid-dev"],
+        "libmicrohttpd": ["libmicrohttpd-dev", "ccache"],
+        "qt": ["libqt5gui5", "qtbase5-dev", "qt5-default"],
+        "qtwebengine": ["qtwebengine5-dev"],
+        "aria2": ["aria2"],
+    },
+    "debian_native_static": {
+        "COMMON": _debian_common + ["libbz2-dev", "libmagic-dev"],
+    },
+    "debian_i586_dyn": {
+        "COMMON": _debian_common
+        + ["libc6-dev-i386", "lib32stdc++6", "gcc-multilib", "g++-multilib"],
+    },
+    "debian_i586_static": {
+        "COMMON": _debian_common
+        + ["libc6-dev-i386", "lib32stdc++6", "gcc-multilib", "g++-multilib"],
+    },
+    "debian_armhf_static": {
+        "COMMON": _debian_common,
+    },
+    "debian_armhf_dyn": {
+        "COMMON": _debian_common,
+    },
+    "debian_android": {
+        "COMMON": _debian_common,
+    },
+    "Darwin_native_dyn": {
+        "COMMON": ["autoconf", "automake", "libtool", "cmake", "pkg-config"],
+        "file": ["libmagic"],
+    },
+    "Darwin_iOS": {
+        "COMMON": ["autoconf", "automake", "libtool", "cmake", "pkg-config"],
+        "file": ["libmagic"],
+    },
+}

kiwixbuild/patches/android-ndk-r21e-linux-x86_64-python3.12+.patch

@@ -0,0 +1,33 @@
+diff -ur android-ndk-r21e/build/tools/make_standalone_toolchain.py android-ndk-r21e.patched/build/tools/make_standalone_toolchain.py
+--- android-ndk-r21e/build/tools/make_standalone_toolchain.py	2025-03-04 20:48:14.681288830 +0400
++++ android-ndk-r21e.patched/build/tools/make_standalone_toolchain.py	2025-03-05 12:10:47.252578915 +0400
+@@ -1,4 +1,4 @@
+-#!/usr/bin/env python
++#!/usr/bin/env python3
+ #
+ # Copyright (C) 2016 The Android Open Source Project
+ #
+@@ -21,7 +21,6 @@
+ """
+ import argparse
+ import atexit
+-from distutils.dir_util import copy_tree
+ import inspect
+ import logging
+ import os
+@@ -221,13 +220,13 @@
+ 
+ def create_toolchain(install_path, arch, api, toolchain_path, host_tag):
+     """Create a standalone toolchain."""
+-    copy_tree(toolchain_path, install_path)
++    shutil.copytree(toolchain_path, install_path)
+     triple = get_triple(arch)
+     make_clang_scripts(install_path, arch, api, host_tag == 'windows-x86_64')
+     replace_gcc_wrappers(install_path, triple, host_tag == 'windows-x86_64')
+ 
+     prebuilt_path = os.path.join(NDK_DIR, 'prebuilt', host_tag)
+-    copy_tree(prebuilt_path, install_path)
++    shutil.copytree(prebuilt_path, install_path, dirs_exist_ok=True)
+ 
+     gdbserver_path = os.path.join(
+         NDK_DIR, 'prebuilt', 'android-' + arch, 'gdbserver')

kiwixbuild/patches/docopt_meson_install_pkgconfig.patch

@@ -0,0 +1,31 @@
+diff '--color=auto' -ur docoptcpp-0.6.2/meson.build docoptcpp-0.6.2_patched/meson.build
+--- docoptcpp-0.6.2/meson.build	2024-08-26 14:28:47.553448529 +0200
++++ docoptcpp-0.6.2_patched/meson.build	2024-08-26 14:10:47.232603427 +0200
+@@ -10,11 +10,25 @@
+ 
+ # bug with missing dllexport. fixed in next version.
+ if cpp.get_argument_syntax() == 'msvc'
+-  doclib = static_library('docopt', 'docopt.cpp')
++  doclib = static_library('docopt', 'docopt.cpp', install: true)
+ else
+-  doclib = library('docopt', 'docopt.cpp')
++  doclib = library('docopt', 'docopt.cpp', install: true)
+ endif
+ 
+ executable('docopt_example', 'examples/naval_fate.cpp', link_with: doclib)
+ docopt_dep = declare_dependency(include_directories: include_directories('.'),
+     link_with: doclib)
++
++install_headers(
++  'docopt.h',
++  'docopt_value.h',
++  subdir: 'docopt'
++)
++
++pkg_mod = import('pkgconfig')
++pkg_mod.generate(
++  doclib,
++  version: meson.project_version(),
++  name: 'docopt',
++  filebase: 'docopt'
++)

kiwixbuild/patches/docopt_meson_use_boostregex.patch

@@ -0,0 +1,14 @@
+diff -ur docoptcpp-0.6.2/meson.build docoptcpp-0.6.2_boostregex/meson.build
+--- docoptcpp-0.6.2/meson.build	2024-08-28 17:22:46.256716100 +0200
++++ docoptcpp-0.6.2_boostregex/meson.build	2024-08-28 17:02:47.932681000 +0200
+@@ -8,6 +8,10 @@
+   add_project_arguments('-DDOCOPT_DLL', '-DDOCOPT_EXPORTS', language: 'cpp')
+ endif
+ 
++if cpp.get_id() =='msvc'
++  add_project_arguments('-DDOCTOPT_USE_BOOST_REGEX', '-DBOOST_REGEX_STANDALONE', language: 'cpp')
++endif
++
+ # bug with missing dllexport. fixed in next version.
+ if cpp.get_argument_syntax() == 'msvc'
+   doclib = static_library('docopt', 'docopt.cpp', install: true)

kiwixbuild/patches/icu4c_build_config.patch

@@ -0,0 +1,36 @@
+diff -ur icu4c/source/common/unicode/platform.h icu4c.patched/source/common/unicode/platform.h
+--- icu4c/source/common/unicode/platform.h	2018-04-17 17:20:31.048946098 +0200
++++ icu4c.patched/source/common/unicode/platform.h	2018-06-11 11:22:40.692327158 +0200
+@@ -644,7 +644,7 @@
+ #elif U_PLATFORM == U_PF_ANDROID || U_PLATFORM_IS_DARWIN_BASED
+ #   define U_CHARSET_IS_UTF8 1
+ #else
+-#   define U_CHARSET_IS_UTF8 0
++#   define U_CHARSET_IS_UTF8 1
+ #endif
+ 
+ /** @} */
+diff -ur icu4c/source/common/unicode/uconfig.h icu4c.patched/source/common/unicode/uconfig.h
+--- icu4c/source/common/unicode/uconfig.h	2018-04-17 17:20:31.072946266 +0200
++++ icu4c.patched/source/common/unicode/uconfig.h	2018-06-11 11:26:24.512936322 +0200
+@@ -107,7 +107,7 @@
+       defined(U_TOOLUTIL_IMPLEMENTATION)
+ #   define U_NO_DEFAULT_INCLUDE_UTF_HEADERS 1
+ #else
+-#   define U_NO_DEFAULT_INCLUDE_UTF_HEADERS 0
++#   define U_NO_DEFAULT_INCLUDE_UTF_HEADERS 1
+ #endif
+ 
+ /**
+diff -ur icu4c/source/common/unicode/uversion.h icu4c.patched/source/common/unicode/uversion.h
+--- icu4c/source/common/unicode/uversion.h	2018-04-17 17:20:31.086946363 +0200
++++ icu4c.patched/source/common/unicode/uversion.h	2018-06-11 11:20:27.093375709 +0200
+@@ -122,7 +122,7 @@
+ #   define U_NAMESPACE_QUALIFIER U_ICU_NAMESPACE::
+ 
+ #   ifndef U_USING_ICU_NAMESPACE
+-#       define U_USING_ICU_NAMESPACE 1
++#       define U_USING_ICU_NAMESPACE 0
+ #   endif
+ #   if U_USING_ICU_NAMESPACE
+         U_NAMESPACE_USE

kiwixbuild/patches/icu4c_custom_data.patch

@@ -34,13 +34,12 @@ diff -ur icu4c/source/data/Makefile.in icu4c.patched/source/data/Makefile.in
  ## RES files
 --include $(LOCSRCDIR)/resfiles.mk
 --include $(CURRSRCDIR)/resfiles.mk
---include $(LANGSRCDIR)/resfiles.mk
++#-include $(LOCSRCDIR)/resfiles.mk
++#-include $(CURRSRCDIR)/resfiles.mk
+ -include $(LANGSRCDIR)/resfiles.mk
 --include $(REGIONSRCDIR)/resfiles.mk
 --include $(ZONESRCDIR)/resfiles.mk
 --include $(UNITSRCDIR)/resfiles.mk
-+#-include $(LOCSRCDIR)/resfiles.mk
-+#-include $(CURRSRCDIR)/resfiles.mk
-+#-include $(LANGSRCDIR)/resfiles.mk
 +#-include $(REGIONSRCDIR)/resfiles.mk
 +#-include $(ZONESRCDIR)/resfiles.mk
 +#-include $(UNITSRCDIR)/resfiles.mk

kiwixbuild/patches/icu4c_rpath.patch

@@ -0,0 +1,13 @@
+diff -ur icu4c/source/config/mh-linux icu4c.rpath/source/config/mh-linux
+--- icu4c/source/config/mh-linux	2018-04-17 11:31:50.674012676 +0200
++++ icu4c.rpath/source/config/mh-linux	2018-04-17 11:28:57.776134587 +0200
+@@ -19,7 +19,7 @@
+ LIBCPPFLAGS =
+ 
+ ## Compiler switch to embed a runtime search path
+-LD_RPATH= -Wl,-zorigin,-rpath,'$$'ORIGIN 
++LD_RPATH= '-Wl,-zorigin,-rpath,\$ORIGIN' 
+ LD_RPATH_PRE = -Wl,-rpath,
+ 
+ ## These are the library specific LDFLAGS
+Les fichiers binaires icu4c/.svn/wc.db et icu4c.rpath/.svn/wc.db sont différents

kiwixbuild/patches/icu4c_wasm.patch

@@ -0,0 +1,153 @@
+diff '--color=auto' -ur icu4c-71.1/source/config/mh-unknown icu4c-71.1.wasm/source/config/mh-unknown
+--- icu4c-71.1/source/config/mh-unknown	2022-04-08 00:41:55.000000000 +0200
++++ icu4c-71.1.wasm/source/config/mh-unknown	2023-10-25 14:00:18.035718690 +0200
+@@ -1,29 +1,87 @@
+ ## -*-makefile-*-
+ ## Copyright (C) 2016 and later: Unicode, Inc. and others.
+ ## License & terms of use: http://www.unicode.org/copyright.html
+-## Copyright (c) 2003, International Business Machines Corporation and
++## Linux-specific setup
++## Copyright (c) 1999-2013, International Business Machines Corporation and
+ ## others. All Rights Reserved.
+-##
+ 
+-# Note, this is not a real mh- file. You got here because configure
+-# (specifically, aclocal.m4) could not determine a suitable mh- file.
+-#
+-# Perhaps your platform wasn't detected- try changing aclocal.m4 and
+-# re-running autoconf. 
+-#
+-# If your platform is truly new/different:
+-# As a start, try copying mh-linux (which is fairly generic) over this
+-# file, and re-run config.status.
+-
+-%.$(STATIC_O) %.o %.$(STATIC_O) %.o ../data/%.o %.d %.d %.$(SO).$(SO_TARGET_VERSION_MAJOR) %.$(SO):
+-	@echo 
+-	@echo 
+-	@echo "*** ERROR - configure could not detect your platform"
+-	@echo "*** see the readme.html"
+-	@echo "*** or, try copying icu/source/config/mh-linux to mh-unknown"
+-	@echo "***   and editing it."
+-	@echo 
+-	@echo 
+-	exit 1
++## Commands to generate dependency files
++GEN_DEPS.c=  $(CC) -E -MM $(DEFS) $(CPPFLAGS)
++GEN_DEPS.cc= $(CXX) -E -MM $(DEFS) $(CPPFLAGS) $(CXXFLAGS)
+ 
++## Flags for position independent code
++SHAREDLIBCFLAGS = -fPIC
++SHAREDLIBCXXFLAGS = -fPIC
++SHAREDLIBCPPFLAGS = -DPIC
++
++## Additional flags when building libraries and with threads
++THREADSCPPFLAGS = -D_REENTRANT
++LIBCPPFLAGS =
++
++## Compiler switch to embed a runtime search path
++LD_RPATH= -Wl,-zorigin,-rpath,'$$'ORIGIN 
++LD_RPATH_PRE = -Wl,-rpath,
++
++## These are the library specific LDFLAGS
++LDFLAGSICUDT=-nodefaultlibs -nostdlib
++
++## Compiler switch to embed a library name
++# The initial tab in the next line is to prevent icu-config from reading it.
++	LD_SONAME = -Wl,-soname -Wl,$(notdir $(MIDDLE_SO_TARGET))
++#SH# # We can't depend on MIDDLE_SO_TARGET being set.
++#SH# LD_SONAME=
++
++## Shared library options
++LD_SOOPTIONS= -Wl,-Bsymbolic
++
++## Shared object suffix
++SO = so
++## Non-shared intermediate object suffix
++STATIC_O = ao
++
++## Compilation rules
++%.$(STATIC_O): $(srcdir)/%.c
++	$(call SILENT_COMPILE,$(strip $(COMPILE.c) $(STATICCPPFLAGS) $(STATICCFLAGS)) -o $@ $<)
++%.o: $(srcdir)/%.c
++	$(call SILENT_COMPILE,$(strip $(COMPILE.c) $(DYNAMICCPPFLAGS) $(DYNAMICCFLAGS)) -o $@ $<)
++
++%.$(STATIC_O): $(srcdir)/%.cpp
++	$(call SILENT_COMPILE,$(strip $(COMPILE.cc) $(STATICCPPFLAGS) $(STATICCXXFLAGS)) -o $@ $<)
++%.o: $(srcdir)/%.cpp
++	$(call SILENT_COMPILE,$(strip $(COMPILE.cc) $(DYNAMICCPPFLAGS) $(DYNAMICCXXFLAGS)) -o $@ $<)
++
++
++## Dependency rules
++%.d: $(srcdir)/%.c
++	$(call ICU_MSG,(deps)) $<
++	@$(SHELL) -ec '$(GEN_DEPS.c) $< \
++		| sed '\''s%\($*\)\.o[ :]*%\1.o $@ : %g'\'' > $@; \
++		[ -s $@ ] || rm -f $@'
++
++%.d: $(srcdir)/%.cpp
++	$(call ICU_MSG,(deps)) $<
++	@$(SHELL) -ec '$(GEN_DEPS.cc) $< \
++		| sed '\''s%\($*\)\.o[ :]*%\1.o $@ : %g'\'' > $@; \
++		[ -s $@ ] || rm -f $@'
++
++## Versioned libraries rules
++
++%.$(SO).$(SO_TARGET_VERSION_MAJOR): %.$(SO).$(SO_TARGET_VERSION)
++	$(RM) $@ && ln -s ${<F} $@
++%.$(SO): %.$(SO).$(SO_TARGET_VERSION_MAJOR)
++	$(RM) $@ && ln -s ${*F}.$(SO).$(SO_TARGET_VERSION) $@
++
++##  Bind internal references
++
++# LDflags that pkgdata will use
++BIR_LDFLAGS= -Wl,-Bsymbolic
++
++# Dependencies [i.e. map files] for the final library
++BIR_DEPS=
++
++## Remove shared library 's'
++STATIC_PREFIX_WHEN_USED = 
++STATIC_PREFIX = 
++
++## End Linux-specific setup
+ 
+diff '--color=auto' -ur icu4c-71.1/source/config.sub icu4c-71.1.wasm/source/config.sub
+--- icu4c-71.1/source/config.sub	2022-04-08 00:41:55.000000000 +0200
++++ icu4c-71.1.wasm/source/config.sub	2023-10-25 13:58:23.711645708 +0200
+@@ -312,7 +312,7 @@
+ 	| ubicom32 \
+ 	| v850 | v850e | v850e1 | v850e2 | v850es | v850e2v3 \
+ 	| visium \
+-	| wasm32 \
++	| wasm32 | wasm64 \
+ 	| x86 | xc16x | xstormy16 | xtensa \
+ 	| z8k | z80)
+ 		basic_machine=$basic_machine-unknown
+@@ -443,7 +443,7 @@
+ 	| v850-* | v850e-* | v850e1-* | v850es-* | v850e2-* | v850e2v3-* \
+ 	| vax-* \
+ 	| visium-* \
+-	| wasm32-* \
++	| wasm32-* | wasm64-* \
+ 	| we32k-* \
+ 	| x86-* | x86_64-* | xc16x-* | xps100-* \
+ 	| xstormy16-* | xtensa*-* \
+@@ -1247,6 +1247,9 @@
+ 	wasm32)
+ 		basic_machine=wasm32-unknown
+ 		;;
++    wasm64)
++        basic_machine=wasm64-unknown
++        ;;
+ 	w65*)
+ 		basic_machine=w65-wdc
+ 		os=-none
+@@ -1416,7 +1419,7 @@
+ 	      | -morphos* | -superux* | -rtmk* | -rtmk-nova* | -windiss* \
+ 	      | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly* \
+ 	      | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es* \
+-	      | -onefs* | -tirtos* | -phoenix* | -fuchsia* | -redox*)
++	      | -onefs* | -tirtos* | -phoenix* | -fuchsia* | -redox* | -emscripten*)
+ 	# Remember, each alternative MUST END IN *, to match a version number.
+ 		;;
+ 	-qnx*)

kiwixbuild/patches/libmicrohttpd_meson_pkgconfig.patch

@@ -0,0 +1,29 @@
+diff '--color=auto' -ur libmicrohttpd-0.9.76_orig/meson.build libmicrohttpd-0.9.76/meson.build
+--- libmicrohttpd-0.9.76_orig/meson.build	2024-08-18 14:55:29.372805433 +0200
++++ libmicrohttpd-0.9.76/meson.build	2024-08-18 17:19:44.087444728 +0200
+@@ -6,6 +6,8 @@
+   default_options: ['warning_level=1'],
+ )
+ 
++pkg = import('pkgconfig')
++
+ add_project_arguments('-D_GNU_SOURCE', language: 'c')
+ 
+ incdirs = include_directories('src/include')
+@@ -267,6 +269,16 @@
+   install: true,
+ )
+ 
++install_headers(
++  'src/include/microhttpd.h',
++)
++
++pkg.generate(
++  libmicrohttpd,
++  description: 'Libmicrohttpd',
++  name: 'libmicrohttpd',
++)
++
+ depinc = include_directories('.', 'src/include')
+ libmicrohttpd_dep = declare_dependency(
+   include_directories: depinc,

kiwixbuild/patches/libmicrohttpd_meson_timeval_tvsec_size.patch

@@ -0,0 +1,31 @@
+diff '--color=auto' -ur libmicrohttpd-0.9.76/meson.build libmicrohttpd-0.9.76_patched/meson.build
+--- libmicrohttpd-0.9.76/meson.build	2024-08-22 15:17:59.217715872 +0200
++++ libmicrohttpd-0.9.76_patched/meson.build	2024-08-22 15:20:23.755358647 +0200
+@@ -126,7 +126,26 @@
+ foreach s : sizes
+   cdata.set('SIZEOF_@0@'.format(s.underscorify().to_upper()), cc.sizeof(s))
+ endforeach
+-cdata.set('SIZEOF_STRUCT_TIMEVAL_TV_SEC', cc.sizeof('time_t'))
++
++cdata.set(
++  'SIZEOF_STRUCT_TIMEVAL_TV_SEC',
++  cc.sizeof(
++    'test_var.tv_sec',
++    prefix: '''#ifdef HAVE_SYS_TIME_H
++#include <sys/time.h>
++#endif /* HAVE_SYS_TIME_H */
++#ifdef HAVE_TIME_H
++#include <time.h>
++#endif /* HAVE_TIME_H */
++#if HAVE_SYS_TYPES_H
++#include <sys/types.h>
++#endif /* HAVE_SYS_TYPES_H */
++#ifdef _WIN32
++#include <winsock.h>
++#endif /* _WIN32 */
++struct timeval test_var;'''
++  )
++)
+ cdata.set('SIZEOF_UINT64_T', 8)
+ 
+ cdata.set('HAVE_PIPE2_FUNC', cc.has_function('pipe2'))

kiwixbuild/patches/libmicrohttpd_meson_winet6.patch

@@ -0,0 +1,15 @@
+--- libmicrohttpd-0.9.76_orig/meson.build	2024-10-08 15:53:53.370828250 +0400
++++ libmicrohttpd-0.9.76/meson.build	2024-10-08 16:23:24.985668690 +0400
+@@ -77,7 +77,11 @@
+ endforeach
+
+ cdata.set('HAVE_ASSERT', cc.has_header_symbol('assert.h', 'assert'))
+-cdata.set10('HAVE_INET6', cc.has_header_symbol('netinet/in.h', 'struct in6_addr'))
++if host_machine.system() == 'windows'
++  cdata.set10('HAVE_INET6', 1)
++else
++  cdata.set10('HAVE_INET6', cc.has_header_symbol('netinet/in.h', 'struct in6_addr'))
++endif
+
+ functions = [
+   'accept4',

kiwixbuild/patches/lzma_meson_install.patch

@@ -0,0 +1,54 @@
+diff '--color=auto' -ur lzma-5.2.6_orig/src/liblzma/meson.build lzma-5.2.6/src/liblzma/meson.build
+--- lzma-5.2.6_orig/src/liblzma/meson.build	2023-11-23 14:31:26.110195070 +0100
++++ lzma-5.2.6/src/liblzma/meson.build	2023-12-06 17:04:49.325148650 +0100
+@@ -1,3 +1,5 @@
++pkg = import('pkgconfig')
++
+ lzma_sources = [
+   '../common/tuklib_physmem.c',
+   'common/common.c',
+@@ -121,12 +123,44 @@
+ lzmainc = include_directories('api', 'common',
+   'check', 'lz', 'rangecoder', 'lzma', 'delta', 'simple', '../common')
+ 
++
++install_headers(
++  'api/lzma.h',
++)
++
++install_headers(
++  'api/lzma/version.h',
++  'api/lzma/base.h',
++  'api/lzma/vli.h',
++  'api/lzma/check.h',
++  'api/lzma/filter.h',
++  'api/lzma/bcj.h',
++  'api/lzma/delta.h',
++  'api/lzma/lzma12.h',
++  'api/lzma/container.h',
++  'api/lzma/stream_flags.h',
++  'api/lzma/block.h',
++  'api/lzma/index.h',
++  'api/lzma/index_hash.h',
++  'api/lzma/hardware.h',
++  subdir: 'lzma'
++)
++
+ liblzma = library('lzma', lzma_sources,
+   main_dec_sources, main_enc_sources, check_sources,
+   simplefilter_sources, lzma1_sources,
+   lz_sources, delta_sources,
+   include_directories : [confinc, lzmainc],
+   c_args : ['-DHAVE_CONFIG_H', '-DTUKLIB_SYMBOL_PREFIX=lzma_'],
++  install: true
++)
++
++pkg.generate(liblzma,
++  name: 'liblzma',
++  filebase: 'liblzma',
++  description: 'The liblzma compression library',
++  version: meson.project_version(),
++  url: 'http://tukaani.org/xz/'
+ )
+ 
+ lzma_dep = declare_dependency(link_with : liblzma,

kiwixbuild/templates/cmake_android_cross_file.txt

@@ -0,0 +1,8 @@
+SET(CMAKE_SYSTEM_NAME {host_machine[system]})
+
+SET(CMAKE_ANDROID_STANDALONE_TOOLCHAIN {install_path})
+
+SET(CMAKE_C_COMPILER "{binaries[CC]}")
+SET(CMAKE_CXX_COMPILER "{binaries[CXX]}")
+
+SET(CMAKE_FIND_ROOT_PATH {root_path})