.github/configs/aqt.ini

@@ -1,46 +0,0 @@
-[DEFAULTS]
-
-[aqt]
-concurrency: 4
-baseurl: https://qt.mirror.constant.com/
-7zcmd: 7z
-print_stacktrace_on_error: False
-always_keep_archives: False
-archive_download_location: .
-min_module_size: 41
-
-[requests]
-connection_timeout: 3.5
-response_timeout: 30
-max_retries_on_connection_error: 5
-retry_backoff: 0.1
-max_retries_on_checksum_error: 5
-max_retries_to_retrieve_hash: 5
-hash_algorithm: sha256
-INSECURE_NOT_FOR_PRODUCTION_ignore_hash: False
-
-[mirrors]
-trusted_mirrors:
-    https://download.qt.io
-    https://qt.mirror.constant.com/
-    https://ftp.fau.de/qtproject/
-blacklist:
-    http://mirrors.ocf.berkeley.edu
-    http://mirrors.tuna.tsinghua.edu.cn
-    http://mirrors.geekpie.club
-fallbacks:
-    https://qtproject.mirror.liquidtelecom.com/
-    https://mirrors.aliyun.com/qt/
-    https://mirrors.ustc.edu.cn/qtproject/
-    https://ftp.jaist.ac.jp/pub/qtproject/
-    https://ftp.yz.yamagata-u.ac.jp/pub/qtproject/
-    https://qt-mirror.dannhauer.de/
-    https://ftp.fau.de/qtproject/
-    https://mirror.netcologne.de/qtproject/
-    https://mirrors.dotsrc.org/qtproject/
-    https://www.nic.funet.fi/pub/mirrors/download.qt-project.org/
-    https://master.qt.io/
-    https://mirrors.ukfast.co.uk/sites/qt.io/
-    https://ftp2.nluug.nl/languages/qt/
-    https://ftp1.nluug.nl/languages/qt/
-    https://qt.mirror.constant.com/

.github/scripts/build_definition.py

@@ -1,161 +0,0 @@
-from typing import NamedTuple
-import csv, io, re
-
-# Definition of what to build.
-# Array is read line by line.
-# Empty cells under (OS_NAME, COMPILE_CONFIG) mean "always match" (catch all, or `.*` regex)
-# Once a cell doesn't match, skip to the next line.
-# Once a line matches, other lines are not read, so put more specific combination first.
-# Lines composed of `-` , or `=`, or starting by `#` are ignored.
-# 'B' letter means that the project is build in the CI
-# 'd' letter means that the project's dependencies are build and published to be used by the project's CI.
-# 'P' letter means that (build) project must be publish when we do a release.
-#     (This is used to avoid two publication of the same archive)
-# 'S' letter means that source code must be publish (almost by definition, S should be put only with a P)
-# 'D' letter means we trigger the docker forkflow to build the docker image.
-# If a cell contains several letters, all are done.
-BUILD_DEF = """
-    | OS_NAME   | COMPILE_CONFIG     | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | platform_name           | dependency_name        |
-    ==============================================================================================================================================
-# manylinux is a special case as we need to compile libzim on old arch for python
-    | manylinux | native_mixed       | BP     |          |           |             |               | linux-x86_64-manylinux  |                        |
-    | manylinux | aarch64_mixed      | BP     |          |           |             |               | linux-aarch64-manylinux |                        |
-    ----------------------------------------------------------------------------------------------------------------------------------------------
-# On Windows, we build only libzim for now. And only native_mixed as xapian doesn't compile as dll
-    | windows   | native_static      | Bd     | d        | BPd       | BPd         |               | win-x86_64              | win-x86_64-static      |
-    | windows   | native_dyn         | Bd     |          |           |             |               | win-x86_64              | win-x86_64-dyn         |
-    | windows   | native_mixed       | BPd    | d        |           |             | BPd           | win-x86_64              | win-x86_64-mixed       |
-    ----------------------------------------------------------------------------------------------------------------------------------------------
-# Osx builds, build binaries on native_dyn and native_static. On anyother things, build only the libraries
-    | macos     | native_dyn         | d      | d        | dB        | B           |               |                         | macos-x86_64-dyn       |
-    | macos     | native_static      |        |          | BP        | BP          |               | macos-x86_64            |                        |
-    | macos     | native_mixed       | BP     | BP       |           |             |               | macos-x86_64            |                        |
-    | macos     | iOS_arm64          | dB     | dB       |           |             |               |                         | ios-arm64-dyn          |
-    | macos     | iOSSimulator_x86_64| dB     | dB       |           |             |               |                         | ios-x86_64-dyn         |
-    | macos     | iOSSimulator_arm64 | B      | B        |           |             |               |                         |                        |
-    | macos     | macOS_arm64_static |        |          | BP        | BP          |               | macos-arm64             |                        |
-    | macos     | macOS_arm64_mixed  | dBP    | dBP      | d         |             |               | macos-arm64             | macos-aarch64-dyn      |
-    | macos     | macOS_x86_64       | B      | B        |           |             |               |                         |                        |
-    | macos     | apple_all_static   |        | BP       |           |             |               | xcframework             |                        |
-    ----------------------------------------------------------------------------------------------------------------------------------------------
-    | focal     | flatpak            |        |          |           |             | BP            |                         |                        |
-    | focal     | native_static      | d      | d        | dBPSD     | dBPSD       |               | linux-x86_64            | linux-x86_64-static    |
-    | focal     | native_mixed       | BPS    | BPS      |           |             |               | linux-x86_64            |                        |
-    | focal     | native_dyn         | d      | d        | dB        | dB          |               |                         | linux-x86_64-dyn       |
-    | jammy     | native_dyn         |        |          |           |             | dBPS          |                         | linux-x86_64-dyn       |
-    # libzim CI is building alpine_dyn but not us
-    | focal     | android_arm        | dBP    | dBP      |           |             |               | android-arm             | android-arm            |
-    | focal     | android_arm64      | dBP    | dBP      |           |             |               | android-arm64           | android-arm64          |
-    | focal     | android_x86        | BP     | BP       |           |             |               | android-x86             |                        |
-    | focal     | android_x86_64     | BP     | BP       |           |             |               | android-x86_64          |                        |
-    | focal     | armv6_static       |        |          | BP        | BP          |               | linux-armv6             |                        |
-    | focal     | armv6_mixed        | BP     |          |           |             |               | linux-armv6             |                        |
-    | focal     | armv6_dyn          |        |          | B         | B           |               |                         |                        |
-    | focal     | armv8_static       |        |          | BP        | BP          |               | linux-armv8             |                        |
-    | focal     | armv8_mixed        | BP     |          |           |             |               | linux-armv8             |                        |
-    | focal     | armv8_dyn          |        |          | B         | B           |               |                         |                        |
-    | focal     | aarch64_static     |        |          | BP        | BP          |               | linux-aarch64           |                        |
-    | focal     | aarch64_mixed      | BP     |          |           |             |               | linux-aarch64           |                        |
-    | focal     | aarch64_dyn        | d      |          | B         | B           |               |                         | linux-aarch64-dyn      |
-    | focal     | aarch64_musl_static|        |          | BP        | BP          |               | linux-aarch64-musl      |                        |
-    | focal     | aarch64_musl_mixed | BP     |          |           |             |               | linux-aarch64-musl      |                        |
-    | focal     | aarch64_musl_dyn   | d      |          | B         | B           |               |                         | linux-aarch64-musl-dyn |
-    | focal     | x86-64_musl_static |        |          | BP        | BP          |               | linux-x86_64-musl       |                        |
-    | focal     | x86-64_musl_mixed  | BP     |          |           |             |               | linux-x86_64-musl       |                        |
-    | focal     | i586_static        |        |          | BP        | BP          |               | linux-i586              |                        |
-    | focal     | i586_dyn           |        |          | B         | B           |               |                         |                        |
-    | focal     | wasm               | dBP    |          |           |             |               | wasm-emscripten         | wasm                   |
-"""
-
-
-class TableDialect(csv.Dialect):
-    delimiter = "|"
-    quoting = csv.QUOTE_NONE
-    lineterminator = "\n"
-
-
-def strip_array(array_str):
-    """Return a iterable of lines, skiping "decorative lines" and with all values in the line's cells stripped"""
-    for line in array_str.splitlines():
-        line = line.strip()
-        line_set = set(line)
-        if (
-            not line
-            or line.startswith("#")
-            or (len(line_set) == 1 and line_set.pop() in "-=")
-        ):
-            continue
-        yield "|".join(c.strip() for c in line.split("|"))
-
-
-def selector_match(selector, value):
-    if not selector:
-        return True
-    return re.fullmatch(selector, value) is not None
-
-
-class Context(NamedTuple):
-    OS_NAME: str
-    COMPILE_CONFIG: str
-
-    def match(self, row):
-        for key in ["OS_NAME", "COMPILE_CONFIG"]:
-            context_value = getattr(self, key)
-            selector = row[key]
-            if not selector_match(selector, context_value):
-                return False
-        return True
-
-
-BUILD = "B"
-PUBLISH = "P"
-SOURCE_PUBLISH = "S"
-DEPS = "d"
-DOCKER = "D"
-
-
-def select_build_targets(criteria):
-    from common import COMPILE_CONFIG, OS_NAME
-
-    context = Context(COMPILE_CONFIG=COMPILE_CONFIG, OS_NAME=OS_NAME)
-
-    reader = csv.DictReader(strip_array(BUILD_DEF), dialect=TableDialect())
-    for row in reader:
-        if context.match(row):
-            build_order = [
-                k
-                for k in (
-                    "libzim",
-                    "libkiwix",
-                    "zim-tools",
-                    "kiwix-tools",
-                    "kiwix-desktop",
-                )
-                if criteria in row[k]
-            ]
-            print(build_order)
-            return build_order
-
-    raise ValueError("No definition match with current context.")
-
-
-def get_column_value(column_name):
-    from common import COMPILE_CONFIG, OS_NAME
-
-    context = Context(COMPILE_CONFIG=COMPILE_CONFIG, OS_NAME=OS_NAME)
-
-    reader = csv.DictReader(strip_array(BUILD_DEF), dialect=TableDialect())
-    for row in reader:
-        if context.match(row):
-            name = row[column_name]
-            return name or None
-
-    raise ValueError("No definition match with current context.")
-
-
-def get_platform_name():
-    return get_column_value("platform_name")
-
-
-def get_dependency_archive_name():
-    return get_column_value("dependency_name")

.github/scripts/build_projects.py

@@ -1,24 +0,0 @@
-#!/usr/bin/env python3
-
-from build_definition import select_build_targets, BUILD
-from common import (
-    run_kiwix_build,
-    make_archive,
-    create_desktop_image,
-    fix_macos_rpath,
-    upload_archive,
-    OS_NAME,
-    COMPILE_CONFIG,
-    DEV_BRANCH,
-)
-
-for target in select_build_targets(BUILD):
-    run_kiwix_build(target, config=COMPILE_CONFIG)
-    if target == "kiwix-desktop":
-        archive = create_desktop_image(make_release=False)
-    else:
-        if COMPILE_CONFIG == "native_mixed" and OS_NAME == "macos":
-            fix_macos_rpath(target)
-        archive = make_archive(target, make_release=False)
-    if archive and DEV_BRANCH:
-        upload_archive(archive, target, make_release=False, dev_branch=DEV_BRANCH)

.github/scripts/build_release_nightly.py

@@ -1,75 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-
-from common import (
-    run_kiwix_build,
-    main_project_versions,
-    release_versions,
-    make_archive,
-    create_desktop_image,
-    update_flathub_git,
-    upload_archive,
-    fix_macos_rpath,
-    BASE_DIR,
-    OS_NAME,
-    COMPILE_CONFIG,
-    MAKE_RELEASE,
-    notarize_macos_build,
-)
-
-from build_definition import select_build_targets, BUILD, PUBLISH, SOURCE_PUBLISH
-
-
-def release_filter(project):
-    return release_versions.get(project) is not None
-
-
-# Filter what to build if we are doing a release.
-TARGETS = select_build_targets(PUBLISH)
-
-if MAKE_RELEASE:
-    TARGETS = tuple(filter(release_filter, TARGETS))
-
-for target in TARGETS:
-    run_kiwix_build(target, config=COMPILE_CONFIG, make_release=MAKE_RELEASE)
-    if target == "kiwix-desktop":
-        archive = create_desktop_image(make_release=MAKE_RELEASE)
-    else:
-        if OS_NAME == "macos" and COMPILE_CONFIG.endswith("_mixed"):
-            fix_macos_rpath(target)
-            notarize_macos_build(target)
-        archive = make_archive(target, make_release=MAKE_RELEASE)
-    if archive:
-        upload_archive(archive, target, make_release=MAKE_RELEASE)
-
-# We have few more things to do for release:
-if MAKE_RELEASE:
-    # Publish source archives
-    source_published_targets = select_build_targets(SOURCE_PUBLISH)
-    for target in TARGETS:
-        # Looping on TARGETS instead of source_published_targets ensures we are
-        # publishing sources only for target we are building.
-        # (source_published_targets must be a subset of TARGETS)
-        if release_versions.get(target) != 0:
-            continue
-        if target not in source_published_targets:
-            continue
-        run_kiwix_build(
-            target, config=COMPILE_CONFIG, make_release=MAKE_RELEASE, make_dist=True
-        )
-        full_target_name = "{}-{}".format(target, main_project_versions[target])
-        if target == "kiwix-desktop":
-            archive = BASE_DIR / full_target_name / "{}.tar.gz".format(full_target_name)
-        else:
-            archive = (
-                BASE_DIR
-                / full_target_name
-                / "meson-dist"
-                / "{}.tar.xz".format(full_target_name)
-            )
-        upload_archive(archive, target, make_release=MAKE_RELEASE)
-
-    # Publish flathub
-    if COMPILE_CONFIG == "flatpak" and "kiwix-desktop" in TARGETS:
-        update_flathub_git()

.github/scripts/common.py

@@ -1,771 +0,0 @@
-import os
-from os import environ as _environ
-from pathlib import Path, PurePosixPath
-from datetime import date
-import tarfile
-import zipfile
-import subprocess
-import re
-import shutil
-import platform
-
-import requests
-
-from build_definition import get_platform_name, get_dependency_archive_name
-
-from kiwixbuild.dependencies.apple_xcframework import AppleXCFramework
-from kiwixbuild.versions import (
-    main_project_versions,
-    release_versions,
-    base_deps_versions,
-)
-
-
-def get_build_dir(config) -> Path:
-    command = ["kiwix-build"]
-    command.extend(["--config", config])
-    command.append("--get-build-dir")
-    command.append("--use-target-arch-name")
-    return Path(
-        subprocess.run(command, cwd=str(HOME), check=True, stdout=subprocess.PIPE)
-        .stdout.strip()
-        .decode("utf8")
-    )
-
-
-COMPILE_CONFIG = _environ["COMPILE_CONFIG"]
-OS_NAME = _environ["OS_NAME"]
-HOME = Path(os.path.expanduser("~"))
-
-BASE_DIR = get_build_dir(COMPILE_CONFIG)
-SOURCE_DIR = HOME / "SOURCE"
-ARCHIVE_DIR = HOME / "ARCHIVE"
-TOOLCHAIN_DIR = BASE_DIR / "TOOLCHAINS"
-INSTALL_DIR = BASE_DIR / "INSTALL"
-default_tmp_dir = os.getenv("TEMP") if platform.system() == "Windows" else "/tmp"
-TMP_DIR = Path(os.getenv("TMP_DIR", default_tmp_dir))
-if platform.system() == "Windows":
-    KBUILD_SOURCE_DIR = Path(_environ["GITHUB_WORKSPACE"])
-    BIN_EXT = ".exe"
-else:
-    KBUILD_SOURCE_DIR = HOME / "kiwix-build"
-    BIN_EXT = ""
-
-
-_ref = _environ.get("GITHUB_REF", "").split("/")[-1]
-MAKE_RELEASE = re.fullmatch(r"r_[0-9]+", _ref) is not None
-MAKE_RELEASE = MAKE_RELEASE and (_environ.get("GITHUB_EVENT_NAME") != "schedule")
-
-if not MAKE_RELEASE and _ref != "main":
-    DEV_BRANCH = _ref
-else:
-    DEV_BRANCH = None
-
-FLATPAK_HTTP_GIT_REMOTE = "https://github.com/flathub/org.kiwix.desktop.git"
-FLATPAK_GIT_REMOTE = "git@github.com:flathub/org.kiwix.desktop.git"
-
-
-def major_version(version: str) -> str:
-    return version.split(".")[0]
-
-
-# Depending of base distribution, libraries are in "lib64" (redhat base) or "lib/<arch>" (debian base).
-# On top of that, when cross-compiling, libraries are always put in `lib/<arch>`.
-# As we use this as glob regex to select which files to add to archive, this is not a problem to have both.
-def lib_prefix(file):
-    yield "lib64/" + file
-    yield "lib/*/" + file
-
-
-# We have build everything. Now create archives for public deployement.
-EXPORT_FILES = {
-    "kiwix-tools": (
-        INSTALL_DIR / "bin",
-        [f + BIN_EXT for f in ("kiwix-manage", "kiwix-search", "kiwix-serve")]
-        + ["icu*.dll"],
-    ),
-    "zim-tools": (
-        INSTALL_DIR / "bin",
-        [
-            f + BIN_EXT
-            for f in (
-                "zimbench",
-                "zimcheck",
-                "zimdump",
-                "zimsearch",
-                "zimdiff",
-                "zimpatch",
-                "zimsplit",
-                "zimwriterfs",
-                "zimrecreate",
-            )
-        ]
-        + ["icu*.dll"],
-    ),
-    "libzim": (
-        INSTALL_DIR,
-        (
-            ## Linux
-            # We need to package all dependencies (`*.a`) on wasm
-            *lib_prefix("libzim.a" if COMPILE_CONFIG != "wasm" else "*.a"),
-            *lib_prefix("libzim.so"),
-            *lib_prefix(
-                "libzim.so.{version}".format(version=main_project_versions["libzim"])
-            ),
-            *lib_prefix(
-                "libzim.so.{version}".format(
-                    version=major_version(main_project_versions["libzim"])
-                )
-            ),
-            ## MacOS
-            "lib/libzim.{}.dylib".format(
-                major_version(main_project_versions["libzim"])
-            ),
-            "lib/libzim.dylib",
-            "lib/*/libzim.pc",
-            ## Windows
-            "bin/zim-{version}.dll".format(
-                version=major_version(main_project_versions["libzim"])
-            ),
-            "bin/icu*.dll",
-            "bin/zim-{version}.pdb".format(
-                version=major_version(main_project_versions["libzim"])
-            ),
-            "lib/zim.lib",
-            ## Includes and others
-            "include/zim/**/*.h",
-            "share/icu/{}/icudt{}l.dat".format(
-                base_deps_versions["icu4c"], major_version(base_deps_versions["icu4c"])
-            ),
-        ),
-    ),
-    "libkiwix": (
-        INSTALL_DIR,
-        (
-            "lib/CoreKiwix.xcframework/",
-            "lib/*/libkiwix.so",
-            "lib/*/libkiwix.so.{version}".format(
-                version=main_project_versions["libkiwix"]
-            ),
-            "lib/*/libkiwix.so.{version}".format(
-                version=major_version(main_project_versions["libkiwix"])
-            ),
-            "lib/libkiwix.{}.dylib".format(
-                major_version(main_project_versions["libkiwix"])
-            ),
-            "bin/kiwix-{version}.dll".format(
-                version=major_version(main_project_versions["libkiwix"])
-            ),
-            "bin/icu*.dll",
-            "bin/kiwix-{version}.pdb".format(
-                version=major_version(main_project_versions["libkiwix"])
-            ),
-            "lib/kiwix.lib",
-            "lib/libkiwix.dylib",
-            "lib/*/libkiwix.pc",
-            "include/kiwix/**/*.h",
-            "share/icu/{}/icudt{}l.dat".format(
-                base_deps_versions["icu4c"], major_version(base_deps_versions["icu4c"])
-            ),
-        ),
-    ),
-}
-
-DATE = date.today().isoformat()
-
-
-def print_message(message, *args, **kwargs):
-    message = message.format(*args, **kwargs)
-    message = "{0} {1} {0}".format("-" * 3, message)
-    print(message, flush=True)
-
-
-MANIFEST_TEMPLATE = """{archive_name}
-***************************
-
-Dependencies archive for {target} using config {config}
-Generated at {date}
-"""
-
-
-def write_manifest(manifest_file, archive_name, target, config):
-    with manifest_file.open(mode="w") as f:
-        f.write(
-            MANIFEST_TEMPLATE.format(
-                archive_name=archive_name,
-                target=target,
-                config=config,
-                date=DATE,
-            )
-        )
-
-
-def run_kiwix_build(
-    target,
-    config,
-    build_deps_only=False,
-    target_only=False,
-    make_release=False,
-    make_dist=False,
-):
-    command = ["kiwix-build"]
-    command.append(target)
-    command.append("--hide-progress")
-    command.append("--fast-clone")
-    command.append("--assume-packages-installed")
-    command.append("--use-target-arch-name")
-    command.extend(["--config", config])
-    if build_deps_only:
-        command.append("--build-deps-only")
-    if target_only:
-        command.append("--build-nodeps")
-    if make_release:
-        command.append("--make-release")
-    if make_dist:
-        command.append("--make-dist")
-    print_message(
-        "Build {} (deps={}, release={}, dist={})",
-        target,
-        build_deps_only,
-        make_release,
-        make_dist,
-    )
-    env = dict(_environ)
-    env["SKIP_BIG_MEMORY_TEST"] = "1"
-    subprocess.check_call(command, cwd=str(HOME), env=env)
-    print_message("Build ended")
-
-
-try:
-    import paramiko
-
-    def upload(file_to_upload, host, dest_path):
-        if not file_to_upload.exists():
-            print_message("No {} to upload!", file_to_upload)
-            return
-
-        if ":" in host:
-            host, port = host.split(":", 1)
-        else:
-            port = "22"
-        if "@" in host:
-            user, host = host.split("@", 1)
-        else:
-            user = None
-
-        from contextlib import contextmanager
-
-        @contextmanager
-        def get_client():
-            client = paramiko.client.SSHClient()
-            client.set_missing_host_key_policy(paramiko.client.WarningPolicy)
-            print_message(f"Connect to {host}:{port}")
-            client.connect(
-                host,
-                port=port,
-                username=user,
-                key_filename=_environ.get("SSH_KEY"),
-                look_for_keys=False,
-                compress=True,
-            )
-            try:
-                yield client
-            finally:
-                client.close()
-
-        @contextmanager
-        def get_sftp():
-            with get_client() as client:
-                sftp = client.open_sftp()
-                try:
-                    yield sftp
-                finally:
-                    sftp.close()
-
-        dest_path = PurePosixPath(dest_path)
-        remote_file = dest_path.joinpath(file_to_upload.name)
-
-        with get_sftp() as sftp:
-            for part in list(reversed(dest_path.parents)) + [dest_path]:
-                part = str(part)
-                try:
-                    sftp.stat(part)
-                except FileNotFoundError:
-                    sftp.mkdir(part)
-
-            print_message(f"Sending archive {file_to_upload} to {remote_file}")
-            sftp.put(str(file_to_upload), str(remote_file), confirm=True)
-
-except ModuleNotFoundError:
-    # On old system (bionic) paramiko is really complex to install
-    # Keep the old implementaion on sush system.
-
-    def upload(file_to_upload, host, dest_path):
-        if not file_to_upload.exists():
-            print_message("No {} to upload!", file_to_upload)
-            return
-
-        if ":" in host:
-            host, port = host.split(":", 1)
-        else:
-            port = "22"
-
-        # Using SFTP to create the directory hierarchy because we can not
-        # use SSH (no shell for this user); and then scp to upload the file.
-        #
-        # Sending SFTP mkdir command to the SFTP interactive mode and not batch (-b) mode
-        # as the latter would exit on any mkdir error while it is most likely
-        # the first parts of the destination is already present and thus can't be created
-        sftp_commands = "\n".join(
-            [
-                f"mkdir {part}"
-                for part in list(reversed(Path(dest_path).parents)) + [dest_path]
-            ]
-        )
-        command = [
-            "sftp",
-            "-c",
-            "aes128-ctr",
-            "-i",
-            _environ.get("SSH_KEY"),
-            "-P",
-            port,
-            "-o",
-            "StrictHostKeyChecking=no",
-            host,
-        ]
-        print_message("Creating dest path {}", dest_path)
-        subprocess.run(command, input=sftp_commands.encode("utf-8"), check=True)
-
-        command = [
-            "scp",
-            "-c",
-            "aes128-ctr",
-            "-rp",
-            "-P",
-            port,
-            "-i",
-            _environ.get("SSH_KEY"),
-            "-o",
-            "StrictHostKeyChecking=no",
-            str(file_to_upload),
-            "{}:{}".format(host, dest_path),
-        ]
-        print_message("Sending archive with command {}", command)
-        subprocess.check_call(command)
-
-
-def upload_archive(archive, project, make_release, dev_branch=None):
-    if not archive.exists():
-        print_message("No archive {} to upload!", archive)
-        return
-
-    if project.startswith("kiwix-") or project in ["libkiwix"]:
-        host = "ci@master.download.kiwix.org:30022"
-        dest_path = "/data/download/"
-    else:
-        host = "ci@download.openzim.org:30022"
-        dest_path = "/data/openzim/"
-
-    if make_release:
-        dest_path = dest_path + "release/" + project
-    else:
-        dest_path = dest_path + "nightly/" + DATE
-
-    if dev_branch:
-        dest_path = "/data/tmp/ci/dev_preview/" + dev_branch
-    else:
-        # Make the archive read only. This way, scp will preserve rights.
-        # If somehow we try to upload twice the same archive, scp will fails.
-        archive.chmod(0o444)
-
-    upload(archive, host, dest_path)
-
-
-# This remove "share/doc" and "share/man" from the thing to copy in the deps archive
-def filter_install_dir(path):
-    for dir in path.glob("*"):
-        if dir.name not in ["share"]:
-            yield dir
-        else:
-            for sub_dir in dir.glob("*"):
-                if sub_dir.name not in ["doc", "man"]:
-                    yield sub_dir
-
-
-# Full: True if we are creating a full archive to be used as cache by kiwix-build (base_deps_{os}_{config}_{base_deps_version}.tar.gz)
-# Full: False if we are creating a archive to be used as pre-cached dependencies for project's CI (deps_{config}_{target}.tar.gz)
-def make_deps_archive(target=None, name=None, full=False):
-    archive_name = name or "deps_{}_{}.tar.gz".format(
-        get_dependency_archive_name(), target
-    )
-    print_message("Create archive {}.", archive_name)
-    files_to_archive = list(filter_install_dir(INSTALL_DIR))
-    files_to_archive += HOME.glob("BUILD_*/LOGS")
-    if COMPILE_CONFIG == "apple_all_static":
-        for subconfig in AppleXCFramework.subConfigNames:
-            base_dir = get_build_dir(subconfig)
-            files_to_archive += filter_install_dir(base_dir / "INSTALL")
-            if (base_dir / "meson_cross_file.txt").exists():
-                files_to_archive.append(base_dir / "meson_cross_file.txt")
-
-    if COMPILE_CONFIG.endswith("_mixed"):
-        static_config = COMPILE_CONFIG.replace("_mixed", "_static")
-        files_to_archive += filter_install_dir(get_build_dir(static_config) / "INSTALL")
-    if COMPILE_CONFIG.startswith("android_"):
-        files_to_archive += filter_install_dir(HOME / "BUILD_neutral" / "INSTALL")
-        base_dir = get_build_dir(COMPILE_CONFIG)
-        if (base_dir / "meson_cross_file.txt").exists():
-            files_to_archive.append(base_dir / "meson_cross_file.txt")
-    # Copy any toolchain
-    files_to_archive += [TOOLCHAIN_DIR]
-    files_to_archive += HOME.glob("BUILD_neutral/TOOLCHAINS/*")
-    if (BASE_DIR / "meson_cross_file.txt").exists():
-        files_to_archive.append(BASE_DIR / "meson_cross_file.txt")
-
-    manifest_file = BASE_DIR / "manifest.txt"
-    write_manifest(manifest_file, archive_name, target, COMPILE_CONFIG)
-    files_to_archive.append(manifest_file)
-
-    relative_path = HOME
-    if full:
-        files_to_archive += ARCHIVE_DIR.glob(".*_ok")
-        files_to_archive += BASE_DIR.glob("*/.*_ok")
-        # Add also static build for mixed target
-        if COMPILE_CONFIG.endswith("_mixed"):
-            static_config = COMPILE_CONFIG.replace("_mixed", "_static")
-            files_to_archive += get_build_dir(static_config).glob("*/.*_ok")
-        # Native dyn and static is needed for potential cross compilation that use native tools (icu)
-        files_to_archive += get_build_dir("native_dyn").glob("*/.*_ok")
-        files_to_archive += get_build_dir("native_static").glob("*/.*_ok")
-        files_to_archive += HOME.glob("BUILD_*android*/**/.*_ok")
-        files_to_archive += HOME.glob("BUILD_*apple-macos*/**/.*_ok")
-        files_to_archive += HOME.glob("BUILD_*apple-ios*/**/.*_ok")
-        files_to_archive += SOURCE_DIR.glob("*/.*_ok")
-        files_to_archive += SOURCE_DIR.glob("zim-testing-suite-*/*")
-
-    archive_file = TMP_DIR / archive_name
-    with tarfile.open(str(archive_file), "w:gz") as tar:
-        for name in set(files_to_archive):
-            print(".{}".format(name), flush=True)
-            tar.add(str(name), arcname=str(name.relative_to(relative_path)))
-
-    return archive_file
-
-
-def get_postfix(project):
-    postfix = main_project_versions[project]
-    extra = release_versions.get(project)
-    if extra:
-        postfix = "{}-{}".format(postfix, extra)
-    return postfix
-
-
-def sign_binary(path):
-    # We assume here that signtool and certificate are properly configured.
-    # Env var `SIGNTOOL_THUMBPRINT` must contain thumbprint of the certificate to use.
-    command = [
-        os.getenv("SIGNTOOL_PATH", "signtool.exe"),
-        "sign",
-        "/fd",
-        "sha256",
-        "/tr",
-        "http://ts.ssl.com",
-        "/td",
-        "sha256",
-        "/sha1",
-        os.environ["SIGNTOOL_THUMBPRINT"],
-        str(path),
-    ]
-    subprocess.run(command, check=True)
-
-
-def make_archive(project, make_release):
-    platform_name = get_platform_name()
-    if not platform_name:
-        return None
-
-    try:
-        base_dir, export_files = EXPORT_FILES[project]
-    except KeyError:
-        # No binary files to export
-        return None
-
-    if make_release:
-        postfix = get_postfix(project)
-    else:
-        postfix = DATE
-
-    archive_name = "{}_{}-{}".format(project, platform_name, postfix)
-
-    files_to_archive = []
-    for export_file in export_files:
-        files_to_archive.extend(base_dir.glob(export_file))
-
-    if make_release and platform.system() == "Windows":
-        for file in files_to_archive:
-            if str(file).endswith(".exe"):
-                sign_binary(file)
-
-    if platform_name == "win-i686" or platform.system() == "Windows":
-        open_archive = lambda a: zipfile.ZipFile(
-            str(a), "w", compression=zipfile.ZIP_DEFLATED
-        )
-        archive_add = lambda a, f: a.write(str(f), arcname=str(f.relative_to(base_dir)))
-        archive_ext = ".zip"
-    else:
-        open_archive = lambda a: tarfile.open(str(a), "w:gz")
-        archive_add = lambda a, f: a.add(
-            str(f), arcname="{}/{}".format(archive_name, str(f.relative_to(base_dir)))
-        )
-        archive_ext = ".tar.gz"
-
-    archive = TMP_DIR / "{}{}".format(archive_name, archive_ext)
-    print_message("create archive {} with {}", archive, files_to_archive)
-    with open_archive(archive) as arch:
-        for f in files_to_archive:
-            archive_add(arch, f)
-    return archive
-
-
-def create_desktop_image(make_release):
-    print_message("creating desktop image")
-    if make_release:
-        postfix = get_postfix("kiwix-desktop")
-        src_dir = SOURCE_DIR / "kiwix-desktop_release"
-    else:
-        postfix = DATE
-        src_dir = SOURCE_DIR / "kiwix-desktop"
-
-    if COMPILE_CONFIG == "flatpak":
-        build_path = BASE_DIR / "org.kiwix.desktop.flatpak"
-        app_name = "org.kiwix.desktop.{}.flatpak".format(postfix)
-        print_message("archive is {}", build_path)
-    elif platform.system() == "Windows":
-        archive_basename = "kiwix-desktop_windows_x64_{}".format(postfix)
-        working_dir = INSTALL_DIR / archive_basename
-        build_path = Path(str(working_dir) + ".zip")
-        app_name = build_path.name
-        command = [
-            "python",
-            KBUILD_SOURCE_DIR / "scripts" / "package_kiwix-desktop_windows.py",
-            str(INSTALL_DIR),
-            str(working_dir),
-            str(build_path),
-        ]
-        if make_release:
-            command += ["-s"]
-        print_message("Package archive of kiwix-desktop")
-        subprocess.check_call(command, cwd=str(HOME))
-    else:
-        build_path = HOME / "Kiwix-{}-x86_64.AppImage".format(postfix)
-        app_name = "kiwix-desktop_x86_64_{}.appimage".format(postfix)
-        command = [
-            KBUILD_SOURCE_DIR / "scripts" / "create_kiwix-desktop_appImage.sh",
-            str(INSTALL_DIR),
-            str(src_dir),
-            str(HOME / "AppDir"),
-        ]
-        env = dict(os.environ)
-        env["VERSION"] = postfix
-        print_message("Build AppImage of kiwix-desktop")
-        subprocess.check_call(command, cwd=str(HOME), env=env)
-
-    print_message("Copy Build to {}".format(TMP_DIR / app_name))
-    shutil.copy(str(build_path), str(TMP_DIR / app_name))
-    return TMP_DIR / app_name
-
-
-def update_flathub_git():
-    git_repo_dir = TMP_DIR / "org.kiwix.desktop"
-    env = dict(os.environ)
-    env["GIT_AUTHOR_NAME"] = env["GIT_COMMITTER_NAME"] = "KiwixBot"
-    env["GIT_AUTHOR_EMAIL"] = env["GIT_COMMITTER_EMAIL"] = "release@kiwix.org"
-
-    def call(command, cwd=None):
-        cwd = cwd or git_repo_dir
-        print_message("call {}", command)
-        subprocess.check_call(command, env=env, cwd=str(cwd))
-
-    command = ["git", "clone", FLATPAK_HTTP_GIT_REMOTE]
-    call(command, cwd=TMP_DIR)
-
-    branch_name = "v_{}".format(get_postfix("kiwix-desktop"))
-    command = ["git", "checkout", "-b", branch_name]
-    call(command)
-    shutil.copy(str(BASE_DIR / "org.kiwix.desktop.json"), str(git_repo_dir))
-    patch_dir = KBUILD_SOURCE_DIR / "kiwixbuild" / "patches"
-    for dep in ["pugixml", "libmicrohttpd"]:
-        for f in patch_dir.glob("{}_*.patch".format(dep)):
-            shutil.copy(str(f), str(git_repo_dir / "patches"))
-    command = ["git", "add", "-A", "."]
-    call(command)
-    command = [
-        "git",
-        "commit",
-        "-m",
-        "Update to version {}".format(get_postfix("kiwix-desktop")),
-    ]
-    try:
-        call(command)
-    except subprocess.CalledProcessError:
-        # This may fail if there is nothing to commit (a rebuild of the CI for exemple)
-        return
-    command = ["git", "config", "remote.origin.pushurl", FLATPAK_GIT_REMOTE]
-    call(command)
-    command = ["git", "push", "origin", branch_name]
-    env["GIT_SSH_COMMAND"] = "ssh -o StrictHostKeyChecking=no -i " + _environ.get(
-        "SSH_KEY"
-    )
-    call(command)
-
-
-def fix_macos_rpath(project):
-    base_dir, export_files = EXPORT_FILES[project]
-    for file in filter(lambda f: f.endswith(".dylib"), export_files):
-        lib = base_dir / file
-        if lib.is_symlink():
-            continue
-        command = ["install_name_tool", "-id", lib.name, str(lib)]
-        print_message("call {}", " ".join(command))
-        subprocess.check_call(command, env=os.environ)
-
-
-def trigger_workflow(repo, workflow="docker.yml", ref="main", inputs=None):
-    """triggers a `workflow_dispatch` event to the specified workflow on its repo
-
-    repo: {user}/{repo} format
-    workflow: workflow ID or workflow file name
-    ref: branch or tag name
-    inputs: dict of inputs to pass to the workflow"""
-    print_message(
-        "triggering workflow `{workflow}` on {repo}@{ref} " "with inputs={inputs}",
-        workflow=workflow,
-        repo=repo,
-        ref=ref,
-        inputs=inputs,
-    )
-
-    url = "{base_url}/repos/{repo}/actions/workflows/{workflow}/dispatches".format(
-        base_url=os.getenv("GITHUB_API_URL", "https://api.github.com"),
-        repo=repo,
-        workflow=workflow,
-    )
-
-    resp = requests.post(
-        url,
-        headers={
-            "Content-Type": "application/json",
-            "Authorization": "token {token}".format(token=os.getenv("GITHUB_PAT", "")),
-            "Accept": "application/vnd.github.v3+json",
-        },
-        json={"ref": ref, "inputs": inputs},
-        timeout=5,
-    )
-    if resp.status_code != 204:
-        raise ValueError(
-            "Unexpected HTTP {code}: {reason}".format(
-                code=resp.status_code, reason=resp.reason
-            )
-        )
-
-
-def trigger_docker_publish(target):
-    if target not in ("zim-tools", "kiwix-tools"):
-        return
-
-    version = get_postfix(target)
-    repo = {"zim-tools": "openzim/zim-tools", "kiwix-tools": "kiwix/kiwix-tools"}.get(
-        target
-    )
-
-    try:
-        trigger_workflow(
-            repo, workflow="docker.yml", ref="main", inputs={"version": version}
-        )
-        print_message("triggered docker workflow on {repo}", repo=repo)
-    except Exception as exc:
-        print_message("Error triggering workflow: {exc}", exc=exc)
-        raise exc
-
-
-def notarize_macos_build(project):
-    """sign and notarize files for macOS
-
-    Expects the following environment:
-    - `SIGNING_IDENTITY` environ with Certificate name/identity
-    - `KEYCHAIN` environ with path to the keychain storing credentials
-    - `KEYCHAIN_PROFILE` environ with name of the profile in that keychain
-    - `KEYCHAIN_PASSWORD` environ with password to unlock the keychain
-    """
-    if project != "libzim":
-        return
-
-    # currently only supports libzim use case: sign every dylib
-    base_dir, export_files = EXPORT_FILES[project]
-    filepaths = [
-        base_dir.joinpath(file)
-        for file in filter(lambda f: f.endswith(".dylib"), export_files)
-        if not base_dir.joinpath(file).is_symlink()
-    ]
-
-    if not filepaths:
-        return
-
-    for filepath in filepaths:
-        subprocess.check_call(
-            [
-                "/usr/bin/codesign",
-                "--force",
-                "--sign",
-                os.getenv("SIGNING_IDENTITY", "no-signing-ident"),
-                "--keychain",
-                os.getenv("KEYCHAIN", "no-keychain-path"),
-                str(filepath),
-                "--deep",
-                "--timestamp",
-            ],
-            env=os.environ,
-        )
-
-    # create a zip of the dylibs and upload for notarization
-    zip_name = "{}.zip".format(project)
-    subprocess.check_call(
-        ["/usr/bin/ditto", "-c", "-k", "--keepParent"]
-        + [str(f) for f in filepaths]
-        + [zip_name],
-        env=os.environ,
-    )
-
-    # make sure keychain is unlocked
-    subprocess.check_call(
-        [
-            "/usr/bin/security",
-            "unlock-keychain",
-            "-p",
-            os.getenv("KEYCHAIN_PASSWORD", "no-keychain-password"),
-            os.getenv("KEYCHAIN", "no-keychain-path"),
-        ],
-        env=os.environ,
-    )
-
-    subprocess.check_call(
-        [
-            "/usr/bin/xcrun",
-            "notarytool",
-            "submit",
-            "--keychain",
-            os.getenv("KEYCHAIN", "no-keychain-path"),
-            "--keychain-profile",
-            os.getenv("KEYCHAIN_PROFILE", "no-keychain-profile"),
-            "--wait",
-            str(zip_name),
-        ],
-        env=os.environ,
-    )
-
-    # check notarization of a file (should be in-progress atm and this != 0)
-    subprocess.call(
-        ["/usr/sbin/spctl", "--assess", "-vv", "--type", "install", filepaths[-1]],
-        env=os.environ,
-    )

.github/scripts/compile_all_deps.py

@@ -1,22 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-
-from common import (
-    run_kiwix_build,
-    make_deps_archive,
-    upload,
-    COMPILE_CONFIG,
-    DEV_BRANCH,
-)
-from build_definition import select_build_targets, DEPS
-
-for target in select_build_targets(DEPS):
-    run_kiwix_build(target, config=COMPILE_CONFIG, build_deps_only=True)
-    archive_file = make_deps_archive(target=target)
-    if DEV_BRANCH:
-        destination = "/data/tmp/ci/dev_preview/" + DEV_BRANCH
-    else:
-        destination = "/data/tmp/ci"
-    upload(archive_file, "ci@tmp.kiwix.org:30022", destination)
-    os.remove(str(archive_file))

.github/scripts/ensure_base_deps.py

@@ -1,72 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import tarfile
-from urllib.request import urlopen
-from urllib.error import URLError
-
-from kiwixbuild.versions import base_deps_meta_version
-
-from common import (
-    print_message,
-    run_kiwix_build,
-    upload,
-    make_deps_archive,
-    HOME,
-    COMPILE_CONFIG,
-    OS_NAME,
-    MAKE_RELEASE,
-)
-
-
-def download_base_archive(base_name):
-    url = "http://tmp.kiwix.org/ci/{}".format(base_name)
-    file_path = str(HOME / base_name)
-    batch_size = 1024 * 1024 * 8
-    with urlopen(url) as resource, open(file_path, "wb") as file:
-        while True:
-            batch = resource.read(batch_size)
-            if not batch:
-                break
-            print(".", end="", flush=True)
-            file.write(batch)
-    return file_path
-
-
-def get_archive_name():
-    ARCHIVE_NAME_TEMPLATE = "base_deps_{os}_{config}_{version}.tar.gz"
-
-    if COMPILE_CONFIG == "flatpak":
-        return "base_deps_flatpak.tar.gz"
-
-    return ARCHIVE_NAME_TEMPLATE.format(
-        os=OS_NAME,
-        config=COMPILE_CONFIG,
-        version=base_deps_meta_version,
-    )
-
-
-def main():
-    base_dep_archive_name = get_archive_name()
-    print_message("Getting archive {}", base_dep_archive_name)
-    try:
-        local_filename = download_base_archive(base_dep_archive_name)
-        with tarfile.open(local_filename) as f:
-            f.extractall(str(HOME))
-        os.remove(str(local_filename))
-    except URLError:
-        if COMPILE_CONFIG == "flatpak":
-            print_message("Cannot get archive. Move on")
-        else:
-            print_message("Cannot get archive. Build dependencies")
-            run_kiwix_build("alldependencies", config=COMPILE_CONFIG)
-            archive_file = make_deps_archive(name=base_dep_archive_name, full=True)
-            upload(archive_file, "ci@tmp.kiwix.org:30022", "/data/tmp/ci")
-            os.remove(str(archive_file))
-
-
-if __name__ == "__main__":
-    if MAKE_RELEASE:
-        print_message("We are building release. Don't download deps archive.")
-    else:
-        main()

.github/scripts/trigger_docker_workflow.py

@@ -1,17 +0,0 @@
-#!/usr/bin/env python3
-
-from common import (
-    trigger_docker_publish,
-    MAKE_RELEASE,
-)
-
-from build_definition import select_build_targets, DOCKER
-
-# Filter what to build if we are doing a release.
-if MAKE_RELEASE:
-    docker_trigger = select_build_targets(DOCKER)
-else:
-    docker_trigger = []
-
-for target in docker_trigger:
-    trigger_docker_publish(target)

.github/scripts/upload_failure_logs.py

@@ -1,18 +0,0 @@
-#!/usr/bin/env python3
-
-import tarfile
-from pathlib import Path
-from common import upload, OS_NAME, COMPILE_CONFIG, HOME
-
-ARCHIVE_NAME = Path(f"fail_log_{OS_NAME}_{COMPILE_CONFIG}.tar.gz")
-
-
-files_to_archive = []
-files_to_archive += HOME.glob("BUILD_*")
-files_to_archive += [HOME / "SOURCE", HOME / "LOGS", HOME / "TOOLCHAINS"]
-
-with tarfile.open(ARCHIVE_NAME, "w:xz") as tar:
-    for name in set(files_to_archive):
-        tar.add(str(name))
-
-upload(ARCHIVE_NAME, "ci@tmp.kiwix.org:30022", "/data/tmp/ci")

.github/workflows/cd.yml

@@ -1,326 +0,0 @@
-name: CD
-
-on:
- push:
-   tags:
-     - r_[0-9]+
- schedule:
-   - cron: '0 3 * * *'
-
-jobs:
-  Windows:
-    strategy:
-      fail-fast: false
-      matrix:
-        config:
-          - native_mixed
-          - native_dyn
-          - native_static
-    runs-on: windows-2022
-    env:
-      OS_NAME: windows
-      COMPILE_CONFIG: ${{matrix.config}}
-      HOME: 'C:\\Users\\runneradmin'
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-    - name: Setup python 3.12
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.12'
-    - name: Install packages
-      run: |
-        choco.exe install pkgconfiglite ninja
-    - name: Install python modules
-      shell: bash
-      run: |
-        pip3 install meson pytest requests distro paramiko
-        pip3 install --no-deps $GITHUB_WORKSPACE
-    - name: Install QT
-      if: ${{ matrix.config == 'native_mixed' }}
-      uses: jurplel/install-qt-action@v4
-      with:
-        version: 5.15.2
-        modules: "qtwebengine"
-        setup-python: false
-      env:
-        AQT_CONFIG: ${{ github.workspace }}/.github/configs/aqt.ini
-    - name: Setup MSVC compiler
-      uses: bus1/cabuild/action/msdevshell@v1
-      with:
-        architecture: x64
-    - name: secret
-      shell: bash
-      run: |
-        echo "${{secrets.ssh_key}}" > $SSH_KEY
-      env:
-        SSH_KEY: ${{ runner.temp }}/id_rsa
-    - name: Install and configure eSigner CKA and Windows SDK
-      if: github.event_name == 'push'
-      env:
-        ESIGNER_URL: https://github.com/SSLcom/eSignerCKA/releases/download/v1.0.7/SSL.COM-eSigner-CKA_1.0.7.zip
-      run: |
-          Set-StrictMode -Version 'Latest'
-
-          # Download and Unzip eSignerCKA Setup
-          Invoke-WebRequest -OutFile eSigner_CKA_Setup.zip "$env:ESIGNER_URL"
-          Expand-Archive -Force eSigner_CKA_Setup.zip
-          Remove-Item eSigner_CKA_Setup.zip
-          Move-Item -Destination “eSigner_CKA_Installer.exe” -Path “eSigner_CKA_*\*.exe”
-
-          # Install eSignerCKA
-          New-Item -ItemType Directory -Force -Path "C:\esigner"
-          ./eSigner_CKA_Installer.exe /CURRENTUSER /VERYSILENT /SUPPRESSMSGBOXES /DIR=”C:\esigner” /TYPE=automatic | Out-Null
-          Remove-Item "eSigner_CKA_Installer.exe"
-
-          # Configure the CKA with SSL.com credentials
-          C:\esigner\eSignerCKATool.exe config -mode product -user "${{ secrets.ESIGNER_USERNAME }}" -pass "${{ secrets.ESIGNER_PASSWORD }}" -totp "${{ secrets.ESIGNER_TOTP_SECRET }}" -key "C:\esigner\master.key" -r
-          C:\esigner\eSignerCKATool.exe unload
-          C:\esigner\eSignerCKATool.exe load
-
-          # Find certificate
-          $CodeSigningCert = Get-ChildItem Cert:\CurrentUser\My -CodeSigningCert | Select-Object -First 1
-          echo Certificate: $CodeSigningCert
-
-          # Extract thumbprint and subject name
-          $Thumbprint = $CodeSigningCert.Thumbprint
-          echo "SIGNTOOL_THUMBPRINT=$Thumbprint" >> $env:GITHUB_ENV
-
-    - name: Ensure base deps
-      run: |
-        python .github\\scripts\\ensure_base_deps.py
-      env:
-        SSH_KEY: ${{ runner.temp }}/id_rsa
-    - name: Build Release
-      run: |
-        python .github\\scripts\\build_release_nightly.py
-      env:
-        SIGNTOOL_PATH: "C:/Program Files (x86)/Windows Kits/10/bin/10.0.22621.0/x86/signtool.exe"
-        SSH_KEY: ${{ runner.temp }}/id_rsa
-    - name: Upload failure logs
-      if: failure()
-      run: |
-        python .github\\scripts\\upload_failure_logs.py
-      env:
-        SSH_KEY: ${{ runner.temp }}/id_rsa
-
-  Linux:
-    strategy:
-      fail-fast: false
-      matrix:
-        config:
-          - native_static
-          - native_mixed
-          - native_dyn
-          - wasm
-          - armv6_static
-          - armv6_mixed
-          - armv8_static
-          - armv8_mixed
-          - aarch64_static
-          - aarch64_mixed
-          - aarch64_musl_static
-          - aarch64_musl_mixed
-          - x86-64_musl_static
-          - x86-64_musl_mixed
-          - i586_static
-          - android_arm
-          - android_arm64
-          - android_x86
-          - android_x86_64
-        image_variant: ['focal']
-        include:
-          - config: native_mixed
-            image_variant: manylinux
-          - config: aarch64_mixed
-            image_variant: manylinux
-          - config: native_dyn
-            image_variant: jammy
-    env:
-      HOME: /home/runner
-      SSH_KEY: /tmp/id_rsa
-    runs-on: ubuntu-22.04
-    container:
-      image: "ghcr.io/kiwix/kiwix-build_ci_${{matrix.image_variant}}:2024-11-30"
-      options: "--device /dev/fuse --privileged"
-    steps:
-    - name: Checkout code
-      shell: bash
-      run: |
-        cd $HOME
-        git clone https://github.com/${REP}
-        cd ./${REP##*/}
-        git checkout --force ${GITHUB_SHA}
-        pip3 install --user --no-deps .
-      env:
-        REP: ${{github.repository}}
-    - name: secret
-      shell: bash
-      run: |
-        echo "${{secrets.ssh_key}}" > $SSH_KEY
-        chmod 600 $SSH_KEY
-    - name: Ensure base deps
-      shell: bash
-      run: |
-        cd $HOME
-        kiwix-build/.github/scripts/ensure_base_deps.py
-      env:
-        COMPILE_CONFIG: ${{matrix.config}}
-    - name: Build release
-      shell: bash
-      run: |
-        cd $HOME
-        kiwix-build/.github/scripts/build_release_nightly.py
-      env:
-        COMPILE_CONFIG: ${{matrix.config}}
-    - name: Upload failure logs
-      if: failure()
-      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py
-      env:
-        COMPILE_CONFIG: ${{matrix.config}}
-
-  Flatpak:
-    strategy:
-      fail-fast: false
-    env:
-      HOME: /home/runner
-      SSH_KEY: /tmp/id_rsa
-      COMPILE_CONFIG: flatpak
-      OS_NAME: focal
-    runs-on: ubuntu-22.04
-    steps:
-    - name: Checkout code
-      shell: bash
-      run: |
-        cd $HOME
-        git clone https://github.com/${REP}
-        cd ./${REP##*/}
-        git checkout --force ${GITHUB_SHA}
-        pip3 install --user --no-deps .
-      env:
-        REP: ${{github.repository}}
-    - name: Install flatpak tools
-      run: |
-        sudo apt-get update
-        sudo apt-get install flatpak-builder ninja-build meson
-    - name: secret
-      shell: bash
-      run: |
-        echo "${{secrets.ssh_key}}" > $SSH_KEY
-        chmod 600 $SSH_KEY
-    - name: Ensure base deps
-      shell: bash
-      run: |
-        cd $HOME
-        kiwix-build/.github/scripts/ensure_base_deps.py
-    - name: Build release
-      shell: bash
-      run: |
-        cd $HOME
-        kiwix-build/.github/scripts/build_release_nightly.py
-    - name: Upload failure logs
-      if: failure()
-      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py
-
-  Macos:
-    strategy:
-      fail-fast: false
-      matrix:
-        config:
-          - native_dyn
-          - native_static
-          - native_mixed
-          - macOS_arm64_static
-          - macOS_arm64_mixed
-          - apple_all_static
-    runs-on: macos-13
-    env:
-      SSH_KEY: /tmp/id_rsa
-      OS_NAME: macos
-      CERTIFICATE: /tmp/wmch-devid.p12
-      SIGNING_IDENTITY: ${{ secrets.APPLE_SIGNING_IDENTITY }}
-      KEYCHAIN: /Users/runner/build.keychain-db
-      KEYCHAIN_PASSWORD: mysecretpassword
-      KEYCHAIN_PROFILE: build-profile
-    steps:
-    - name: Set Xcode version (15.0.1)
-      # https://github.com/actions/runner-images/blob/main/images/macos/macos-13-Readme.md#xcode
-      run: sudo xcode-select -s /Applications/Xcode_15.0.1.app
-    - name: Checkout code
-      uses: actions/checkout@v4
-    - name: Setup python 3.10
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.10'
-    - name: Install packages
-      run:
-        brew install automake # ninja
-    - name: Install python modules
-      run: |
-        pip3 install meson pytest requests distro
-        pip3 install --no-deps $GITHUB_WORKSPACE
-    - name: secret
-      shell: bash
-      run: |
-        echo "${{secrets.ssh_key}}" > $SSH_KEY
-        chmod 600 $SSH_KEY
-    - name: install Apple certificate
-      shell: bash
-      run: |
-        echo "${{ secrets.APPLE_SIGNING_CERTIFICATE }}" | base64 --decode -o $CERTIFICATE
-        security create-keychain -p $KEYCHAIN_PASSWORD $KEYCHAIN
-        security default-keychain -s $KEYCHAIN
-        security set-keychain-settings $KEYCHAIN
-        security unlock-keychain -p $KEYCHAIN_PASSWORD $KEYCHAIN
-        security import $CERTIFICATE -k $KEYCHAIN -P "${{ secrets.APPLE_SIGNING_P12_PASSWORD }}" -A -T "/usr/bin/codesign"
-        rm $CERTIFICATE
-        security set-key-partition-list -S apple-tool:,apple: -s -k $KEYCHAIN_PASSWORD $KEYCHAIN
-        security find-identity -v $KEYCHAIN
-        xcrun notarytool store-credentials \
-            --apple-id "${{ secrets.APPLE_SIGNING_ALTOOL_USERNAME }}" \
-            --password "${{ secrets.APPLE_SIGNING_ALTOOL_PASSWORD }}" \
-            --team-id "${{ secrets.APPLE_SIGNING_TEAM }}" \
-            --validate \
-            --keychain $KEYCHAIN \
-            $KEYCHAIN_PROFILE
-    - name: Ensure base deps
-      shell: bash
-      run: |
-        cd $HOME
-        $GITHUB_WORKSPACE/.github/scripts/ensure_base_deps.py
-      env:
-        COMPILE_CONFIG: ${{matrix.config}}
-    - name: Build release
-      shell: bash
-      run: |
-        cd $HOME
-        $GITHUB_WORKSPACE/.github/scripts/build_release_nightly.py
-      env:
-        COMPILE_CONFIG: ${{matrix.config}}
-    - name: Upload failure logs
-      if: failure()
-      run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.py
-      env:
-        COMPILE_CONFIG: ${{matrix.config}}
-
-
-  Trigger_Docker:
-    needs: [Linux]
-    runs-on: ubuntu-22.04
-    env:
-      COMPILE_CONFIG: native_static
-      OS_NAME: focal
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-    - name: Install python modules
-      shell: bash
-      run: |
-        pip3 install --user --no-deps $GITHUB_WORKSPACE
-    - name: Trigger docker workflow
-      shell: bash
-      run: |
-        cd $HOME
-        $GITHUB_WORKSPACE/.github/scripts/trigger_docker_workflow.py
-      env:
-        GITHUB_PAT: ${{secrets.DOCKER_TRIGGER_GITHUB_PAT}}

.github/workflows/ci.yml

@@ -1,280 +0,0 @@
-name: CI
-
-on:
- push:
- schedule:
-   - cron: '0 1 * * *'
-
-jobs:
-  Windows:
-    strategy:
-      fail-fast: false
-      matrix:
-        config:
-          - native_mixed
-          - native_dyn
-          - native_static
-    runs-on: windows-2022
-    env:
-      OS_NAME: windows
-      COMPILE_CONFIG: ${{matrix.config}}
-      HOME: 'C:\\Users\\runneradmin'
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-    - name: Setup python 3.12
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.12'
-    - name: Install packages
-      run: |
-        choco.exe install pkgconfiglite ninja
-    - name: Install python modules
-      shell: bash
-      run: |
-        pip3 install meson pytest requests distro paramiko
-        pip3 install --no-deps $GITHUB_WORKSPACE
-    - name: Install QT
-      if: ${{ matrix.config == 'native_mixed' }}
-      uses: jurplel/install-qt-action@v4
-      with:
-        version: 6.4.3
-        modules: "qtwebengine qtwebchannel qtpositioning"
-        setup-python: false
-      env:
-        AQT_CONFIG: ${{ github.workspace }}/.github/configs/aqt.ini
-    - name: Setup MSVC compiler
-      uses: bus1/cabuild/action/msdevshell@v1
-      with:
-        architecture: x64
-    - name: secret
-      shell: bash
-      run: |
-        echo "${{secrets.ssh_key}}" > $SSH_KEY
-      env:
-        SSH_KEY: ${{ runner.temp }}/id_rsa
-    - name: Ensure base deps
-      run: |
-        python .github\\scripts\\ensure_base_deps.py
-      env:
-        SSH_KEY: ${{ runner.temp }}/id_rsa
-    - name: Compile all deps
-      run: |
-        python .github\\scripts\\compile_all_deps.py
-      env:
-        SSH_KEY: ${{ runner.temp }}/id_rsa
-    - name: Build projects
-      run: |
-        python .github\\scripts\\build_projects.py
-      env:
-        SSH_KEY: ${{ runner.temp }}/id_rsa
-    - name: Upload failure logs
-      if: failure()
-      run: |
-        python .github\\scripts\\upload_failure_logs.py
-      env:
-        SSH_KEY: ${{ runner.temp }}/id_rsa
-
-  Linux:
-    strategy:
-      fail-fast: false
-      matrix:
-        config:
-          - native_static
-          - native_dyn
-          - native_mixed
-          - wasm
-          - armv6_static
-          - armv6_dyn
-          - armv6_mixed
-          - armv8_static
-          - armv8_dyn
-          - armv8_mixed
-          - aarch64_static
-          - aarch64_dyn
-          - aarch64_mixed
-          - aarch64_musl_static
-          - aarch64_musl_dyn
-          - aarch64_musl_mixed
-          - x86-64_musl_static
-          - x86-64_musl_mixed
-          - i586_static
-          - i586_dyn
-          - android_arm
-          - android_arm64
-          - android_x86
-          - android_x86_64
-        image_variant: ['focal']
-        include:
-          - config: native_mixed
-            image_variant: manylinux
-          - config: aarch64_mixed
-            image_variant: manylinux
-          - config: native_dyn
-            image_variant: jammy
-    env:
-      HOME: /home/runner
-      SSH_KEY: /tmp/id_rsa
-      OS_NAME: ${{matrix.image_variant}}
-    runs-on: ubuntu-22.04
-    container:
-      image: "ghcr.io/kiwix/kiwix-build_ci_${{matrix.image_variant}}:2024-11-30"
-      options: "--device /dev/fuse --privileged"
-    steps:
-    - name: Checkout code
-      shell: bash
-      run: |
-        cd $HOME
-        git clone https://github.com/${REP}
-        cd ./${REP##*/}
-        git checkout --force ${GITHUB_SHA}
-        pip3 install --user --no-deps .
-      env:
-        REP: ${{github.repository}}
-    - name: Install paramiko
-      if: ${{matrix.image_variant != 'bionic' }}
-      shell: bash
-      run: pip3 install --user paramiko
-    - name: secret
-      shell: bash
-      run: |
-        echo "${{secrets.ssh_key}}" > $SSH_KEY
-        chmod 600 $SSH_KEY
-    - name: Ensure base deps
-      shell: bash
-      run: |
-        cd $HOME
-        kiwix-build/.github/scripts/ensure_base_deps.py
-      env:
-        COMPILE_CONFIG: ${{matrix.config}}
-    - name: Compile all deps
-      shell: bash
-      run: |
-        cd $HOME
-        kiwix-build/.github/scripts/compile_all_deps.py
-      env:
-        COMPILE_CONFIG: ${{matrix.config}}
-    - name: Build projects
-      shell: bash
-      run: |
-        cd $HOME
-        kiwix-build/.github/scripts/build_projects.py
-      env:
-        COMPILE_CONFIG: ${{matrix.config}}
-    - name: Upload failure logs
-      if: failure()
-      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py
-      env:
-        COMPILE_CONFIG: ${{matrix.config}}
-
-  Flatpak:
-    strategy:
-      fail-fast: false
-    env:
-      HOME: /home/runner
-      SSH_KEY: /tmp/id_rsa
-      COMPILE_CONFIG: flatpak
-      OS_NAME: focal
-    runs-on: ubuntu-22.04
-    steps:
-    - name: Checkout code
-      shell: bash
-      run: |
-        cd $HOME
-        git clone https://github.com/${REP}
-        cd ./${REP##*/}
-        git checkout --force ${GITHUB_SHA}
-        pip3 install --user paramiko
-        pip3 install --user --no-deps .
-      env:
-        REP: ${{github.repository}}
-    - name: Install flatpak tools
-      run: |
-        sudo apt-get update
-        sudo apt-get install flatpak-builder ninja-build meson
-    - name: secret
-      shell: bash
-      run: |
-        echo "${{secrets.ssh_key}}" > $SSH_KEY
-        chmod 600 $SSH_KEY
-    - name: Ensure base deps
-      shell: bash
-      run: |
-        cd $HOME
-        kiwix-build/.github/scripts/ensure_base_deps.py
-    - name: Build projects
-      shell: bash
-      run: |
-        cd $HOME
-        kiwix-build/.github/scripts/build_projects.py
-    - name: Upload failure logs
-      if: failure()
-      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py
-
-  Macos:
-    strategy:
-      fail-fast: false
-      matrix:
-        config:
-          - native_dyn
-          - native_static
-          - native_mixed
-          - iOS_arm64
-          - iOSSimulator_x86_64
-          - iOSSimulator_arm64
-          - macOS_arm64_static
-          - macOS_arm64_mixed
-          - macOS_x86_64
-          - apple_all_static
-    runs-on: macos-13
-    env:
-      SSH_KEY: /tmp/id_rsa
-      OS_NAME: macos
-    steps:
-    - name: Set Xcode version (15.0.1)
-      # https://github.com/actions/runner-images/blob/main/images/macos/macos-13-Readme.md#xcode
-      run: sudo xcode-select -s /Applications/Xcode_15.0.1.app
-    - name: Checkout code
-      uses: actions/checkout@v4
-    - name: Setup Python 3.10
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.10'
-    - name: Install packages
-      run:
-        brew install automake # ninja
-    - name: Install python modules
-      run: |
-        pip3 install meson pytest requests distro paramiko
-        pip3 install --no-deps $GITHUB_WORKSPACE
-    - name: secret
-      shell: bash
-      run: |
-        echo "${{secrets.ssh_key}}" > $SSH_KEY
-        chmod 600 $SSH_KEY
-    - name: Ensure base deps
-      shell: bash
-      run: |
-        cd $HOME
-        $GITHUB_WORKSPACE/.github/scripts/ensure_base_deps.py
-      env:
-        COMPILE_CONFIG: ${{matrix.config}}
-    - name: Compile all deps
-      shell: bash
-      run: |
-        cd $HOME
-        $GITHUB_WORKSPACE/.github/scripts/compile_all_deps.py
-      env:
-        COMPILE_CONFIG: ${{matrix.config}}
-    - name: Build projects
-      shell: bash
-      run: |
-        cd $HOME
-        $GITHUB_WORKSPACE/.github/scripts/build_projects.py
-      env:
-        COMPILE_CONFIG: ${{matrix.config}}
-    - name: Upload failure logs
-      if: failure()
-      run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.py
-      env:
-        COMPILE_CONFIG: ${{matrix.config}}

.gitignore

@@ -109,4 +109,3 @@ ENV/
 # mypy
 .mypy_cache/
 
-.idea

.travis.yml

@@ -0,0 +1,91 @@
+language: cpp
+dist: trusty
+sudo: required
+branches:
+  only:
+  - master
+  - /\d+\.\d+\.\d+$/
+before_install:
+- eval "${MATRIX_EVAL}"
+- ${CXX} --version
+- openssl aes-256-cbc -K $encrypted_eba2f7543984_key -iv $encrypted_eba2f7543984_iv
+  -in travis/travisci_builder_id_key.enc -out travis/travisci_builder_id_key -d
+- chmod 600 travis/travisci_builder_id_key
+before_cache:
+  - rm -f $HOME/.gradle/caches/modules-2/modules-2.lock
+  - rm -fr $HOME/.gradle/caches/*/plugin-resolution/
+cache:
+  ccache: true
+  directories:
+    - $HOME/.cache/pip
+    - $HOME/.gradle/caches/
+    - $HOME/.gradle/wrapper/
+    - $HOME/.android/build-cache
+install: travis/install_extra_deps.sh
+script: travis/compile_all.py
+after_failure: travis/upload_all_log.sh
+deploy:
+  - provider: script
+    skip_cleanup: true
+    script: travis/deploy.sh
+    on:
+      tags: true
+  - provider: script
+    skip_cleanup: true
+    script: travis/deploy.sh
+    on:
+      branch: master
+      condition: $TRAVIS_EVENT_TYPE = cron
+env:
+  global:
+    - MATRIX_EVAL="CC=gcc-5 && CXX=g++-5"
+  matrix:
+    - PLATFORM="native_dyn"
+    - PLATFORM="native_static"
+    - PLATFORM="win32_dyn"
+    - PLATFORM="win32_static"
+    - PLATFORM="armhf_dyn"
+    - PLATFORM="armhf_static"
+    - PLATFORM="android_arm"
+    - PLATFORM="android_arm64"
+    - PLATFORM="android_mips"
+    - PLATFORM="android_mips64"
+    - PLATFORM="android_x86"
+    - PLATFORM="android_x86_64"
+matrix:
+  allow_failures:
+    - env: PLATFORM="android_arm"
+    - env: PLATFORM="android_arm64"
+    - env: PLATFORM="android_mips"
+    - env: PLATFORM="android_mips64"
+    - env: PLATFORM="android_x86"
+    - env: PLATFORM="android_x86_64"
+
+notifications:
+  irc:
+    channels:
+     - "chat.freenode.net#kiwix"
+    on_success: change
+    on_failure: always
+addons:
+  ssh_known_hosts:
+  - download.kiwix.org
+  - download.openzim.org
+  apt:
+    sources:
+      - ubuntu-toolchain-r-test
+    packages:
+      - g++-5
+      - cmake
+      - python3-pip
+      - zlib1g-dev
+      - libjpeg-dev
+      - libmagic-dev
+      - ctpp2-utils
+      - libctpp2-dev
+      - libmicrohttpd-dev
+      - g++-mingw-w64-i686
+      - gcc-mingw-w64-i686
+      - gcc-mingw-w64-base
+      - mingw-w64-tools
+      - default-jdk

MANIFEST.in

@@ -1,3 +0,0 @@
-include kiwixbuild/templates/*_cross_file.txt
-include kiwixbuild/patches/*.patch
-include kiwixbuild/dependencies/icu4c_data_filter.json

README.md

@@ -1,181 +1,104 @@
-# Kiwix Build
+Build status: [![Build Status](https://travis-ci.org/kiwix/kiwix-build.svg?branch=master)](https://travis-ci.org/kiwix/kiwix-build)
 
-Kiwix Build provides advanced tools to (cross-)compile easily
-[Kiwix](https://kiwix.org) & [openZIM](https://openzim.org) softwares
-and libraries and deploy them. They have been tested on
-[Fedora](https://getfedora.org) 35+ & [Ubuntu](https://ubuntu.com)
-20.04+.
+Kiwix is an offline reader for web content. It's especially thought to
+make Wikipedia available offline. This is done by reading the content
+of the project stored in a file format ZIM, a high compressed open
+format with additional meta-data.
 
-Kiwix Build audience is:
-* Advanced users who don't want/can handle all the dependencies
-  compilations manually
-* Kiwix developer team for its own CI/CD
+This repository contains advanced tools to (cross-)compile easily
+Kiwix softwares and library and deploy them. They have been tested on
+Fedora 23 and Ubuntu 16.10.
 
-[![CI Build Status](https://github.com/kiwix/kiwix-build/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/kiwix/kiwix-build/actions/workflows/ci.yml?query=branch%3Amain)
-[![CD Build Status](https://github.com/kiwix/kiwix-build/actions/workflows/cd.yml/badge.svg?branch=main)](https://github.com/kiwix/kiwix-build/actions/workflows/cd.yml?query=branch%3Amain)
-[![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
+# Prerequesites
 
-Prerequisites
--------------
-
-You will need a recent version of [Meson](https://mesonbuild.com/) (>=
-0.34) and [Ninja](https://ninja-build.org) (>= 1.6) If your
-distribution provides a recent enough versions for them, just install
-them with your package manager. Continue to read the instructions
-otherwise.
+You will need a recent version of `meson` (0.34) and `ninja` (1.6) If
+your distribution provides a recent enough versions for them, just
+install them with your package manager. Continue to read the
+instructions otherwise.
 
 Before anything else you need to install Python3 related tools. On Debian
 based systems:
-```bash
+
+```
 sudo apt-get install python3-pip virtualenv
 ```
 
-Create a virtual environment to install python module in it instead
-of modifying the system.
-```bash
-virtualenv -p python3 ./ # Create virtualenv
-source bin/activate      # Activate the virtualenv
+Then install Meson:
 ```
-
-Then, download and install kiwix-build and its dependencies:
-```bash
-git clone https://github.com/kiwix/kiwix-build.git
-cd kiwix-build
-pip3 install .
-hash -r # Refresh bash paths
-```
-
-Compilation
------------
-
-The compilation is handled by the `kiwix-build` command. It will compile
-everything. If you are using a supported platform (Redhat or Debian
-based) it will install missing packages using `sudo`. You can get
-`kiwix-build` usage like this:
-```bash
-kiwix-build --help
-```
-
-#### Target
-
-You may want to compile a specific target so you will have to specify it on the
-
-command line:
-```bash
-kiwix-build libkiwix # will build kiwix-build and its dependencies
-kiwix-build kiwix-desktop # will build kiwix-desktop and its dependencies
-kiwix-build zim-tools # will build zim-tools and its dependencies
-```
-
-By default, `kiwix-build` will build `kiwix-tools` .
-
-To see the whole list of available targets run with non existing target, ex:
-
-```bash
-kiwix-build not-existing-target
-...
-invalid choice: 'not-existing-target' (choose from 'alldependencies', 'android-ndk',
-...
-```
-
-#### Config
-
-If no config is specified, the default will be `native_dyn`.
-
-You can select another config using the option
-`--config`. For now, there is ten different supported
-platforms:
-
-- native_dyn
-- native_mixed
-- native_static
-- android
-- android_arm
-- android_arm64
-- android_x86
-- android_x86_64
-- flatpak
-
-All `native_*` config means using the native compiler without any cross-compilation option.
-Other may simply use cross-compilation or may download a specific toolchain to use.
-
-Android
--------
-
-`kiwix-android` (https://github.com/kiwix/kiwix-android) depends of
-the `libkiwix` project.
-
-When building `libkiwix`, you should directly use the
-target-platform `android_<arch>`:
-```bash
-kiwix-build libkiwix --config android_arm
-```
-
-You may directly use the special config `android` which will build different android architectures
-```bash
-kiwix-build --config android libkiwix
-```
-
-By default, it will build for all android architecture,
-you can limit this with option `--android-arch`:
-```bash
-kiwix-build libkiwix --config android --android-arch arm # aar with arm architecture
-kiwix-build libkiwix --config android --android-arch arm --android-arch arm64 # aan with arm and arm64 architectures
-```
-
-To build `kiwix-android` itself, you should see the documentation of `kiwix-android`.
-
-iOS
----
-
-When building for ios, we may want to compile a "fat library", a library
-for several architectures.
-
-To do so, you should directly use the target-platfrom `ios_multi`.
-As for `android`, `kiwix-build` will build the library several times
-(once for each platform) and then create the fat library.
-```bash
-kiwix-build --config iOS_multi libkiwix
-```
-
-You can specify the supported architectures with the option `--ios-arch`:
-```bash
-kiwix-build --config iOS_multi libkiwix # all architetures
-kiwix-build --config iOS_multi --ios-arch arm --ios-arch arm64 # arm and arm64 arch only
-```
-
-Outputs
--------
-
-Kiwix-build.py will create several directories:
-- `ARCHIVES`: All the downloaded archives go there.
-- `SOURCES`: All the sources (extracted from archives and patched) go there.
-- `BUILD_<config>`: All the build files go there.
-- `BUILD_<config>/INSTALL`: The installed files go there.
-- `BUILD_<config>/LOGS`: The logs files of the build.
-
-If you want to install all those directories elsewhere, you can pass the
-`--working-dir` option to `kiwix-build`:
-
-Troubleshooting
----------------
-
-If you need to install [Meson](https://mesonbuild.com/) "manually":
-```bash
 virtualenv -p python3 ./ # Create virtualenv
 source bin/activate      # Activate the virtualenv
 pip3 install meson       # Install Meson
 hash -r                  # Refresh bash paths
 ```
 
-If you need to install [Ninja](https://ninja-build.org) "manually":
-```bash
-wget https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip
-unzip ninja-linux.zip ninja -d $HOME/bin
+Finally we need the Ninja tool (available in the $PATH). Here is a
+solution to download, build and install it locally:
+
+```
+git clone git://github.com/ninja-build/ninja.git
+cd ninja
+git checkout release
+./configure.py --bootstrap
+mkdir ../bin
+cp ninja ../bin
+cd ..
 ```
 
-License
--------
+# Compilation
 
-[GPLv3](https://www.gnu.org/licenses/gpl-3.0) or later, see
-[LICENSE](LICENSE) for more details.
+The compilation is handled by `kiwix-build.py`. It will compile
+everything. If you are using a supported platform (Redhat or Debian
+based) it will install missing packages using `sudo`. You can get
+`kiwix-build.py` usage like this:
+
+```
+./kiwix-build.py -h
+```
+
+## Target
+
+By default, `kiwix-build.py` will build `kiwix-tools`. If you want to
+compile another target only (let's said kiwixlib), you can specify it:
+
+```
+./kiwix-build Kiwixlib
+```
+
+## Target platform
+
+By default, `kiwix-build.py` will build everything for the current (native)
+platform using dynamic linkage (hence the `native_dyn` of the
+BUILD_native_dyn directory).
+
+But you can select another target platform using the option
+`target-platform`. For now, there is ten different supported
+platforms :
+
+- native_dyn
+- native_static
+- win32_dyn
+- win32_static
+- android_arm
+- android_arm64
+- android_mips
+- android_mips64
+- android_x86
+- android_x86_64
+
+So, if you want to compile for win32 using static linkage:
+
+```
+./kiwix-build.py --target-platform win32_dyn
+```
+
+# Outputs
+
+Kiwix-build.py will create several directories:
+- `ARCHIVES`: All the downloaded archives go there.
+- `SOURCES`: All the sources (extracted from archives and patched) go there.
+- `BUILD_native_dyn`: All the build files go there.
+- `BUILD_native_dyn/INSTALL`: The installed files go there.
+- `BUILD_native_dyn/LOGS`: The logs files of the build.
+
+If you want to install all those directories elsewhere, you can pass the
+`--working-dir` option to `kiwix-build.py`:

actions/dl_deps_archive/.gitignore

@@ -1 +0,0 @@
-node_modules

actions/dl_deps_archive/README.md

@@ -1,55 +0,0 @@
-# Download dependencies archive action
-
-This action download dependencies archive made by kiwix-build.
-It is intended to be used in projects using theses dependencies
-
-## Inputs
-
-### `base_url`
-
-The base url where we can download the archive.
-This input is provided for greater customization but you probably shouldn't set it.
-
-### `os_name`
-
-The os "name" on which the compilation is done.
-By default this use the `OS_NAME` env var, which is set in the docker file.
-
-
-### `target_platform`
-
-**Required** The targeted platform. Must be provided. Values are kind of :
-- native_dyn
-- android_arm
-- ...
-
-### `project`
-
-The name of the project being compiled.
-By default, the name of the repository.
-
-
-### `branch`
-
-The name of the "branch" to try to download (`/dev_preview/<branch>`).
-By default, the current branch on which the action is run.
-
-### `extract_dir`
-
-Where to extract the dependencies archive. By default it is `$HOME`
-
-
-## Example usage
-
-```yaml
-uses: kiwix/kiwix-build/actions/dl_deps_archive@main
-with:
-  target_platform: ${{ matrix.target_platform }}
-```
-
-```
-uses: kiwix/kiwix-build/actions/dl_deps_archive@main
-with:
-  target_platform: native_mixed
-  os_name: windows
-```

actions/dl_deps_archive/action.yml

@@ -1,25 +0,0 @@
-name: 'DL deps archive'
-description: 'Download the dependencies archive of the project'
-inputs:
-  base_url:  # id of input
-    description: 'What is the base url to download the archive'
-    required: false
-    default: 'https://tmp.kiwix.org/ci'
-  os_name:
-    description: "On which os are we running. [Default to env var `OS_NAME`]"
-    required: false
-  target_platform:
-    description: "What platform are we targetting"
-    required: true
-  project:
-    description: "What project are we building"
-    required: false
-  branch:
-    description: "On which branch are we ?"
-    required: false
-  extract_dir:
-    description: "Where to extract our dependencies. [Default to env var `HOME`]"
-    required: false
-runs:
-  using: 'node20'
-  main: 'dist/index.js'

actions/dl_deps_archive/index.js

@@ -1,59 +0,0 @@
-const tc = require("@actions/tool-cache");
-const core = require("@actions/core");
-const path = require("path");
-const os = require("os");
-
-function getInput(name, dflt) {
-  const val = process.env[`INPUT_${name.replace(/ /g, "_").toUpperCase()}`];
-  if (!val) {
-    return dflt;
-  }
-  return val;
-}
-
-function addLocalPath(inputPath) {
-  process.env["PATH"] = `${inputPath}${path.delimiter}${process.env["PATH"]}`;
-}
-
-async function run() {
-  try {
-    const base_url = core.getInput("base_url");
-    const target = core.getInput("target_platform");
-    const project = getInput(
-      "project",
-      process.env["GITHUB_REPOSITORY"].split("/")[1],
-    );
-    const branch = getInput(
-      "branch",
-      process.env["GITHUB_HEAD_REF"] || process.env["GITHUB_REF_NAME"],
-    );
-    const extract_dir = getInput(
-      "extract_dir",
-      process.env["HOME"] || process.env["GITHUB_WORKSPACE"],
-    );
-
-    let archivePath;
-    try {
-      const archive_url = `${base_url}/dev_preview/${branch}/deps_${target}_${project}.tar.gz`;
-      process.stdout.write("Downloading " + archive_url + "\n");
-      archivePath = await tc.downloadTool(archive_url);
-    } catch (error) {
-      const archive_url = `${base_url}/deps_${target}_${project}.tar.gz`;
-      process.stdout.write("Downloading " + archive_url + "\n");
-      archivePath = await tc.downloadTool(archive_url);
-    }
-
-    process.stdout.write("Extracting " + archivePath + " to " + extract_dir);
-    const archive_dir = await tc.extractTar(archivePath, extract_dir);
-    process.stdout.write("Extracted to " + archive_dir);
-  } catch (error) {
-    core.setFailed(error.message);
-  }
-}
-
-if (os.platform() === "win32") {
-  addLocalPath("C:\\Program Files\\Git\\usr\\bin");
-}
-core.setCommandEcho(true);
-
-run();

actions/dl_deps_archive/package-lock.json

@@ -1,144 +0,0 @@
-{
-  "name": "dl_deps_archive",
-  "version": "1.0.0",
-  "lockfileVersion": 3,
-  "requires": true,
-  "packages": {
-    "": {
-      "name": "dl_deps_archive",
-      "version": "1.0.0",
-      "license": "ISC",
-      "dependencies": {
-        "@actions/core": "^1.10.1",
-        "@actions/tool-cache": "^2.0.1",
-        "path": "^0.12.7"
-      }
-    },
-    "node_modules/@actions/core": {
-      "version": "1.10.1",
-      "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.10.1.tgz",
-      "integrity": "sha512-3lBR9EDAY+iYIpTnTIXmWcNbX3T2kCkAEQGIQx4NVQ0575nk2k3GRZDTPQG+vVtS2izSLmINlxXf0uLtnrTP+g==",
-      "dependencies": {
-        "@actions/http-client": "^2.0.1",
-        "uuid": "^8.3.2"
-      }
-    },
-    "node_modules/@actions/exec": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/@actions/exec/-/exec-1.1.1.tgz",
-      "integrity": "sha512-+sCcHHbVdk93a0XT19ECtO/gIXoxvdsgQLzb2fE2/5sIZmWQuluYyjPQtrtTHdU1YzTZ7bAPN4sITq2xi1679w==",
-      "dependencies": {
-        "@actions/io": "^1.0.1"
-      }
-    },
-    "node_modules/@actions/http-client": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.2.0.tgz",
-      "integrity": "sha512-q+epW0trjVUUHboliPb4UF9g2msf+w61b32tAkFEwL/IwP0DQWgbCMM0Hbe3e3WXSKz5VcUXbzJQgy8Hkra/Lg==",
-      "dependencies": {
-        "tunnel": "^0.0.6",
-        "undici": "^5.25.4"
-      }
-    },
-    "node_modules/@actions/io": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/@actions/io/-/io-1.1.3.tgz",
-      "integrity": "sha512-wi9JjgKLYS7U/z8PPbco+PvTb/nRWjeoFlJ1Qer83k/3C5PHQi28hiVdeE2kHXmIL99mQFawx8qt/JPjZilJ8Q=="
-    },
-    "node_modules/@actions/tool-cache": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/@actions/tool-cache/-/tool-cache-2.0.1.tgz",
-      "integrity": "sha512-iPU+mNwrbA8jodY8eyo/0S/QqCKDajiR8OxWTnSk/SnYg0sj8Hp4QcUEVC1YFpHWXtrfbQrE13Jz4k4HXJQKcA==",
-      "dependencies": {
-        "@actions/core": "^1.2.6",
-        "@actions/exec": "^1.0.0",
-        "@actions/http-client": "^2.0.1",
-        "@actions/io": "^1.1.1",
-        "semver": "^6.1.0",
-        "uuid": "^3.3.2"
-      }
-    },
-    "node_modules/@actions/tool-cache/node_modules/uuid": {
-      "version": "3.4.0",
-      "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
-      "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==",
-      "deprecated": "Please upgrade  to version 7 or higher.  Older versions may use Math.random() in certain circumstances, which is known to be problematic.  See https://v8.dev/blog/math-random for details.",
-      "bin": {
-        "uuid": "bin/uuid"
-      }
-    },
-    "node_modules/@fastify/busboy": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.0.tgz",
-      "integrity": "sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==",
-      "engines": {
-        "node": ">=14"
-      }
-    },
-    "node_modules/inherits": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
-      "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw=="
-    },
-    "node_modules/path": {
-      "version": "0.12.7",
-      "resolved": "https://registry.npmjs.org/path/-/path-0.12.7.tgz",
-      "integrity": "sha512-aXXC6s+1w7otVF9UletFkFcDsJeO7lSZBPUQhtb5O0xJe8LtYhj/GxldoL09bBj9+ZmE2hNoHqQSFMN5fikh4Q==",
-      "dependencies": {
-        "process": "^0.11.1",
-        "util": "^0.10.3"
-      }
-    },
-    "node_modules/process": {
-      "version": "0.11.10",
-      "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz",
-      "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==",
-      "engines": {
-        "node": ">= 0.6.0"
-      }
-    },
-    "node_modules/semver": {
-      "version": "6.3.1",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
-      "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
-      "bin": {
-        "semver": "bin/semver.js"
-      }
-    },
-    "node_modules/tunnel": {
-      "version": "0.0.6",
-      "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz",
-      "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==",
-      "engines": {
-        "node": ">=0.6.11 <=0.7.0 || >=0.7.3"
-      }
-    },
-    "node_modules/undici": {
-      "version": "5.28.4",
-      "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.4.tgz",
-      "integrity": "sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g==",
-      "dependencies": {
-        "@fastify/busboy": "^2.0.0"
-      },
-      "engines": {
-        "node": ">=14.0"
-      }
-    },
-    "node_modules/util": {
-      "version": "0.10.4",
-      "resolved": "https://registry.npmjs.org/util/-/util-0.10.4.tgz",
-      "integrity": "sha512-0Pm9hTQ3se5ll1XihRic3FDIku70C+iHUdT/W926rSgHV5QgXsYbKZN8MSC3tJtSkhuROzvsQjAaFENRXr+19A==",
-      "dependencies": {
-        "inherits": "2.0.3"
-      }
-    },
-    "node_modules/uuid": {
-      "version": "8.3.2",
-      "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
-      "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
-      "bin": {
-        "uuid": "dist/bin/uuid"
-      }
-    }
-  }
-}

actions/dl_deps_archive/package.json

@@ -1,17 +0,0 @@
-{
-  "name": "dl_deps_archive",
-  "version": "1.0.0",
-  "description": "",
-  "main": "index.js",
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
-  },
-  "keywords": [],
-  "author": "",
-  "license": "ISC",
-  "dependencies": {
-    "@actions/core": "^1.10.1",
-    "@actions/tool-cache": "^2.0.1",
-    "path": "^0.12.7"
-  }
-}

build_custom_app.pl

@@ -0,0 +1,97 @@
+#!/usr/bin/perl
+
+use utf8;
+use strict;
+use warnings;
+use Getopt::Long;
+
+# get the params
+my $zim_url;
+my $custom_app;
+my $keystore;
+my $api_key;
+my $version = "0";
+my $cmd;
+
+# Get console line arguments
+GetOptions('zim_url=s' => \$zim_url,
+	   'custom_app=s' => \$custom_app,
+	   'keystore=s' => \$keystore,
+	   'api_key=s' => \$api_key,
+	   'version=s' => \$version
+	   );
+
+# Print usage() if necessary
+if (!$zim_url || !$custom_app || !$keystore || !$api_key) {
+    print "usage: ./build_custom_app.pl --keystore=kiwix-android.keystore --api_key=google.json --zim_url=\"https://download.kiwix.org/zim/wikipedia_en_medicine_novid.zim\" --custom_app=wikimed [--version=1]\n";
+    exit;
+}
+
+# Clean signed ap
+$cmd = "rm ./signed_apks/*apk"; `$cmd`;
+
+# Download ZIM file
+$cmd = "wget \"$zim_url\" -O content.zim"; `$cmd`;
+
+# Get ZIM file size
+$cmd = "stat -c %s content.zim";
+my $zim_size = `$cmd` =~ s/\n//gr ;
+$ENV{ZIM_SIZE} = $zim_size;
+
+# Compute version code base
+$cmd = "date +%y%j";
+my $version_code_base = `$cmd` =~ s/\n//gr . $version;
+
+# Compute content version code
+my $content_version_code = $version_code_base;
+$ENV{CONTENT_VERSION_CODE} = $content_version_code;
+
+# Compute custom app date
+$cmd = "date +%Y-%m";
+my $date = `$cmd` =~ s/\n//gr;
+$ENV{VERSION_NAME} = $date;
+
+# Compile apps
+$ENV{VERSION_CODE} = $version_code_base;
+$cmd = "./kiwix-build.py --target-platform android_arm --android-custom-app $custom_app --zim-file-size $zim_size kiwix-android-custom"; system $cmd;
+
+$ENV{VERSION_CODE} = "1" . $version_code_base;
+$cmd = "./kiwix-build.py --target-platform android_arm64 --android-custom-app $custom_app --zim-file-size $zim_size kiwix-android-custom"; system $cmd;
+
+$ENV{VERSION_CODE} = "2" . $version_code_base;
+$cmd = "./kiwix-build.py --target-platform android_x86 --android-custom-app $custom_app --zim-file-size $zim_size kiwix-android-custom"; system $cmd;
+
+$ENV{VERSION_CODE} = "3" . $version_code_base;
+$cmd = "./kiwix-build.py --target-platform android_x86_64 --android-custom-app $custom_app --zim-file-size $zim_size kiwix-android-custom"; system $cmd;
+
+$ENV{VERSION_CODE} = "4" . $version_code_base;
+$cmd = "./kiwix-build.py --target-platform android_mips --android-custom-app $custom_app --zim-file-size $zim_size kiwix-android-custom"; system $cmd;
+
+$ENV{VERSION_CODE} = "5" . $version_code_base;
+$cmd = "./kiwix-build.py --target-platform android_mips64 --android-custom-app $custom_app --zim-file-size $zim_size kiwix-android-custom"; system $cmd;
+
+# Sign apps
+$cmd = "./TOOLCHAINS/android-sdk-r25.2.3/build-tools/25.0.2/apksigner sign -ks \"${keystore}\" --out signed_apks/app-${version_code_base}-release-signed.apk BUILD_android_arm/kiwix-android-custom_${custom_app}/app/build/outputs/apk/${custom_app}/release/app-${custom_app}-release-unsigned.apk";
+system $cmd;
+
+$cmd = "./TOOLCHAINS/android-sdk-r25.2.3/build-tools/25.0.2/apksigner sign -ks \"${keystore}\" --out signed_apks/app-1${version_code_base}-release-signed.apk BUILD_android_arm64/kiwix-android-custom_${custom_app}/app/build/outputs/apk/${custom_app}/release/app-${custom_app}-release-unsigned.apk";
+system $cmd;
+
+$cmd = "./TOOLCHAINS/android-sdk-r25.2.3/build-tools/25.0.2/apksigner sign -ks \"${keystore}\" --out signed_apks/app-2${version_code_base}-release-signed.apk BUILD_android_x86/kiwix-android-custom_${custom_app}/app/build/outputs/apk/${custom_app}/release/app-${custom_app}-release-unsigned.apk";
+system $cmd;
+
+$cmd = "./TOOLCHAINS/android-sdk-r25.2.3/build-tools/25.0.2/apksigner sign -ks \"${keystore}\" --out signed_apks/app-3${version_code_base}-release-signed.apk BUILD_android_x86_64/kiwix-android-custom_${custom_app}/app/build/outputs/apk/${custom_app}/release/app-${custom_app}-release-unsigned.apk";
+system $cmd;
+
+$cmd = "./TOOLCHAINS/android-sdk-r25.2.3/build-tools/25.0.2/apksigner sign -ks \"${keystore}\" --out signed_apks/app-4${version_code_base}-release-signed.apk BUILD_android_mips/kiwix-android-custom_${custom_app}/app/build/outputs/apk/${custom_app}/release/app-${custom_app}-release-unsigned.apk";
+system $cmd;
+
+$cmd = "./TOOLCHAINS/android-sdk-r25.2.3/build-tools/25.0.2/apksigner sign -ks \"${keystore}\" --out signed_apks/app-5${version_code_base}-release-signed.apk BUILD_android_mips64/kiwix-android-custom_${custom_app}/app/build/outputs/apk/${custom_app}/release/app-${custom_app}-release-unsigned.apk";
+system $cmd;
+
+# Upload
+
+$cmd = "./build_custom_app.py --step publish --custom-app ${custom_app} --google-api-key ${api_key} --zim-path content.zim --apks-dir signed_apks --content-version-code ${content_version_code}";
+system $cmd;
+
+exit;

build_custom_app.py

@@ -0,0 +1,430 @@
+#!/usr/bin/env python3
+
+import \
+    argparse, \
+    datetime, \
+    glob, \
+    json, \
+    os, \
+    ssl, \
+    sys, \
+    tempfile, \
+    time, \
+    urllib
+from uuid import uuid4
+from contextlib import contextmanager
+from urllib.parse import urlparse
+
+from utils import (
+    Remotefile,
+    download_remote
+)
+
+import requests
+import httplib2
+from apiclient.discovery import build
+from apiclient.errors import HttpError
+from oauth2client import client
+from oauth2client.service_account import ServiceAccountCredentials
+
+tmpl_request_url = "https://api.travis-ci.org/repo/{organisation}%2F{repository}/{endpoint}"
+tmpl_message = """Build of custom app {app} with zim file {zim}.
+
+UUID:#{uuid}#"""
+
+description = """Launch a custom application build.
+This command will launch a custom application build on Travis-CI.
+Travis-CI jobs will compile the application and upload the build apks on
+google play, using tha 'alpha' track of the application.
+
+You will need to have a valid TRAVIS_TOKEN associated to your personnal account
+and the kiwix-build repository on travis.
+Use the 'travis' command line tool (https://github.com/travis-ci/travis.rb)
+to generate a token.
+"""
+
+def parse_args():
+    parser = argparse.ArgumentParser(description=description,
+        formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument('--custom-app')
+    parser.add_argument('--travis-token')
+
+    advance = parser.add_argument_group('advance', "Some advanced options.")
+    advance.add_argument('--extra-code', type=int, default=0)
+    advance.add_argument('--version-name', default=None,
+                         help="The version of the application (seen by user). Get from json info file by default.")
+    advance.add_argument('--check-certificate', default=True)
+    advance.add_argument('--zim-url', default=None, help="Get from json info file by default.")
+    advance.add_argument('--no-android-upload', action='store_false', dest='android_upload')
+
+    # Hidden options
+    parser.add_argument('--step', default='launch', choices=['launch', 'publish'], help=argparse.SUPPRESS)
+    parser.add_argument('--apks-dir', help=argparse.SUPPRESS)
+    parser.add_argument('--zim-path', default=None, help=argparse.SUPPRESS)
+    parser.add_argument('--content-version-code', type=int, help=argparse.SUPPRESS)
+    parser.add_argument('--package-name', default=None, help=argparse.SUPPRESS)
+    parser.add_argument('--google-api-key', help=argparse.SUPPRESS)
+
+    options = parser.parse_args()
+
+    if (not options.package_name
+     or not (options.zim_url or options.zim_path)
+     or not options.version_name):
+        if not options.package_name:
+            print("Try to get package name from info.json file")
+        if not options.zim_url:
+            print("Try to get zim url from info.json file")
+        if not options.version_name:
+            print("Try to get version_name form info.json file")
+        request_url = ('https://raw.githubusercontent.com/kiwix/kiwix-android-custom/master/{}/info.json'
+                      .format(options.custom_app))
+        json_request = requests.get(request_url)
+        if json_request.status_code != 200:
+            print("Error while getting json file.")
+            print("Reason is '{}'".format(json_request.reason))
+            sys.exit(-1)
+        json_data = json.loads(json_request.text)
+        if not options.package_name:
+            print("Found package_name '{}'".format(json_data['package']))
+            options.package_name = json_data['package']
+        if not options.zim_url:
+            print("Found zim_url '{}'".format(json_data['zim_url']))
+            options.zim_url = json_data['zim_url']
+        if not options.version_name:
+            print("Found version_name '{}'".format(json_data['version_name']))
+            options.version_name = json_data['version_name']
+
+    options.base_version = "{}{}".format(
+        datetime.date.today().strftime('%y%j'),
+        options.extra_code)
+
+    return options
+
+
+def download_zim_file(zim_url, dest_dir=None):
+    if dest_dir is None:
+        dest_dir = os.getcwd()
+    out_filename = urlparse(zim_url).path
+    out_filename = os.path.basename(out_filename)
+    zim_file = Remotefile(out_filename, '', zim_url)
+    download_remote(zim_file, dest_dir)
+    return os.path.join(dest_dir, out_filename)
+
+
+def get_zim_size(*, zim_url=None, zim_path=None, check_certificate=True):
+    print("Try to get zim size")
+    if not zim_path:
+        if not check_certificate:
+            context = ssl.create_default_context()
+            context.check_hostname = False
+            context.verify_mode = ssl.CERT_NONE
+        else:
+            context = None
+        extra_args = {'context':context} if sys.version_info >= (3, 4, 3) else {}
+        with urllib.request.urlopen(zim_url, **extra_args) as resource:
+            size = resource.getheader('Content-Length', None)
+        if size is not None:
+            size = int(size)
+            print("Zim size is {}".format(size))
+            return size
+        else:
+            print("No 'Content-Length' header in http answer from the server.\n"
+                  "We need to download the zim file to get its size.")
+            zim_path = download_zim_file(zim_url, tempfile.gettempdir())
+
+    size = os.path.getsize(zim_path)
+    print("Zim size is {}".format(size))
+    return size
+
+
+def do_launch(options):
+    if options.zim_path:
+        zim_size = get_zim_size(zim_path=options.zim_path)
+    else:
+        zim_size = get_zim_size(zim_url=options.zim_url,
+                                check_certificate=options.check_certificate)
+    travis_launch_build('kiwix', 'kiwix-build', options, zim_size)
+    print("Travis build has been launch.")
+
+
+def do_publish(options):
+    zim_path = options.zim_path or download_zim_file(options.zim_url)
+    googleService = Google(options)
+    with googleService.new_request():
+        versionCodes = []
+        for apk in glob.iglob(options.apks_dir+"/*.apk"):
+            result = googleService.upload_apk(apk)
+            versionCodes.append(result['versionCode'])
+        googleService.upload_expansionfile(
+            zim_path, options.content_version_code, versionCodes)
+        googleService.publish_release(options, versionCodes)
+
+
+def travis_launch_build(organisation, repository, options, zim_size):
+    request_url = tmpl_request_url.format(
+        organisation=organisation,
+        repository=repository,
+        endpoint='requests')
+    headers = {
+        "Travis-API-Version": "3",
+        "Content-Type": "application/json",
+        "Authorization": "token {}".format(options.travis_token),
+        "User-Agent": "kiwix-build"
+    }
+    uuid = uuid4()
+    envs = []
+    for platform_index, platform in enumerate(['arm', 'arm64', 'x86', 'x86_64', 'mips', 'mips64']):
+        d = {
+            'PLATFORM': "android_{}".format(platform),
+            'VERSION_CODE': gen_version_code(platform_index, options.base_version)
+        }
+        envs.append(d)
+
+    global_env = [
+        { 'CUSTOM_APP': options.custom_app},
+        { 'ZIM_FILE_SIZE': zim_size},
+        { 'PACKAGE_NAME': options.package_name},
+        { 'ZIM_URL': options.zim_url},
+        { 'EXTRA_CODE': options.extra_code},
+        { 'CONTENT_VERSION_CODE': gen_version_code(0, options.base_version)},
+        { 'VERSION_NAME': options.version_name},
+        # google_key
+        { 'secure': ('VAgKBMx0KEIyJlSnpM4YrHKLALIbaibkhlsgiv19ITa6dODoEIqeYHz'
+                     'wFTiL3mRHU6HwtXtdNb/JeMle9NfHJVFSV56ZgFzX7ev9zr0YG0qZQv'
+                     'tl8vHQlFPBErARder/L2tblOTM194/TiJk/q89a0XWDanKswXExwjcW'
+                     'Z0tnDYQXTHSAKEt+xW8hjbnhqqB/v16lX6dUjZI+sVlsw+qAM4VT/qf'
+                     'FCyDO5eJCzWIEL2LDUWI7jKSETNih5hl5fMMvCCNRPnkgGnytw5kF/t'
+                     'Lw8YAbLRxkGsO4FCx5mB7HF5pNHyWOCCalMTKheyg/qUV/VcXW9Unlr'
+                     'puMu0+d3hpLZESplS/NkvDxSrx16ank7EORS8OxLOufu56TW2hDuBzz'
+                     'w1CBAj1p6s+Z6Kc4RMYYdxRgR1TjXg/ZVUn3T69d9igdS/5lAPFx2Ww'
+                     '8x82FWCLSaiXymxXRNsNcKx5ifuvtv307r4yh31QjlKFYwadOCaCHHZ'
+                     'zGE1mXcOu3j6W9WIaZfYRTpxmOrcfDIHxZSdLf11hOSZEUUFpj9hQlV'
+                     'Op0RHkDEJUMNs6vkXUhZq9yPuqgrcb6GaN+UhOT8iHlijKmlG8NJEPk'
+                     'Hp8RnL1hsr44N57ZzLqmSUZtvC83u/5e+YUb7beUDGsMyJV/fcMiGMM'
+                     'LVtRnuPCFyNVNQUf2CphtG0=') },
+        # pass
+        { 'secure': ('AtbgKUukES2uJPpEWNEDHLg0WcghLlCGL171Ah3+4CckBI8y1Fn+VpH'
+                     'U2vEXzsV8tKoxX1IyB2tFivzuyo6CQXHSuWGJYQexwkBeGCgOfzKJLj'
+                     'MAy75ATYA6JnFrikV+UcqdEz/9Dow3J1K7Slp3jpsQhERHbNeqkr4I+'
+                     'XCL1LLnpewfOZo9OIEu93p6b6YlqvIPXJHyQe5xnMd8jFWg3/uIYqFn'
+                     'XPvigeZqC2lhNp48mj4JdwwF2tmiArgyXOmgxiuHJNVVI7okbhc7kmI'
+                     'Y3MmCSFgG0XPUEBU3Kdr4o/2hy8DDP6Gff+rUZW8nPI/2UWXRLWtOxv'
+                     'XGGRyjHHTxGWzI4JyZbli9dls5M32MMjsXVKtciSFVwsMM8qn7wFnRi'
+                     'q248a1Sg5fDNX/WYowmsHlWjffHZ7+UqUqJxAKtZ0vpQL+4SPIALPnK'
+                     'V6j6CoorQp+VhMF01EFlZ0c8bkNmk4YW7R7RyNLIcaHKfd1ud8QF9PD'
+                     'AnQ7Jr1GRBxzkjHvHfFrE14WPUu+FjVvDO7UPVMNQX7RS1IVACpKSRu'
+                     '7N8KnIK3vSnLpn5GXKsbx0JB2vtyoTaFZvC9c3qyAw1nlpn7Lp3sPs3'
+                     'bgIBU4tWOzg5g46eHbc4ad5nyB9Soz715lbMdECvKs2HHJUG3tubLKj'
+                     'L0S/LRGRQ+IDgC7xrjQj8aA=') },
+    ]
+
+    env = {
+        'matrix': envs,
+        'global': global_env
+    }
+
+    data = {
+        'request': {
+            'message' : tmpl_message.format(app=options.custom_app, zim=options.zim_url, uuid=uuid),
+            'branch'  : "custom_app",
+            'config'  : {
+                'before_install' : [
+                    ( 'pip3 install pyOpenSSl google-api-python-client'
+                      ' httplib2 apiclient requests'),
+                    ( 'openssl aes-256-cbc -k $google_key'
+                      ' -in travis/googleplay_android_developer-5a411156212c.json.enc'
+                      ' -out travis/googleplay_android_developer-5a411156212c.json'
+                      ' -d'),
+                    ( 'openssl aes-256-cbc -k $google_key'
+                      ' -in travis/kiwix-android.keystore.enc'
+                      ' -out travis/kiwix-android.keystore -d'),
+                    ( 'openssl aes-256-cbc -K $encrypted_eba2f7543984_key'
+                      ' -iv $encrypted_eba2f7543984_iv'
+                      ' -in travis/travisci_builder_id_key.enc'
+                      ' -out travis/travisci_builder_id_key -d'),
+                    'chmod 600 travis/travisci_builder_id_key'
+                ],
+                'env' : env,
+                'script' : 'travis_wait 30 travis/compile_custom_app.sh',
+                'deploy' : {
+                     'provider': 'script',
+                     'skip_cleanup': True,
+                     'script': 'travis/deploy_apk.sh',
+                     'on': {
+                         'branch': 'custom_app'
+                     }
+                }
+            }
+        }
+    }
+
+    if options.android_upload:
+        data['request']['config']['jobs'] = {
+            'include': [
+                {
+                    'stage' : 'make_release',
+                    'install': 'pip3 install -r requirements_build_custom_app.txt',
+                    'script': True,
+                    'env': global_env,
+                    'deploy' : {
+                        'provider': 'script',
+                        'skip_cleanup': True,
+                        'script': 'travis/make_release.sh',
+                        'on': {
+                            'branch': 'custom_app'
+                        }
+                    }
+                }
+            ]
+        }
+        global_env.append({
+            'DEPLOY_DIR' : '/home/nightlybot/apks/{}_{}'.format(
+                options.custom_app, options.base_version)
+        })
+    else:
+        global_env.append({
+            'DEPLOY_DIR' : '/var/www/tmp.kiwix.org/custom_apps/{}_{}'.format(
+                options.custom_app, options.base_version)
+        })
+
+
+    r = requests.post(request_url, headers=headers, json=data)
+    if r.status_code != 202:
+        print("Error while requesting build:")
+        print(r.reason)
+        print("Have you forget to give the travis token ?")
+        sys.exit(-1)
+    else:
+        request_id = r.json()['request']['id']
+        print("Request {} has been schedule.".format(request_id))
+        request_left = 10
+        found = False
+        request_url = tmpl_request_url.format(
+            organisation=organisation,
+            repository=repository,
+            endpoint='builds')
+        while request_left:
+            time.sleep(1)
+            print("Try to get associated build.")
+            r = requests.get(request_url, headers=headers)
+            json_data = json.loads(r.text)
+            for build in json_data['builds']:
+                if build['event_type'] != 'api':
+                    continue
+                message = build['commit']['message']
+                if str(uuid) in message:
+                    found = True
+                    break
+            if found:
+                break
+            print("Cannot find build. Wait 1 second and try again")
+            print("{} tries left".format(request_left))
+            request_left -= 1
+        if found:
+            print("Associated build found:  {}.".format(build['number']))
+            print("https://travis-ci.org/kiwix/kiwix-build/builds/{}".format(build['id']))
+        else:
+            print("Request has been accepted by travis-ci but I cannot found "
+                  "the associated build. Have a look here "
+                  "https://travis-ci.org/kiwix/kiwix-build/builds"
+                  "if you found it.")
+        if not options.android_upload:
+            print(("Automatic upload to android play store has been deactivated.\n"
+                   "You will find the apks at this address once they have been compiled :"
+                   " http://tmp.kiwix.org/custom_apps/{}_{}").format(
+                       options.custom_app, options.base_version))
+
+
+ERROR_MSG_EDIT_CHANGE = "A change was made to the application outside of this Edit, please create a new edit."
+
+class Google:
+    def __init__(self, options):
+        scope = 'https://www.googleapis.com/auth/androidpublisher'
+        key = options.google_api_key
+        credentials = ServiceAccountCredentials.from_json_keyfile_name(
+            key,
+            scopes=[scope])
+
+        http = httplib2.Http()
+        http = credentials.authorize(http)
+
+        self.service = build('androidpublisher', 'v2', http=http)
+        self.packageName = options.package_name
+        self.edit_id = None
+
+    @contextmanager
+    def new_request(self):
+        edit_request = self.service.edits().insert(
+            packageName=self.packageName,
+            body={})
+        result = edit_request.execute()
+        print("create", result)
+        self.edit_id = result['id']
+
+        yield
+
+        commit_request = self.service.edits().commit(
+            editId=self.edit_id,
+            packageName=self.packageName)
+        result = commit_request.execute()
+        print("commit", result)
+
+        self.edit_id = None
+
+    def make_request(self, section, method, **kwargs):
+        request_content = self._build_request_content(kwargs)
+        _section = getattr(self._edits, section)()
+        _method = getattr(_section, method)
+        print(">", request_content)
+        request = _method(**request_content)
+        result = request.execute()
+        print("<", result)
+        return result
+
+    def _build_request_content(self, kwargs):
+        d = kwargs.copy()
+        d['editId'] = self.edit_id
+        d['packageName'] = self.packageName
+        return d
+
+    @property
+    def _edits(self):
+        return self.service.edits()
+
+    def upload_expansionfile(self, comp_file, contentVersionCode, versionCodes):
+        versionCodes = [int(v) for v in versionCodes]
+        self.make_request('expansionfiles', 'upload',
+            expansionFileType='main',
+            apkVersionCode=contentVersionCode,
+            media_body=comp_file,
+            media_mime_type='application/octet-stream')
+        for versionCode in versionCodes:
+            if versionCode == contentVersionCode:
+                continue
+            self.make_request('expansionfiles', 'update',
+                expansionFileType='main',
+                apkVersionCode=versionCode,
+                body={'referencesVersion': contentVersionCode}
+            )
+
+    def upload_apk(self, apk_file):
+        return self.make_request('apks', 'upload',
+            media_body=apk_file)
+
+    def publish_release(self, options, versionCodes):
+        return self.make_request('tracks', 'update',
+            track="alpha",
+            body={'versionCodes': versionCodes})
+
+
+def gen_version_code(platform_index, base_version):
+    str_version = "{platform}{base_version}".format(
+        platform=platform_index,
+        base_version=base_version
+    )
+    return int(str_version)
+
+if __name__ == "__main__":
+    options = parse_args()
+    func = globals()['do_{}'.format(options.step)]
+    func(options)

dependencies.py

@@ -0,0 +1,547 @@
+import shutil, os, json
+from urllib.parse import urlparse
+
+from dependency_utils import (
+    Dependency,
+    ReleaseDownload,
+    GitClone,
+    SvnClone,
+    NoopSource,
+    MakeBuilder,
+    CMakeBuilder,
+    MesonBuilder,
+    GradleBuilder,
+    NoopBuilder,
+    Builder as BaseBuilder)
+
+from utils import Remotefile, pj, SkipCommand, copy_tree, add_execution_right
+
+# *************************************
+# Missing dependencies
+# Is this ok to assume that those libs
+# exist in your "distri" (linux/mac) ?
+# If not, we need to compile them here
+# *************************************
+# aria2
+# Argtable
+# MSVirtual
+# Android
+# libiconv
+# gettext
+# *************************************
+
+
+class zlib(Dependency):
+    name = 'zlib'
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('zlib-1.2.8.tar.gz',
+                             '36658cb768a54c1d4dec43c3116c27ed893e88b02ecfcb44f2166f9c0b7f2a0d')
+        patches = ['zlib_std_libname.patch']
+
+    class Builder(MakeBuilder):
+        dynamic_configure_option = "--shared"
+        static_configure_option = "--static"
+
+        def _pre_build_script(self, context):
+            context.try_skip(self.build_path)
+            shutil.copytree(self.source_path, self.build_path)
+
+        @property
+        def all_configure_option(self):
+            return '--static' if self.buildEnv.platform_info.static else '--shared'
+
+        @property
+        def configure_option(self):
+            options = "-DINSTALL_PKGCONFIG_DIR={}".format(pj(self.buildEnv.install_dir, self.buildEnv.libprefix, 'pkgconfig'))
+            if self.buildEnv.platform_info.static:
+                options += " -DBUILD_SHARED_LIBS=false"
+            else:
+                options += " -DBUILD_SHARED_LIBS=true"
+            return options
+
+        def _configure(self, context):
+            if self.buildEnv.platform_info.build == 'win32':
+                raise SkipCommand()
+            return super()._configure(context)
+
+        @property
+        def make_option(self):
+            if self.buildEnv.platform_info.build == 'win32':
+                return "--makefile win32/Makefile.gcc PREFIX={host}- SHARED_MODE={static} INCLUDE_PATH={include_path} LIBRARY_PATH={library_path} BINARY_PATH={binary_path}".format(
+                    host='i686-w64-mingw32',
+                    static="0" if self.buildEnv.platform_info.static else "1",
+                    include_path=pj(self.buildEnv.install_dir, 'include'),
+                    library_path=pj(self.buildEnv.install_dir, self.buildEnv.libprefix),
+                    binary_path=pj(self.buildEnv.install_dir, 'bin'),
+                    )
+            return ""
+
+class lzma(Dependency):
+    name = 'lzma'
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('xz-5.0.4.tar.bz2',
+                             '5cd9b060d3a1ad396b3be52c9b9311046a1c369e6062aea752658c435629ce92')
+
+    class Builder(MakeBuilder):
+        @property
+        def configure_option(self):
+            return "--disable-assembler"
+
+class UUID(Dependency):
+    name = 'uuid'
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('e2fsprogs-libs-1.43.4.tar.gz',
+                             'eed4516325768255c9745e7b82c9d7d0393abce302520a5b2cde693204b0e419',
+                             'https://www.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v1.43.4/e2fsprogs-libs-1.43.4.tar.gz')
+        extract_dir = 'e2fsprogs-libs-1.43.4'
+
+    class Builder(MakeBuilder):
+        configure_option = ("--enable-libuuid --disable-debugfs --disable-imager --disable-resizer --disable-defrag --enable-fsck"
+                            " --disable-uuidd")
+        configure_env = {'_format_CFLAGS': "{env.CFLAGS} -fPIC"}
+        static_configure_option = dynamic_configure_option = ""
+        make_target = 'libs'
+        make_install_target = 'install-libs'
+
+
+class Xapian(Dependency):
+    name = "xapian-core"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('xapian-core-1.4.5.tar.xz',
+                             '85b5f952de9df925fd13e00f6e82484162fd506d38745613a50b0a2064c6b02b')
+
+    class Builder(MakeBuilder):
+        configure_option = "--disable-sse --disable-backend-inmemory --disable-documentation"
+        configure_env = {'_format_LDFLAGS': "-L{buildEnv.install_dir}/{buildEnv.libprefix}",
+                         '_format_CXXFLAGS': "-I{buildEnv.install_dir}/include"}
+
+    @property
+    def dependencies(self):
+        deps = ['zlib', 'lzma']
+        if (self.buildEnv.platform_info.build == 'win32'
+         or self.buildEnv.distname == 'Darwin'):
+            return deps
+        return deps + ['uuid']
+
+
+class CTPP2(Dependency):
+    name = "ctpp2"
+
+    class Source(ReleaseDownload):
+        name = "ctpp2"
+        source_dir = "ctpp2-2.8.3"
+        archive = Remotefile('ctpp2-2.8.3.tar.gz',
+                             'a83ffd07817adb575295ef40fbf759892512e5a63059c520f9062d9ab8fb42fc')
+        patches = ["ctpp2_include.patch",
+                   "ctpp2_no_src_modification.patch",
+                   "ctpp2_fix-static-libname.patch",
+                   "ctpp2_mingw32.patch",
+                   "ctpp2_dll_export_VMExecutable.patch",
+                   "ctpp2_win_install_lib_in_lib_dir.patch",
+                   "ctpp2_iconv_support.patch",
+                   "ctpp2_compile_ctpp2c_static.patch",
+                  ]
+
+    class Builder(CMakeBuilder):
+        configure_option = "-DMD5_SUPPORT=OFF"
+
+
+class CTPP2C(CTPP2):
+    name = "ctpp2c"
+    force_native_build = True
+
+    class Builder(CTPP2.Builder):
+        make_target = "ctpp2c"
+
+        @property
+        def build_path(self):
+            return super().build_path+"_native"
+
+        def _install(self, context):
+            context.try_skip(self.build_path)
+            command = "cp {ctpp2c}* {install_dir}".format(
+                ctpp2c=pj(self.build_path, 'ctpp2c'),
+                install_dir=pj(self.buildEnv.install_dir, 'bin')
+            )
+            self.buildEnv.run_command(command, self.build_path, context)
+
+
+class Pugixml(Dependency):
+    name = "pugixml"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('pugixml-1.2.tar.gz',
+                             '0f422dad86da0a2e56a37fb2a88376aae6e931f22cc8b956978460c9db06136b')
+        patches = ["pugixml_meson.patch"]
+
+    Builder = MesonBuilder
+
+
+class MicroHttpd(Dependency):
+    name = "libmicrohttpd"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('libmicrohttpd-0.9.46.tar.gz',
+                             '06dbd2654f390fa1e8196fe063fc1449a6c2ed65a38199a49bf29ad8a93b8979',
+                             'http://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.46.tar.gz')
+
+    class Builder(MakeBuilder):
+        configure_option = "--disable-https --without-libgcrypt --without-libcurl"
+
+
+class Gumbo(Dependency):
+    name = "gumbo"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('gumbo-0.10.1.tar.gz',
+                             '28463053d44a5dfbc4b77bcf49c8cee119338ffa636cc17fc3378421d714efad',
+                             'https://github.com/google/gumbo-parser/archive/v0.10.1.tar.gz')
+
+        def _post_prepare_script(self, context):
+            context.try_skip(self.extract_path)
+            command = "./autogen.sh"
+            self.buildEnv.run_command(command, self.extract_path, context)
+
+    Builder = MakeBuilder
+
+
+class Icu(Dependency):
+    name = "icu4c"
+
+    class Source(SvnClone):
+        name = "icu4c"
+        svn_remote = "http://source.icu-project.org/repos/icu/tags/release-58-2/icu4c"
+        svn_dir = "icu4c"
+
+        patches = ["icu4c_fix_static_lib_name_mingw.patch",
+                   "icu4c_android_elf64_st_info.patch",
+                   "icu4c_custom_data.patch",
+                   "icu4c_noxlocale.patch"]
+
+
+    class Builder(MakeBuilder):
+        subsource_dir = "source"
+
+        @property
+        def configure_option(self):
+            options = "--disable-samples --disable-tests --disable-extras --disable-dyload"
+            if self.buildEnv.platform_info.build == 'android':
+                options += " --with-data-packaging=archive"
+            return options
+
+
+class Icu_native(Icu):
+    name = "icu4c_native"
+    force_native_build = True
+
+    class Builder(Icu.Builder):
+        @property
+        def build_path(self):
+            return super().build_path+"_native"
+
+        def _install(self, context):
+            raise SkipCommand()
+
+
+class Icu_cross_compile(Icu):
+    name = "icu4c_cross-compile"
+    dependencies = ['icu4c_native']
+
+    class Builder(Icu.Builder):
+        @property
+        def configure_option(self):
+            icu_native_dep = self.buildEnv.targetsDict['icu4c_native']
+            return super().configure_option + " --with-cross-build=" + icu_native_dep.builder.build_path
+
+
+class Libzim(Dependency):
+    name = "libzim"
+
+    @property
+    def dependencies(self):
+        base_dependencies = ['zlib', 'lzma', 'xapian-core']
+        if self.buildEnv.platform_info.build != 'native':
+            return base_dependencies + ["icu4c_cross-compile"]
+        else:
+            return base_dependencies + ["icu4c"]
+
+    class Source(GitClone):
+        git_remote = "https://github.com/openzim/libzim.git"
+        git_dir = "libzim"
+
+    class Builder(MesonBuilder):
+        test_option = "-t 8"
+
+
+class ZimTools(Dependency):
+    name = "zim-tools"
+    dependencies = ['libzim']
+
+    class Source(GitClone):
+        git_remote = "https://github.com/openzim/zim-tools.git"
+        git_dir = "zim-tools"
+
+    class Builder(MesonBuilder):
+        @property
+        def configure_option(self):
+            if self.buildEnv.platform_info.static:
+                return "-Dstatic-linkage=true"
+            return ""
+
+
+class Zimwriterfs(Dependency):
+    name = "zimwriterfs"
+    extra_packages = ['file']
+
+    @property
+    def dependencies(self):
+        base_dependencies = ['libzim', 'zlib', 'lzma', 'xapian-core', 'gumbo']
+        if self.buildEnv.platform_info.build != 'native':
+            return base_dependencies + ["icu4c_cross-compile"]
+        else:
+            return base_dependencies + ["icu4c"]
+
+    class Source(GitClone):
+        git_remote = "https://github.com/openzim/zimwriterfs.git"
+        git_dir = "zimwriterfs"
+        release_git_ref = "1.1"
+
+        def _post_prepare_script(self, context):
+            context.try_skip(self.git_path)
+            command = "./autogen.sh"
+            self.buildEnv.run_command(command, self.git_path, context)
+
+    Builder = MakeBuilder
+
+
+class Kiwixlib(Dependency):
+    name = "kiwix-lib"
+
+    @property
+    def dependencies(self):
+        base_dependencies = ["pugixml", "libzim", "zlib", "lzma"]
+        if ( self.buildEnv.platform_info.build != 'android'
+          and self.buildEnv.distname != 'Darwin'):
+            base_dependencies += ['ctpp2c', 'ctpp2']
+        if self.buildEnv.platform_info.build != 'native':
+            return base_dependencies + ["icu4c_cross-compile"]
+        else:
+            return base_dependencies + ["icu4c"]
+
+    class Source(GitClone):
+        git_remote = "https://github.com/kiwix/kiwix-lib.git"
+        git_dir = "kiwix-lib"
+
+    class Builder(MesonBuilder):
+        @property
+        def configure_option(self):
+            base_option = "-Dctpp2-install-prefix={buildEnv.install_dir}"
+            if self.buildEnv.platform_info.build == 'android':
+                base_option += ' -Dandroid=true'
+            return base_option
+
+        @property
+        def library_type(self):
+            if self.buildEnv.platform_info.build == 'android':
+                return 'shared'
+            return super().library_type
+
+
+class KiwixTools(Dependency):
+    name = "kiwix-tools"
+    dependencies = ["kiwix-lib", "libmicrohttpd", "zlib"]
+
+    class Source(GitClone):
+        git_remote = "https://github.com/kiwix/kiwix-tools.git"
+        git_dir = "kiwix-tools"
+
+    class Builder(MesonBuilder):
+        @property
+        def configure_option(self):
+            if self.buildEnv.platform_info.static:
+                return "-Dstatic-linkage=true"
+            return ""
+
+
+class Gradle(Dependency):
+    name = "Gradle"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('gradle-4.1-bin.zip',
+                             'd55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43',
+                             'https://services.gradle.org/distributions/gradle-4.1-bin.zip')
+
+    class Builder(BaseBuilder):
+        def build(self):
+            self.command('install', self._install)
+
+        def _install(self, context):
+            copy_tree(
+                pj(self.source_path, "bin"),
+                pj(self.buildEnv.install_dir, "bin"),
+                post_copy_function = add_execution_right)
+            copy_tree(
+                pj(self.source_path, "lib"),
+                pj(self.buildEnv.install_dir, "lib"))
+
+
+class AllBaseDependencies(Dependency):
+    name = "alldependencies"
+
+    @property
+    def dependencies(self):
+        base_deps = ['zlib', 'lzma', 'xapian-core', 'gumbo', 'pugixml', 'libmicrohttpd']
+        if self.buildEnv.platform_info.build != 'native':
+            base_deps += ["icu4c_cross-compile"]
+        else:
+            base_deps += ["icu4c"]
+        if ( self.buildEnv.platform_info.build != 'android'
+           and self.buildEnv.distname != 'Darwin'):
+            base_deps += ['ctpp2c', 'ctpp2']
+        if self.buildEnv.platform_info.build == 'android':
+            base_deps += ['Gradle']
+
+        return base_deps
+
+
+    Source = NoopSource
+    Builder = NoopBuilder
+
+
+class KiwixAndroid(Dependency):
+    name = "kiwix-android"
+    dependencies = ["Gradle", "kiwix-lib"]
+
+    class Source(GitClone):
+        git_remote = "https://github.com/kiwix/kiwix-android"
+        git_dir = "kiwix-android"
+
+    class Builder(GradleBuilder):
+        def build(self):
+            if self.buildEnv.options.targets == 'kiwix-android-custom':
+                print("SKIP")
+            else:
+                super().build()
+
+        def _configure(self, context):
+            if not os.path.exists(self.build_path):
+                shutil.copytree(self.source_path, self.build_path, symlinks=True)
+            try:
+                shutil.rmtree(pj(self.build_path, 'kiwixlib', 'src', 'main'))
+            except FileNotFoundError:
+                pass
+            shutil.copytree(pj(self.buildEnv.install_dir, 'kiwix-lib'),
+                            pj(self.build_path, 'kiwixlib', 'src', 'main'))
+            os.makedirs(
+                pj(self.build_path, 'app', 'src', 'main', 'assets', 'icu'),
+                exist_ok=True)
+            shutil.copy2(pj(self.buildEnv.install_dir, 'share', 'icu', '58.2',
+                            'icudt58l.dat'),
+                         pj(self.build_path, 'app', 'src', 'main', 'assets',
+                            'icu', 'icudt58l.dat'))
+
+
+class KiwixCustomApp(Dependency):
+    name = "kiwix-android-custom"
+    dependencies = ["kiwix-android", "kiwix-lib"]
+
+    def __init__(self, buildEnv):
+        super().__init__(buildEnv)
+        self.custom_name = buildEnv.options.android_custom_app
+
+    class Source(GitClone):
+        git_remote = "https://github.com/kiwix/kiwix-android-custom"
+        git_dir = "kiwix-android-custom"
+
+    class Builder(GradleBuilder):
+        @property
+        def gradle_target(self):
+            return "assemble{}".format(self.target.custom_name)
+
+        @property
+        def gradle_option(self):
+            template = ("-i -P customDir={customDir}"
+                        " -P zim_file_size={zim_size}"
+                        " -P version_code={version_code}"
+                        " -P version_name={version_name}"
+                        " -P content_version_code={content_version_code}")
+            return template.format(
+                customDir=pj(self.build_path, 'custom'),
+                zim_size=self._get_zim_size(),
+                version_code=os.environ['VERSION_CODE'],
+                version_name=os.environ['VERSION_NAME'],
+                content_version_code=os.environ['CONTENT_VERSION_CODE'])
+
+        @property
+        def build_path(self):
+            return pj(self.buildEnv.build_dir, "{}_{}".format(self.target.full_name, self.target.custom_name))
+
+        @property
+        def custom_build_path(self):
+            return pj(self.build_path, 'custom', self.target.custom_name)
+
+        def _get_zim_size(self):
+            try:
+                zim_size = self.buildEnv.options.zim_file_size
+            except AttributeError:
+                with open(pj(self.source_path, self.target.custom_name, 'info.json')) as f:
+                    app_info = json.load(f)
+                zim_size = os.path.getsize(pj(self.custom_build_path, app_info['zim_file']))
+            return zim_size
+
+        def build(self):
+            self.command('configure', self._configure)
+            self.command('download_zim', self._download_zim)
+            self.command('compile', self._compile)
+
+        def _download_zim(self, context):
+            zim_url = self.buildEnv.options.zim_file_url
+            if zim_url is None:
+                raise SkipCommand()
+            with open(pj(self.source_path, self.target.custom_name, 'info.json')) as f:
+                app_info = json.load(f)
+            zim_url = app_info.get('zim_url', zim_url)
+            out_filename = urlparse(zim_url).path
+            out_filename = os.path.basename(out_filename)
+            zim_file = Remotefile(out_filename, '', zim_url)
+            self.buildEnv.download(zim_file)
+            shutil.copy(pj(self.buildEnv.archive_dir, out_filename),
+                        pj(self.custom_build_path, app_info['zim_file']))
+
+        def _configure(self, context):
+            # Copy kiwix-android in build dir.
+            kiwix_android_dep = self.buildEnv.targetsDict['kiwix-android']
+            if not os.path.exists(self.build_path):
+                shutil.copytree(kiwix_android_dep.source_path, self.build_path)
+
+            # Copy kiwix-lib application in build dir
+            try:
+                shutil.rmtree(pj(self.build_path, 'kiwixlib', 'src', 'main'))
+            except FileNotFoundError:
+                pass
+            shutil.copytree(pj(self.buildEnv.install_dir, 'kiwix-lib'),
+                            pj(self.build_path, 'kiwixlib', 'src', 'main'))
+            os.makedirs(
+                pj(self.build_path, 'app', 'src', 'main', 'assets', 'icu'),
+                exist_ok=True)
+            shutil.copy2(pj(self.buildEnv.install_dir, 'share', 'icu', '58.2',
+                            'icudt58l.dat'),
+                         pj(self.build_path, 'app', 'src', 'main', 'assets',
+                            'icu', 'icudt58l.dat'))
+
+            # Generate custom directory
+            try:
+                shutil.rmtree(pj(self.build_path, 'custom'))
+            except FileNotFoundError:
+                pass
+            os.makedirs(pj(self.build_path, 'custom'))
+            command = "./gen-custom-android-directory.py {custom_name} --output-dir {custom_dir}"
+            command = command.format(
+                custom_name=self.target.custom_name,
+                custom_dir=pj(self.build_path, 'custom', self.target.custom_name)
+            )
+            self.buildEnv.run_command(command, self.source_path, context)

dependency_utils.py

@@ -0,0 +1,431 @@
+import subprocess
+import os
+import shutil
+
+from utils import pj, Context, SkipCommand, extract_archive, Defaultdict, StopBuild
+from dependency_versions import main_project_versions, base_deps_versions
+
+SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
+
+
+class _MetaDependency(type):
+    def __new__(cls, name, bases, dct):
+        _class = type.__new__(cls, name, bases, dct)
+        if name != 'Dependency':
+            dep_name = dct['name']
+            Dependency.all_deps[dep_name] = _class
+        return _class
+
+
+class Dependency(metaclass=_MetaDependency):
+    all_deps = {}
+    dependencies = []
+    force_native_build = False
+
+    def __init__(self, buildEnv):
+        self.buildEnv = buildEnv
+        self.source = self.Source(self)
+        self.builder = self.Builder(self)
+        self.skip = False
+
+    @property
+    def version(self):
+        return base_deps_versions.get(self.name, None)
+
+    @property
+    def full_name(self):
+        if self.version:
+            return "{}-{}".format(self.name, self.version)
+        return self.name
+
+    @property
+    def source_path(self):
+        return pj(self.buildEnv.source_dir, self.source.source_dir)
+
+    @property
+    def _log_dir(self):
+        return self.buildEnv.log_dir
+
+    def command(self, name, function, *args):
+        print("  {} {} : ".format(name, self.name), end="", flush=True)
+        log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
+        context = Context(name, log, self.force_native_build)
+        try:
+            ret = function(*args, context=context)
+            context._finalise()
+            print("OK")
+            return ret
+        except SkipCommand:
+            print("SKIP")
+        except subprocess.CalledProcessError:
+            print("ERROR")
+            try:
+                with open(log, 'r') as f:
+                    print(f.read())
+            except:
+                pass
+            raise StopBuild()
+        except:
+            print("ERROR")
+            raise
+
+
+class Source:
+    """Base Class to the real preparator
+       A source preparator must install source in the self.source_dir attribute
+       inside the buildEnv.source_dir."""
+    def __init__(self, target):
+        self.target = target
+        self.buildEnv = target.buildEnv
+
+    @property
+    def name(self):
+        return self.target.name
+
+    @property
+    def source_dir(self):
+        return self.target.full_name
+
+    def _patch(self, context):
+        source_path = pj(self.buildEnv.source_dir, self.source_dir)
+        context.try_skip(source_path)
+        context.force_native_build = True
+        for p in self.patches:
+            with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input:
+                self.buildEnv.run_command("patch -p1", source_path, context, input=patch_input.read())
+
+    def command(self, *args, **kwargs):
+        return self.target.command(*args, **kwargs)
+
+
+class NoopSource(Source):
+    def prepare(self):
+        pass
+
+
+class ReleaseDownload(Source):
+    archive_top_dir = None
+
+    @property
+    def extract_path(self):
+        return pj(self.buildEnv.source_dir, self.source_dir)
+
+    def _download(self, context):
+        context.try_skip(self.buildEnv.archive_dir, self.name)
+        self.buildEnv.download(self.archive)
+
+    def _extract(self, context):
+        context.try_skip(self.extract_path)
+        if os.path.exists(self.extract_path):
+            shutil.rmtree(self.extract_path)
+        extract_archive(pj(self.buildEnv.archive_dir, self.archive.name),
+                        self.buildEnv.source_dir,
+                        topdir=self.archive_top_dir,
+                        name=self.source_dir)
+
+    def prepare(self):
+        self.command('download', self._download)
+        self.command('extract', self._extract)
+        if hasattr(self, 'patches'):
+            self.command('patch', self._patch)
+        if hasattr(self, '_post_prepare_script'):
+            self.command('post_prepare_script', self._post_prepare_script)
+
+
+class GitClone(Source):
+    base_git_ref = "master"
+
+    @property
+    def release_git_ref(self):
+        return main_project_versions.get(self.name, "master")
+
+    @property
+    def source_dir(self):
+        if self.buildEnv.make_release:
+            return "{}_release".format(self.git_dir)
+        else:
+            return self.git_dir
+
+    @property
+    def git_path(self):
+        return pj(self.buildEnv.source_dir, self.source_dir)
+
+    @property
+    def git_ref(self):
+        if self.buildEnv.make_release:
+            return self.release_git_ref
+        else:
+            return self.base_git_ref
+
+    def _git_clone(self, context):
+        context.force_native_build = True
+        if os.path.exists(self.git_path):
+            raise SkipCommand()
+        command = "git clone --depth=1 --branch {} {} {}".format(
+            self.git_ref, self.git_remote, self.source_dir)
+        self.buildEnv.run_command(command, self.buildEnv.source_dir, context)
+
+    def _git_update(self, context):
+        context.force_native_build = True
+        command = "git fetch origin {}".format(
+            self.git_ref)
+        self.buildEnv.run_command(command, self.git_path, context)
+        self.buildEnv.run_command("git checkout "+self.git_ref, self.git_path, context)
+
+    def prepare(self):
+        self.command('gitclone', self._git_clone)
+        self.command('gitupdate', self._git_update)
+        if hasattr(self, '_post_prepare_script'):
+            self.command('post_prepare_script', self._post_prepare_script)
+
+
+class SvnClone(Source):
+    @property
+    def source_dir(self):
+        return self.svn_dir
+
+    @property
+    def svn_path(self):
+        return pj(self.buildEnv.source_dir, self.svn_dir)
+
+    def _svn_checkout(self, context):
+        context.force_native_build = True
+        if os.path.exists(self.svn_path):
+            raise SkipCommand()
+        command = "svn checkout {} {}".format(self.svn_remote, self.svn_dir)
+        self.buildEnv.run_command(command, self.buildEnv.source_dir, context)
+
+    def _svn_update(self, context):
+        context.force_native_build = True
+        self.buildEnv.run_command("svn update", self.svn_path, context)
+
+    def prepare(self):
+        self.command('svncheckout', self._svn_checkout)
+        self.command('svnupdate', self._svn_update)
+        if hasattr(self, 'patches'):
+            self.command('patch', self._patch)
+
+
+class Builder:
+    subsource_dir = None
+
+    def __init__(self, target):
+        self.target = target
+        self.buildEnv = target.buildEnv
+
+    @property
+    def name(self):
+        return self.target.name
+
+    @property
+    def source_path(self):
+        base_source_path = self.target.source_path
+        if self.subsource_dir:
+            return pj(base_source_path, self.subsource_dir)
+        return base_source_path
+
+    @property
+    def build_path(self):
+        return pj(self.buildEnv.build_dir, self.target.full_name)
+
+    def command(self, *args, **kwargs):
+        return self.target.command(*args, **kwargs)
+
+    def build(self):
+        if hasattr(self, '_pre_build_script'):
+            self.command('pre_build_script', self._pre_build_script)
+        self.command('configure', self._configure)
+        self.command('compile', self._compile)
+        if hasattr(self, '_test'):
+            self.command('test', self._test)
+        self.command('install', self._install)
+        if hasattr(self, '_post_build_script'):
+            self.command('post_build_script', self._post_build_script)
+
+    def make_dist(self):
+        if hasattr(self, '_pre_build_script'):
+            self.command('pre_build_script', self._pre_build_script)
+        self.command('configure', self._configure)
+        self.command('make_dist', self._make_dist)
+
+
+class NoopBuilder(Builder):
+    def build(self):
+        pass
+
+    def make_dist(self):
+        pass
+
+
+class MakeBuilder(Builder):
+    configure_option = ""
+    dynamic_configure_option = "--enable-shared --disable-static"
+    static_configure_option = "--enable-static --disable-shared"
+    make_option = ""
+    install_option = ""
+    configure_script = "configure"
+    configure_env = None
+    make_target = ""
+    make_install_target = "install"
+
+    @property
+    def all_configure_option(self):
+        return "{} {} {}".format(
+            self.configure_option,
+            self.static_configure_option if self.buildEnv.platform_info.static else self.dynamic_configure_option,
+            self.buildEnv.configure_option if not self.target.force_native_build else "")
+
+    def _configure(self, context):
+        context.try_skip(self.build_path)
+        command = "{configure_script} {configure_option} --prefix {install_dir} --libdir {libdir}"
+        command = command.format(
+            configure_script=pj(self.source_path, self.configure_script),
+            configure_option=self.all_configure_option,
+            install_dir=self.buildEnv.install_dir,
+            libdir=pj(self.buildEnv.install_dir, self.buildEnv.libprefix)
+        )
+        env = Defaultdict(str, os.environ)
+        if self.buildEnv.platform_info.static:
+            env['CFLAGS'] = env['CFLAGS'] + ' -fPIC'
+        if self.configure_env:
+            for k in self.configure_env:
+                if k.startswith('_format_'):
+                    v = self.configure_env.pop(k)
+                    v = v.format(buildEnv=self.buildEnv, env=env)
+                    self.configure_env[k[8:]] = v
+            env.update(self.configure_env)
+        self.buildEnv.run_command(command, self.build_path, context, env=env)
+
+    def _compile(self, context):
+        context.try_skip(self.build_path)
+        command = "make -j4 {make_target} {make_option}".format(
+            make_target=self.make_target,
+            make_option=self.make_option
+        )
+        self.buildEnv.run_command(command, self.build_path, context)
+
+    def _install(self, context):
+        context.try_skip(self.build_path)
+        command = "make {make_install_target} {make_option}".format(
+            make_install_target=self.make_install_target,
+            make_option=self.make_option
+        )
+        self.buildEnv.run_command(command, self.build_path, context)
+
+    def _make_dist(self, context):
+        context.try_skip(self.build_path)
+        command = "make dist"
+        self.buildEnv.run_command(command, self.build_path, context)
+
+
+class CMakeBuilder(MakeBuilder):
+    def _configure(self, context):
+        context.try_skip(self.build_path)
+        cross_option = ""
+        if not self.target.force_native_build and self.buildEnv.cmake_crossfile:
+            cross_option = "-DCMAKE_TOOLCHAIN_FILE={}".format(self.buildEnv.cmake_crossfile)
+        command = ("cmake {configure_option}"
+                   " -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON"
+                   " -DCMAKE_INSTALL_PREFIX={install_dir}"
+                   " -DCMAKE_INSTALL_LIBDIR={libdir}"
+                   " {source_path}"
+                   " {cross_option}")
+        command = command.format(
+            configure_option="{} {}".format(self.buildEnv.cmake_option, self.configure_option),
+            install_dir=self.buildEnv.install_dir,
+            libdir=self.buildEnv.libprefix,
+            source_path=self.source_path,
+            cross_option=cross_option
+        )
+        env = Defaultdict(str, os.environ)
+        if self.buildEnv.platform_info.static:
+            env['CFLAGS'] = env['CFLAGS'] + ' -fPIC'
+        if self.configure_env:
+            for k in self.configure_env:
+                if k.startswith('_format_'):
+                    v = self.configure_env.pop(k)
+                    v = v.format(buildEnv=self.buildEnv, env=env)
+                    self.configure_env[k[8:]] = v
+            env.update(self.configure_env)
+        self.buildEnv.run_command(command, self.build_path, context, env=env, cross_env_only=True)
+
+
+class MesonBuilder(Builder):
+    configure_option = ""
+    test_option = ""
+
+    @property
+    def library_type(self):
+        return 'static' if self.buildEnv.platform_info.static else 'shared'
+
+    def _configure(self, context):
+        context.try_skip(self.build_path)
+        if os.path.exists(self.build_path):
+            shutil.rmtree(self.build_path)
+        os.makedirs(self.build_path)
+        configure_option = self.configure_option.format(buildEnv=self.buildEnv)
+        cross_option = ""
+        if not self.target.force_native_build and self.buildEnv.meson_crossfile:
+            cross_option = "--cross-file {}".format(
+                self.buildEnv.meson_crossfile)
+        command = ("{command} . {build_path}"
+                   " --default-library={library_type}"
+                   " {configure_option}"
+                   " --prefix={buildEnv.install_dir}"
+                   " --libdir={buildEnv.libprefix}"
+                   " {cross_option}")
+        command = command.format(
+            command=self.buildEnv.meson_command,
+            library_type=self.library_type,
+            configure_option=configure_option,
+            build_path=self.build_path,
+            buildEnv=self.buildEnv,
+            cross_option=cross_option
+        )
+        self.buildEnv.run_command(command, self.source_path, context, cross_env_only=True)
+
+    def _compile(self, context):
+        command = "{} -v".format(self.buildEnv.ninja_command)
+        self.buildEnv.run_command(command, self.build_path, context)
+
+    def _test(self, context):
+        if ( self.buildEnv.platform_info.build == 'android'
+             or (self.buildEnv.platform_info.build != 'native'
+                 and not self.buildEnv.platform_info.static)
+           ):
+            raise SkipCommand()
+        command = "{} --verbose {}".format(self.buildEnv.mesontest_command, self.test_option)
+        self.buildEnv.run_command(command, self.build_path, context)
+
+    def _install(self, context):
+        command = "{} -v install".format(self.buildEnv.ninja_command)
+        self.buildEnv.run_command(command, self.build_path, context)
+
+    def _make_dist(self, context):
+        command = "{} -v dist".format(self.buildEnv.ninja_command)
+        self.buildEnv.run_command(command, self.build_path, context)
+
+
+class GradleBuilder(Builder):
+    gradle_target = "build"
+    gradle_option = "-i"
+
+    def build(self):
+        self.command('configure', self._configure)
+        if hasattr(self, '_pre_compile_script'):
+            self.command('pre_compile_script', self._pre_compile_script)
+        self.command('compile', self._compile)
+
+    def _configure(self, context):
+        # We don't have a lot to configure by itself
+        context.try_skip(self.build_path)
+        if os.path.exists(self.build_path):
+            shutil.rmtree(self.build_path)
+        shutil.copytree(self.source_path, self.build_path)
+
+    def _compile(self, context):
+        command = "gradle {gradle_target} {gradle_option}"
+        command = command.format(
+            gradle_target=self.gradle_target,
+            gradle_option=self.gradle_option)
+        self.buildEnv.run_command(command, self.build_path, context)

dependency_versions.py

@@ -0,0 +1,27 @@
+
+main_project_versions = {
+    'kiwix-lib': '1.0.2',
+    'kiwix-tools': '0.3.0',
+    'libzim': '3.0.0',
+    'zim-tools': '0.0.1',
+    'zimwriterfs': '1.1',
+}
+
+
+# This is the "version" of the whole base_deps_versions dict.
+# Change this when you change base_deps_versions.
+base_deps_meta_version = '0'
+
+
+base_deps_versions = {
+  'zlib' : '1.2.8',
+  'lzma' : '5.0.4',
+  'uuid' : '1.43.4',
+  'xapian-core' : '1.4.5',
+  'ctpp2' : '2.8.3',
+  'pugixml' : '1.2',
+  'libmicrohttpd' : '0.9.46',
+  'gumbo' : '0.10.1',
+  'icu4c' : '58.2',
+  'Gradle' : '3.4',
+}

kiwix-deploy.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+
+import os, sys, stat
+import argparse
+import datetime
+import subprocess
+import tarfile
+import zipfile
+
+pj = os.path.join
+
+FILES_TO_UPLOAD = [
+    'kiwix-index',
+    'kiwix-install',
+    'kiwix-manage',
+    'kiwix-read',
+    'kiwix-search',
+    'kiwix-serve'
+]
+
+class Archiver:
+    def __init__(self, options):
+        self.options = options
+        self.working_directory = "{:%Y-%m-%d}".format(datetime.date.today())
+        os.makedirs(self.working_directory, exist_ok=True)
+        self.archive_basename = "kiwix-tools_{:%Y-%m-%d}".format(datetime.date.today())
+        self.files_to_upload = list(self._gen_file_list())
+
+    def _gen_file_list(self):
+        bin_dir = pj(self.options.install_dir, 'bin')
+        for filename in os.listdir(bin_dir):
+            basename, _ = os.path.splitext(filename)
+            if basename in FILES_TO_UPLOAD:
+                yield pj(bin_dir, filename), pj(self.archive_basename, filename)
+
+    def build_tar(self):
+        archive_name = "{}.tar.gz".format(self.archive_basename)
+        archive_name = pj(self.working_directory, archive_name)
+        with tarfile.open(archive_name, "w:gz") as archive:
+            for filename, arcname in self.files_to_upload:
+                archive.add(filename, arcname=arcname)
+        return archive_name
+
+    def build_zip(self):
+        archive_name = "{}.zip".format(self.archive_basename)
+        archive_name = pj(self.working_directory, archive_name)
+        with zipfile.ZipFile(archive_name, "w") as archive:
+            for filename, arcname in self.files_to_upload:
+                archive.write(filename, arcname)
+        return archive_name
+
+
+class Deployer:
+    def __init__(self, options):
+        self.options = options
+
+    def deploy(self, directory):
+        if not os.path.isdir(directory):
+            return
+        command = "scp -v -r -p -i {id_file} {directory} {host_addr}".format(
+            id_file=self.options.ssh_private_key,
+            directory=directory,
+            host_addr="{}:{}".format(self.options.server, self.options.base_path)
+        )
+        return subprocess.check_call(command, shell=True)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('install_dir')
+    parser.add_argument('--deploy', action="store_true")
+    group = parser.add_argument_group('deploy options')
+    group.add_argument('--ssh_private_key')
+    group.add_argument('--server')
+    group.add_argument('--base_path')
+    parser.add_argument('--tar', action="store_true")
+    parser.add_argument('--zip', action="store_true")
+    parser.add_argument('--verbose', '-v', action="store_true",
+                        help=("Print all logs on stdout instead of in specific"
+                              " log files per commands"))
+    return parser.parse_args()
+
+if __name__ == "__main__":
+    options = parse_args()
+    options.install_dir = os.path.abspath(options.install_dir)
+
+    archiver = Archiver(options)
+    archive_list = []
+    if options.tar:
+        print("Generating tar archive")
+        archive_list.append(archiver.build_tar())
+    if options.zip:
+        print("Generating zip archive")
+        archive_list.append(archiver.build_zip())
+
+    if options.deploy:
+       deployer = Deployer(options)
+       deployer.deploy(archiver.working_directory)

kiwixbuild/__init__.py

@@ -1,166 +0,0 @@
-#!/usr/bin/env python3
-
-import os, sys
-import argparse
-
-from .dependencies import Dependency
-from .configs import ConfigInfo
-from .builder import Builder
-from .flatpak_builder import FlatpakBuilder
-from . import _global
-
-
-def parse_args():
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "target",
-        default="kiwix-tools",
-        nargs="?",
-        metavar="TARGET",
-        choices=Dependency.all_deps.keys(),
-    )
-    parser.add_argument(
-        "--working-dir",
-        default=".",
-        help=(
-            "Directory where kiwix-build puts all its files "
-            "(source, archive and build)\n"
-            "working-dir can be absolute path or a relative (to cwd) one."
-        ),
-    )
-    parser.add_argument(
-        "--build-dir",
-        default=".",
-        help=(
-            "Directory where kiwix-build puts all build files.\n"
-            "build-dir can be absolute path or a relative (to working-dir) one."
-        ),
-    )
-    parser.add_argument("--libprefix", default=None)
-    parser.add_argument(
-        "--config", choices=ConfigInfo.all_configs, default="native_dyn"
-    )
-    parser.add_argument(
-        "--verbose",
-        "-v",
-        action="store_true",
-        help=(
-            "Print all logs on stdout instead of in specific" " log files per commands"
-        ),
-    )
-    parser.add_argument(
-        "--hide-progress",
-        action="store_false",
-        dest="show_progress",
-        help="Hide intermediate progress information.",
-    )
-    parser.add_argument(
-        "--skip-source-prepare",
-        action="store_true",
-        help="Skip the source download part",
-    )
-    parser.add_argument(
-        "--build-deps-only",
-        action="store_true",
-        help="Build only the dependencies of the specified target.",
-    )
-    parser.add_argument(
-        "--build-nodeps",
-        action="store_true",
-        help="Build only the target, not its dependencies.",
-    )
-    parser.add_argument(
-        "--make-dist",
-        action="store_true",
-        help="Build distrubution (dist) source archive",
-    )
-    parser.add_argument(
-        "--make-release", action="store_true", help="Build a release version"
-    )
-    subgroup = parser.add_argument_group("advanced")
-    subgroup.add_argument(
-        "--no-cert-check",
-        action="store_true",
-        help="Skip SSL certificate verification during download",
-    )
-    subgroup.add_argument(
-        "--clean-at-end",
-        action="store_true",
-        help="Clean all intermediate files after the (successfull) build",
-    )
-    subgroup.add_argument(
-        "--dont-install-packages",
-        action="store_true",
-        help="Do not try to install packages before compiling",
-    )
-    subgroup.add_argument(
-        "--assume-packages-installed",
-        action="store_true",
-        help="Assume the package to install to be aleady installed",
-    )
-    subgroup.add_argument(
-        "--android-arch",
-        action="append",
-        help=(
-            "Specify the architecture to build for android application/libraries.\n"
-            "Can be specified several times to build for several architectures.\n"
-            "If not specified, all architectures will be build."
-        ),
-    )
-    subgroup.add_argument(
-        "--ios-arch",
-        action="append",
-        help=(
-            "Specify the architecture to build for ios application/libraries.\n"
-            "Can be specified several times to build for several architectures.\n"
-            "If not specified, all architectures will be build."
-        ),
-    )
-    subgroup.add_argument(
-        "--fast-clone",
-        action="store_true",
-        help=(
-            "Do not clone the whole repository.\n"
-            "This is useful for one shot build but it is not recommended if you want "
-            "to develop with the cloned sources."
-        ),
-    )
-    subgroup.add_argument(
-        "--use-target-arch-name",
-        action="store_true",
-        help=(
-            "Name the build directory using the arch name instead of the config name.\n"
-            "Different configs may create binary for the same arch so this option is "
-            "not recommended when working with several config on the same computer.\n"
-            "However, when generating dependencies for other it is better to have a "
-            "directory named using the target instead of the used config.\n"
-            "Intended to be used in CI only."
-        ),
-    )
-    subgroup.add_argument(
-        "--get-build-dir", action="store_true", help="Print the output directory."
-    )
-    options = parser.parse_args()
-
-    if not options.android_arch:
-        options.android_arch = ["arm", "arm64", "x86", "x86_64"]
-    if not options.ios_arch:
-        options.ios_arch = ["arm64", "x86_64"]
-
-    return options
-
-
-def main():
-    options = parse_args()
-    options.working_dir = os.path.abspath(options.working_dir)
-    _global.set_options(options)
-    neutralEnv = buildenv.NeutralEnv(options.get_build_dir)
-    _global.set_neutralEnv(neutralEnv)
-    if options.config == "flatpak":
-        builder = FlatpakBuilder()
-    else:
-        builder = Builder()
-    if options.get_build_dir:
-        print(ConfigInfo.get_config(options.config).buildEnv.build_dir)
-    else:
-        builder.run()

kiwixbuild/__main__.py

@@ -1,7 +0,0 @@
-#!/usr/bin/env python3
-
-from . import main
-
-
-if __name__ == "__main__":
-    main()

kiwixbuild/_global.py

@@ -1,40 +0,0 @@
-from collections import OrderedDict as _OrderedDict
-import platform
-
-_neutralEnv = None
-_options = None
-_target_steps = _OrderedDict()
-
-
-def set_neutralEnv(env):
-    global _neutralEnv
-    _neutralEnv = env
-
-
-def neutralEnv(what):
-    return getattr(_neutralEnv, what)
-
-
-def set_options(options):
-    global _options
-    _options = options
-
-
-def option(what):
-    return getattr(_options, what)
-
-
-def add_target_step(key, what):
-    _target_steps[key] = what
-
-
-def get_target_step(key, default_context=None):
-    if isinstance(key, tuple):
-        context, target = key
-    else:
-        context, target = default_context, key
-    return _target_steps[(context, target)]
-
-
-def target_steps():
-    return _target_steps

kiwixbuild/buildenv.py

@@ -1,183 +0,0 @@
-import os, sys, shutil
-import subprocess
-import platform
-import distro
-
-from .utils import pj, download_remote, escape_path
-from ._global import neutralEnv, option
-
-
-class NeutralEnv:
-    def __init__(self, dummy_run):
-        self.working_dir = option("working_dir")
-        self.source_dir = pj(self.working_dir, "SOURCE")
-        self.archive_dir = pj(self.working_dir, "ARCHIVE")
-        self.toolchain_dir = pj(self.working_dir, "TOOLCHAINS")
-        self.log_dir = pj(self.working_dir, "LOGS")
-        for d in (self.source_dir, self.archive_dir, self.toolchain_dir, self.log_dir):
-            os.makedirs(d, exist_ok=True)
-        self.detect_platform()
-        if dummy_run:
-            # If this is for a dummy run, we will not run anything.
-            # To check for command (and so, don't enforce their presence)
-            return
-        self.ninja_command = self._detect_command(
-            "ninja", default=[["ninja"], ["ninja-build"]]
-        )
-        self.meson_command = self._detect_command(
-            "meson", default=[["meson.py"], ["meson"]]
-        )
-        self.mesontest_command = [*self.meson_command, "test"]
-        self.patch_command = self._detect_command("patch")
-        self.git_command = self._detect_command("git")
-        if platform.system() == "Windows":
-            self.make_command = self._detect_command("nmake", options=["/?", "/NOLOGO"])
-        else:
-            self.make_command = self._detect_command("make")
-        self.cmake_command = self._detect_command("cmake")
-        self.qmake_command = self._detect_command(
-            "qmake", required=False, default=[["qmake"], ["qmake-qt5"]]
-        )
-
-    def detect_platform(self):
-        _platform = platform.system()
-        self.distname = _platform
-        if _platform == "Linux":
-            self.distname = distro.id()
-            if self.distname == "ubuntu":
-                self.distname = "debian"
-
-    def download(self, what, where=None):
-        where = where or self.archive_dir
-        download_remote(what, where)
-
-    def _detect_command(self, name, default=None, options=["--version"], required=True):
-        if default is None:
-            default = [[name]]
-        env_key = "KBUILD_{}_COMMAND".format(name.upper())
-        if env_key in os.environ:
-            default = [os.environ[env_key].split()] + default
-        for command in default:
-            try:
-                retcode = subprocess.check_call(
-                    command + options, stdout=subprocess.DEVNULL
-                )
-            except (FileNotFoundError, PermissionError, OSError):
-                # Doesn't exist in PATH or isn't executable
-                continue
-            if retcode == 0:
-                return command
-        else:
-            if required:
-                sys.exit("ERROR: {} command not found".format(name))
-            else:
-                print("WARNING: {} command not found".format(name), file=sys.stderr)
-                return ["{}_NOT_FOUND".format(name.upper())]
-
-
-class BuildEnv:
-    def __init__(self, configInfo):
-        self.configInfo = configInfo
-        self.base_build_dir = pj(option("working_dir"), option("build_dir"))
-        build_dir = (
-            configInfo.arch_name if option("use_target_arch_name") else configInfo.name
-        )
-        build_dir = f"BUILD_{build_dir}"
-        self.build_dir = pj(self.base_build_dir, build_dir)
-        self.install_dir = pj(self.build_dir, "INSTALL")
-        self.toolchain_dir = pj(self.build_dir, "TOOLCHAINS")
-        self.log_dir = pj(self.build_dir, "LOGS")
-        for d in (self.build_dir, self.install_dir, self.toolchain_dir, self.log_dir):
-            os.makedirs(d, exist_ok=True)
-
-        self.libprefix = option("libprefix") or self._detect_libdir()
-
-    def clean_intermediate_directories(self):
-        for subdir in os.listdir(self.build_dir):
-            subpath = pj(self.build_dir, subdir)
-            if subpath == self.install_dir:
-                continue
-            if os.path.isdir(subpath):
-                shutil.rmtree(subpath)
-            else:
-                os.remove(subpath)
-
-    def _is_debianlike(self):
-        return os.path.isfile("/etc/debian_version")
-
-    def _detect_libdir(self):
-        if self.configInfo.libdir is not None:
-            return self.configInfo.libdir
-        if self._is_debianlike():
-            try:
-                pc = subprocess.Popen(
-                    ["dpkg-architecture", "-qDEB_HOST_MULTIARCH"],
-                    stdout=subprocess.PIPE,
-                    stderr=subprocess.DEVNULL,
-                )
-                (stdo, _) = pc.communicate()
-                if pc.returncode == 0:
-                    archpath = stdo.decode().strip()
-                    return "lib/" + archpath
-            except Exception:
-                pass
-        if os.path.isdir("/usr/lib64") and not os.path.islink("/usr/lib64"):
-            return "lib64"
-        return "lib"
-
-    def get_env(self, *, cross_comp_flags, cross_compilers, cross_path):
-        env = self.configInfo.get_env()
-        pkgconfig_path = pj(self.install_dir, self.libprefix, "pkgconfig")
-        env["PKG_CONFIG_PATH"].append(pkgconfig_path)
-
-        env["PATH"].insert(0, pj(self.install_dir, "bin"))
-
-        env["LD_LIBRARY_PATH"].extend(
-            [
-                pj(self.install_dir, "lib"),
-                pj(self.install_dir, self.libprefix),
-            ]
-        )
-
-        env["QMAKE_CXXFLAGS"] = " ".join(
-            [escape_path("-I" + pj(self.install_dir, "include")), env["QMAKE_CXXFLAGS"]]
-        )
-        env["CPPFLAGS"] = " ".join(
-            [escape_path("-I" + pj(self.install_dir, "include")), env["CPPFLAGS"]]
-        )
-        env["QMAKE_LFLAGS"] = " ".join(
-            [
-                escape_path("-L" + pj(self.install_dir, "lib")),
-                escape_path("-L" + pj(self.install_dir, self.libprefix)),
-                env["QMAKE_LFLAGS"],
-            ]
-        )
-        env["LDFLAGS"] = " ".join(
-            [
-                escape_path("-L" + pj(self.install_dir, "lib")),
-                escape_path("-L" + pj(self.install_dir, self.libprefix)),
-                env["LDFLAGS"],
-            ]
-        )
-
-        if cross_comp_flags:
-            self.configInfo.set_comp_flags(env)
-        if cross_compilers:
-            self.configInfo.set_compiler(env)
-        if cross_path:
-            env["PATH"][0:0] = self.configInfo.get_bin_dir()
-        return env
-
-    @property
-    def configure_wrapper(self):
-        try:
-            yield self.configInfo.configure_wrapper
-        except AttributeError:
-            pass
-
-    @property
-    def make_wrapper(self):
-        try:
-            yield self.configInfo.make_wrapper
-        except AttributeError:
-            pass

kiwixbuild/builder.py

@@ -1,213 +0,0 @@
-import sys
-from collections import OrderedDict
-from .buildenv import *
-
-from .configs import ConfigInfo
-from .utils import remove_duplicates, StopBuild, colorize
-from .dependencies import Dependency
-from .packages import PACKAGE_NAME_MAPPERS
-from ._global import (
-    neutralEnv,
-    option,
-    add_target_step,
-    get_target_step,
-    target_steps,
-)
-from . import _global
-
-
-class Builder:
-    def __init__(self):
-        self._targets = {}
-        ConfigInfo.get_config("neutral", self._targets)
-
-        config_name = option("config")
-        config = ConfigInfo.get_config(config_name, self._targets)
-        if neutralEnv("distname") not in config.compatible_hosts:
-            print(
-                (
-                    colorize("ERROR") + ": The config {} cannot be build on host {}.\n"
-                    "Select another config or change your host system."
-                ).format(config.name, neutralEnv("distname"))
-            )
-        self.targetDefs = config.add_targets(option("target"), self._targets)
-
-    def finalize_target_steps(self):
-        steps = []
-        for targetDef in self.targetDefs:
-            steps += self.order_steps(targetDef)
-        steps = list(remove_duplicates(steps))
-
-        if option("build_nodeps"):
-            # add all config steps
-            for dep in steps:
-                stepClass = Dependency.all_deps[dep[1]]
-                if stepClass.dont_skip:
-                    add_target_step(dep, self._targets[dep])
-
-            src_targetDef = ("source", targetDef[1])
-            add_target_step(src_targetDef, self._targets[src_targetDef])
-            add_target_step(targetDef, self._targets[targetDef])
-        else:
-            for dep in steps:
-                if option("build_deps_only") and dep[1] == targetDef[1]:
-                    continue
-                add_target_step(dep, self._targets[dep])
-        self.instanciate_steps()
-
-    def order_steps(self, targetDef):
-        for cfgName in ConfigInfo.all_running_configs:
-            cfg = ConfigInfo.all_configs[cfgName]
-            for tlcName in cfg.toolchain_names:
-                tlc = Dependency.all_deps[tlcName]
-                yield ("source", tlcName)
-                yield ("neutral" if tlc.neutral else cfgName, tlcName)
-        _targets = dict(self._targets)
-        yield from self.order_dependencies(targetDef, _targets)
-
-    def order_dependencies(self, targetDef, targets):
-        targetConfigName, targetName = targetDef
-        if targetConfigName == "source":
-            # Do not try to order sources, they will be added as dep by the
-            # build step two lines later.
-            return
-        try:
-            target = targets.pop(targetDef)
-        except KeyError:
-            return
-
-        targetConfig = ConfigInfo.get_config(targetConfigName)
-        for dep in target.get_dependencies(targetConfig, True):
-            depConfig, depName = targetConfig.get_fully_qualified_dep(dep)
-            if (depConfig, depName) in targets:
-                yield from self.order_dependencies((depConfig, depName), targets)
-        yield ("source", targetName)
-        yield targetDef
-
-    def instanciate_steps(self):
-        for stepDef in list(target_steps()):
-            stepConfig, stepName = stepDef
-            stepClass = Dependency.all_deps[stepName]
-            if stepConfig == "source":
-                source = get_target_step(stepDef)(stepClass)
-                add_target_step(stepDef, source)
-            else:
-                source = get_target_step(stepName, "source")
-                env = ConfigInfo.get_config(stepConfig).buildEnv
-                builder = get_target_step(stepDef)(stepClass, source, env)
-                add_target_step(stepDef, builder)
-
-    def prepare_sources(self):
-        if option("skip_source_prepare"):
-            print(colorize("SKIP"))
-            return
-
-        sourceDefs = remove_duplicates(
-            tDef for tDef in target_steps() if tDef[0] == "source"
-        )
-        for sourceDef in sourceDefs:
-            print("prepare sources {} :".format(sourceDef[1]))
-            source = get_target_step(sourceDef)
-            source.prepare()
-
-    def build(self):
-        builderDefs = (tDef for tDef in target_steps() if tDef[0] != "source")
-        for builderDef in builderDefs:
-            builder = get_target_step(builderDef)
-            if option("make_dist") and builderDef[1] == option("target"):
-                print("make dist {} ({}):".format(builder.name, builderDef[0]))
-                builder.make_dist()
-                continue
-            print("build {} ({}):".format(builder.name, builderDef[0]))
-            add_target_step(builderDef, builder)
-            builder.build()
-
-    def _get_packages(self):
-        packages_list = []
-        for config in ConfigInfo.all_running_configs.values():
-            mapper_name = "{host}_{config}".format(
-                host=neutralEnv("distname"), config=config
-            )
-            package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {})
-            packages_list += package_name_mapper.get("COMMON", [])
-
-        to_drop = []
-        for builderDef in self._targets:
-            configName, builderName = builderDef
-            mapper_name = "{host}_{config}".format(
-                host=neutralEnv("distname"), config=configName
-            )
-            package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {})
-            packages = package_name_mapper.get(builderName)
-            if packages:
-                to_drop.append(builderDef)
-                if packages is not True:
-                    # True means "assume the dependency is install but do not try to install anything for it"
-                    packages_list += packages
-        for dep in to_drop:
-            del self._targets[dep]
-        return packages_list
-
-    def install_packages(self):
-        packages_to_have = self._get_packages()
-        packages_to_have = remove_duplicates(packages_to_have)
-
-        if option("assume_packages_installed"):
-            print(colorize("SKIP") + ", Assume package installed")
-            return
-
-        distname = neutralEnv("distname")
-        if distname in ("fedora", "redhat", "centos"):
-            package_installer = "sudo dnf install {}"
-            package_checker = "rpm -q --quiet {}"
-        elif distname in ("debian", "Ubuntu"):
-            package_installer = "sudo apt-get install {}"
-            package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1'
-        elif distname == "Darwin":
-            package_installer = "brew install {}"
-            package_checker = "brew ls --version {} > /dev/null"
-
-        packages_to_install = []
-        for package in packages_to_have:
-            print(" - {} : ".format(package), end="")
-            command = package_checker.format(package)
-            try:
-                subprocess.check_call(command, shell=True)
-            except subprocess.CalledProcessError:
-                print(colorize("NEEDED"))
-                packages_to_install.append(package)
-            else:
-                print(colorize("SKIP"))
-
-        if packages_to_install:
-            command = package_installer.format(" ".join(packages_to_install))
-            print(command)
-            subprocess.check_call(command, shell=True)
-        else:
-            print(colorize("SKIP") + ", No package to install.")
-
-    def run(self):
-        try:
-            print("[INSTALL PACKAGES]")
-            if option("dont_install_packages"):
-                print(colorize("SKIP"))
-            else:
-                self.install_packages()
-            self.finalize_target_steps()
-            print("[SETUP TOOLCHAINS]")
-            for config in ConfigInfo.all_running_configs.values():
-                config.finalize_setup()
-            print("[PREPARE]")
-            self.prepare_sources()
-            print("[BUILD]")
-            self.build()
-            # No error, clean intermediate file at end of build if needed.
-            print("[CLEAN]")
-            if option("clean_at_end"):
-                for config in ConfigInfo.all_running_configs.values():
-                    config.clean_intermediate_directories()
-            else:
-                print(colorize("SKIP"))
-        except StopBuild as e:
-            print(e)
-            sys.exit("Stopping build due to errors")

kiwixbuild/configs/__init__.py

@@ -1,3 +0,0 @@
-from .base import *
-
-from . import android, armhf, musl, flatpak, i586, ios, native, neutral, wasm

kiwixbuild/configs/android.py

@@ -1,169 +0,0 @@
-from .base import ConfigInfo, MetaConfigInfo
-from kiwixbuild.utils import pj
-from kiwixbuild._global import get_target_step, option
-
-
-class AndroidConfigInfo(ConfigInfo):
-    build = "android"
-    static = True
-    toolchain_names = ["android-ndk"]
-    compatible_hosts = ["fedora", "debian"]
-
-    def __str__(self):
-        return "android"
-
-    @property
-    def libdir(self):
-        return "lib/{}".format(self.arch_full)
-
-    @property
-    def binaries_name(self):
-        arch_full = self.arch_full
-        return {
-            "CC": "{}-{}".format(arch_full, "clang"),
-            "CXX": "{}-{}".format(arch_full, "clang++"),
-            "AR": "{}-{}".format(arch_full, "ar"),
-            "STRIP": "{}-{}".format(arch_full, "strip"),
-            "RANLIB": "{}-{}".format(arch_full, "ranlib"),
-            "LD": "{}-{}".format(arch_full, "ld"),
-        }
-
-    def binaries(self):
-        install_path = self.install_path
-        binaries = {
-            k: pj(install_path, "bin", v) for k, v in self.binaries_name.items()
-        }
-        binaries["PKGCONFIG"] = "pkg-config"
-        return binaries
-
-    @property
-    def ndk_builder(self):
-        return get_target_step("android-ndk", self.name)
-
-    @property
-    def install_path(self):
-        return self.ndk_builder.install_path
-
-    def get_cross_config(self):
-        extra_libs = [
-            "-llog",
-            "-Wl,--exclude-libs,libgcc.a",
-            "-Wl,--exclude-libs,libunwind.a",
-        ]
-        extra_cflags = [
-            "-I{}".format(include_dir) for include_dir in self.get_include_dirs()
-        ]
-        if hasattr(self, "march"):
-            extra_libs.append("-march={}".format(self.march))
-            extra_cflags.append("-march={}".format(self.march))
-        return {
-            "exe_wrapper_def": "",
-            "install_path": self.install_path,
-            "binaries": self.binaries(),
-            "root_path": pj(self.install_path, "sysroot"),
-            "extra_libs": extra_libs,
-            "extra_cflags": extra_cflags,
-            "host_machine": {
-                "system": "Android",
-                "lsystem": "android",
-                "cpu_family": self.arch,
-                "cpu": self.cpu,
-                "endian": "little",
-                "abi": self.abi,
-            },
-        }
-
-    def get_env(self):
-        env = super().get_env()
-        root_path = pj(self.install_path, "sysroot")
-        env["PKG_CONFIG_LIBDIR"] = pj(root_path, "lib", "pkgconfig")
-        env["NDK_DEBUG"] = "0"
-        return env
-
-    def get_bin_dir(self):
-        return [pj(self.install_path, "bin")]
-
-    def set_comp_flags(self, env):
-        super().set_comp_flags(env)
-        root_path = pj(self.install_path, "sysroot")
-        march = "-march={}".format(self.march) if hasattr(self, "march") else ""
-        env["CFLAGS"] = (
-            "-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} {} ".format(
-                root_path, march
-            )
-            + env["CFLAGS"]
-        )
-        env["CXXFLAGS"] = (
-            "-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} {} ".format(
-                root_path, march
-            )
-            + env["CXXFLAGS"]
-        )
-        env["LDFLAGS"] = "--sysroot={} {} ".format(root_path, march) + env["LDFLAGS"]
-
-    def set_compiler(self, env):
-        binaries = self.binaries()
-        for k, v in binaries.items():
-            env[k] = v
-
-    @property
-    def configure_options(self):
-        yield "--host={}".format(self.arch_full)
-
-    def finalize_setup(self):
-        super().finalize_setup()
-        self.buildEnv.cmake_crossfile = self._gen_crossfile(
-            "cmake_android_cross_file.txt", "cmake_cross_file.txt"
-        )
-        self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
-
-
-class AndroidArm(AndroidConfigInfo):
-    name = "android_arm"
-    arch = cpu = "arm"
-    arch_full = "arm-linux-androideabi"
-    abi = "armeabi-v7a"
-    march = "armv7-a"
-
-
-class AndroidArm64(AndroidConfigInfo):
-    name = "android_arm64"
-    arch = "arm64"
-    arch_full = "aarch64-linux-android"
-    cpu = "aarch64"
-    abi = "arm64-v8a"
-
-
-class AndroidX86(AndroidConfigInfo):
-    name = "android_x86"
-    arch = abi = "x86"
-    arch_full = "i686-linux-android"
-    cpu = "i686"
-
-
-class AndroidX8664(AndroidConfigInfo):
-    name = "android_x86_64"
-    arch = cpu = abi = "x86_64"
-    arch_full = "x86_64-linux-android"
-
-
-class Android(MetaConfigInfo):
-    name = "android"
-    compatible_hosts = ["fedora", "debian"]
-
-    @property
-    def arch_name(self):
-        return "multi-linux-android"
-
-    @property
-    def subConfigNames(self):
-        return ["android_{}".format(arch) for arch in option("android_arch")]
-
-    def add_targets(self, targetName, targets):
-        return super().add_targets(targetName, targets)
-
-    def __str__(self):
-        return self.name
-
-    def set_comp_flags(self, env):
-        pass

kiwixbuild/configs/armhf.py

@@ -1,183 +0,0 @@
-from .base import ConfigInfo, MixedMixin
-
-from kiwixbuild.utils import pj
-from kiwixbuild._global import get_target_step
-
-
-# Base config for arm
-class ArmConfigInfo(ConfigInfo):
-    compatible_hosts = ["fedora", "debian", "almalinux"]
-
-    def get_cross_config(self):
-        return {
-            "binaries": self.binaries,
-            "exe_wrapper_def": "",
-            "root_path": self.root_path,
-            "extra_libs": [],
-            "extra_cflags": [
-                "-I{}".format(include_dir) for include_dir in self.get_include_dirs()
-            ],
-            "host_machine": {
-                "system": "linux",
-                "lsystem": "linux",
-                "cpu_family": self.cpu_family,
-                "cpu": self.cpu,
-                "endian": "little",
-                "abi": "",
-            },
-        }
-
-    @property
-    def libdir(self):
-        return "lib/{}".format(self.arch_full)
-
-    @property
-    def toolchain(self):
-        return get_target_step(self.build, "neutral")
-
-    @property
-    def root_path(self):
-        return self.toolchain.build_path
-
-    @property
-    def binaries(self):
-        binaries = (
-            (k, "{}-{}".format(self.arch_full, v))
-            for k, v in (
-                ("CC", "gcc"),
-                ("CXX", "g++"),
-                ("AR", "ar"),
-                ("STRIP", "strip"),
-                ("WINDRES", "windres"),
-                ("RANLIB", "ranlib"),
-                ("LD", "ld"),
-                ("LDSHARED", "g++ -shared"),
-            )
-        )
-        binaries = {k: pj(self.root_path, "bin", v) for k, v in binaries}
-        binaries["PKGCONFIG"] = "pkg-config"
-        return binaries
-
-    @property
-    def exe_wrapper_def(self):
-        try:
-            which("qemu-arm")
-        except subprocess.CalledProcessError:
-            return ""
-        else:
-            return "exe_wrapper = 'qemu-arm'"
-
-    @property
-    def configure_options(self):
-        yield "--host={}".format(self.arch_full)
-
-    def get_bin_dir(self):
-        return [pj(self.root_path, "bin")]
-
-    def get_env(self):
-        env = super().get_env()
-        env["LD_LIBRARY_PATH"][0:0] = [
-            pj(self.root_path, self.arch_full, "lib64"),
-            pj(self.root_path, "lib"),
-        ]
-        env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig")
-        env["QEMU_LD_PREFIX"] = pj(self.root_path, self.arch_full, "libc")
-        env["QEMU_SET_ENV"] = "LD_LIBRARY_PATH={}".format(
-            ":".join(
-                [pj(self.root_path, self.arch_full, "lib"), str(env["LD_LIBRARY_PATH"])]
-            )
-        )
-        return env
-
-    def set_comp_flags(self, env):
-        super().set_comp_flags(env)
-        env["CFLAGS"] = (
-            " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
-            + env["CFLAGS"]
-        )
-        env["CXXFLAGS"] = (
-            " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
-            + env["CXXFLAGS"]
-        )
-
-    def set_compiler(self, env):
-        for k, v in self.binaries.items():
-            env[k] = v
-
-    def finalize_setup(self):
-        super().finalize_setup()
-        self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt")
-        self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
-
-
-class Armv6(ArmConfigInfo):
-    build = "armv6"
-    arch_full = "armv6-rpi-linux-gnueabihf"
-    toolchain_names = ["armv6"]
-    cpu_family = "arm"
-    cpu = "armv6"
-
-
-class Armv6Dyn(Armv6):
-    name = "armv6_dyn"
-    static = False
-
-
-class Armv6Static(Armv6):
-    name = "armv6_static"
-    static = True
-
-
-class Armv6Mixed(MixedMixin("armv6_static"), Armv6):
-    name = "armv6_mixed"
-    static = False
-
-
-class Armv8(ArmConfigInfo):
-    build = "armv8"
-    arch_full = "armv8-rpi3-linux-gnueabihf"
-    toolchain_names = ["armv8"]
-    cpu_family = "arm"
-    cpu = "armv8"
-
-
-class Armv8Dyn(Armv8):
-    name = "armv8_dyn"
-    static = False
-
-
-class Armv8Static(Armv8):
-    name = "armv8_static"
-    static = True
-
-
-class Armv8Mixed(MixedMixin("armv8_static"), Armv8):
-    name = "armv8_mixed"
-    static = False
-
-
-class Aarch64(ArmConfigInfo):
-    build = "aarch64"
-    arch_full = "aarch64-linux-gnu"
-    toolchain_names = ["aarch64"]
-    cpu_family = "aarch64"
-    cpu = "aarch64"
-
-    @property
-    def root_path(self):
-        return self.toolchain.build_path
-
-
-class Aarch64Dyn(Aarch64):
-    name = "aarch64_dyn"
-    static = False
-
-
-class Aarch64Static(Aarch64):
-    name = "aarch64_static"
-    static = True
-
-
-class Aarch64Mixed(MixedMixin("aarch64_static"), Aarch64):
-    name = "aarch64_mixed"
-    static = False

kiwixbuild/configs/base.py

@@ -1,179 +0,0 @@
-import os, sys
-import subprocess
-
-from kiwixbuild.dependencies import Dependency
-from kiwixbuild.utils import pj, remove_duplicates, DefaultEnv
-from kiwixbuild.buildenv import BuildEnv
-from kiwixbuild._global import neutralEnv, option, target_steps
-
-_SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
-TEMPLATES_DIR = pj(os.path.dirname(_SCRIPT_DIR), "templates")
-
-
-class _MetaConfig(type):
-    def __new__(cls, name, bases, dct):
-        _class = type.__new__(cls, name, bases, dct)
-        if name not in ("ConfigInfo", "MetaConfigInfo") and "name" in dct:
-            dep_name = dct["name"]
-            ConfigInfo.all_configs[dep_name] = _class
-        return _class
-
-
-class ConfigInfo(metaclass=_MetaConfig):
-    all_configs = {}
-    all_running_configs = {}
-    toolchain_names = []
-    configure_options = []
-    mixed = False
-    libdir = None
-
-    @property
-    def arch_name(self):
-        return self.arch_full
-
-    @classmethod
-    def get_config(cls, name, targets=None):
-        if name not in cls.all_running_configs:
-            if targets is None:
-                print("Should not got there.")
-                print(cls.all_running_configs)
-                raise KeyError(name)
-            cls.all_running_configs[name] = cls.all_configs[name](targets)
-        return cls.all_running_configs[name]
-
-    def __init__(self, targets):
-        self.all_running_configs[self.name] = self
-        self.buildEnv = BuildEnv(self)
-        self.setup_toolchains(targets)
-
-    def __str__(self):
-        return "{}_{}".format(self.build, "static" if self.static else "dyn")
-
-    def setup_toolchains(self, targets):
-        for tlc_name in self.toolchain_names:
-            ToolchainClass = Dependency.all_deps[tlc_name]
-            targets[("source", tlc_name)] = ToolchainClass.Source
-            cfg_name = "neutral" if ToolchainClass.neutral else self.name
-            targets[(cfg_name, tlc_name)] = ToolchainClass.Builder
-
-    def add_targets(self, targetName, targets):
-        if (self.name, targetName) in targets:
-            return []
-        targetClass = Dependency.all_deps[targetName]
-        targets[("source", targetName)] = targetClass.Source
-        targets[(self.name, targetName)] = targetClass.Builder
-        for dep in targetClass.Builder.get_dependencies(self, False):
-            if isinstance(dep, tuple):
-                depConfigName, depName = dep
-            else:
-                depConfigName, depName = self.name, dep
-            depConfig = self.get_config(depConfigName, targets)
-            depConfig.add_targets(depName, targets)
-        return [(self.name, targetName)]
-
-    def get_fully_qualified_dep(self, dep):
-        if isinstance(dep, tuple):
-            return dep
-        else:
-            return self.name, dep
-
-    def get_cross_config(self):
-        return {}
-
-    def get_include_dirs(self):
-        return [pj(self.buildEnv.install_dir, "include")]
-
-    def get_env(self):
-        return DefaultEnv()
-
-    def get_bin_dir(self):
-        return []
-
-    def set_compiler(self, env):
-        pass
-
-    def set_comp_flags(self, env):
-        if self.static:
-            env["CFLAGS"] = env["CFLAGS"] + " -fPIC"
-            env["CXXFLAGS"] = env["CXXFLAGS"] + " -fPIC"
-
-    def _gen_crossfile(self, name, outname=None):
-        if outname is None:
-            outname = name
-        crossfile = pj(self.buildEnv.build_dir, outname)
-        template_file = pj(TEMPLATES_DIR, name)
-        with open(template_file, "r") as f:
-            template = f.read()
-        content = template.format(**self.get_cross_config())
-        with open(crossfile, "w") as outfile:
-            outfile.write(content)
-        return crossfile
-
-    def finalize_setup(self):
-        self.buildEnv.cross_config = self.get_cross_config()
-        self.buildEnv.meson_crossfile = None
-        self.buildEnv.cmake_crossfile = None
-
-    def clean_intermediate_directories(self):
-        self.buildEnv.clean_intermediate_directories()
-
-
-class MetaConfigInfo(ConfigInfo):
-    subConfigNames = []
-
-    def add_targets(self, targetName, targets):
-        targetDefs = []
-        for configName in self.subConfigNames:
-            config = self.get_config(configName, targets)
-            targetDefs += config.add_targets(targetName, targets)
-        return targetDefs
-
-
-def MixedMixin(static_name):
-    class MixedMixinClass:
-        mixed = True
-        static = False
-
-        def add_targets(self, targetName, targets):
-            if option("target") == targetName:
-                return super().add_targets(targetName, targets)
-            else:
-                static_config = self.get_config(static_name, targets)
-                return static_config.add_targets(targetName, targets)
-
-        def get_fully_qualified_dep(self, dep):
-            if isinstance(dep, tuple):
-                return dep
-            if option("target") == dep:
-                return self.name, dep
-            return static_name, dep
-
-        @property
-        def static_buildEnv(self):
-            static_config = self.get_config(static_name)
-            return static_config.buildEnv
-
-        def get_include_dirs(self):
-            return [
-                pj(self.buildEnv.install_dir, "include"),
-                pj(self.static_buildEnv.install_dir, "include"),
-            ]
-
-        def get_env(self):
-            env = super().get_env()
-            env["PATH"].insert(0, pj(self.static_buildEnv.install_dir, "bin"))
-            pkgconfig_path = pj(
-                self.static_buildEnv.install_dir,
-                self.static_buildEnv.libprefix,
-                "pkgconfig",
-            )
-            env["PKG_CONFIG_PATH"].append(pkgconfig_path)
-            env["CPPFLAGS"] = " ".join(
-                [
-                   "-I" + pj(self.static_buildEnv.install_dir, "include"),
-                    env["CPPFLAGS"],
-                ]
-            )
-            return env
-
-    return MixedMixinClass

kiwixbuild/configs/flatpak.py

@@ -1,19 +0,0 @@
-from .base import ConfigInfo
-from kiwixbuild._global import option, neutralEnv
-
-
-class FlatpakConfigInfo(ConfigInfo):
-    name = "flatpak"
-    arch_name = "flatpak"
-    build = "flatpak"
-    static = ""
-    toolchain_names = ["org.kde", "io.qt.qtwebengine"]
-    compatible_hosts = ["debian", "fedora"]
-
-    def __str__(self):
-        return "flatpak"
-
-    def get_env(self):
-        env = super().get_env()
-        env["FLATPAK_USER_DIR"] = self.buildEnv.build_dir
-        return env

kiwixbuild/configs/i586.py

@@ -1,79 +0,0 @@
-import os
-
-from .base import ConfigInfo
-from kiwixbuild.utils import which, pj
-
-
-class I586ConfigInfo(ConfigInfo):
-    build = "i586"
-    arch_full = "i586-linux-gnu"
-    compatible_hosts = ["fedora", "debian"]
-
-    def get_cross_config(self):
-        return {
-            "binaries": self.binaries,
-            "exe_wrapper_def": "",
-            "extra_libs": ["-m32", "-march=i586", "-mno-sse"],
-            "extra_cflags": [
-                "-m32",
-                "-march=i586",
-                "-mno-sse",
-                *(
-                    "-I{}".format(include_dir)
-                    for include_dir in self.get_include_dirs()
-                ),
-            ],
-            "host_machine": {
-                "system": "linux",
-                "lsystem": "linux",
-                "cpu_family": "x86",
-                "cpu": "i586",
-                "endian": "little",
-                "abi": "",
-            },
-        }
-
-    @property
-    def configure_options(self):
-        yield f"--host={self.arch_full}"
-
-    @property
-    def binaries(self):
-        return {
-            k: which(v)
-            for k, v in (
-                ("CC", os.environ.get("CC", "gcc")),
-                ("CXX", os.environ.get("CXX", "g++")),
-                ("AR", "ar"),
-                ("STRIP", "strip"),
-                ("RANLIB", "ranlib"),
-                ("LD", "ld"),
-                ("PKGCONFIG", "pkg-config"),
-            )
-        }
-
-    def set_comp_flags(self, env):
-        super().set_comp_flags(env)
-        env["CFLAGS"] = "-m32 -march=i586 -mno-sse " + env["CFLAGS"]
-        env["CXXFLAGS"] = "-m32 -march=i586 -mno-sse " + env["CXXFLAGS"]
-        env["LDFLAGS"] = "-m32 -march=i586 -mno-sse " + env["LDFLAGS"]
-
-    def get_bin_dir(self):
-        return []
-
-    def finalize_setup(self):
-        super().finalize_setup()
-        self.buildEnv.cmake_crossfile = self._gen_crossfile(
-            "cmake_i586_cross_file.txt", "cmake_cross_file.txt"
-        )
-        self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
-
-
-class I586Dyn(I586ConfigInfo):
-    name = "i586_dyn"
-    static = False
-
-
-class I586Static(I586ConfigInfo):
-    name = "i586_static"
-    static = True

kiwixbuild/configs/ios.py

@@ -1,249 +0,0 @@
-import subprocess
-
-from kiwixbuild._global import option
-from kiwixbuild.utils import pj, xrun_find
-from .base import ConfigInfo, MetaConfigInfo, MixedMixin
-from kiwixbuild.dependencies.apple_xcframework import AppleXCFramework
-
-
-MIN_MACOS_VERSION = "12.0"
-
-
-class AppleConfigInfo(ConfigInfo):
-    build = "iOS"
-    static = True
-    compatible_hosts = ["Darwin"]
-    arch = None
-    host = None
-    target = None
-    sdk_name = None
-    min_iphoneos_version = None
-    min_macos_version = None
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self._root_path = None
-
-    @property
-    def arch_name(self):
-        return self.target
-
-    @property
-    def root_path(self):
-        if self._root_path is None:
-            command = "xcrun --sdk {} --show-sdk-path".format(self.sdk_name)
-            self._root_path = subprocess.check_output(command, shell=True)[:-1].decode()
-        return self._root_path
-
-    def __str__(self):
-        return "iOS"
-
-    def finalize_setup(self):
-        super().finalize_setup()
-        self.buildEnv.cmake_crossfile = self._gen_crossfile(
-            "cmake_ios_cross_file.txt", "cmake_cross_file.txt"
-        )
-        self.buildEnv.meson_crossfile = self._gen_crossfile(
-            "meson_ios_cross_file.txt", "meson_cross_file.txt"
-        )
-
-    def get_cross_config(self):
-        config = {
-            "root_path": self.root_path,
-            "binaries": self.binaries,
-            "exe_wrapper_def": "",
-            "extra_libs": [
-                "-isysroot",
-                self.root_path,
-                "-arch",
-                self.arch,
-                "-target",
-                self.target,
-            ],
-            "extra_cflags": [
-                "-isysroot",
-                self.root_path,
-                "-arch",
-                self.arch,
-                "-target",
-                self.target,
-                *(
-                    "-I{}".format(include_dir)
-                    for include_dir in self.get_include_dirs()
-                ),
-            ],
-            "host_machine": {
-                "system": "Darwin",
-                "lsystem": "darwin",
-                "cpu_family": self.arch,
-                "cpu": self.cpu,
-                "endian": "",
-                "abi": "",
-            },
-        }
-        if self.min_iphoneos_version:
-            config["extra_libs"].append(
-                "-miphoneos-version-min={}".format(self.min_iphoneos_version)
-            )
-            config["extra_cflags"].append(
-                "-miphoneos-version-min={}".format(self.min_iphoneos_version)
-            )
-        if self.min_macos_version:
-            config["extra_libs"].append(
-                "-mmacosx-version-min={}".format(self.min_macos_version)
-            )
-            config["extra_cflags"].append(
-                "-mmacosx-version-min={}".format(self.min_macos_version)
-            )
-        return config
-
-    def get_env(self):
-        env = super().get_env()
-        cflags = [env["CFLAGS"]]
-        if self.min_iphoneos_version:
-            cflags.append("-miphoneos-version-min={}".format(self.min_iphoneos_version))
-        if self.min_macos_version:
-            cflags.append("-mmacosx-version-min={}".format(self.min_macos_version))
-        env["CFLAGS"] = " ".join(cflags)
-        return env
-
-    def set_comp_flags(self, env):
-        super().set_comp_flags(env)
-        cflags = [
-            "-isysroot {}".format(self.root_path),
-            "-arch {}".format(self.arch),
-            "-target {}".format(self.target),
-            env["CFLAGS"],
-        ]
-        if self.min_iphoneos_version:
-            cflags.append("-miphoneos-version-min={}".format(self.min_iphoneos_version))
-        env["CFLAGS"] = " ".join(cflags)
-        env["CXXFLAGS"] = " ".join(
-            [
-                env["CFLAGS"],
-                "-std=c++11",
-                env["CXXFLAGS"],
-            ]
-        )
-        env["LDFLAGS"] = " ".join(
-            [
-                " -arch {}".format(self.arch),
-                "-isysroot {}".format(self.root_path),
-            ]
-        )
-
-    def get_bin_dir(self):
-        return [pj(self.root_path, "bin")]
-
-    @property
-    def binaries(self):
-        return {
-            "CC": xrun_find("clang"),
-            "CXX": xrun_find("clang++"),
-            "AR": xrun_find("ar"),
-            "STRIP": xrun_find("strip"),
-            "RANLIB": xrun_find("ranlib"),
-            "LD": xrun_find("ld"),
-            "PKGCONFIG": "pkg-config",
-        }
-
-    @property
-    def configure_options(self):
-        yield f"--host={self.host}"
-
-
-class iOSArm64(AppleConfigInfo):
-    name = "iOS_arm64"
-    arch = cpu = "arm64"
-    host = "arm-apple-darwin"
-    target = "aarch64-apple-ios"
-    sdk_name = "iphoneos"
-    min_iphoneos_version = "15.0"
-
-
-class iOSx64Simulator(AppleConfigInfo):
-    name = "iOSSimulator_x86_64"
-    arch = cpu = "x86_64"
-    host = "x86_64-apple-darwin"
-    target = "x86-apple-ios-simulator"
-    sdk_name = "iphonesimulator"
-    min_iphoneos_version = "15.0"
-
-
-class iOSArm64Simulator(AppleConfigInfo):
-    name = "iOSSimulator_arm64"
-    arch = cpu = "arm64"
-    host = "arm-apple-darwin"
-    target = "aarch64-apple-ios-simulator"
-    sdk_name = "iphonesimulator"
-    min_iphoneos_version = "15.0"
-
-
-class macOSArm64(AppleConfigInfo):
-    name = "macOS_arm64_static"
-    arch = cpu = "arm64"
-    host = "aarch64-apple-darwin"
-    target = "arm64-apple-macos"
-    sdk_name = "macosx"
-    min_iphoneos_version = None
-    min_macos_version = MIN_MACOS_VERSION
-
-
-class macOSArm64Mixed(MixedMixin("macOS_arm64_static"), AppleConfigInfo):
-    name = "macOS_arm64_mixed"
-    arch = cpu = "arm64"
-    host = "aarch64-apple-darwin"
-    target = "arm64-apple-macos"
-    sdk_name = "macosx"
-    min_iphoneos_version = None
-    min_macos_version = MIN_MACOS_VERSION
-
-
-class macOSx64(AppleConfigInfo):
-    name = "macOS_x86_64"
-    arch = cpu = "x86_64"
-    host = "x86_64-apple-darwin"
-    target = "x86_64-apple-macos"
-    sdk_name = "macosx"
-    min_iphoneos_version = None
-    min_macos_version = MIN_MACOS_VERSION
-
-
-class IOS(MetaConfigInfo):
-    name = "iOS_multi"
-    compatible_hosts = ["Darwin"]
-
-    @property
-    def arch_name(self):
-        return self.name
-
-    @property
-    def subConfigNames(self):
-        return ["iOS_{}".format(arch) for arch in option("ios_arch")]
-
-    def add_targets(self, targetName, targets):
-        super().add_targets(targetName, targets)
-        return ConfigInfo.add_targets(self, "_ios_fat_lib", targets)
-
-    def __str__(self):
-        return self.name
-
-
-class AppleStaticAll(MetaConfigInfo):
-    name = "apple_all_static"
-    compatible_hosts = ["Darwin"]
-
-    @property
-    def arch_name(self):
-        return self.name
-
-    @property
-    def subConfigNames(self):
-        return AppleXCFramework.subConfigNames
-
-    def add_targets(self, targetName, targets):
-        super().add_targets(targetName, targets)
-        return ConfigInfo.add_targets(self, "apple_xcframework", targets)
-
-    def __str__(self):
-        return self.name

kiwixbuild/configs/musl.py

@@ -1,157 +0,0 @@
-from .base import ConfigInfo, MixedMixin
-
-from kiwixbuild.utils import pj
-from kiwixbuild._global import get_target_step
-
-
-class MuslConfigInfo(ConfigInfo):
-    compatible_hosts = ["fedora", "debian"]
-
-    def get_cross_config(self):
-        return {
-            "binaries": self.binaries,
-            "exe_wrapper_def": "",
-            "root_path": self.root_path,
-            "extra_libs": [],
-            "extra_cflags": [
-                "-I{}".format(include_dir) for include_dir in self.get_include_dirs()
-            ],
-            "host_machine": {
-                "system": "linux",
-                "lsystem": "linux",
-                "cpu_family": self.cpu_family,
-                "cpu": self.cpu,
-                "endian": "little",
-                "abi": "",
-            },
-        }
-
-    @property
-    def toolchain(self):
-        return get_target_step(self.build, "neutral")
-
-    @property
-    def root_path(self):
-        return self.toolchain.build_path
-
-    @property
-    def binaries(self):
-        binaries = (
-            (k, "{}-{}".format(self.arch_full, v))
-            for k, v in (
-                ("CC", "gcc"),
-                ("CXX", "g++"),
-                ("AR", "ar"),
-                ("STRIP", "strip"),
-                ("WINDRES", "windres"),
-                ("RANLIB", "ranlib"),
-                ("LD", "ld"),
-                ("LDSHARED", "g++ -shared"),
-            )
-        )
-        binaries = {k: pj(self.root_path, "bin", v) for k, v in binaries}
-        binaries["PKGCONFIG"] = "pkg-config"
-        return binaries
-
-    @property
-    def exe_wrapper_def(self):
-        try:
-            which(self.qemu)
-        except subprocess.CalledProcessError:
-            return ""
-        except AttributeError:
-            return ""
-        else:
-            return f"exe_wrapper = '{self.qemu}'"
-
-    @property
-    def configure_options(self):
-        return [f"--host={self.arch_full}"]
-
-    def get_bin_dir(self):
-        return [pj(self.root_path, "bin")]
-
-    def get_env(self):
-        env = super().get_env()
-        env["LD_LIBRARY_PATH"][0:0] = [
-            pj(self.root_path, self.arch_full, "lib64"),
-            pj(self.root_path, "lib"),
-        ]
-        env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig")
-        env["QEMU_LD_PREFIX"] = pj(self.root_path, self.arch_full, "libc")
-        env["QEMU_SET_ENV"] = "LD_LIBRARY_PATH={}".format(
-            ":".join(
-                [pj(self.root_path, self.arch_full, "lib"), str(env["LD_LIBRARY_PATH"])]
-            )
-        )
-        return env
-
-    def set_comp_flags(self, env):
-        super().set_comp_flags(env)
-        env["LD_LIBRARY_PATH"].insert(0, pj(self.root_path, self.arch_full, "lib"))
-        env["CFLAGS"] = (
-            " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
-            + env["CFLAGS"]
-        )
-        env["CXXFLAGS"] = (
-            " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
-            + env["CXXFLAGS"]
-        )
-
-    def set_compiler(self, env):
-        for k, v in self.binaries.items():
-            env[k] = v
-
-    def finalize_setup(self):
-        super().finalize_setup()
-        self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt")
-        self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
-
-
-class Aarch64MuslConfigInfo(MuslConfigInfo):
-    build = "aarch64_musl"
-    arch_full = "aarch64-linux-musl"
-    toolchain_names = ["aarch64_musl"]
-    libdir = "lib/aarch64-linux-musl"
-    cpu_family = "arm"
-    cpu = "armhf"
-    qemu = "qemu-arm"
-
-
-class Aarch64MuslDyn(Aarch64MuslConfigInfo):
-    name = "aarch64_musl_dyn"
-    static = False
-
-
-class Aarch64MuslStatic(Aarch64MuslConfigInfo):
-    name = "aarch64_musl_static"
-    static = True
-
-
-class Aarch64MuslMixed(MixedMixin("aarch64_musl_static"), Aarch64MuslConfigInfo):
-    name = "aarch64_musl_mixed"
-    static = False
-
-
-class X86_64MuslConfigInfo(MuslConfigInfo):
-    build = "x86-64_musl"
-    arch_full = "x86_64-linux-musl"
-    toolchain_names = ["x86-64_musl"]
-    libdir = "lib/x86_64-linux-musl"
-    cpu_family = "x86_64"
-    cpu = "x86_64"
-
-
-class X86_64MuslDyn(X86_64MuslConfigInfo):
-    name = "x86-64_musl_dyn"
-    static = False
-
-
-class X86_64MuslStatic(X86_64MuslConfigInfo):
-    name = "x86-64_musl_static"
-    static = True
-
-
-class x86_64MuslMixed(MixedMixin("x86-64_musl_static"), X86_64MuslConfigInfo):
-    name = "x86-64_musl_mixed"
-    static = False

kiwixbuild/configs/native.py

@@ -1,44 +0,0 @@
-from .base import ConfigInfo, MixedMixin
-
-from kiwixbuild.utils import pj
-from kiwixbuild._global import option, neutralEnv
-from kiwixbuild.configs.ios import MIN_MACOS_VERSION
-import sysconfig
-import platform
-import sys
-
-
-class NativeConfigInfo(ConfigInfo):
-    build = "native"
-
-    def get_env(self):
-        env = super().get_env()
-        if neutralEnv("distname") == "fedora":
-            env["QT_SELECT"] = "5-64"
-        if neutralEnv("distname") == "Darwin":
-            env["CFLAGS"] += f"-mmacosx-version-min={MIN_MACOS_VERSION}"
-        return env
-
-    @property
-    def arch_name(self):
-        if sys.platform == "darwin":
-            return f"{platform.machine()}-apple-darwin"
-        return sysconfig.get_platform()
-
-
-class NativeDyn(NativeConfigInfo):
-    name = "native_dyn"
-    static = False
-    compatible_hosts = ["fedora", "debian", "Darwin", "almalinux", "Windows"]
-
-
-class NativeStatic(NativeConfigInfo):
-    name = "native_static"
-    static = True
-    compatible_hosts = ["fedora", "debian", "Darwin", "almalinux", "Windows"]
-
-
-class NativeMixed(MixedMixin("native_static"), NativeConfigInfo):
-    name = "native_mixed"
-    static = False
-    compatible_hosts = ["fedora", "debian", "Darwin", "almalinux", "Windows"]

kiwixbuild/configs/neutral.py

@@ -1,11 +0,0 @@
-from .base import ConfigInfo
-
-
-class NeutralConfigInfo(ConfigInfo):
-    name = "neutral"
-    arch_name = "neutral"
-    static = ""
-    compatible_hosts = ["fedora", "debian", "Darwin"]
-
-    def __str__(self):
-        return "neutral"

kiwixbuild/configs/wasm.py

@@ -1,113 +0,0 @@
-from .base import ConfigInfo
-
-from kiwixbuild.utils import pj
-from kiwixbuild._global import get_target_step
-
-
-class WasmConfigInfo(ConfigInfo):
-    name = "wasm"
-    static = True
-    build = "wasm"
-    arch_full = "wasm64-emscripten"
-    libdir = "lib"
-    # arch_full = 'wasm64-linux'
-    toolchain_names = ["emsdk"]
-    compatible_hosts = ["fedora", "debian"]
-    exe_wrapper_def = ""
-
-    def get_cross_config(self):
-        return {
-            "binaries": self.binaries,
-            "exe_wrapper_def": "",
-            "root_path": self.root_path,
-            "extra_libs": [],
-            "extra_cflags": [],
-            "host_machine": {
-                "system": "emscripten",
-                "lsystem": "emscripten",
-                "cpu_family": "wasm64",
-                "cpu": "wasm64",
-                "endian": "little",
-                "abi": "",
-            },
-        }
-
-    @property
-    def wasm_sdk(self):
-        return get_target_step("emsdk", self.name)
-
-    @property
-    def install_path(self):
-        return self.wasm_sdk.install_path
-
-    @property
-    def root_path(self):
-        return self.install_path
-
-    @property
-    def binaries(self):
-        binaries = (
-            ("CC", "emcc"),
-            ("CXX", "em++"),
-            ("AR", "emar"),
-            ("STRIP", "emstrip"),
-            ("WINDRES", "windres"),
-            ("RANLIB", "emranlib"),
-            ("LD", "wasm-ld"),
-        )
-        binaries = {
-            k: pj(self.install_path, "upstream", "emscripten", v) for k, v in binaries
-        }
-        binaries["PKGCONFIG"] = "pkg-config"
-        return binaries
-
-    @property
-    def configure_options(self):
-        # return ""
-        return [f"--host={self.arch_full}"]
-
-    @property
-    def configure_wrapper(self):
-        return "emconfigure"
-
-    @property
-    def make_wrapper(self):
-        return "emmake"
-
-    def get_bin_dir(self):
-        return [pj(self.install_path, "bin")]
-
-    def get_env(self):
-        env = super().get_env()
-        env["PATH"].extend(
-            [
-                self.install_path,
-                pj(self.install_path, "upstream", "emscripten"),
-                pj(self.install_path, "node", "14.18.2_64bit", "bin"),
-            ]
-        )
-        env["EMSDK"] = self.install_path
-        env["EMSDK_NODE"] = pj(
-            self.install_path, "node", "14.18.2_64bit", "bin", "node"
-        )
-        return env
-
-    def set_comp_flags(self, env):
-        super().set_comp_flags(env)
-        env["CFLAGS"] = (
-            " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
-            + env["CFLAGS"]
-        )
-        env["CXXFLAGS"] = (
-            " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
-            + env["CXXFLAGS"]
-        )
-
-    def set_compiler(self, env):
-        for k, v in self.binaries.items():
-            env[k] = v
-
-    def finalize_setup(self):
-        super().finalize_setup()
-        self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt")
-        self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")

kiwixbuild/configs/win64.py

@@ -1,103 +0,0 @@
-import subprocess
-
-from .base import ConfigInfo
-from kiwixbuild.utils import which, pj
-from kiwixbuild._global import neutralEnv
-
-
-class Win64ConfigInfo(ConfigInfo):
-    extra_libs = [
-        "-lmingw32",
-        "-lwinmm",
-        "-lws2_32",
-        "-lshlwapi",
-        "-lrpcrt4",
-        "-lmsvcr100",
-        "-liphlpapi",
-        "-lshell32",
-        "-lkernel32",
-    ]
-    build = "win64"
-    compatible_hosts = ["fedora", "debian"]
-    arch_full = "x86_64-w64-mingw32"
-
-    def get_cross_config(self):
-        return {
-            "exe_wrapper_def": self.exe_wrapper_def,
-            "binaries": self.binaries,
-            "root_path": self.root_path,
-            "extra_libs": self.extra_libs,
-            "extra_cflags": ["-DWIN32"],
-            "host_machine": {
-                "system": "Windows",
-                "lsystem": "windows",
-                "cpu_family": "x86_64",
-                "cpu": "x86_64",
-                "endian": "little",
-                "abi": "",
-            },
-        }
-
-    def finalize_setup(self):
-        super().finalize_setup()
-        self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt")
-        self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
-
-    @property
-    def root_path(self):
-        root_paths = {
-            "fedora": "/usr/x86_64-w64-mingw32/sys-root/mingw",
-            "debian": "/usr/x86_64-w64-mingw32",
-        }
-        return root_paths[neutralEnv("distname")]
-
-    @property
-    def binaries(self):
-        return {
-            k: which("{}-{}".format(self.arch_full, v))
-            for k, v in (
-                ("CC", "gcc"),
-                ("CXX", "g++"),
-                ("AR", "ar"),
-                ("STRIP", "strip"),
-                ("WINDRES", "windres"),
-                ("RANLIB", "ranlib"),
-                ("PKGCONFIG", "pkg-config"),
-            )
-        }
-
-    @property
-    def exe_wrapper_def(self):
-        try:
-            which("wine")
-        except subprocess.CalledProcessError:
-            return ""
-        else:
-            return "exe_wrapper = 'wine'"
-
-    @property
-    def configure_options(self):
-        return [f"--host={self.arch_full}"]
-
-    def set_compiler(self, env):
-        for k, v in self.binaries.items():
-            env[k] = v
-
-    def get_bin_dir(self):
-        return [pj(self.root_path, "bin")]
-
-    def get_env(self):
-        env = super().get_env()
-        env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig")
-        env["LIBS"] = " ".join(self.extra_libs) + " " + env["LIBS"]
-        return env
-
-
-class Win64Dyn(Win64ConfigInfo):
-    name = "win64_dyn"
-    static = False
-
-
-class Win64Static(Win64ConfigInfo):
-    name = "win64_static"
-    static = True

kiwixbuild/dependencies/__init__.py

@@ -1,32 +0,0 @@
-from .base import *
-from . import (
-    all_dependencies,
-    boostregex,
-    tc_android_ndk,
-    aria2,
-    tc_armhf,
-    tc_musl,
-    docoptcpp,
-    tc_emsdk,
-    tc_flatpak,
-    gumbo,
-    icu4c,
-    ios_fat_lib,
-    mustache,
-    kiwix_desktop,
-    kiwix_tools,
-    libcurl,
-    libkiwix,
-    libmagic,
-    libmicrohttpd,
-    libzim,
-    lzma,
-    qt,
-    pugixml,
-    uuid,
-    xapian,
-    zim_tools,
-    zim_testing_suite,
-    zlib,
-    zstd,
-)

kiwixbuild/dependencies/all_dependencies.py

@@ -1,61 +0,0 @@
-from os import environ
-
-from .base import Dependency, NoopSource, NoopBuilder
-
-from kiwixbuild._global import neutralEnv
-
-
-class AllBaseDependencies(Dependency):
-    name = "alldependencies"
-
-    Source = NoopSource
-
-    class Builder(NoopBuilder):
-        @classmethod
-        def get_dependencies(cls, configInfo, allDeps):
-            if configInfo.build == "wasm" or environ.get("OS_NAME") == "manylinux":
-                return ["zlib", "lzma", "zstd", "icu4c", "xapian-core"]
-
-            if neutralEnv("distname") == "Windows":
-                base_deps = [
-                    "zlib",
-                    "zstd",
-                    "xapian-core",
-                    "zim-testing-suite",
-                    "icu4c",
-                    "boostregex",
-                    "docoptcpp",
-                ]
-
-                if not configInfo.name.endswith("_dyn"):
-                    base_deps += [
-                        "pugixml",
-                        "libcurl",
-                        "mustache",
-                        "libmicrohttpd",
-                    ]
-            else:
-                base_deps = [
-                    "zlib",
-                    "lzma",
-                    "zstd",
-                    "xapian-core",
-                    "pugixml",
-                    "libcurl",
-                    "icu4c",
-                    "mustache",
-                    "libmicrohttpd",
-                    "zim-testing-suite",
-                ]
-                # Add specific dependencies depending of the config
-                if configInfo.build not in ("android", "iOS"):
-                    # For zimtools
-                    base_deps += ["docoptcpp", "libmagic", "gumbo"]
-                    if (
-                        configInfo.build == "native"
-                        and neutralEnv("distname") != "Darwin"
-                    ):
-                        # We compile kiwix-desktop only on native and not on `Darwin`
-                        # So we need aria2 only there
-                        base_deps += ["aria2"]
-            return base_deps

kiwixbuild/dependencies/apple_xcframework.py

@@ -1,126 +0,0 @@
-import os
-import shutil
-from pathlib import Path
-
-from kiwixbuild.configs import ConfigInfo
-from kiwixbuild.utils import pj, run_command
-from .base import Dependency, NoopSource, Builder as BaseBuilder
-
-
-class AppleXCFramework(Dependency):
-    name = "apple_xcframework"
-    subConfigNames = [
-        "macOS_x86_64",
-        "macOS_arm64_static",
-        "iOS_arm64",
-        "iOSSimulator_x86_64",
-        "iOSSimulator_arm64",
-    ]
-    Source = NoopSource
-
-    class Builder(BaseBuilder):
-        @property
-        def all_subconfigs(self):
-            return self.buildEnv.configInfo.subConfigNames
-
-        @property
-        def macos_subconfigs(self):
-            return [
-                target for target in self.all_subconfigs if target.startswith("macOS")
-            ]
-
-        @property
-        def iossimulator_subconfigs(self):
-            return [
-                target
-                for target in self.all_subconfigs
-                if target.startswith("iOSSimulator")
-            ]
-
-        @property
-        def ios_subconfigs(self):
-            return [
-                target for target in self.all_subconfigs if target.startswith("iOS_")
-            ]
-
-        @classmethod
-        def get_dependencies(cls, configInfo, alldeps):
-            return [(target, "libkiwix") for target in AppleXCFramework.subConfigNames]
-
-        @property
-        def final_path(self):
-            return pj(self.buildEnv.install_dir, "lib", "CoreKiwix.xcframework")
-
-        def _remove_if_exists(self, context):
-            if not os.path.exists(self.final_path):
-                return
-
-            shutil.rmtree(self.final_path)
-
-        def _merge_libs(self, context):
-            """create merged.a in all targets to bundle all static archives"""
-            xcf_libs = []
-            for target in self.all_subconfigs:
-                static_ars = []
-
-                cfg = ConfigInfo.get_config(target)
-                lib_dir = pj(cfg.buildEnv.install_dir, "lib")
-                static_ars = [str(f) for f in Path(lib_dir).glob("*.a")]
-
-                # create merged.a from all *.a in install_dir/lib
-                command = ["libtool", "-static", "-o", "merged.a", *static_ars]
-                run_command(command, lib_dir, context)
-
-                # will be included in xcframework
-                if target in self.ios_subconfigs:
-                    xcf_libs.append(pj(lib_dir, "merged.a"))
-
-            return xcf_libs
-
-        def make_fat_with(self, configs, folder_name, context):
-            """create fat merged.a in {folder_name} install/lib with {configs}"""
-            libs = []
-            for target in configs:
-                cfg = ConfigInfo.get_config(target)
-                libs.append(pj(cfg.buildEnv.install_dir, "lib", "merged.a"))
-
-            fat_dir = pj(self.buildEnv.build_dir, folder_name)
-            os.makedirs(fat_dir, exist_ok=True)
-
-            output_merged = pj(fat_dir, "merged.a")
-            command = ["lipo", "-create", "-output", output_merged, *libs]
-            run_command(command, self.buildEnv.build_dir, context)
-
-            return [output_merged]
-
-        def _build_xcframework(self, xcf_libs, context):
-            # create xcframework
-            ref_conf = ConfigInfo.get_config(self.macos_subconfigs[0])
-            command = ["xcodebuild", "-create-xcframework"]
-            for lib in xcf_libs:
-                command += [
-                    "-library",
-                    lib,
-                    "-headers",
-                    pj(ref_conf.buildEnv.install_dir, "include"),
-                ]
-            command += ["-output", self.final_path]
-            run_command(command, self.buildEnv.build_dir, context)
-
-        def build(self):
-            xcf_libs = []
-            self.command("remove_if_exists", self._remove_if_exists)
-            xcf_libs += self.command("merge_libs", self._merge_libs)
-            xcf_libs += self.command(
-                "make_macos_fat",
-                self.make_fat_with,
-                self.macos_subconfigs,
-                "macOS_fat",
-            )
-            xcf_libs += self.command(
-                "make_simulator_fat",
-                self.make_fat_with,
-                self.iossimulator_subconfigs,
-                "iOS-simulator_fat",
-            )
-            self.command("build_xcframework", self._build_xcframework, xcf_libs)

kiwixbuild/dependencies/aria2.py

@@ -1,53 +0,0 @@
-from .base import Dependency, ReleaseDownload, MakeBuilder, NoopBuilder
-
-from kiwixbuild.utils import Remotefile, run_command, pj
-import platform
-from shutil import copy2
-
-# Important: in case of aria2c update,
-# 'scripts/create_kiwix-desktop_appImage.sh' should not be forgotten!
-
-class Aria2(Dependency):
-    name = "aria2"
-
-    if platform.system() == "Windows":
-
-        class Source(ReleaseDownload):
-            archive = Remotefile(
-                "aria2-1.37.0-win-64bit-build1.zip",
-                "67d015301eef0b612191212d564c5bb0a14b5b9c4796b76454276a4d28d9b288",
-                "https://dev.kiwix.org/kiwix-desktop/aria2-1.37.0-win-64bit-build1.zip",
-            )
-
-        class Builder(NoopBuilder):
-            def build(self):
-                self.command("copy_binary", self._copy_binary)
-
-            def _copy_binary(self, context):
-                context.try_skip(self.build_path)
-                copy2(
-                    pj(self.source_path, "aria2c.exe"),
-                    pj(self.buildEnv.install_dir, "bin"),
-                )
-
-    else:
-
-        class Source(ReleaseDownload):
-            archive = Remotefile(
-                "aria2-1.37.0.tar.xz",
-                "60a420ad7085eb616cb6e2bdf0a7206d68ff3d37fb5a956dc44242eb2f79b66b",
-                "https://dev.kiwix.org/kiwix-desktop/aria2-1.37.0.tar.xz",
-            )
-
-            def _post_prepare_script(self, context):
-                context.try_skip(self.extract_path)
-                command = ["autoreconf", "-i"]
-                run_command(command, self.extract_path, context)
-
-        class Builder(MakeBuilder):
-            dependencies = ["zlib"]
-            configure_options = [
-                "--disable-libaria2",
-                "--disable-websocket",
-                "--without-sqlite3",
-            ]

kiwixbuild/dependencies/base.py

@@ -1,615 +0,0 @@
-import subprocess
-import os
-import shutil
-import time
-import platform
-
-from kiwixbuild.utils import (
-    pj,
-    Context,
-    SkipCommand,
-    WarningMessage,
-    extract_archive,
-    StopBuild,
-    run_command,
-    colorize,
-    copy_tree,
-)
-from kiwixbuild.versions import main_project_versions, base_deps_versions
-from kiwixbuild._global import neutralEnv, option, get_target_step
-
-SCRIPT_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-
-
-class _MetaDependency(type):
-    def __new__(cls, name, bases, dct):
-        _class = type.__new__(cls, name, bases, dct)
-        if name != "Dependency":
-            dep_name = dct["name"]
-            Dependency.all_deps[dep_name] = _class
-        return _class
-
-
-class Dependency(metaclass=_MetaDependency):
-    all_deps = {}
-    force_build = False
-    force_native_build = False
-    dont_skip = False
-
-    @classmethod
-    def version(cls):
-        if cls.name in base_deps_versions:
-            return base_deps_versions[cls.name]
-        elif option("make_release"):
-            return main_project_versions.get(cls.name, None)
-        return None
-
-    @classmethod
-    def full_name(cls):
-        if cls.version():
-            return "{}-{}".format(cls.name, cls.version())
-        return cls.name
-
-
-class Source:
-    """Base Class to the real preparator
-    A source preparator must install source in the self.source_dir attribute
-    inside the neutralEnv.source_dir."""
-
-    def __init__(self, target):
-        self.target = target
-
-    @property
-    def name(self):
-        return self.target.name
-
-    @property
-    def full_name(self):
-        return self.target.full_name()
-
-    @property
-    def source_dir(self):
-        return self.target.full_name()
-
-    @property
-    def source_path(self):
-        return pj(neutralEnv("source_dir"), self.source_dir)
-
-    @property
-    def _log_dir(self):
-        return neutralEnv("log_dir")
-
-    def _patch(self, context):
-        context.try_skip(self.source_path)
-        for p in self.patches:
-            patch_file_path = pj(SCRIPT_DIR, "patches", p)
-            patch_command = [*neutralEnv("patch_command"), "-p1", "-i", patch_file_path]
-            run_command(patch_command, self.source_path, context)
-
-    def command(self, name, function, *args):
-        print("  {} {} : ".format(name, self.name), end="", flush=True)
-        log = pj(self._log_dir, "cmd_{}_{}.log".format(name, self.name))
-        context = Context(name, log, True)
-        try:
-            start_time = time.time()
-            ret = function(*args, context=context)
-            context._finalise()
-            duration = time.time() - start_time
-            print(colorize("OK"), "({:.1f}s)".format(duration))
-            return ret
-        except WarningMessage as e:
-            print(e)
-        except SkipCommand as e:
-            print(e)
-        except subprocess.CalledProcessError:
-            print(colorize("ERROR"))
-            try:
-                with open(log, "r") as f:
-                    print(f.read())
-            except:
-                pass
-            raise StopBuild()
-        except:
-            print(colorize("ERROR"))
-            raise
-
-
-class NoopSource(Source):
-    def prepare(self):
-        pass
-
-
-class ReleaseDownload(Source):
-    archive_top_dir = None
-
-    @property
-    def archives(self):
-        return (self.archive,)
-
-    @property
-    def extract_path(self):
-        return pj(neutralEnv("source_dir"), self.source_dir)
-
-    def _download(self, context):
-        context.try_skip(neutralEnv("archive_dir"), self.full_name)
-        archive_iter = iter(self.archives)
-        archive = next(archive_iter, None)
-        while archive:
-            try:
-                neutralEnv("download")(archive)
-            except SkipCommand as e:
-                archive = next(archive_iter, None)
-                if not archive:
-                    raise e
-                continue
-            archive = next(archive_iter, None)
-
-    def _extract(self, context):
-        context.try_skip(self.extract_path)
-        if os.path.exists(self.extract_path):
-            shutil.rmtree(self.extract_path)
-        for archive in self.archives:
-            extract_archive(
-                pj(neutralEnv("archive_dir"), archive.name),
-                neutralEnv("source_dir"),
-                topdir=self.archive_top_dir,
-                name=self.source_dir,
-            )
-
-    def prepare(self):
-        self.command("download", self._download)
-        self.command("extract", self._extract)
-        if hasattr(self, "patches"):
-            self.command("patch", self._patch)
-        if hasattr(self, "_post_prepare_script"):
-            self.command("post_prepare_script", self._post_prepare_script)
-
-
-class GitClone(Source):
-    base_git_ref = "main"
-    force_full_clone = False
-
-    @property
-    def release_git_ref(self):
-        return main_project_versions.get(self.name, self.base_git_ref)
-
-    @property
-    def source_dir(self):
-        if option("make_release"):
-            return "{}_release".format(self.git_dir)
-        else:
-            return self.git_dir
-
-    @property
-    def git_path(self):
-        return pj(neutralEnv("source_dir"), self.source_dir)
-
-    @property
-    def git_ref(self):
-        if option("make_release"):
-            return self.release_git_ref
-        else:
-            return self.base_git_ref
-
-    def _git_init(self, context):
-        if option("fast_clone") and self.force_full_clone == False:
-            command = [
-                *neutralEnv("git_command"),
-                "clone",
-                "--depth=1",
-                "--branch",
-                self.git_ref,
-                self.git_remote,
-                self.source_dir,
-            ]
-            run_command(command, neutralEnv("source_dir"), context)
-        else:
-            command = [
-                *neutralEnv("git_command"),
-                "clone",
-                self.git_remote,
-                self.source_dir,
-            ]
-            run_command(command, neutralEnv("source_dir"), context)
-            command = [*neutralEnv("git_command"), "checkout", self.git_ref]
-            run_command(command, self.git_path, context)
-
-    def _git_update(self, context):
-        command = [*neutralEnv("git_command"), "fetch", "origin", self.git_ref]
-        run_command(command, self.git_path, context)
-        try:
-            command = [
-                *neutralEnv("git_command"),
-                "merge",
-                "--ff-only",
-                f"origin/{self.git_ref}",
-            ]
-            run_command(command, self.git_path, context)
-        except subprocess.CalledProcessError:
-            raise WarningMessage("Cannot update, please check log for information")
-
-    def prepare(self):
-        if not os.path.exists(self.git_path):
-            self.command("gitinit", self._git_init)
-        else:
-            self.command("gitupdate", self._git_update)
-        if hasattr(self, "_post_prepare_script"):
-            self.command("post_prepare_script", self._post_prepare_script)
-
-
-class Builder:
-    subsource_dir = None
-    dependencies = []
-
-    def __init__(self, target, source, buildEnv):
-        self.target = target
-        self.source = source
-        self.buildEnv = buildEnv
-
-    @classmethod
-    def get_dependencies(cls, configInfo, allDeps):
-        return cls.dependencies
-
-    @property
-    def name(self):
-        return self.target.name
-
-    @property
-    def source_path(self):
-        base_source_path = self.source.source_path
-        if self.subsource_dir:
-            return pj(base_source_path, self.subsource_dir)
-        return base_source_path
-
-    @property
-    def build_path(self):
-        return pj(self.buildEnv.build_dir, self.target.full_name())
-
-    @property
-    def _log_dir(self):
-        return self.buildEnv.log_dir
-
-    def command(self, name, function, *args):
-        print("  {} {} : ".format(name, self.name), end="", flush=True)
-        log = pj(self._log_dir, "cmd_{}_{}.log".format(name, self.name))
-        context = Context(name, log, self.target.force_native_build)
-        if self.target.force_build:
-            context.no_skip = True
-        try:
-            start_time = time.time()
-            ret = function(*args, context=context)
-            context._finalise()
-            duration = time.time() - start_time
-            print(colorize("OK"), "({:.1f}s)".format(duration))
-            return ret
-        except SkipCommand as e:
-            print(e)
-        except WarningMessage as e:
-            print(e)
-        except subprocess.CalledProcessError:
-            print(colorize("ERROR"))
-            try:
-                with open(log, "r") as f:
-                    print(f.read())
-            except:
-                pass
-            raise StopBuild()
-        except:
-            print(colorize("ERROR"))
-            raise
-
-    def build(self):
-        if hasattr(self, "_pre_build_script"):
-            self.command("pre_build_script", self._pre_build_script)
-        self.command("configure", self._configure)
-        if hasattr(self, "_post_configure_script"):
-            self.command("post_configure_script", self._post_configure_script)
-        self.command("compile", self._compile)
-        if hasattr(self, "_test"):
-            self.command("test", self._test)
-        self.command("install", self._install)
-        if hasattr(self, "_post_build_script"):
-            self.command("post_build_script", self._post_build_script)
-
-    def make_dist(self):
-        if hasattr(self, "_pre_build_script"):
-            self.command("pre_build_script", self._pre_build_script)
-        self.command("configure", self._configure)
-        self.command("make_dist", self._make_dist)
-
-    def set_flatpak_buildsystem(self, module):
-        if getattr(self, "flatpak_buildsystem", None):
-            module["buildsystem"] = self.flatpak_buildsystem
-        if getattr(self, "subsource_dir", None):
-            module["subdir"] = self.subsource_dir
-        if getattr(self, "flatpack_build_options", None):
-            module["build-options"] = self.flatpack_build_options
-        if getattr(self, "configure_options", ""):
-            module["config-opts"] = list(self.configure_options)
-
-    def get_env(self, *, cross_comp_flags, cross_compilers, cross_path):
-        env = self.buildEnv.get_env(
-            cross_comp_flags=cross_comp_flags,
-            cross_compilers=cross_compilers,
-            cross_path=cross_path,
-        )
-        for dep in self.get_dependencies(self.buildEnv.configInfo, False):
-            try:
-                builder = get_target_step(dep, self.buildEnv.configInfo.name)
-                builder.set_env(env)
-            except KeyError:
-                # Some target may be missing (installed by a package, ...)
-                pass
-        return env
-
-    def set_env(self, env):
-        pass
-
-
-class NoopBuilder(Builder):
-    def build(self):
-        pass
-
-    def make_dist(self):
-        pass
-
-
-class TcCopyBuilder(Builder):
-    src_subdir = None
-
-    @property
-    def build_path(self):
-        return pj(self.buildEnv.toolchain_dir, self.target.full_name())
-
-    def build(self):
-        self.command("copy", self._copy)
-
-    def _copy(self, context):
-        context.try_skip(self.build_path)
-        if self.src_subdir:
-            source_path = pj(self.source_path, self.src_subdir)
-        else:
-            source_path = self.source_path
-        copy_tree(source_path, self.build_path)
-
-    def make_dist(self):
-        pass
-
-
-class MakeBuilder(Builder):
-    configure_options = []
-    dynamic_configure_options = ["--enable-shared", "--disable-static"]
-    static_configure_options = ["--enable-static", "--disable-shared"]
-    make_options = ["-j4"]
-    install_options = []
-    configure_script = "configure"
-    configure_env = {
-        "_format_CFLAGS": "{env[CFLAGS]} -O3",
-        "_format_CXXFLAGS": "{env[CXXFLAGS]} -O3",
-    }
-    make_targets = []
-    flatpak_buildsystem = None
-
-    @property
-    def make_install_targets(self):
-        if self.buildEnv.configInfo.build in ("iOS", "wasm"):
-            yield "install"
-        else:
-            yield "install-strip"
-
-    @property
-    def all_configure_options(self):
-        yield from self.configure_options
-        if self.buildEnv.configInfo.static:
-            yield from self.static_configure_options
-        else:
-            yield from self.dynamic_configure_options
-        if not self.target.force_native_build:
-            yield from self.buildEnv.configInfo.configure_options
-        yield from ("--prefix", self.buildEnv.install_dir)
-        yield from ("--libdir", pj(self.buildEnv.install_dir, self.buildEnv.libprefix))
-
-    def set_configure_env(self, env):
-        dep_conf_env = self.configure_env
-        if not dep_conf_env:
-            return
-        for k, v in dep_conf_env.items():
-            if k.startswith("_format_"):
-                v = v.format(buildEnv=self.buildEnv, env=env)
-                env[k[8:]] = v
-            else:
-                env[k] = v
-
-    def _configure(self, context):
-        context.try_skip(self.build_path)
-        command = [
-            *self.buildEnv.configure_wrapper,
-            pj(self.source_path, self.configure_script),
-            *self.all_configure_options,
-        ]
-        env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
-        self.set_configure_env(env)
-        run_command(command, self.build_path, context, env=env)
-
-    def _compile(self, context):
-        context.try_skip(self.build_path)
-        command = [
-            *self.buildEnv.make_wrapper,
-            *neutralEnv("make_command"),
-            *self.make_targets,
-            *self.make_options,
-        ]
-        env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
-        run_command(command, self.build_path, context, env=env)
-
-    def _install(self, context):
-        context.try_skip(self.build_path)
-        command = [
-            *self.buildEnv.make_wrapper,
-            *neutralEnv("make_command"),
-            *self.make_install_targets,
-            *self.make_options,
-        ]
-        env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
-        run_command(command, self.build_path, context, env=env)
-
-    def _make_dist(self, context):
-        context.try_skip(self.build_path)
-        command = [*self.buildEnv.make_wrapper, *neutralEnv("make_command"), "dist"]
-        env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
-        run_command(command, self.build_path, context, env=env)
-
-
-class CMakeBuilder(MakeBuilder):
-    flatpak_buildsystem = "cmake"
-
-    def _configure(self, context):
-        context.try_skip(self.build_path)
-        cross_options = []
-        if not self.target.force_native_build and self.buildEnv.cmake_crossfile:
-            cross_options += [f"-DCMAKE_TOOLCHAIN_FILE={self.buildEnv.cmake_crossfile}"]
-        command = [
-            *neutralEnv("cmake_command"),
-            *self.configure_options,
-            "-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON",
-            f"-DCMAKE_INSTALL_PREFIX={self.buildEnv.install_dir}",
-            f"-DCMAKE_INSTALL_LIBDIR={self.buildEnv.libprefix}",
-            self.source_path,
-            *cross_options,
-        ]
-        env = self.get_env(
-            cross_comp_flags=True, cross_compilers=False, cross_path=True
-        )
-        self.set_configure_env(env)
-        run_command(command, self.build_path, context, env=env)
-
-    def set_flatpak_buildsystem(self, module):
-        super().set_flatpak_buildsystem(module)
-        module["buildir"] = True
-
-
-class QMakeBuilder(MakeBuilder):
-    qmake_targets = []
-    flatpak_buildsystem = "qmake"
-
-    @property
-    def make_options(self):
-        if platform.system() == "Windows":
-            return []
-        return super().make_options
-
-    @property
-    def env_options(self):
-        if "QMAKE_CC" in os.environ:
-            yield f"QMAKE_CC={os.environ['QMAKE_CC']}"
-        if "QMAKE_CXX" in os.environ:
-            yield f"QMAKE_CXX={os.environ['QMAKE_CXX']}"
-
-    def _configure(self, context):
-        context.try_skip(self.build_path)
-        command = [
-            *neutralEnv("qmake_command"),
-            *self.configure_options,
-            *self.env_options,
-            self.source_path,
-        ]
-        env = self.get_env(
-            cross_comp_flags=True, cross_compilers=False, cross_path=True
-        )
-        self.set_configure_env(env)
-        run_command(command, self.build_path, context, env=env)
-
-    def _make_dist(self, context):
-        command = [
-            *neutralEnv("git_command"),
-            "archive",
-            "-o",
-            f"{self.build_path}/{self.target.full_name()}.tar.gz",
-            f"--prefix={self.target.full_name()}/",
-            "HEAD",
-        ]
-        run_command(command, self.source_path, context)
-
-
-class MesonBuilder(Builder):
-    configure_options = []
-    test_options = []
-    flatpak_buildsystem = "meson"
-
-    @property
-    def build_type(self):
-        if platform.system() == "Windows":
-            return "release"
-
-        return "release" if option("make_release") else "debug"
-
-    @property
-    def strip_options(self):
-        if option("make_release"):
-            yield "--strip"
-
-    @property
-    def library_type(self):
-        return "static" if self.buildEnv.configInfo.static else "shared"
-
-    def _configure(self, context):
-        context.try_skip(self.build_path)
-        if os.path.exists(self.build_path):
-            shutil.rmtree(self.build_path)
-        os.makedirs(self.build_path)
-        cross_options = []
-        if not self.target.force_native_build and self.buildEnv.meson_crossfile:
-            cross_options += ["--cross-file", self.buildEnv.meson_crossfile]
-        command = [
-            *neutralEnv("meson_command"),
-            "setup",
-            ".",
-            self.build_path,
-            f"--buildtype={self.build_type}",
-            *self.strip_options,
-            f"--default-library={self.library_type}",
-            *self.configure_options,
-            f"--prefix={self.buildEnv.install_dir}",
-            f"--libdir={self.buildEnv.libprefix}",
-            *cross_options,
-        ]
-        env = self.get_env(
-            cross_comp_flags=False, cross_compilers=False, cross_path=True
-        )
-        run_command(command, self.source_path, context, env=env)
-
-    def _compile(self, context):
-        context.try_skip(self.build_path)
-        command = [*neutralEnv("ninja_command"), "-v"]
-        env = self.get_env(
-            cross_comp_flags=False, cross_compilers=False, cross_path=True
-        )
-        run_command(command, self.build_path, context, env=env)
-
-    def _test(self, context):
-        context.try_skip(self.build_path)
-        if self.buildEnv.configInfo.build == "android" or (
-            self.buildEnv.configInfo.build != "native"
-            and not self.buildEnv.configInfo.static
-        ):
-            raise SkipCommand()
-        command = [*neutralEnv("mesontest_command"), "--verbose", *self.test_options]
-        env = self.get_env(
-            cross_comp_flags=False, cross_compilers=False, cross_path=True
-        )
-        run_command(command, self.build_path, context, env=env)
-
-    def _install(self, context):
-        context.try_skip(self.build_path)
-        command = [*neutralEnv("ninja_command"), "-v", "install"]
-        env = self.get_env(
-            cross_comp_flags=False, cross_compilers=False, cross_path=True
-        )
-        run_command(command, self.build_path, context, env=env)
-
-    def _make_dist(self, context):
-        command = [*neutralEnv("ninja_command"), "-v", "dist"]
-        env = self.get_env(
-            cross_comp_flags=False, cross_compilers=False, cross_path=True
-        )
-        run_command(command, self.build_path, context, env=env)

kiwixbuild/dependencies/boostregex.py

@@ -1,27 +0,0 @@
-from .base import Dependency, ReleaseDownload, Builder as BaseBuilder
-
-from kiwixbuild.utils import Remotefile, pj
-from shutil import copytree
-
-
-class BoostRegex(Dependency):
-    name = "boostregex"
-
-    class Source(ReleaseDownload):
-        archive = Remotefile(
-            "regex-boost-1.86.0.zip",
-            "",
-            "https://codeload.github.com/boostorg/regex/zip/refs/tags/boost-1.86.0",
-        )
-
-    class Builder(BaseBuilder):
-        def build(self):
-            self.command("copy_headers", self._copy_headers)
-
-        def _copy_headers(self, context):
-            context.try_skip(self.build_path)
-            copytree(
-                pj(self.source_path, "include", "boost"),
-                pj(self.buildEnv.install_dir, "include", "boost"),
-                dirs_exist_ok=True,
-            )

kiwixbuild/dependencies/docoptcpp.py

@@ -1,34 +0,0 @@
-from .base import Dependency, ReleaseDownload, MesonBuilder
-
-from kiwixbuild.utils import Remotefile
-from kiwixbuild._global import neutralEnv
-
-class docoptcpp(Dependency):
-    name = "docoptcpp"
-
-    class Source(ReleaseDownload):
-        name = "docoptcpp"
-        src_archive = Remotefile(
-            "v0.6.3.tar.gz",
-            "28af5a0c482c6d508d22b14d588a3b0bd9ff97135f99c2814a5aa3cbff1d6632",
-            "https://github.com/docopt/docopt.cpp/archive/v0.6.3.tar.gz",
-        )
-
-        meson_archive = Remotefile(
-            "docopt_0.6.3-3_patch.zip",
-            "1f641187f9d3f35b0a5ebd2011876ef8e9b04b69b7b163095dd7dfa16219ad01",
-            "https://wrapdb.mesonbuild.com/v2/docopt_0.6.3-3/get_patch",
-        )
-
-        archives = [src_archive, meson_archive]
-        patches = [
-            "docopt_meson_install_pkgconfig.patch",
-            "docopt_meson_use_boostregex.patch",
-        ]
-
-    class Builder(MesonBuilder):
-        @classmethod
-        def get_dependencies(cls, configInfo, allDeps):
-            if neutralEnv("distname") == "Windows":
-                return ["boostregex"]
-            return []

kiwixbuild/dependencies/gumbo.py

@@ -1,21 +0,0 @@
-from .base import Dependency, ReleaseDownload, MakeBuilder
-
-from kiwixbuild.utils import Remotefile, run_command
-
-
-class Gumbo(Dependency):
-    name = "gumbo"
-
-    class Source(ReleaseDownload):
-        archive = Remotefile(
-            "gumbo-parser-0.12.1.tar.gz",
-            "c0bb5354e46539680724d638dbea07296b797229a7e965b13305c930ddc10d82",
-            "https://dev.kiwix.org/kiwix-build/gumbo-parser-0.12.1.tar.gz",
-        )
-
-        def _post_prepare_script(self, context):
-            context.try_skip(self.extract_path)
-            command = ["./autogen.sh"]
-            run_command(command, self.extract_path, context)
-
-    Builder = MakeBuilder

kiwixbuild/dependencies/icu4c.py

@@ -1,191 +0,0 @@
-from .base import (
-    Dependency,
-    ReleaseDownload,
-    MakeBuilder,
-    Builder as BaseBuilder,
-)
-
-from kiwixbuild.utils import pj, SkipCommand, Remotefile, extract_archive
-from kiwixbuild._global import get_target_step, neutralEnv
-import os, shutil
-import fileinput
-import platform
-
-if platform.system() == "Windows":
-
-    class Icu(Dependency):
-        name = "icu4c"
-
-        class Source(ReleaseDownload):
-            archive = Remotefile(
-                "icu4c-74_1-Win64-MSVC2022.zip",
-                "",
-                "https://github.com/unicode-org/icu/releases/download/release-74-1/icu4c-74_1-Win64-MSVC2022.zip",
-            )
-
-        class Builder(BaseBuilder):
-            def build(self):
-                self.command("copy_headers", self._copy_headers)
-                self.command("copy_bins", self._copy_bin)
-                self.command("generate_pkg_config", self._generate_pkg_config)
-
-            def _copy_headers(self, context):
-                context.try_skip(self.build_path)
-                shutil.copytree(
-                    pj(self.source_path, "include", "unicode"),
-                    pj(self.buildEnv.install_dir, "include", "unicode"),
-                )
-
-            def _copy_bin(self, context):
-                context.try_skip(self.build_path)
-                shutil.copytree(
-                    pj(self.source_path, "lib64"),
-                    pj(self.buildEnv.install_dir, "lib"),
-                    dirs_exist_ok=True,
-                )
-
-                def ignore_non_dll(path, names):
-                    return [n for n in names if not n.endswith(".dll")]
-
-                shutil.copytree(
-                    pj(self.source_path, "bin64"),
-                    pj(self.buildEnv.install_dir, "bin"),
-                    ignore=ignore_non_dll,
-                    dirs_exist_ok=True,
-                )
-
-            def _generate_pkg_config(self, context):
-                context.try_skip(self.build_path)
-
-                pkg_config_template = """ Copyright (C) 2016 and later: Unicode, Inc. and others.
-# License & terms of use: http://www.unicode.org/copyright.html
-# Copyright (C) 2010-2013, International Business Machines Corporation. All Rights Reserved.
-
-# CFLAGS contains only anything end users should set
-CFLAGS = 
-# CXXFLAGS contains only anything end users should set
-CXXFLAGS =  -std=c++11
-# DEFS only contains those UCONFIG_CPPFLAGS which are not auto-set by platform.h
-DEFS = 
-prefix = {prefix}
-exec_prefix = ${{prefix}}
-libdir = ${{exec_prefix}}/lib
-includedir = ${{prefix}}/include
-baselibs = -lpthread -lm 
-UNICODE_VERSION=15.0
-ICUPREFIX=icu
-ICULIBSUFFIX=
-LIBICU=lib${{ICUPREFIX}}
-pkglibdir=${{libdir}}/icu${{ICULIBSUFFIX}}/73.1
-ICUDATA_NAME = icudt73l
-ICUDESC=International Components for Unicode
-
-Version: 73.1
-Cflags: -I${{includedir}}
-Description: International Components for Unicode: Internationalization library
-Name: icu-i18n
-Libs: -L${{libdir}} -licuin -licuuc -licudt"""
-
-                pkg_config_content = pkg_config_template.format(
-                    prefix=self.buildEnv.install_dir
-                )
-
-                os.makedirs(
-                    pj(self.buildEnv.install_dir, "lib", "pkgconfig"), exist_ok=True
-                )
-                with open(
-                    pj(self.buildEnv.install_dir, "lib", "pkgconfig", "icu-i18n.pc"),
-                    mode="w",
-                ) as f:
-                    f.write(pkg_config_content)
-
-else:
-
-    class Icu(Dependency):
-        name = "icu4c"
-
-        class Source(ReleaseDownload):
-            archive_src = Remotefile(
-                "icu4c-73_2-src.tgz",
-                "818a80712ed3caacd9b652305e01afc7fa167e6f2e94996da44b90c2ab604ce1",
-                "https://github.com/unicode-org/icu/releases/download/release-73-2/icu4c-73_2-src.tgz",
-            )
-            archive_data = Remotefile(
-                "icu4c-73_2-data.zip",
-                "ca1ee076163b438461e484421a7679fc33a64cd0a54f9d4b401893fa1eb42701",
-                "https://github.com/unicode-org/icu/releases/download/release-73-2/icu4c-73_2-data.zip",
-            )
-
-            archives = [archive_src, archive_data]
-
-            def _extract(self, context):
-                context.try_skip(self.extract_path)
-                if os.path.exists(self.extract_path):
-                    shutil.rmtree(self.extract_path)
-                extract_archive(
-                    pj(neutralEnv("archive_dir"), self.archive_src.name),
-                    neutralEnv("source_dir"),
-                    topdir=None,
-                    name=self.source_dir,
-                )
-                shutil.rmtree(
-                    pj(neutralEnv("source_dir"), self.source_dir, "source", "data")
-                )
-                extract_archive(
-                    pj(neutralEnv("archive_dir"), self.archive_data.name),
-                    pj(neutralEnv("source_dir"), self.source_dir, "source"),
-                    topdir="data",
-                    name="data",
-                )
-
-            patches = [
-                "icu4c_fix_static_lib_name_mingw.patch",
-                "icu4c_rpath.patch",
-                "icu4c_wasm.patch",
-            ]
-
-        class Builder(MakeBuilder):
-            subsource_dir = "source"
-            make_install_targets = ["install"]
-
-            @classmethod
-            def get_dependencies(cls, configInfo, allDeps):
-                plt = "native_static" if configInfo.static else "native_dyn"
-                return [(plt, "icu4c")]
-
-            @property
-            def configure_options(self):
-                yield "--disable-samples"
-                yield "--disable-tests"
-                yield "--disable-extras"
-                yield "--disable-dyload"
-                yield "--enable-rpath"
-                yield "--disable-icuio"
-                yield "--disable-layoutex"
-                configInfo = self.buildEnv.configInfo
-                if configInfo.build != "native":
-                    icu_native_builder = get_target_step(
-                        "icu4c", "native_static" if configInfo.static else "native_dyn"
-                    )
-                    yield f"--with-cross-build={icu_native_builder.build_path}"
-                    yield "--disable-tools"
-                if configInfo.build in ("android", "wasm"):
-                    yield "--with-data-packaging=archive"
-
-            def set_env(self, env):
-                env["ICU_DATA_FILTER_FILE"] = pj(
-                    os.path.dirname(os.path.realpath(__file__)),
-                    "icu4c_data_filter.json",
-                )
-
-            def _post_configure_script(self, context):
-                if self.buildEnv.configInfo.build != "wasm":
-                    context.skip()
-                context.try_skip(self.build_path)
-                for line in fileinput.input(
-                    pj(self.build_path, "Makefile"), inplace=True
-                ):
-                    if line == "#DATASUBDIR = data\n":
-                        print("DATASUBDIR = data")
-                    else:
-                        print(line, end="")

kiwixbuild/dependencies/icu4c_data_filter.json

@@ -1,14 +0,0 @@
-{
-    "strategy": "additive",
-    "featureFilters": {
-        "lang_tree": "include",
-        "locales_tree": "include",
-        "translit": "include",
-        "misc": {
-            "includelist": [
-               "likelySubtags",
-               "metadata"
-            ]
-        }
-    }
-}

kiwixbuild/dependencies/ios_fat_lib.py

@@ -1,50 +0,0 @@
-import os
-
-from kiwixbuild.configs import ConfigInfo
-from kiwixbuild.utils import pj, copy_tree, run_command
-from kiwixbuild._global import option
-from .base import Dependency, NoopSource, Builder as BaseBuilder
-
-
-class IOSFatLib(Dependency):
-    name = "_ios_fat_lib"
-
-    Source = NoopSource
-
-    class Builder(BaseBuilder):
-        @classmethod
-        def get_dependencies(self, platfomInfo, alldeps):
-            base_target = option("target")
-            return [("iOS_{}".format(arch), base_target) for arch in option("ios_arch")]
-
-        def _copy_headers(self, context):
-            plt = ConfigInfo.get_config("iOS_{}".format(option("ios_arch")[0]))
-            include_src = pj(plt.buildEnv.install_dir, "include")
-            include_dst = pj(self.buildEnv.install_dir, "include")
-            copy_tree(include_src, include_dst)
-
-        def _merge_libs(self, context):
-            lib_dirs = []
-            for arch in option("ios_arch"):
-                plt = ConfigInfo.get_config("iOS_{}".format(arch))
-                lib_dirs.append(pj(plt.buildEnv.install_dir, "lib"))
-            libs = []
-            for f in os.listdir(lib_dirs[0]):
-                if os.path.islink(pj(lib_dirs[0], f)):
-                    continue
-                if f.endswith(".a") or f.endswith(".dylib"):
-                    libs.append(f)
-            os.makedirs(pj(self.buildEnv.install_dir, "lib"), exist_ok=True)
-            for l in libs:
-                command = [
-                    "lipo",
-                    "-create",
-                    *[pj(d, l) for d in lib_dirs],
-                    "-output",
-                    pj(self.buildEnv.install_dir, "lib", l),
-                ]
-                run_command(command, self.buildEnv.install_dir, context)
-
-        def build(self):
-            self.command("copy_headers", self._copy_headers)
-            self.command("merge_libs", self._merge_libs)

kiwixbuild/dependencies/kiwix_desktop.py

@@ -1,39 +0,0 @@
-from kiwixbuild._global import option
-from .base import Dependency, GitClone, QMakeBuilder
-import platform
-
-
-class KiwixDesktop(Dependency):
-    name = "kiwix-desktop"
-    force_build = True
-
-    class Source(GitClone):
-        git_remote = "https://github.com/kiwix/kiwix-desktop.git"
-        git_dir = "kiwix-desktop"
-
-    class Builder(QMakeBuilder):
-        dependencies = ["qt", "qtwebengine", "libkiwix", "aria2"]
-        configure_env = None
-
-        flatpack_build_options = {"env": {"QMAKEPATH": "/app"}}
-
-        @property
-        def make_targets(self):
-            if platform.system() == "Windows":
-                yield "release-all"
-            else:
-                yield from super().make_targets
-
-        @property
-        def make_install_targets(self):
-            if platform.system() == "Windows":
-                yield "release-install"
-            else:
-                yield "install"
-
-        @property
-        def configure_options(self):
-            if self.buildEnv.configInfo.name != "flatpak":
-                yield f"PREFIX={self.buildEnv.install_dir}"
-            if self.buildEnv.configInfo.static:
-                yield "CONFIG+=static"

kiwixbuild/dependencies/kiwix_tools.py

@@ -1,18 +0,0 @@
-from .base import Dependency, GitClone, MesonBuilder
-
-
-class KiwixTools(Dependency):
-    name = "kiwix-tools"
-    force_build = True
-
-    class Source(GitClone):
-        git_remote = "https://github.com/kiwix/kiwix-tools.git"
-        git_dir = "kiwix-tools"
-
-    class Builder(MesonBuilder):
-        dependencies = ["libkiwix", "docoptcpp"]
-
-        @property
-        def configure_options(self):
-            if self.buildEnv.configInfo.static:
-                yield "-Dstatic-linkage=true"

kiwixbuild/dependencies/libcurl.py

@@ -1,61 +0,0 @@
-import os
-
-from .base import (
-    Dependency,
-    ReleaseDownload,
-    MesonBuilder,
-)
-
-from kiwixbuild.utils import Remotefile, pj, Defaultdict, SkipCommand, run_command
-from kiwixbuild._global import get_target_step
-
-
-class LibCurl(Dependency):
-    name = "libcurl"
-
-    class Source(ReleaseDownload):
-        name = "libcurl"
-        src_archive = Remotefile(
-            "curl-8.4.0.tar.xz",
-            "16c62a9c4af0f703d28bda6d7bbf37ba47055ad3414d70dec63e2e6336f2a82d",
-            "https://curl.se/download/curl-8.4.0.tar.xz",
-        )
-        meson_archive = Remotefile(
-            "curl_8.4.0-2_patch.zip",
-            "bbb6ae75225c36ef9bb336cface729794c7c070c623a003fff40bd416042ff6e",
-            "https://dev.kiwix.org/libkiwix/curl_8.4.0-2_patch.zip",
-        )
-        archives = [src_archive, meson_archive]
-
-    class Builder(MesonBuilder):
-        dependencies = ["zlib"]
-        configure_options = [
-            f"-D{p}=disabled"
-            for p in (
-                "psl",
-                "kerberos-auth",
-                "gss-api",
-                "ssh",
-                "rtmp",
-                "http2",
-                "idn",
-                "brotli",
-                "ftp",
-                "file",
-                "ldap",
-                "ldaps",
-                "rtsp",
-                "dict",
-                "telnet",
-                "tftp",
-                "pop3",
-                "imap",
-                "smb",
-                "smtp",
-                "gopher",
-                "tool",
-            )
-        ]
-
-        def _test(self, context):
-            context.skip("No Test")

kiwixbuild/dependencies/libkiwix.py

@@ -1,52 +0,0 @@
-import shutil, os
-
-from .base import Dependency, GitClone, MesonBuilder
-from kiwixbuild.utils import pj, copy_tree
-from kiwixbuild._global import option, get_target_step, neutralEnv
-
-
-class Libkiwix(Dependency):
-    name = "libkiwix"
-    force_build = True
-
-    class Source(GitClone):
-        git_remote = "https://github.com/kiwix/libkiwix.git"
-        git_dir = "libkiwix"
-
-    class Builder(MesonBuilder):
-        dependencies = [
-            "pugixml",
-            "libzim",
-            "zlib",
-            "libcurl",
-            "libmicrohttpd",
-            "icu4c",
-            "mustache",
-            "xapian-core",
-        ]
-        strip_options = []
-
-        @property
-        def build_type(self):
-            if self.buildEnv.configInfo.build == "android":
-                return "debug"
-            return super().build_type
-
-        @property
-        def configure_options(self):
-            configInfo = self.buildEnv.configInfo
-            if configInfo.build == "android":
-                yield "-Dstatic-linkage=true"
-                yield "-Dwerror=false"
-            if configInfo.build == "iOS":
-                yield "-Db_bitcode=true"
-            if configInfo.name == "flatpak":
-                yield "--wrap-mode=nodownload"
-            if configInfo.mixed and option("target") == "libkiwix":
-                yield "-Dstatic-linkage=true"
-
-        @property
-        def library_type(self):
-            if self.buildEnv.configInfo.build == "android":
-                return "shared"
-            return super().library_type

kiwixbuild/dependencies/libmagic.py

@@ -1,50 +0,0 @@
-import os
-
-from .base import (
-    Dependency,
-    ReleaseDownload,
-    MakeBuilder,
-)
-
-from kiwixbuild.utils import Remotefile, pj, SkipCommand, run_command
-from kiwixbuild._global import get_target_step
-
-
-class LibMagic(Dependency):
-    name = "libmagic"
-
-    class Source(ReleaseDownload):
-        name = "libmagic"
-        source_dir = "libmagic"
-        archive_top_dir = "file-5.44"
-        archive = Remotefile(
-            "file-5.44.tar.gz",
-            "3751c7fba8dbc831cb8d7cc8aff21035459b8ce5155ef8b0880a27d028475f3b",
-        )
-
-    class Builder(MakeBuilder):
-        @property
-        def configure_options(self):
-            yield "--disable-bzlib"
-            yield "--disable-xzlib"
-            yield "--disable-zstdlib"
-            yield "--disable-lzlib"
-
-        @classmethod
-        def get_dependencies(cls, configInfo, allDeps):
-            if configInfo.build != "native":
-                return [("native_static", "libmagic")]
-            return []
-
-        def _compile(self, context):
-            configInfo = self.buildEnv.configInfo
-            if configInfo.build == "native":
-                return super()._compile(context)
-            context.try_skip(self.build_path)
-            command = ["make", "-j4", *self.make_targets, *self.make_options]
-            env = self.buildEnv.get_env(
-                cross_comp_flags=True, cross_compilers=True, cross_path=True
-            )
-            libmagic_native_builder = get_target_step("libmagic", "native_static")
-            env["PATH"].insert(0, pj(libmagic_native_builder.build_path, "src"))
-            run_command(command, self.build_path, context, env=env)

kiwixbuild/dependencies/libmicrohttpd.py

@@ -1,33 +0,0 @@
-from .base import Dependency, ReleaseDownload, MesonBuilder
-
-from kiwixbuild.utils import Remotefile
-
-
-class MicroHttpd(Dependency):
-    name = "libmicrohttpd"
-
-    class Source(ReleaseDownload):
-        src_archive = Remotefile(
-            "libmicrohttpd-0.9.76.tar.gz",
-            "f0b1547b5a42a6c0f724e8e1c1cb5ce9c4c35fb495e7d780b9930d35011ceb4c",
-            "https://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.76.tar.gz",
-        )
-        meson_archive = Remotefile(
-            "libmicrohttpd_0.9.16-3_patch.zip",
-            "0954c094a0d4cfe0dd799d8df8a04face6669f7b4d51a7386a9c3e2d37b9c3b3",
-            "https://wrapdb.mesonbuild.com/v2/libmicrohttpd_0.9.76-3/get_patch",
-        )
-        archives = [src_archive, meson_archive]
-        patches = [
-            "libmicrohttpd_meson_pkgconfig.patch",
-            "libmicrohttpd_meson_timeval_tvsec_size.patch",
-            "libmicrohttpd_meson_winet6.patch",
-        ]
-
-    class Builder(MesonBuilder):
-        configure_options = [
-            "-Dgnutls=disabled",
-            "-Dgcrypt=disabled",
-            "-Dcurl=disabled",
-            "-Dexpat=disabled",
-        ]

kiwixbuild/dependencies/libzim.py

@@ -1,57 +0,0 @@
-from .base import Dependency, GitClone, MesonBuilder
-from kiwixbuild._global import option, get_target_step, neutralEnv
-
-
-class Libzim(Dependency):
-    name = "libzim"
-    force_build = True
-
-    class Source(GitClone):
-        git_remote = "https://github.com/openzim/libzim.git"
-        git_dir = "libzim"
-
-    class Builder(MesonBuilder):
-        test_options = ["-t", "8"]
-        strip_options = []
-
-        @property
-        def build_type(self):
-            if self.buildEnv.configInfo.build == "android":
-                return "debug"
-            return super().build_type
-
-        @classmethod
-        def get_dependencies(cls, configInfo, allDeps):
-            if neutralEnv("distname") == "Windows":
-                return ["zstd", "xapian-core", "icu4c", "zim-testing-suite"]
-            deps = ["lzma", "zstd", "xapian-core", "icu4c"]
-            if configInfo.name not in ("flatpak", "wasm"):
-                deps.append("zim-testing-suite")
-            return deps
-
-        @property
-        def configure_options(self):
-            configInfo = self.buildEnv.configInfo
-            if neutralEnv("distname") == "Windows":
-                yield "-Dwith_xapian_fuller=false"
-                yield "-Dwerror=false"
-            if configInfo.build == "android":
-                yield "-DUSE_BUFFER_HEADER=false"
-                yield "-Dstatic-linkage=true"
-            if configInfo.mixed and option("target") == "libzim":
-                yield "-Dstatic-linkage=true"
-            if configInfo.name == "flatpak":
-                yield "--wrap-mode=nodownload"
-                yield "-Dtest_data_dir=none"
-            if configInfo.name == "wasm":
-                yield "-Dexamples=false"
-                yield "-DUSE_MMAP=false"
-            if configInfo.name not in ("flatpak", "wasm"):
-                zim_testing_suite = get_target_step("zim-testing-suite", "source")
-                yield "-Dtest_data_dir={}".format(zim_testing_suite.source_path)
-
-        @property
-        def library_type(self):
-            if self.buildEnv.configInfo.build == "android":
-                return "shared"
-            return super().library_type

kiwixbuild/dependencies/lzma.py

@@ -1,26 +0,0 @@
-from .base import (
-    Dependency,
-    ReleaseDownload,
-    MesonBuilder)
-
-from kiwixbuild.utils import Remotefile
-
-
-class lzma(Dependency):
-    name = 'lzma'
-
-    class Source(ReleaseDownload):
-        src_archive = Remotefile(
-            "xz-5.2.6.tar.gz",
-            "a2105abee17bcd2ebd15ced31b4f5eda6e17efd6b10f921a01cda4a44c91b3a0",
-            "https://altushost-swe.dl.sourceforge.net/project/lzmautils/xz-5.2.6.tar.gz",
-        )
-        meson_patch = Remotefile(
-            "liblzma_5.2.6-3_patch.zip",
-            "1c71536d364e1a3ce6bea61266576f89cc5cce4d3b9e11f3494417dafa29780b",
-            "https://wrapdb.mesonbuild.com/v2/liblzma_5.2.6-3/get_patch",
-        )
-        archives = [src_archive, meson_patch]
-        patches = ['lzma_meson_install.patch']
-
-    Builder = MesonBuilder

kiwixbuild/dependencies/mustache.py

@@ -1,30 +0,0 @@
-from .base import Dependency, ReleaseDownload, Builder as BaseBuilder
-
-from kiwixbuild.utils import Remotefile, pj
-from shutil import copy2
-
-
-class Mustache(Dependency):
-    name = "mustache"
-
-    class Source(ReleaseDownload):
-        archive = Remotefile(
-            "Mustache-4.1.tar.gz",
-            "acd66359feb4318b421f9574cfc5a511133a77d916d0b13c7caa3783c0bfe167",
-            "https://dev.kiwix.org/kiwix-build/mustache-4.1.tar.gz",
-        )
-
-    class Builder(BaseBuilder):
-        def build(self):
-            self.command("copy_header", self._copy_header)
-
-        def _copy_header(self, context):
-            context.try_skip(self.build_path)
-            copy2(
-                pj(self.source_path, "mustache.hpp"),
-                pj(self.buildEnv.install_dir, "include"),
-            )
-
-        def set_flatpak_buildsystem(self, module):
-            module["buildsystem"] = "simple"
-            module["build-commands"] = ["cp mustache.hpp /app/include"]

kiwixbuild/dependencies/pugixml.py

@@ -1,18 +0,0 @@
-from .base import Dependency, ReleaseDownload, MesonBuilder
-
-from kiwixbuild.utils import Remotefile
-
-
-class Pugixml(Dependency):
-    name = "pugixml"
-
-    class Source(ReleaseDownload):
-        archive = Remotefile(
-            "pugixml-1.2.tar.gz",
-            "0f422dad86da0a2e56a37fb2a88376aae6e931f22cc8b956978460c9db06136b",
-        )
-        patches = ["pugixml_meson.patch"]
-        flatpak_dest = "src"
-
-    class Builder(MesonBuilder):
-        strip_options = []

kiwixbuild/dependencies/qt.py

@@ -1,25 +0,0 @@
-import shutil
-
-from .base import Dependency, NoopBuilder, NoopSource
-
-from kiwixbuild.utils import SkipCommand, colorize
-
-
-class Qt(Dependency):
-    name = "qt"
-
-    Source = NoopSource
-
-    class Builder(NoopBuilder):
-        def build(self):
-            error_msg = f"""WARNING: kiwix-build cannot build {self.name} for you.
-You must install it yourself using official Qt installer or your distribution system."""
-            print(colorize(error_msg, "WARNING"))
-
-
-class QtWebEngine(Dependency):
-    name = "qtwebengine"
-
-    Source = NoopSource
-
-    Builder = Qt.Builder

kiwixbuild/dependencies/tc_android_ndk.py

@@ -1,93 +0,0 @@
-import os
-
-from .base import Dependency, ReleaseDownload, Builder
-from kiwixbuild.utils import Remotefile, add_execution_right, run_command
-
-pj = os.path.join
-
-
-class android_ndk(Dependency):
-    dont_skip = True
-    neutral = False
-    name = "android-ndk"
-    gccver = "4.9.x"
-    api = "24"
-
-    class Source(ReleaseDownload):
-        archive = Remotefile(
-            "android-ndk-r21e-linux-x86_64.zip",
-            "ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e",
-            "https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip",
-        )
-
-        @property
-        def source_dir(self):
-            return self.target.full_name()
-
-        patches = [
-            "android-ndk-r21e-linux-x86_64-python3.12+.patch",
-        ]
-
-    class Builder(Builder):
-        @property
-        def install_path(self):
-            return pj(self.buildEnv.toolchain_dir, self.target.full_name())
-
-        @property
-        def api(self):
-            return self.target.api
-
-        @property
-        def config(self):
-            return "android-" + self.api
-
-        @property
-        def arch(self):
-            return self.buildEnv.configInfo.arch
-
-        @property
-        def arch_full(self):
-            return self.buildEnv.configInfo.arch_full
-
-        def _build_toolchain(self, context):
-            context.try_skip(self.build_path)
-            script = pj(self.source_path, "build/tools/make_standalone_toolchain.py")
-            add_execution_right(script)
-            command = [
-                script,
-                f"--arch={self.arch}",
-                f"--api={self.api}",
-                f"--install-dir={self.install_path}",
-                "--force",
-            ]
-            env = self.buildEnv.get_env(
-                cross_comp_flags=False, cross_compilers=False, cross_path=False
-            )
-            run_command(command, self.build_path, context, env=env)
-
-        def _fix_permission_right(self, context):
-            context.try_skip(self.build_path)
-            bin_dirs = [
-                pj(self.install_path, "bin"),
-                pj(self.install_path, self.arch_full, "bin"),
-                pj(
-                    self.install_path,
-                    "libexec",
-                    "gcc",
-                    self.arch_full,
-                    self.target.gccver,
-                ),
-            ]
-            for root, dirs, files in os.walk(self.install_path):
-                if not root in bin_dirs:
-                    continue
-
-                for file_ in files:
-                    file_path = pj(root, file_)
-                    if os.path.islink(file_path):
-                        continue
-                    add_execution_right(file_path)
-
-        def build(self):
-            self.command("build_toolchain", self._build_toolchain)
-            self.command("fix_permission_right", self._fix_permission_right)

kiwixbuild/dependencies/tc_armhf.py

@@ -1,55 +0,0 @@
-from .base import Dependency, ReleaseDownload, TcCopyBuilder
-from kiwixbuild.utils import Remotefile
-
-# The arm toolchains
-# This is based on toolchains published here : https://github.com/tttapa/docker-arm-cross-toolchain
-
-base_url = (
-    "https://github.com/tttapa/docker-arm-cross-toolchain/releases/download/0.1.0/"
-)
-
-
-class armv6_toolchain(Dependency):
-    dont_skip = True
-    neutral = True
-    name = "armv6"
-
-    class Source(ReleaseDownload):
-        archive = Remotefile(
-            "x-tools-armv6-rpi-linux-gnueabihf.tar.xz",
-            "4c371c4c5b55ebd1f3d7dd26b14703632d9ba47423f901bcd9303d83ad444434",
-            base_url + "x-tools-armv6-rpi-linux-gnueabihf.tar.xz",
-        )
-
-    class Builder(TcCopyBuilder):
-        src_subdir = "armv6-rpi-linux-gnueabihf"
-
-
-class armv8_toolchain(Dependency):
-    dont_skip = True
-    neutral = True
-    name = "armv8"
-
-    class Source(ReleaseDownload):
-        archive = Remotefile(
-            "x-tools-armv8-rpi-linux-gnueabihf.tar.xz",
-            "cc28f5c3f6a3e7d9985f98779c4e72224b4eb5a7e4dc2bcdefd90cb241fb94a5",
-            base_url + "x-tools-armv8-rpi3-linux-gnueabihf.tar.xz",
-        )
-
-    class Builder(TcCopyBuilder):
-        src_subdir = "armv8-rpi3-linux-gnueabihf"
-
-
-class aarch64_toolchain(Dependency):
-    dont_skip = True
-    neutral = True
-    name = "aarch64"
-
-    class Source(ReleaseDownload):
-        archive = Remotefile(
-            "cross-gcc-6.3.0-pi_64.tar.gz",
-            "1b048bb8886ad63d21797cd9129fc37b9ea0dfaac7e3c36f888aa16fbec1d320",
-        )
-
-    Builder = TcCopyBuilder

kiwixbuild/dependencies/tc_emsdk.py

@@ -1,47 +0,0 @@
-import os
-
-from .base import Dependency, ReleaseDownload, Builder
-from kiwixbuild.utils import Remotefile, run_command, copy_tree
-
-pj = os.path.join
-
-
-class emsdk(Dependency):
-    dont_skip = True
-    neutral = False
-    name = "emsdk"
-
-    class Source(ReleaseDownload):
-        archive = Remotefile(
-            "emsdk-3.1.41.tar.gz",
-            "147a2d72df34227bdb4ffedc587a8cb674a42269c40458f3f69ae37e8966cdc6",
-            "https://codeload.github.com/emscripten-core/emsdk/tar.gz/refs/tags/3.1.41",
-        )
-
-        @property
-        def source_dir(self):
-            return self.target.full_name()
-
-    class Builder(Builder):
-        @property
-        def install_path(self):
-            return pj(self.buildEnv.toolchain_dir, self.target.full_name())
-
-        def _copy_source(self, context):
-            context.try_skip(self.build_path)
-            copy_tree(self.source_path, self.install_path)
-
-        def _install(self, context):
-            context.try_skip(self.build_path)
-            command = ["./emsdk", "install", "3.1.24"]
-            run_command(command, self.install_path, context)
-
-        def _activate(self, context):
-            context.try_skip(self.build_path)
-            command = ["./emsdk", "activate", "3.1.24"]
-            run_command(command, self.install_path, context)
-
-        def build(self):
-            self.command("copy_source", self._copy_source)
-            self.command("install", self._install)
-            self.command("activate", self._activate)

kiwixbuild/dependencies/tc_flatpak.py

@@ -1,89 +0,0 @@
-import os
-
-from .base import Dependency, NoopSource, Builder
-from kiwixbuild.utils import Remotefile, add_execution_right, run_command
-
-pj = os.path.join
-
-
-class org_kde(Dependency):
-    neutral = False
-    name = "org.kde"
-
-    Source = NoopSource
-
-    class Builder(Builder):
-        def _setup_remote(self, context):
-            command = [
-                "flatpak",
-                "--user",
-                "remote-add",
-                "--if-not-exists",
-                "flathub",
-                "https://flathub.org/repo/flathub.flatpakrepo",
-            ]
-            env = self.buildEnv.get_env(
-                cross_comp_flags=False, cross_compilers=False, cross_path=False
-            )
-            run_command(command, self.buildEnv.build_dir, context, env=env)
-
-        def _install_sdk(self, context):
-            command = [
-                "flatpak",
-                "--user",
-                "install",
-                "--noninteractive",
-                "--verbose",
-                "-y",
-                "flathub",
-                f"{self.target.name}.Sdk//{self.target.version()}",
-                f"{self.target.name}.Platform//{self.target.version()}",
-            ]
-            env = self.buildEnv.get_env(
-                cross_comp_flags=False, cross_compilers=False, cross_path=False
-            )
-            run_command(command, self.buildEnv.build_dir, context, env=env)
-
-        def build(self):
-            self.command("setup_remote", self._setup_remote)
-            self.command("install_sdk", self._install_sdk)
-
-
-class io_qt_qtwebengine(Dependency):
-    neutral = False
-    name = "io.qt.qtwebengine"
-
-    Source = NoopSource
-
-    class Builder(Builder):
-        def _setup_remote(self, context):
-            command = [
-                "flatpak",
-                "--user",
-                "remote-add",
-                "--if-not-exists",
-                "flathub",
-                "https://flathub.org/repo/flathub.flatpakrepo",
-            ]
-            env = self.buildEnv.get_env(
-                cross_comp_flags=False, cross_compilers=False, cross_path=False
-            )
-            run_command(command, self.buildEnv.build_dir, context, env=env)
-
-        def _install_sdk(self, context):
-            command = [
-                "flatpak",
-                "--user",
-                "install",
-                "-y",
-                "flathub",
-                f"{self.target.name}.BaseApp//{self.target.version()}",
-            ]
-            env = self.buildEnv.get_env(
-                cross_comp_flags=False, cross_compilers=False, cross_path=False
-            )
-            run_command(command, self.buildEnv.build_dir, context, env=env)
-
-        def build(self):
-            self.command("setup_remote", self._setup_remote)
-            self.command("install_sdk", self._install_sdk)

kiwixbuild/dependencies/tc_musl.py

@@ -1,32 +0,0 @@
-from .base import Dependency, ReleaseDownload, TcCopyBuilder
-from kiwixbuild.utils import Remotefile
-
-
-class aarch64_musl_toolchain(Dependency):
-    dont_skip = True
-    neutral = True
-    name = "aarch64_musl"
-
-    class Source(ReleaseDownload):
-        archive = Remotefile(
-            "aarch64-linux-musl-cross.tgz",
-            "0f18a885b161815520bbb5757a4b4ab40d0898c29bebee58d0cddd6112e59cc6",
-            "https://more.musl.cc/10/x86_64-linux-musl/aarch64-linux-musl-cross.tgz",
-        )
-
-    Builder = TcCopyBuilder
-
-
-class x86_64_musl_toolchain(Dependency):
-    dont_skip = True
-    neutral = True
-    name = "x86-64_musl"
-
-    class Source(ReleaseDownload):
-        archive = Remotefile(
-            "x86_64-linux-musl-cross.tgz",
-            "a3d55de8105739fcfb8b10eaa72cdb5d779319726bacff24149388d7608d1ed8",
-            "https://more.musl.cc/10/x86_64-linux-musl/x86_64-linux-musl-cross.tgz",
-        )
-
-    Builder = TcCopyBuilder

kiwixbuild/dependencies/uuid.py

@@ -1,30 +0,0 @@
-from .base import Dependency, ReleaseDownload, MakeBuilder
-
-from kiwixbuild.utils import Remotefile
-
-
-class UUID(Dependency):
-    name = "uuid"
-
-    class Source(ReleaseDownload):
-        archive = Remotefile(
-            "e2fsprogs-libs-1.43.4.tar.gz",
-            "eed4516325768255c9745e7b82c9d7d0393abce302520a5b2cde693204b0e419",
-            "https://www.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v1.43.4/e2fsprogs-libs-1.43.4.tar.gz",
-        )
-        extract_dir = "e2fsprogs-libs-1.43.4"
-
-    class Builder(MakeBuilder):
-        configure_options = [
-            "--enable-libuuid",
-            "--disable-debugfs",
-            "--disable-imager",
-            "--disable-resizer",
-            "--disable-defrag",
-            "--enable-fsck",
-            "--disable-uuidd",
-        ]
-        configure_env = {"_format_CFLAGS": "{env.CFLAGS} -O3 -fPIC"}
-        static_configure_options = dynamic_configure_options = []
-        make_targets = ["libs"]
-        make_install_targets = ["install-libs"]

kiwixbuild/dependencies/xapian.py

@@ -1,55 +0,0 @@
-from .base import Dependency, GitClone, ReleaseDownload, MakeBuilder, MesonBuilder
-
-from kiwixbuild.utils import Remotefile
-from kiwixbuild._global import neutralEnv
-
-import platform
-
-
-class Xapian(Dependency):
-    name = "xapian-core"
-
-    if platform.system() == "Windows":
-
-        class Source(GitClone):
-            git_remote = "https://github.com/openzim/xapian-meson.git"
-            git_dir = "xapian-core"
-
-        class Builder(MesonBuilder):
-            configure_options = [
-                "-Denable-sse=false",
-                "-Denable-backend-chert=false",
-                "-Denable-backend-remote=false",
-            ]
-            subsource_dir = "xapian-core"
-
-            @classmethod
-            def get_dependencies(cls, configInfo, allDeps):
-                return ["zlib"]
-
-    else:
-
-        class Source(ReleaseDownload):
-            archive = Remotefile(
-                "xapian-core-1.4.23.tar.xz",
-                "30d3518172084f310dab86d262b512718a7f9a13635aaa1a188e61dc26b2288c",
-            )
-
-        class Builder(MakeBuilder):
-            configure_options = [
-                "--disable-sse",
-                "--disable-backend-chert",
-                "--disable-backend-remote",
-                "--disable-documentation",
-            ]
-            configure_env = {
-                "_format_LDFLAGS": "{env.LDFLAGS} -L{buildEnv.install_dir}/{buildEnv.libprefix}",
-                "_format_CXXFLAGS": "{env.CXXFLAGS} -I{buildEnv.install_dir}/include",
-            }
-
-            @classmethod
-            def get_dependencies(cls, configInfo, allDeps):
-                deps = ["zlib", "lzma"]
-                if configInfo.build == "wasm" or neutralEnv("distname") == "Darwin":
-                    return deps
-                return deps + ["uuid"]

kiwixbuild/dependencies/zim_testing_suite.py

@@ -1,17 +0,0 @@
-from .base import Dependency, ReleaseDownload, NoopBuilder
-
-from kiwixbuild.utils import Remotefile
-
-
-class ZimTestingSuite(Dependency):
-    name = "zim-testing-suite"
-    dont_skip = True
-
-    class Source(ReleaseDownload):
-        archive = Remotefile(
-            "zim-testing-suite-0.8.0.tar.gz",
-            "28f51449a3f9aea02652ca21f32c5598fd610d6cec3810fa552bd0c0f7a2d5fc",
-            "https://github.com/openzim/zim-testing-suite/releases/download/0.8.0/zim-testing-suite-0.8.0.tar.gz",
-        )
-
-    Builder = NoopBuilder

kiwixbuild/dependencies/zim_tools.py

@@ -1,27 +0,0 @@
-from .base import Dependency, GitClone, MesonBuilder
-from kiwixbuild._global import neutralEnv
-
-
-class ZimTools(Dependency):
-    name = "zim-tools"
-    force_build = True
-
-    class Source(GitClone):
-        git_remote = "https://github.com/openzim/zim-tools.git"
-        git_dir = "zim-tools"
-
-    class Builder(MesonBuilder):
-        @classmethod
-        def get_dependencies(cls, configInfo, allDeps):
-            base_deps = ["libzim", "docoptcpp", "mustache"]
-            if neutralEnv("distname") != "Windows":
-                base_deps += ["libmagic", "gumbo"]
-            return base_deps
-
-        @property
-        def configure_options(self):
-            # We don't build zimwriterfs on Windows, and so we don't have magic
-            if neutralEnv("distname") != "Windows":
-                yield f"-Dmagic-install-prefix={self.buildEnv.install_dir}"
-            if self.buildEnv.configInfo.static:
-                yield "-Dstatic-linkage=true"

kiwixbuild/dependencies/zlib.py

@@ -1,27 +0,0 @@
-import shutil
-
-from .base import (
-    Dependency,
-    ReleaseDownload,
-    MesonBuilder)
-
-from kiwixbuild.utils import Remotefile, pj, SkipCommand
-from kiwixbuild._global import neutralEnv
-
-
-class zlib(Dependency):
-    name = "zlib"
-
-    class Source(ReleaseDownload):
-        src_archive = Remotefile(
-            "zlib-1.2.12.tar.gz",
-            "91844808532e5ce316b3c010929493c0244f3d37593afd6de04f71821d5136d9"
-        )
-        meson_patch = Remotefile(
-            "zlib_1.2.12-1_patch.zip",
-            "8ec8344f3fe7b06ad4be768fd416694bc56cb4545ce78b0f1c18b3e72b3ec936",
-            "https://wrapdb.mesonbuild.com/v2/zlib_1.2.12-1/get_patch")
-        archives = [src_archive, meson_patch]
-        #patches = ['zlib_std_libname.patch']
-
-    Builder = MesonBuilder

kiwixbuild/dependencies/zstd.py

@@ -1,20 +0,0 @@
-from .base import Dependency, ReleaseDownload, MesonBuilder
-
-from kiwixbuild.utils import Remotefile
-
-
-class zstd(Dependency):
-    name = "zstd"
-
-    class Source(ReleaseDownload):
-        archive = Remotefile(
-            "zstd-1.5.5.tar.gz",
-            "98e9c3d949d1b924e28e01eccb7deed865eefebf25c2f21c702e5cd5b63b85e1",
-            "https://github.com/facebook/zstd/archive/refs/tags/v1.5.5.tar.gz",
-        )
-
-    class Builder(MesonBuilder):
-        subsource_dir = "build/meson"
-        build_type = "release"
-        strip_options = []
-        configure_options = ["-Dbin_programs=false", "-Dbin_contrib=false"]

kiwixbuild/flathub.flatpakrepo

@@ -1,8 +0,0 @@
-[Flatpak Repo]
-Title=Flathub
-Url=https://dl.flathub.org/repo/
-Homepage=https://flathub.org/
-Comment=Central repository of Flatpak applications
-Description=Central repository of Flatpak applications
-Icon=https://dl.flathub.org/repo/logo.svg
-GPGKey=mQINBFlD2sABEADsiUZUOYBg1UdDaWkEdJYkTSZD68214m8Q1fbrP5AptaUfCl8KYKFMNoAJRBXn9FbE6q6VBzghHXj/rSnA8WPnkbaEWR7xltOqzB1yHpCQ1l8xSfH5N02DMUBSRtD/rOYsBKbaJcOgW0K21sX+BecMY/AI2yADvCJEjhVKrjR9yfRX+NQEhDcbXUFRGt9ZT+TI5yT4xcwbvvTu7aFUR/dH7+wjrQ7lzoGlZGFFrQXSs2WI0WaYHWDeCwymtohXryF8lcWQkhH8UhfNJVBJFgCY8Q6UHkZG0FxMu8xnIDBMjBmSZKwKQn0nwzwM2afskZEnmNPYDI8nuNsSZBZSAw+ThhkdCZHZZRwzmjzyRuLLVFpOj3XryXwZcSefNMPDkZAuWWzPYjxS80cm2hG1WfqrG0Gl8+iX69cbQchb7gbEb0RtqNskTo9DDmO0bNKNnMbzmIJ3/rTbSahKSwtewklqSP/01o0WKZiy+n/RAkUKOFBprjJtWOZkc8SPXV/rnoS2dWsJWQZhuPPtv3tefdDiEyp7ePrfgfKxuHpZES0IZRiFI4J/nAUP5bix+srcIxOVqAam68CbAlPvWTivRUMRVbKjJiGXIOJ78wAMjqPg3QIC0GQ0EPAWwAOzzpdgbnG7TCQetaVV8rSYCuirlPYN+bJIwBtkOC9SWLoPMVZTwQARAQABtC5GbGF0aHViIFJlcG8gU2lnbmluZyBLZXkgPGZsYXRodWJAZmxhdGh1Yi5vcmc+iQJUBBMBCAA+FiEEblwF2XnHba+TwIE1QYTdTZB6fK4FAllD2sACGwMFCRLMAwAFCwkIBwIGFQgJCgsCBBYCAwECHgECF4AACgkQQYTdTZB6fK5RJQ/+Ptd4sWxaiAW91FFk7+wmYOkEe1NY2UDNJjEEz34PNP/1RoxveHDt43kYJQ23OWaPJuZAbu+fWtjRYcMBzOsMCaFcRSHFiDIC9aTp4ux/mo+IEeyarYt/oyKb5t5lta6xaAqg7rwt65jW5/aQjnS4h7eFZ+dAKta7Y/fljNrOznUp81/SMcx4QA5G2Pw0hs4Xrxg59oONOTFGBgA6FF8WQghrpR7SnEe0FSEOVsAjwQ13Cfkfa7b70omXSWp7GWfUzgBKyoWxKTqzMN3RQHjjhPJcsQnrqH5enUu4Pcb2LcMFpzimHnUgb9ft72DP5wxfzHGAWOUiUXHbAekfq5iFks8cha/RST6wkxG3Rf44Zn09aOxh1btMcGL+5xb1G0BuCQnA0fP/kDYIPwh9z22EqwRQOspIcvGeLVkFeIfubxpcMdOfQqQnZtHMCabV5Q/Rk9K1ZGc8M2hlg8gHbXMFch2xJ0Wu72eXbA/UY5MskEeBgawTQnQOK/vNm7t0AJMpWK26Qg6178UmRghmeZDj9uNRc3EI1nSbgvmGlpDmCxaAGqaGL1zW4KPW5yN25/qeqXcgCvUjZLI9PNq3Kvizp1lUrbx7heRiSoazCucvHQ1VHUzcPVLUKKTkoTP8okThnRRRsBcZ1+jI4yMWIDLOCT7IW3FePr+3xyuy5eEo9a25Ag0EWUPa7AEQALT/CmSyZ8LWlRYQZKYw417p7Z2hxqd6TjwkwM3IQ1irumkWcTZBZIbBgrSOg6CcXD2oWydCQHWi9qaxhuhEl2bJL5LskmBcMxVdQeD0LLHd8QUnbnnIby8ocvWN1alPfvJFjCUTrmD22U1ycOzRw2lIe4kiQONbOZtdWrVImQQSndjFlisitbmlWHvHm2lOOYy8+GJB7YffVV193hmnBSJffCy4bvkuLxsI+n1DhOzc7MPV3z6HGk4HiEcF0yyt9tCYhpsxHFdBoq2h771HfAcS0s98EVAqYMFnf9em+4cnYpdI6mhIfS1FQiKl6DBAYA8tT3ggla00DurPo0JwX/zN+PaO5h/6O9aCZwV7G6rbkgMuqMergXaf8oP38gr0z+MqWnkfM63Bodq68GP4l4hd02BoFBbDf38TMuGQB14+twJMdfbAxo2MbgluvQgfwHfZ2ca6gyEY+9s/YD1gugLjV+S6CB51WkFNe1z4tAPgJZNxUcKCbeaHNbthl8Hks/pY9RCEseX/EdfzF18epbSjJMPh4DPQXbUoFwmyuYcoBOPmvZHNl9hK7B/1RP8w1ZrXk8qdupC0SNbafX7270B7lMMVImzZetGsM9ypXJ6llhp3FwW09iseNyGJGPsr/dvTMGDXqOPfU/9SAS1LSTY4K9PbRtdrBE318YX8mIk5ABEBAAGJBHIEGAEIACYWIQRuXAXZecdtr5PAgTVBhN1NkHp8rgUCWUPa7AIbAgUJEswDAAJACRBBhN1NkHp8rsF0IAQZAQgAHRYhBFSmzd2JGfsgQgDYrFYnAunj7X7oBQJZQ9rsAAoJEFYnAunj7X7oR6AP/0KYmiAFeqx14Z43/6s2gt3VhxlSd8bmcVV7oJFbMhdHBIeWBp2BvsUf00I0Zl14ZkwCKfLwbbORC2eIxvzJ+QWjGfPhDmS4XUSmhlXxWnYEveSek5Tde+fmu6lqKM8CHg5BNx4GWIX/vdLi1wWJZyhrUwwICAxkuhKxuP2Z1An48930eslTD2GGcjByc27+9cIZjHKa07I/aLffo04V+oMT9/tgzoquzgpVV4jwekADo2MJjhkkPveSNI420bgT+Q7Fi1l0X1aFUniBvQMsaBa27PngWm6xE2ZYvh7nWCdd5g0c0eLIHxWwzV1lZ4Ryx4ITO/VL25ItECcjhTRdYa64sA62MYSaB0x3eR+SihpgP3wSNPFu3MJo6FKTFdi4CBAEmpWHFW7FcRmd+cQXeFrHLN3iNVWryy0HK/CUEJmiZEmpNiXecl4vPIIuyF0zgSCztQtKoMr+injpmQGC/rF/ELBVZTUSLNB350S0Ztvw0FKWDAJSxFmoxt3xycqvvt47rxTrhi78nkk6jATKGyvP55sO+K7Q7Wh0DXA69hvPrYW2eu8jGCdVGxi6HX7L1qcfEd0378S71dZ3g9o6KKl1OsDWWQ6MJ6FGBZedl/ibRfs8p5+sbCX3lQSjEFy3rx6n0rUrXx8U2qb+RCLzJlmC5MNBOTDJwHPcX6gKsUcXZrEQALmRHoo3SrewO41RCr+5nUlqiqV3AohBMhnQbGzyHf2+drutIaoh7Rj80XRh2bkkuPLwlNPf+bTXwNVGse4bej7B3oV6Ae1N7lTNVF4Qh+1OowtGjmfJPWo0z1s6HFJVxoIof9z58Msvgao0zrKGqaMWaNQ6LUeC9g9Aj/9Uqjbo8X54aLiYs8Z1WNc06jKP+gv8AWLtv6CR+l2kLez1YMDucjm7v6iuCMVAmZdmxhg5I/X2+OM3vBsqPDdQpr2TPDLX3rCrSBiS0gOQ6DwN5N5QeTkxmY/7QO8bgLo/Wzu1iilH4vMKW6LBKCaRx5UEJxKpL4wkgITsYKneIt3NTHo5EOuaYk+y2+Dvt6EQFiuMsdbfUjs3seIHsghX/cbPJa4YUqZAL8C4OtVHaijwGo0ymt9MWvS9yNKMyT0JhN2/BdeOVWrHk7wXXJn/ZjpXilicXKPx4udCF76meE+6N2u/T+RYZ7fP1QMEtNZNmYDOfA6sViuPDfQSHLNbauJBo/n1sRYAsL5mcG22UDchJrlKvmK3EOADCQg+myrm8006LltubNB4wWNzHDJ0Ls2JGzQZCd/xGyVmUiidCBUrD537WdknOYE4FD7P0cHaM9brKJ/M8LkEH0zUlo73bY4XagbnCqve6PvQb5G2Z55qhWphd6f4B6DGed86zJEa/RhS

kiwixbuild/flatpak_builder.py

@@ -1,321 +0,0 @@
-import sys
-from collections import OrderedDict
-from .buildenv import *
-
-from .configs import ConfigInfo
-from .utils import remove_duplicates, run_command, StopBuild, Context
-from .dependencies import Dependency
-from .packages import PACKAGE_NAME_MAPPERS
-from .versions import base_deps_versions
-from ._global import (
-    neutralEnv,
-    option,
-    add_target_step,
-    get_target_step,
-    target_steps,
-)
-from . import _global
-from .dependencies.base import (
-    Source,
-    Builder,
-    ReleaseDownload,
-    GitClone,
-    MesonBuilder,
-    CMakeBuilder,
-    QMakeBuilder,
-    MakeBuilder,
-    SCRIPT_DIR,
-)
-import json
-from shutil import copyfile
-from urllib.parse import urlparse
-from urllib.request import urlopen
-import json
-
-MANIFEST = {
-    "app-id": "org.kiwix.desktop",
-    "runtime": "org.kde.Platform",
-    "runtime-version": base_deps_versions["org.kde"],
-    "base": "io.qt.qtwebengine.BaseApp",
-    "base-version": base_deps_versions["io.qt.qtwebengine"],
-    "sdk": "org.kde.Sdk",
-    "command": "kiwix-desktop",
-    "rename-icon": "kiwix-desktop",
-    "finish-args": [
-        "--device=dri",
-        "--env=QTWEBENGINEPROCESS_PATH=/app/bin/QtWebEngineProcess",
-        "--socket=wayland",
-        "--socket=fallback-x11",
-        "--socket=pulseaudio",
-        "--share=network",
-        "--share=ipc",
-        "--filesystem=host:ro",
-    ],
-    "cleanup": [
-        "/include",
-        "/lib/pkgconfig",
-        "/lib/cmake",
-        "/lib/*.la",
-        "/bin/curl",
-        "/bin/copydatabase",
-        "/bin/kiwix-compile-resources",
-        "/bin/kiwix-manage",
-        "/bin/kiwix-read",
-        "/bin/kiwix-search",
-        "/bin/quest",
-        "/bin/simple*",
-        "/bin/xapian-*",
-        "/share/aclocal",
-        "/share/doc",
-        "/share/man",
-    ],
-    "cleanup-commands": ["/app/cleanup-BaseApp.sh"],
-}
-
-GET_REF_URL_API_TEMPLATE = "https://api.github.com/repos{repo}/git/refs/tags/{ref}"
-
-
-class FlatpakBuilder:
-    def __init__(self):
-        self._targets = {}
-        ConfigInfo.get_config("neutral", self._targets)
-        self.config = ConfigInfo.get_config("flatpak", self._targets)
-        if neutralEnv("distname") not in self.config.compatible_hosts:
-            print(
-                (
-                    "ERROR: The config {} cannot be build on host {}.\n"
-                    "Select another config or change your host system."
-                ).format(self.config.name, neutralEnv("distname"))
-            )
-        self.targetDefs = self.config.add_targets(option("target"), self._targets)
-
-    def finalize_target_steps(self):
-        steps = []
-        for targetDef in self.targetDefs:
-            steps += self.order_steps(targetDef)
-        steps = list(remove_duplicates(steps))
-
-        for cfgName in ConfigInfo.all_running_configs:
-            cfg = ConfigInfo.all_configs[cfgName]
-            for tlcName in cfg.toolchain_names:
-                tlc = Dependency.all_deps[tlcName]
-                src_cfg_step = ("source", tlcName)
-                add_target_step(src_cfg_step, self._targets[src_cfg_step])
-                blt_cfg_step = ("neutral" if tlc.neutral else cfgName, tlcName)
-                add_target_step(blt_cfg_step, self._targets[blt_cfg_step])
-
-        for dep in steps:
-            add_target_step(dep, self._targets[dep])
-        self.instanciate_steps()
-
-    def order_steps(self, targetDef):
-        _targets = dict(self._targets)
-        yield from self.order_dependencies(targetDef, _targets)
-
-    def order_dependencies(self, targetDef, targets):
-        targetConfigName, targetName = targetDef
-        if targetConfigName == "source":
-            # Do not try to order sources, they will be added as dep by the
-            # build step two lines later.
-            return
-        try:
-            target = targets.pop(targetDef)
-        except KeyError:
-            return
-
-        targetConfig = ConfigInfo.get_config(targetConfigName)
-        for dep in target.get_dependencies(targetConfig, True):
-            if isinstance(dep, tuple):
-                depConfig, depName = dep
-            else:
-                depConfig, depName = targetConfigName, dep
-            if (depConfig, depName) in targets:
-                yield from self.order_dependencies((depConfig, depName), targets)
-        yield ("source", targetName)
-        yield targetDef
-
-    def instanciate_steps(self):
-        for stepDef in list(target_steps()):
-            stepConfig, stepName = stepDef
-            stepClass = Dependency.all_deps[stepName]
-            if stepConfig == "source":
-                source = get_target_step(stepDef)(stepClass)
-                add_target_step(stepDef, source)
-            else:
-                source = get_target_step(stepName, "source")
-                env = ConfigInfo.get_config(stepConfig).buildEnv
-                builder = get_target_step(stepDef)(stepClass, source, env)
-                add_target_step(stepDef, builder)
-
-    def configure(self):
-        steps = remove_duplicates(target_steps())
-        modules = OrderedDict()
-        for stepDef in steps:
-            module = modules.setdefault(stepDef[1], {})
-            module["name"] = stepDef[1]
-            if stepDef[0] == "source":
-                source = get_target_step(stepDef)
-                if getattr(source, "flatpak_no_autogen", False):
-                    module["no-autogen"] = True
-                module_sources = module.setdefault("sources", [])
-                if isinstance(source, ReleaseDownload):
-                    for archive in source.archives:
-                        src = {
-                            "type": "archive",
-                            "dest-filename": archive.name,
-                            "sha256": archive.sha256,
-                            "url": archive.url,
-                        }
-                        if hasattr(source, "flatpak_dest"):
-                            src["dest"] = source.flatpak_dest
-                        module_sources.append(src)
-                elif isinstance(source, GitClone):
-                    src = {
-                        "type": "git",
-                        "url": source.git_remote,
-                        "tag": source.git_ref,
-                    }
-                    try:
-                        parsed = urlparse(source.git_remote)
-                        if parsed.hostname == "github.com":
-                            repo = parsed.path[:-4]
-                            api_url = GET_REF_URL_API_TEMPLATE.format(
-                                repo=repo, ref=source.git_ref
-                            )
-                            with urlopen(api_url) as r:
-                                ret = json.loads(r.read())
-                            src["commit"] = ret["object"]["sha"]
-                    except:
-                        pass
-                    module_sources.append(src)
-                for p in getattr(source, "patches", []):
-                    patch = {"type": "patch", "path": "patches/" + p}
-                    module_sources.append(patch)
-
-                if hasattr(source, "flatpak_command"):
-                    command = {"type": "shell", "commands": [source.flatpak_command]}
-                    module_sources.append(command)
-
-            else:
-                builder = get_target_step(stepDef)
-                builder.set_flatpak_buildsystem(module)
-                print(module["name"])
-
-        manifest = MANIFEST.copy()
-        modules = [m for m in modules.values() if m.get("sources")]
-        for m in modules:
-            temp = m["sources"]
-            del m["sources"]
-            m["sources"] = temp
-        manifest["modules"] = modules
-        manifest_name = "{}.json".format(MANIFEST["app-id"])
-        manifest_path = pj(self.config.buildEnv.build_dir, manifest_name)
-        with open(manifest_path, "w") as f:
-            f.write(json.dumps(manifest, indent=4))
-
-    def copy_patches(self):
-        sourceDefs = (tDef for tDef in target_steps() if tDef[0] == "source")
-        for sourceDef in sourceDefs:
-            source = get_target_step(sourceDef)
-            if not hasattr(source, "patches"):
-                continue
-            for p in source.patches:
-                path = pj(SCRIPT_DIR, "patches", p)
-                os.makedirs(
-                    pj(self.config.buildEnv.build_dir, "patches"), exist_ok=True
-                )
-                dest = pj(self.config.buildEnv.build_dir, "patches", p)
-                copyfile(path, dest)
-
-    def build(self):
-        log = pj(self.config.buildEnv.log_dir, "cmd_build_flatpak.log")
-        context = Context("build", log, False)
-        command = [
-            "flatpak-builder",
-            "--user",
-            "--ccache",
-            "--force-clean",
-            "--keep-build-dirs",
-            "--disable-rofiles-fuse",
-            "--repo=repo",
-            "builddir",
-            f"{MANIFEST['app-id']}.json",
-        ]
-        try:
-            run_command(
-                command,
-                self.config.buildEnv.build_dir,
-                context,
-                env=self.config.get_env(),
-            )
-            context._finalise()
-        except subprocess.CalledProcessError:
-            with open(log, "r") as f:
-                print(f.read())
-            raise StopBuild()
-
-    def bundle(self):
-        log = pj(self.config.buildEnv.log_dir, "cmd_bundle_flatpak.log")
-        context = Context("bundle", log, False)
-        app_id = MANIFEST["app-id"]
-        command = ["flatpak", "build-bundle", "repo", f"{app_id}.flatpak", app_id]
-        try:
-            run_command(
-                command,
-                self.config.buildEnv.build_dir,
-                context,
-                env=self.config.get_env(),
-            )
-            context._finalise()
-        except subprocess.CalledProcessError:
-            with open(log, "r") as f:
-                print(f.read())
-            raise StopBuild()
-
-    def _get_packages(self):
-        package_name_mapper = PACKAGE_NAME_MAPPERS.get("flatpak", {})
-
-        to_drop = []
-        for builderDef in self._targets:
-            configName, builderName = builderDef
-            packages = package_name_mapper.get(builderName)
-            if packages:
-                to_drop.append(builderDef)
-        for dep in to_drop:
-            del self._targets[dep]
-
-    def run(self):
-        try:
-            # This is a small hack, we don't need the list of packages to
-            # install in a flatpak sdk, but _get_packages() will drop the
-            # dependencies we already have in the sdk.
-            self._get_packages()
-            self.finalize_target_steps()
-            print("[SETUP TOOLCHAINS]")
-            for config in ConfigInfo.all_running_configs.values():
-                config.finalize_setup()
-            for cfgName in ConfigInfo.all_running_configs:
-                cfg = ConfigInfo.all_configs[cfgName]
-                for tlcName in cfg.toolchain_names:
-                    tlc = Dependency.all_deps[tlcName]
-                    builderDef = (cfgName, tlcName)
-                    builder = get_target_step(builderDef)
-                    print("build {} ({}):".format(builder.name, cfgName))
-                    add_target_step(builderDef, builder)
-                    builder.build()
-            print("[GENERATE FLATPAK MANIFEST]")
-            self.configure()
-            self.copy_patches()
-            print("[BUILD FLATBACK]")
-            self.build()
-            print("[BUNDLE]")
-            self.bundle()
-            # No error, clean intermediate file at end of build if needed.
-            print("[CLEAN]")
-            if option("clean_at_end"):
-                for config in ConfigInfo.all_running_configs.values():
-                    config.clean_intermediate_directories()
-            else:
-                print("SKIP")
-        except StopBuild:
-            sys.exit("Stopping build due to errors")

kiwixbuild/packages.py

@@ -1,102 +0,0 @@
-_fedora_common = [
-    "automake",
-    "libtool",
-    "cmake",
-    "git",
-    "subversion",
-    "ccache",
-    "pkgconf-pkg-config",
-    "gcc-c++",
-    "gettext-devel",
-]
-_debian_common = [
-    "automake",
-    "libtool",
-    "cmake",
-    "git",
-    "subversion",
-    "ccache",
-    "pkg-config",
-    "gcc",
-    "autopoint",
-]
-PACKAGE_NAME_MAPPERS = {
-    "flatpak": {
-        "zlib": True,
-        "lzma": True,
-        "icu4c": True,
-        "qt": True,
-        "qtwebengine": True,
-        "uuid": True,
-        "libxml2": True,
-        "libssl": True,
-        "libcurl": True,
-    },
-    "fedora_native_dyn": {
-        "COMMON": _fedora_common,
-        "uuid": ["libuuid-devel"],
-        "xapian-core": None,  # Not the right version on fedora 25
-        "pugixml": None,  # ['pugixml-devel'] but package doesn't provide pkg-config file
-        "libmicrohttpd": ["libmicrohttpd-devel"],
-        "zlib": ["zlib-devel"],
-        "lzma": ["xz-devel"],
-        "icu4c": None,
-        "zimlib": None,
-        "file": ["file-devel"],
-        "gumbo": ["gumbo-parser-devel"],
-        "aria2": ["aria2"],
-        "qt": ["qt5-qtbase-devel", "qt5-qtsvg"],
-        "qtwebengine": ["qt5-qtwebengine-devel"],
-    },
-    "fedora_native_static": {
-        "COMMON": _fedora_common + ["glibc-static", "libstdc++-static"],
-        "lzma": ["xz-devel", "xz-static"],
-        # Either there is no packages, or no static or too old
-    },
-    "fedora_i586_dyn": {
-        "COMMON": _fedora_common + ["glibc-devel.i686", "libstdc++-devel.i686"],
-    },
-    "fedora_i586_static": {
-        "COMMON": _fedora_common + ["glibc-devel.i686"],
-    },
-    "fedora_armhf_static": {"COMMON": _fedora_common},
-    "fedora_armhf_dyn": {"COMMON": _fedora_common},
-    "fedora_android": {"COMMON": _fedora_common},
-    "debian_native_dyn": {
-        "COMMON": _debian_common + ["libbz2-dev", "libmagic-dev"],
-        "zlib": ["zlib1g-dev"],
-        "uuid": ["uuid-dev"],
-        "libmicrohttpd": ["libmicrohttpd-dev", "ccache"],
-        "qt": ["libqt5gui5", "qtbase5-dev", "qt5-default"],
-        "qtwebengine": ["qtwebengine5-dev"],
-        "aria2": ["aria2"],
-    },
-    "debian_native_static": {
-        "COMMON": _debian_common + ["libbz2-dev", "libmagic-dev"],
-    },
-    "debian_i586_dyn": {
-        "COMMON": _debian_common
-        + ["libc6-dev-i386", "lib32stdc++6", "gcc-multilib", "g++-multilib"],
-    },
-    "debian_i586_static": {
-        "COMMON": _debian_common
-        + ["libc6-dev-i386", "lib32stdc++6", "gcc-multilib", "g++-multilib"],
-    },
-    "debian_armhf_static": {
-        "COMMON": _debian_common,
-    },
-    "debian_armhf_dyn": {
-        "COMMON": _debian_common,
-    },
-    "debian_android": {
-        "COMMON": _debian_common,
-    },
-    "Darwin_native_dyn": {
-        "COMMON": ["autoconf", "automake", "libtool", "cmake", "pkg-config"],
-        "file": ["libmagic"],
-    },
-    "Darwin_iOS": {
-        "COMMON": ["autoconf", "automake", "libtool", "cmake", "pkg-config"],
-        "file": ["libmagic"],
-    },
-}

kiwixbuild/patches/android-ndk-r21e-linux-x86_64-python3.12+.patch

@@ -1,33 +0,0 @@
-diff -ur android-ndk-r21e/build/tools/make_standalone_toolchain.py android-ndk-r21e.patched/build/tools/make_standalone_toolchain.py
---- android-ndk-r21e/build/tools/make_standalone_toolchain.py	2025-03-04 20:48:14.681288830 +0400
-+++ android-ndk-r21e.patched/build/tools/make_standalone_toolchain.py	2025-03-05 12:10:47.252578915 +0400
-@@ -1,4 +1,4 @@
--#!/usr/bin/env python
-+#!/usr/bin/env python3
- #
- # Copyright (C) 2016 The Android Open Source Project
- #
-@@ -21,7 +21,6 @@
- """
- import argparse
- import atexit
--from distutils.dir_util import copy_tree
- import inspect
- import logging
- import os
-@@ -221,13 +220,13 @@
- 
- def create_toolchain(install_path, arch, api, toolchain_path, host_tag):
-     """Create a standalone toolchain."""
--    copy_tree(toolchain_path, install_path)
-+    shutil.copytree(toolchain_path, install_path)
-     triple = get_triple(arch)
-     make_clang_scripts(install_path, arch, api, host_tag == 'windows-x86_64')
-     replace_gcc_wrappers(install_path, triple, host_tag == 'windows-x86_64')
- 
-     prebuilt_path = os.path.join(NDK_DIR, 'prebuilt', host_tag)
--    copy_tree(prebuilt_path, install_path)
-+    shutil.copytree(prebuilt_path, install_path, dirs_exist_ok=True)
- 
-     gdbserver_path = os.path.join(
-         NDK_DIR, 'prebuilt', 'android-' + arch, 'gdbserver')

kiwixbuild/patches/docopt_meson_install_pkgconfig.patch

@@ -1,31 +0,0 @@
-diff '--color=auto' -ur docoptcpp-0.6.2/meson.build docoptcpp-0.6.2_patched/meson.build
---- docoptcpp-0.6.2/meson.build	2024-08-26 14:28:47.553448529 +0200
-+++ docoptcpp-0.6.2_patched/meson.build	2024-08-26 14:10:47.232603427 +0200
-@@ -10,11 +10,25 @@
- 
- # bug with missing dllexport. fixed in next version.
- if cpp.get_argument_syntax() == 'msvc'
--  doclib = static_library('docopt', 'docopt.cpp')
-+  doclib = static_library('docopt', 'docopt.cpp', install: true)
- else
--  doclib = library('docopt', 'docopt.cpp')
-+  doclib = library('docopt', 'docopt.cpp', install: true)
- endif
- 
- executable('docopt_example', 'examples/naval_fate.cpp', link_with: doclib)
- docopt_dep = declare_dependency(include_directories: include_directories('.'),
-     link_with: doclib)
-+
-+install_headers(
-+  'docopt.h',
-+  'docopt_value.h',
-+  subdir: 'docopt'
-+)
-+
-+pkg_mod = import('pkgconfig')
-+pkg_mod.generate(
-+  doclib,
-+  version: meson.project_version(),
-+  name: 'docopt',
-+  filebase: 'docopt'
-+)

kiwixbuild/patches/docopt_meson_use_boostregex.patch

@@ -1,14 +0,0 @@
-diff -ur docoptcpp-0.6.2/meson.build docoptcpp-0.6.2_boostregex/meson.build
---- docoptcpp-0.6.2/meson.build	2024-08-28 17:22:46.256716100 +0200
-+++ docoptcpp-0.6.2_boostregex/meson.build	2024-08-28 17:02:47.932681000 +0200
-@@ -8,6 +8,10 @@
-   add_project_arguments('-DDOCOPT_DLL', '-DDOCOPT_EXPORTS', language: 'cpp')
- endif
- 
-+if cpp.get_id() =='msvc'
-+  add_project_arguments('-DDOCTOPT_USE_BOOST_REGEX', '-DBOOST_REGEX_STANDALONE', language: 'cpp')
-+endif
-+
- # bug with missing dllexport. fixed in next version.
- if cpp.get_argument_syntax() == 'msvc'
-   doclib = static_library('docopt', 'docopt.cpp', install: true)

kiwixbuild/patches/icu4c_build_config.patch

@@ -1,36 +0,0 @@
-diff -ur icu4c/source/common/unicode/platform.h icu4c.patched/source/common/unicode/platform.h
---- icu4c/source/common/unicode/platform.h	2018-04-17 17:20:31.048946098 +0200
-+++ icu4c.patched/source/common/unicode/platform.h	2018-06-11 11:22:40.692327158 +0200
-@@ -644,7 +644,7 @@
- #elif U_PLATFORM == U_PF_ANDROID || U_PLATFORM_IS_DARWIN_BASED
- #   define U_CHARSET_IS_UTF8 1
- #else
--#   define U_CHARSET_IS_UTF8 0
-+#   define U_CHARSET_IS_UTF8 1
- #endif
- 
- /** @} */
-diff -ur icu4c/source/common/unicode/uconfig.h icu4c.patched/source/common/unicode/uconfig.h
---- icu4c/source/common/unicode/uconfig.h	2018-04-17 17:20:31.072946266 +0200
-+++ icu4c.patched/source/common/unicode/uconfig.h	2018-06-11 11:26:24.512936322 +0200
-@@ -107,7 +107,7 @@
-       defined(U_TOOLUTIL_IMPLEMENTATION)
- #   define U_NO_DEFAULT_INCLUDE_UTF_HEADERS 1
- #else
--#   define U_NO_DEFAULT_INCLUDE_UTF_HEADERS 0
-+#   define U_NO_DEFAULT_INCLUDE_UTF_HEADERS 1
- #endif
- 
- /**
-diff -ur icu4c/source/common/unicode/uversion.h icu4c.patched/source/common/unicode/uversion.h
---- icu4c/source/common/unicode/uversion.h	2018-04-17 17:20:31.086946363 +0200
-+++ icu4c.patched/source/common/unicode/uversion.h	2018-06-11 11:20:27.093375709 +0200
-@@ -122,7 +122,7 @@
- #   define U_NAMESPACE_QUALIFIER U_ICU_NAMESPACE::
- 
- #   ifndef U_USING_ICU_NAMESPACE
--#       define U_USING_ICU_NAMESPACE 1
-+#       define U_USING_ICU_NAMESPACE 0
- #   endif
- #   if U_USING_ICU_NAMESPACE
-         U_NAMESPACE_USE

kiwixbuild/patches/icu4c_rpath.patch

@@ -1,13 +0,0 @@
-diff -ur icu4c/source/config/mh-linux icu4c.rpath/source/config/mh-linux
---- icu4c/source/config/mh-linux	2018-04-17 11:31:50.674012676 +0200
-+++ icu4c.rpath/source/config/mh-linux	2018-04-17 11:28:57.776134587 +0200
-@@ -19,7 +19,7 @@
- LIBCPPFLAGS =
- 
- ## Compiler switch to embed a runtime search path
--LD_RPATH= -Wl,-zorigin,-rpath,'$$'ORIGIN 
-+LD_RPATH= '-Wl,-zorigin,-rpath,\$ORIGIN' 
- LD_RPATH_PRE = -Wl,-rpath,
- 
- ## These are the library specific LDFLAGS
-Les fichiers binaires icu4c/.svn/wc.db et icu4c.rpath/.svn/wc.db sont différents

kiwixbuild/patches/icu4c_wasm.patch

@@ -1,153 +0,0 @@
-diff '--color=auto' -ur icu4c-71.1/source/config/mh-unknown icu4c-71.1.wasm/source/config/mh-unknown
---- icu4c-71.1/source/config/mh-unknown	2022-04-08 00:41:55.000000000 +0200
-+++ icu4c-71.1.wasm/source/config/mh-unknown	2023-10-25 14:00:18.035718690 +0200
-@@ -1,29 +1,87 @@
- ## -*-makefile-*-
- ## Copyright (C) 2016 and later: Unicode, Inc. and others.
- ## License & terms of use: http://www.unicode.org/copyright.html
--## Copyright (c) 2003, International Business Machines Corporation and
-+## Linux-specific setup
-+## Copyright (c) 1999-2013, International Business Machines Corporation and
- ## others. All Rights Reserved.
--##
- 
--# Note, this is not a real mh- file. You got here because configure
--# (specifically, aclocal.m4) could not determine a suitable mh- file.
--#
--# Perhaps your platform wasn't detected- try changing aclocal.m4 and
--# re-running autoconf. 
--#
--# If your platform is truly new/different:
--# As a start, try copying mh-linux (which is fairly generic) over this
--# file, and re-run config.status.
--
--%.$(STATIC_O) %.o %.$(STATIC_O) %.o ../data/%.o %.d %.d %.$(SO).$(SO_TARGET_VERSION_MAJOR) %.$(SO):
--	@echo 
--	@echo 
--	@echo "*** ERROR - configure could not detect your platform"
--	@echo "*** see the readme.html"
--	@echo "*** or, try copying icu/source/config/mh-linux to mh-unknown"
--	@echo "***   and editing it."
--	@echo 
--	@echo 
--	exit 1
-+## Commands to generate dependency files
-+GEN_DEPS.c=  $(CC) -E -MM $(DEFS) $(CPPFLAGS)
-+GEN_DEPS.cc= $(CXX) -E -MM $(DEFS) $(CPPFLAGS) $(CXXFLAGS)
- 
-+## Flags for position independent code
-+SHAREDLIBCFLAGS = -fPIC
-+SHAREDLIBCXXFLAGS = -fPIC
-+SHAREDLIBCPPFLAGS = -DPIC
-+
-+## Additional flags when building libraries and with threads
-+THREADSCPPFLAGS = -D_REENTRANT
-+LIBCPPFLAGS =
-+
-+## Compiler switch to embed a runtime search path
-+LD_RPATH= -Wl,-zorigin,-rpath,'$$'ORIGIN 
-+LD_RPATH_PRE = -Wl,-rpath,
-+
-+## These are the library specific LDFLAGS
-+LDFLAGSICUDT=-nodefaultlibs -nostdlib
-+
-+## Compiler switch to embed a library name
-+# The initial tab in the next line is to prevent icu-config from reading it.
-+	LD_SONAME = -Wl,-soname -Wl,$(notdir $(MIDDLE_SO_TARGET))
-+#SH# # We can't depend on MIDDLE_SO_TARGET being set.
-+#SH# LD_SONAME=
-+
-+## Shared library options
-+LD_SOOPTIONS= -Wl,-Bsymbolic
-+
-+## Shared object suffix
-+SO = so
-+## Non-shared intermediate object suffix
-+STATIC_O = ao
-+
-+## Compilation rules
-+%.$(STATIC_O): $(srcdir)/%.c
-+	$(call SILENT_COMPILE,$(strip $(COMPILE.c) $(STATICCPPFLAGS) $(STATICCFLAGS)) -o $@ $<)
-+%.o: $(srcdir)/%.c
-+	$(call SILENT_COMPILE,$(strip $(COMPILE.c) $(DYNAMICCPPFLAGS) $(DYNAMICCFLAGS)) -o $@ $<)
-+
-+%.$(STATIC_O): $(srcdir)/%.cpp
-+	$(call SILENT_COMPILE,$(strip $(COMPILE.cc) $(STATICCPPFLAGS) $(STATICCXXFLAGS)) -o $@ $<)
-+%.o: $(srcdir)/%.cpp
-+	$(call SILENT_COMPILE,$(strip $(COMPILE.cc) $(DYNAMICCPPFLAGS) $(DYNAMICCXXFLAGS)) -o $@ $<)
-+
-+
-+## Dependency rules
-+%.d: $(srcdir)/%.c
-+	$(call ICU_MSG,(deps)) $<
-+	@$(SHELL) -ec '$(GEN_DEPS.c) $< \
-+		| sed '\''s%\($*\)\.o[ :]*%\1.o $@ : %g'\'' > $@; \
-+		[ -s $@ ] || rm -f $@'
-+
-+%.d: $(srcdir)/%.cpp
-+	$(call ICU_MSG,(deps)) $<
-+	@$(SHELL) -ec '$(GEN_DEPS.cc) $< \
-+		| sed '\''s%\($*\)\.o[ :]*%\1.o $@ : %g'\'' > $@; \
-+		[ -s $@ ] || rm -f $@'
-+
-+## Versioned libraries rules
-+
-+%.$(SO).$(SO_TARGET_VERSION_MAJOR): %.$(SO).$(SO_TARGET_VERSION)
-+	$(RM) $@ && ln -s ${<F} $@
-+%.$(SO): %.$(SO).$(SO_TARGET_VERSION_MAJOR)
-+	$(RM) $@ && ln -s ${*F}.$(SO).$(SO_TARGET_VERSION) $@
-+
-+##  Bind internal references
-+
-+# LDflags that pkgdata will use
-+BIR_LDFLAGS= -Wl,-Bsymbolic
-+
-+# Dependencies [i.e. map files] for the final library
-+BIR_DEPS=
-+
-+## Remove shared library 's'
-+STATIC_PREFIX_WHEN_USED = 
-+STATIC_PREFIX = 
-+
-+## End Linux-specific setup
- 
-diff '--color=auto' -ur icu4c-71.1/source/config.sub icu4c-71.1.wasm/source/config.sub
---- icu4c-71.1/source/config.sub	2022-04-08 00:41:55.000000000 +0200
-+++ icu4c-71.1.wasm/source/config.sub	2023-10-25 13:58:23.711645708 +0200
-@@ -312,7 +312,7 @@
- 	| ubicom32 \
- 	| v850 | v850e | v850e1 | v850e2 | v850es | v850e2v3 \
- 	| visium \
--	| wasm32 \
-+	| wasm32 | wasm64 \
- 	| x86 | xc16x | xstormy16 | xtensa \
- 	| z8k | z80)
- 		basic_machine=$basic_machine-unknown
-@@ -443,7 +443,7 @@
- 	| v850-* | v850e-* | v850e1-* | v850es-* | v850e2-* | v850e2v3-* \
- 	| vax-* \
- 	| visium-* \
--	| wasm32-* \
-+	| wasm32-* | wasm64-* \
- 	| we32k-* \
- 	| x86-* | x86_64-* | xc16x-* | xps100-* \
- 	| xstormy16-* | xtensa*-* \
-@@ -1247,6 +1247,9 @@
- 	wasm32)
- 		basic_machine=wasm32-unknown
- 		;;
-+    wasm64)
-+        basic_machine=wasm64-unknown
-+        ;;
- 	w65*)
- 		basic_machine=w65-wdc
- 		os=-none
-@@ -1416,7 +1419,7 @@
- 	      | -morphos* | -superux* | -rtmk* | -rtmk-nova* | -windiss* \
- 	      | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly* \
- 	      | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es* \
--	      | -onefs* | -tirtos* | -phoenix* | -fuchsia* | -redox*)
-+	      | -onefs* | -tirtos* | -phoenix* | -fuchsia* | -redox* | -emscripten*)
- 	# Remember, each alternative MUST END IN *, to match a version number.
- 		;;
- 	-qnx*)

kiwixbuild/patches/libmicrohttpd_meson_pkgconfig.patch

@@ -1,29 +0,0 @@
-diff '--color=auto' -ur libmicrohttpd-0.9.76_orig/meson.build libmicrohttpd-0.9.76/meson.build
---- libmicrohttpd-0.9.76_orig/meson.build	2024-08-18 14:55:29.372805433 +0200
-+++ libmicrohttpd-0.9.76/meson.build	2024-08-18 17:19:44.087444728 +0200
-@@ -6,6 +6,8 @@
-   default_options: ['warning_level=1'],
- )
- 
-+pkg = import('pkgconfig')
-+
- add_project_arguments('-D_GNU_SOURCE', language: 'c')
- 
- incdirs = include_directories('src/include')
-@@ -267,6 +269,16 @@
-   install: true,
- )
- 
-+install_headers(
-+  'src/include/microhttpd.h',
-+)
-+
-+pkg.generate(
-+  libmicrohttpd,
-+  description: 'Libmicrohttpd',
-+  name: 'libmicrohttpd',
-+)
-+
- depinc = include_directories('.', 'src/include')
- libmicrohttpd_dep = declare_dependency(
-   include_directories: depinc,

kiwixbuild/patches/libmicrohttpd_meson_timeval_tvsec_size.patch

@@ -1,31 +0,0 @@
-diff '--color=auto' -ur libmicrohttpd-0.9.76/meson.build libmicrohttpd-0.9.76_patched/meson.build
---- libmicrohttpd-0.9.76/meson.build	2024-08-22 15:17:59.217715872 +0200
-+++ libmicrohttpd-0.9.76_patched/meson.build	2024-08-22 15:20:23.755358647 +0200
-@@ -126,7 +126,26 @@
- foreach s : sizes
-   cdata.set('SIZEOF_@0@'.format(s.underscorify().to_upper()), cc.sizeof(s))
- endforeach
--cdata.set('SIZEOF_STRUCT_TIMEVAL_TV_SEC', cc.sizeof('time_t'))
-+
-+cdata.set(
-+  'SIZEOF_STRUCT_TIMEVAL_TV_SEC',
-+  cc.sizeof(
-+    'test_var.tv_sec',
-+    prefix: '''#ifdef HAVE_SYS_TIME_H
-+#include <sys/time.h>
-+#endif /* HAVE_SYS_TIME_H */
-+#ifdef HAVE_TIME_H
-+#include <time.h>
-+#endif /* HAVE_TIME_H */
-+#if HAVE_SYS_TYPES_H
-+#include <sys/types.h>
-+#endif /* HAVE_SYS_TYPES_H */
-+#ifdef _WIN32
-+#include <winsock.h>
-+#endif /* _WIN32 */
-+struct timeval test_var;'''
-+  )
-+)
- cdata.set('SIZEOF_UINT64_T', 8)
- 
- cdata.set('HAVE_PIPE2_FUNC', cc.has_function('pipe2'))

kiwixbuild/patches/libmicrohttpd_meson_winet6.patch

@@ -1,15 +0,0 @@
---- libmicrohttpd-0.9.76_orig/meson.build	2024-10-08 15:53:53.370828250 +0400
-+++ libmicrohttpd-0.9.76/meson.build	2024-10-08 16:23:24.985668690 +0400
-@@ -77,7 +77,11 @@
- endforeach
-
- cdata.set('HAVE_ASSERT', cc.has_header_symbol('assert.h', 'assert'))
--cdata.set10('HAVE_INET6', cc.has_header_symbol('netinet/in.h', 'struct in6_addr'))
-+if host_machine.system() == 'windows'
-+  cdata.set10('HAVE_INET6', 1)
-+else
-+  cdata.set10('HAVE_INET6', cc.has_header_symbol('netinet/in.h', 'struct in6_addr'))
-+endif
-
- functions = [
-   'accept4',

kiwixbuild/patches/lzma_meson_install.patch

@@ -1,54 +0,0 @@
-diff '--color=auto' -ur lzma-5.2.6_orig/src/liblzma/meson.build lzma-5.2.6/src/liblzma/meson.build
---- lzma-5.2.6_orig/src/liblzma/meson.build	2023-11-23 14:31:26.110195070 +0100
-+++ lzma-5.2.6/src/liblzma/meson.build	2023-12-06 17:04:49.325148650 +0100
-@@ -1,3 +1,5 @@
-+pkg = import('pkgconfig')
-+
- lzma_sources = [
-   '../common/tuklib_physmem.c',
-   'common/common.c',
-@@ -121,12 +123,44 @@
- lzmainc = include_directories('api', 'common',
-   'check', 'lz', 'rangecoder', 'lzma', 'delta', 'simple', '../common')
- 
-+
-+install_headers(
-+  'api/lzma.h',
-+)
-+
-+install_headers(
-+  'api/lzma/version.h',
-+  'api/lzma/base.h',
-+  'api/lzma/vli.h',
-+  'api/lzma/check.h',
-+  'api/lzma/filter.h',
-+  'api/lzma/bcj.h',
-+  'api/lzma/delta.h',
-+  'api/lzma/lzma12.h',
-+  'api/lzma/container.h',
-+  'api/lzma/stream_flags.h',
-+  'api/lzma/block.h',
-+  'api/lzma/index.h',
-+  'api/lzma/index_hash.h',
-+  'api/lzma/hardware.h',
-+  subdir: 'lzma'
-+)
-+
- liblzma = library('lzma', lzma_sources,
-   main_dec_sources, main_enc_sources, check_sources,
-   simplefilter_sources, lzma1_sources,
-   lz_sources, delta_sources,
-   include_directories : [confinc, lzmainc],
-   c_args : ['-DHAVE_CONFIG_H', '-DTUKLIB_SYMBOL_PREFIX=lzma_'],
-+  install: true
-+)
-+
-+pkg.generate(liblzma,
-+  name: 'liblzma',
-+  filebase: 'liblzma',
-+  description: 'The liblzma compression library',
-+  version: meson.project_version(),
-+  url: 'http://tukaani.org/xz/'
- )
- 
- lzma_dep = declare_dependency(link_with : liblzma,

kiwixbuild/templates/cmake_android_cross_file.txt

@@ -1,8 +0,0 @@
-SET(CMAKE_SYSTEM_NAME {host_machine[system]})
-
-SET(CMAKE_ANDROID_STANDALONE_TOOLCHAIN {install_path})
-
-SET(CMAKE_C_COMPILER "{binaries[CC]}")
-SET(CMAKE_CXX_COMPILER "{binaries[CXX]}")
-
-SET(CMAKE_FIND_ROOT_PATH {root_path})

kiwixbuild/templates/cmake_i586_cross_file.txt

@@ -1,18 +0,0 @@
-SET(CMAKE_SYSTEM_NAME {host_machine[system]})
-SET(CMAKE_SYSTEM_PROCESSOR {host_machine[cpu_family]})
-
-# specify the cross compiler
-SET(CMAKE_C_COMPILER "{binaries[CC]}")
-SET(CMAKE_CXX_COMPILER "{binaries[CXX]}")
-SET(C_FLAGS "-m32 -march=i586")
-SET(CXX_FLAGS "-m32 -march=i586")
-SET(CMAKE_LD_FLAGS "-m32 -march=i586")
-SET(CMAKE_AR:FILEPATH {binaries[AR]})
-SET(CMAKE_RANLIB:FILEPATH {binaries[RANLIB]})
-
-find_program(CCACHE_FOUND ccache)
-if(CCACHE_FOUND)
-        set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
-        set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache)
-endif(CCACHE_FOUND)
-

kiwixbuild/templates/cmake_ios_cross_file.txt

@@ -1,7 +0,0 @@
-SET(CMAKE_SYSTEM_NAME {host_machine[system]})
-
-SET(CMAKE_C_COMPILER "{binaries[CC]}")
-SET(CMAKE_CXX_COMPILER "{binaries[CXX]}")
-
-SET(CMAKE_FIND_ROOT_PATH {root_path})
-

kiwixbuild/templates/meson_ios_cross_file.txt

@@ -1,21 +0,0 @@
-[binaries]
-pkgconfig = '{binaries[PKGCONFIG]}'
-c = '{binaries[CC]}'
-ar = '{binaries[AR]}'
-cpp = '{binaries[CXX]}'
-strip = '{binaries[STRIP]}'
-ranlib = '{binaries[RANLIB]}'
-{exe_wrapper_def}
-
-[properties]
-c_link_args = {extra_libs!r}
-cpp_link_args = {extra_libs!r}
-c_args = {extra_cflags!r}
-cpp_args = {extra_cflags!r}
-needs_exe_wrapper = true
-
-[host_machine]
-cpu_family = '{host_machine[cpu_family]}'
-cpu = '{host_machine[cpu]}'
-system = '{host_machine[lsystem]}'
-endian = '{host_machine[endian]}'

kiwixbuild/utils.py

@@ -1,374 +0,0 @@
-import os.path
-import hashlib
-import tarfile, zipfile
-import tempfile
-import shutil
-import os, stat, sys
-import urllib.request
-import urllib.error
-import ssl
-import subprocess
-import re
-from collections import namedtuple, defaultdict
-
-from kiwixbuild._global import neutralEnv, option
-
-
-def pj(*args):
-    return os.path.normpath(os.path.join(*args))
-
-
-COLORS = {
-    "OK": "\033[92m",
-    "WARNING": "\033[93m",
-    "NEEDED": "\033[93m",
-    "SKIP": "\033[34m",
-    "ERROR": "\033[91m",
-    "": "\033[0m",
-}
-
-
-REMOTE_PREFIX = "https://dev.kiwix.org/kiwix-build/"
-
-
-def which(name):
-    command = "which {}".format(name)
-    output = subprocess.check_output(command, shell=True)
-    return output[:-1].decode()
-
-
-def xrun_find(name):
-    command = "xcrun -find {}".format(name)
-    output = subprocess.check_output(command, shell=True)
-    return output[:-1].decode()
-
-
-regex_space = re.compile(r"((?<!\\) )")
-
-
-def escape_path(path):
-    path = str(path)
-    return regex_space.sub(r"\ ", path)
-
-
-class Defaultdict(defaultdict):
-    def __getattr__(self, name):
-        return self[name]
-
-
-class DefaultEnv(Defaultdict):
-    def __init__(self):
-        super().__init__(str, os.environ)
-
-    def __getitem__(self, name):
-        if name == b"PATH":
-            raise KeyError
-        if name in ["PATH", "PKG_CONFIG_PATH", "LD_LIBRARY_PATH"]:
-            item = super().__getitem__(name)
-            if isinstance(item, PathArray):
-                return item
-            else:
-                item = PathArray(item)
-                self[name] = item
-                return item
-        return super().__getitem__(name)
-
-
-def get_separator():
-    return ";" if neutralEnv("distname") == "Windows" else ":"
-
-
-class PathArray(list):
-    def __init__(self, value):
-        self.separator = get_separator()
-        if not value:
-            super().__init__([])
-        else:
-            super().__init__(value.split(self.separator))
-
-    def __str__(self):
-        return self.separator.join(self)
-
-
-def remove_duplicates(iterable, key_function=None):
-    seen = set()
-    if key_function is None:
-        key_function = lambda e: e
-    for elem in iterable:
-        key = key_function(elem)
-        if key in seen:
-            continue
-        seen.add(key)
-        yield elem
-
-
-def get_sha256(path):
-    progress_chars = "/-\\|"
-    current = 0
-    batch_size = 1024 * 8
-    sha256 = hashlib.sha256()
-    with open(path, "br") as f:
-        while True:
-            batch = f.read(batch_size)
-            if not batch:
-                break
-            sha256.update(batch)
-            print_progress(progress_chars[current])
-            current = (current + 1) % 4
-    return sha256.hexdigest()
-
-
-def colorize(text, color=None):
-    if color is None:
-        color = text
-    return "{}{}{}".format(COLORS[color], text, COLORS[""])
-
-
-def print_progress(progress):
-    if option("show_progress"):
-        text = "{}\033[{}D".format(progress, len(progress))
-        print(text, end="")
-
-
-def add_execution_right(file_path):
-    current_permissions = stat.S_IMODE(os.lstat(file_path).st_mode)
-    os.chmod(file_path, current_permissions | stat.S_IXUSR)
-
-
-def copy_tree(src, dst, post_copy_function=None):
-    os.makedirs(dst, exist_ok=True)
-    for root, dirs, files in os.walk(src):
-        r = os.path.relpath(root, src)
-        dstdir = pj(dst, r)
-        os.makedirs(dstdir, exist_ok=True)
-        for f in files:
-            dstfile = pj(dstdir, f)
-            shutil.copy2(pj(root, f), dstfile, follow_symlinks=False)
-            if post_copy_function is not None:
-                post_copy_function(dstfile)
-
-
-def download_remote(what, where):
-    file_path = pj(where, what.name)
-    if os.path.exists(file_path):
-        if what.sha256 == get_sha256(file_path):
-            raise SkipCommand()
-        os.remove(file_path)
-
-    if option("no_cert_check"):
-        context = ssl.create_default_context()
-        context.check_hostname = False
-        context.verify_mode = ssl.CERT_NONE
-    else:
-        context = None
-    batch_size = 1024 * 8
-    extra_args = {"context": context} if sys.version_info >= (3, 4, 3) else {}
-    progress_chars = "/-\\|"
-    try:
-        with urllib.request.urlopen(what.url, **extra_args) as resource, open(
-            file_path, "wb"
-        ) as file:
-            tsize = resource.info().get("Content-Length", None)
-            if tsize is not None:
-                tsize = int(tsize)
-            current = 0
-            while True:
-                batch = resource.read(batch_size)
-                if not batch:
-                    break
-                if tsize:
-                    current += batch_size
-                    print_progress("{:.2%}".format(current / tsize))
-                else:
-                    print_progress(progress_chars[current])
-                    current = (current + 1) % 4
-                file.write(batch)
-    except urllib.error.URLError as e:
-        print("Cannot download url {}:\n{}".format(what.url, e.reason))
-        raise StopBuild()
-
-    if not what.sha256:
-        print("Sha256 for {} not set, do no verify download".format(what.name))
-    elif what.sha256 != get_sha256(file_path):
-        os.remove(file_path)
-        raise StopBuild("Sha 256 doesn't correspond")
-
-
-class BaseCommandResult(Exception):
-    def __init__(self, msg=""):
-        self.msg = msg
-
-    def __str__(self):
-        return self.msg
-
-
-class SkipCommand(BaseCommandResult):
-    def __str__(self):
-        if self.msg:
-            return colorize("SKIP") + " : {}".format(self.msg)
-        return colorize("SKIP")
-
-
-class WarningMessage(BaseCommandResult):
-    def __str__(self):
-        return colorize("WARNING") + " : {}".format(self.msg)
-
-
-class StopBuild(BaseCommandResult):
-    pass
-
-
-class Remotefile(namedtuple("Remotefile", ("name", "sha256", "url"))):
-    def __new__(cls, name, sha256, url=None):
-        if url is None:
-            url = REMOTE_PREFIX + name
-        return super().__new__(cls, name, sha256, url)
-
-
-class Context:
-    def __init__(self, command_name, log_file, force_native_build):
-        self.command_name = command_name
-        self.log_file = log_file
-        self.force_native_build = force_native_build
-        self.autoskip_file = None
-        self.no_skip = False
-
-    def skip(self, msg=""):
-        raise SkipCommand(msg)
-
-    def try_skip(self, path, extra_name=""):
-        if self.no_skip:
-            return
-        if extra_name:
-            extra_name = "_{}".format(extra_name)
-        self.autoskip_file = pj(path, ".{}{}_ok".format(self.command_name, extra_name))
-        if os.path.exists(self.autoskip_file):
-            raise SkipCommand()
-
-    def _finalise(self):
-        if self.autoskip_file is not None:
-            os.makedirs(os.path.dirname(self.autoskip_file), exist_ok=True)
-            with open(self.autoskip_file, "w"):
-                pass
-
-
-def extract_archive(archive_path, dest_dir, topdir=None, name=None):
-    is_zip_archive = archive_path.endswith(".zip")
-    archive = None
-    try:
-        if is_zip_archive:
-            archive = zipfile.ZipFile(archive_path)
-            members = archive.infolist()
-            getname = lambda info: info.filename
-            isdir = lambda info: info.filename.endswith("/")
-        else:
-            archive = tarfile.open(archive_path)
-            members = archive.getmembers()
-            getname = lambda info: getattr(info, "name")
-            isdir = lambda info: info.isdir()
-        if not topdir:
-            for d in (m for m in members if isdir(m)):
-                _name = getname(d)
-                if _name.endswith("/"):
-                    _name = _name[:-1]
-                if not os.path.dirname(_name):
-                    if topdir:
-                        # There is already a top dir.
-                        # Two topdirs in the same archive.
-                        # Extract all
-                        topdir = None
-                        break
-                    topdir = _name
-        if topdir:
-            members_to_extract = [
-                m for m in members if getname(m).startswith(topdir + "/")
-            ]
-            os.makedirs(dest_dir, exist_ok=True)
-            with tempfile.TemporaryDirectory(
-                prefix=os.path.basename(archive_path), dir=dest_dir
-            ) as tmpdir:
-                if is_zip_archive:
-                    _members_to_extract = [getname(m) for m in members_to_extract]
-                else:
-                    _members_to_extract = members_to_extract
-                archive.extractall(path=tmpdir, members=_members_to_extract)
-                if is_zip_archive:
-                    for member in members_to_extract:
-                        if isdir(member):
-                            continue
-                        perm = (member.external_attr >> 16) & 0x1FF
-                        if perm:
-                            os.chmod(pj(tmpdir, getname(member)), perm)
-                name = name or topdir
-                shutil.copytree(
-                    pj(tmpdir, topdir),
-                    pj(dest_dir, name),
-                    symlinks=True,
-                    dirs_exist_ok=True,
-                )
-                # Be sure that all directory in tmpdir are writable to allow correct suppersion of it
-                for root, dirs, _files in os.walk(tmpdir):
-                    for d in dirs:
-                        os.chmod(
-                            pj(root, d), stat.S_IWRITE | stat.S_IREAD | stat.S_IEXEC
-                        )
-
-        else:
-            if name:
-                dest_dir = pj(dest_dir, name)
-                os.makedirs(dest_dir)
-            archive.extractall(path=dest_dir)
-    finally:
-        if archive is not None:
-            archive.close()
-
-
-def run_command(command, cwd, context, *, env=None, input=None):
-    os.makedirs(cwd, exist_ok=True)
-    if env is None:
-        env = DefaultEnv()
-    log = None
-    try:
-        if not option("verbose"):
-            log = open(context.log_file, "w")
-        print("run command '{}'".format(command), file=log)
-        print("current directory is '{}'".format(cwd), file=log)
-        print("env is :", file=log)
-        env = {k: str(v) for k, v in env.items()}
-        for k, v in env.items():
-            print("  {} : {!r}".format(k, v), file=log)
-
-        if log:
-            log.flush()
-        kwargs = dict()
-        if input:
-            kwargs["stdin"] = subprocess.PIPE
-        process = subprocess.Popen(
-            command,
-            cwd=cwd,
-            env=env,
-            stdout=log or sys.stdout,
-            stderr=subprocess.STDOUT,
-            **kwargs
-        )
-        if input:
-            input = input.encode()
-        while True:
-            try:
-                if input is None:
-                    process.wait(timeout=30)
-                else:
-                    process.communicate(input, timeout=30)
-            except subprocess.TimeoutExpired:
-                # Either `wait` timeout (and `input` is None) or
-                # `communicate` timeout (and we must set `input` to None
-                # to not communicate again).
-                input = None
-                print(".", end="", flush=True)
-            else:
-                break
-        if process.returncode:
-            raise subprocess.CalledProcessError(process.returncode, command)
-    finally:
-        if log:
-            log.close()

kiwixbuild/versions.py

@@ -1,57 +0,0 @@
-# This file reference all the versions of the depedencies we use in kiwix-build.
-
-main_project_versions = {
-    "libzim": "9.3.0",
-    "libkiwix": "14.0.0",
-    "kiwix-tools": "3.7.0",
-    "zim-tools": "3.6.0",
-    "kiwix-desktop": "2.4.1",
-}
-
-# This dictionnary specify what we need to build at each release process.
-# - Values are integer or None
-# - If a project is not in the dict (or None), the project is not released.
-# - If release_versions[project] == 0, this is the first time the project is
-#   build for this release, so publish src and build archives.
-# - If release_versions[project] > 0, release only the build archive with a
-#   build postfix.
-# To change this dictionnary, use the following algorithm:
-# - If project version change, set release_versions[project] = 0
-# - Else
-#    - If project depedencies have not change, set it to None and update the
-#     `(was ...)`.
-#    - Else, increment the value. If no value was present, see `(was ...)`.
-
-release_versions = {
-    "libzim": 1,  # Depends of base deps (was 0)
-    "libkiwix": None,  # Depends of libzim (was 1)
-    "kiwix-tools": None,  # Depends of libkiwix and libzim (was 2)
-    "zim-tools": 0,  # Depends of libzim (was None)
-    "kiwix-desktop": None,  # Depends of libkiwix and libzim (was 0)
-}
-
-
-# This is the "version" of the whole base_deps_versions dict.
-# Change this when you change base_deps_versions.
-base_deps_meta_version = "13"
-
-base_deps_versions = {
-    "zlib": "1.2.12",
-    "lzma": "5.2.6",
-    "zstd": "1.5.2",
-    "docoptcpp": "0.6.2",
-    "uuid": "1.43.4",
-    "xapian-core": "1.4.26",
-    "mustache": "4.1",
-    "pugixml": "1.2",
-    "libmicrohttpd": "0.9.76",
-    "gumbo": "0.12.1",
-    "icu4c": "73.2",
-    "libaria2": "1.37.0",
-    "libmagic": "5.44",
-    "android-ndk": "r21e",
-    "org.kde": "6.7",
-    "io.qt.qtwebengine": "6.7",
-    "zim-testing-suite": "0.8.0",
-    "emsdk": "3.1.41",
-}

patches/CTPP2SourceLoader.hpp.patch

@@ -0,0 +1,6 @@
+34a35,39
+> #ifdef _WIN32
+> #include <Windows.h>
+> #else
+>  #include <unistd.h>
+> #endif

patches/ctpp2_compile_ctpp2c_static.patch

@@ -0,0 +1,37 @@
+diff -u ctpp2-2.8.3/CMakeLists.txt ctpp2-2.8.3-static/CMakeLists.txt
+--- ctpp2-2.8.3/CMakeLists.txt	2017-07-12 11:53:28.656535071 +0200
++++ ctpp2-2.8.3-static/CMakeLists.txt	2017-07-12 11:52:15.358692988 +0200
+@@ -464,7 +464,8 @@
+ 
+ # CTPP Compiler
+ ADD_EXECUTABLE(ctpp2c                       tests/CTPP2Compiler.cpp)
+-TARGET_LINK_LIBRARIES(ctpp2c                ctpp2)
++TARGET_LINK_LIBRARIES(ctpp2c                ctpp2-static)
++TARGET_LINK_LIBRARIES(ctpp2c                "-static")
+ 
+ # CTPP2 Interpreter
+ ADD_EXECUTABLE(ctpp2i                       tests/CTPP2Interpreter.cpp)
+@@ -794,7 +795,6 @@
+             DESTINATION .)
+ 
+     INSTALL(TARGETS ctpp2vm
+-                    ctpp2c
+                     ctpp2i
+                     ctpp2json
+             DESTINATION .)
+@@ -806,7 +806,6 @@
+ # Install Manpages
+ INSTALL(FILES
+               man/ctpp2-config.1
+-              man/ctpp2c.1
+               man/ctpp2i.1
+               man/ctpp2json.1
+               man/ctpp2vm.1
+@@ -904,7 +903,6 @@
+ 
+     # Install binaries
+     INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/ctpp2vm
+-                  ${CMAKE_CURRENT_BINARY_DIR}/ctpp2c
+                   ${CMAKE_CURRENT_BINARY_DIR}/ctpp2i
+                   ${CMAKE_CURRENT_BINARY_DIR}/ctpp2json
+                   ${CMAKE_CURRENT_BINARY_DIR}/ctpp2-config