Merge pull request #819 from kiwix/release-zim-tools-3.6.0

Relase zim-tools 3.6.0

.github/FUNDING.yml

@@ -1,12 +0,0 @@
-# These are supported funding model platforms
-
-github: kiwix # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
-patreon: # Replace with a single Patreon username
-open_collective: # Replace with a single Open Collective username
-ko_fi: # Replace with a single Ko-fi username
-tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
-community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
-liberapay: # Replace with a single Liberapay username
-issuehunt: # Replace with a single IssueHunt username
-otechie: # Replace with a single Otechie username
-custom: # https://kiwix.org/support-us/

.github/configs/aqt.ini

@@ -0,0 +1,46 @@
+[DEFAULTS]
+
+[aqt]
+concurrency: 4
+baseurl: https://qt.mirror.constant.com/
+7zcmd: 7z
+print_stacktrace_on_error: False
+always_keep_archives: False
+archive_download_location: .
+min_module_size: 41
+
+[requests]
+connection_timeout: 3.5
+response_timeout: 30
+max_retries_on_connection_error: 5
+retry_backoff: 0.1
+max_retries_on_checksum_error: 5
+max_retries_to_retrieve_hash: 5
+hash_algorithm: sha256
+INSECURE_NOT_FOR_PRODUCTION_ignore_hash: False
+
+[mirrors]
+trusted_mirrors:
+    https://download.qt.io
+    https://qt.mirror.constant.com/
+    https://ftp.fau.de/qtproject/
+blacklist:
+    http://mirrors.ocf.berkeley.edu
+    http://mirrors.tuna.tsinghua.edu.cn
+    http://mirrors.geekpie.club
+fallbacks:
+    https://qtproject.mirror.liquidtelecom.com/
+    https://mirrors.aliyun.com/qt/
+    https://mirrors.ustc.edu.cn/qtproject/
+    https://ftp.jaist.ac.jp/pub/qtproject/
+    https://ftp.yz.yamagata-u.ac.jp/pub/qtproject/
+    https://qt-mirror.dannhauer.de/
+    https://ftp.fau.de/qtproject/
+    https://mirror.netcologne.de/qtproject/
+    https://mirrors.dotsrc.org/qtproject/
+    https://www.nic.funet.fi/pub/mirrors/download.qt-project.org/
+    https://master.qt.io/
+    https://mirrors.ukfast.co.uk/sites/qt.io/
+    https://ftp2.nluug.nl/languages/qt/
+    https://ftp1.nluug.nl/languages/qt/
+    https://qt.mirror.constant.com/

.github/scripts/build_definition.py

@@ -3,7 +3,7 @@ import csv, io, re
 
 # Definition of what to build.
 # Array is read line by line.
-# Empty cells under (OS_NAME, PLATFORM_TARGET) mean "always match" (catch all, or `.*` regex)
+# Empty cells under (OS_NAME, COMPILE_CONFIG) mean "always match" (catch all, or `.*` regex)
 # Once a cell doesn't match, skip to the next line.
 # Once a line matches, other lines are not read, so put more specific combination first.
 # Lines composed of `-` , or `=`, or starting by `#` are ignored.
@@ -15,50 +15,56 @@ import csv, io, re
 # 'D' letter means we trigger the docker forkflow to build the docker image.
 # If a cell contains several letters, all are done.
 BUILD_DEF = """
-    | OS_NAME | PLATFORM_TARGET    | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | platform_name        |
+    | OS_NAME   | COMPILE_CONFIG     | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | platform_name           | dependency_name        |
-    =====================================================================================================================
+    ==============================================================================================================================================
-# Bionic is a special case as we need to compile libzim on old arch for python
+# manylinux is a special case as we need to compile libzim on old arch for python
-    | bionic  | native_mixed       | BP     |          |           |             |               | linux-x86_64-bionic  |
+    | manylinux | native_mixed       | BP     |          |           |             |               | linux-x86_64-manylinux  |                        |
-    | bionic  | aarch64_mixed      | BP     |          |           |             |               | linux-aarch64-bionic |
+    | manylinux | aarch64_mixed      | BP     |          |           |             |               | linux-aarch64-manylinux |                        |
-    --------------------------------------------------------------------------------------------------------------------
+    ----------------------------------------------------------------------------------------------------------------------------------------------
+# On Windows, we build only libzim for now. And only native_mixed as xapian doesn't compile as dll
+    | windows   | native_static      | Bd     | d        | BPd       | BPd         |               | win-x86_64              | win-x86_64-static      |
+    | windows   | native_dyn         | Bd     |          |           |             |               | win-x86_64              | win-x86_64-dyn         |
+    | windows   | native_mixed       | BPd    | d        |           |             | BPd           | win-x86_64              | win-x86_64-mixed       |
+    ----------------------------------------------------------------------------------------------------------------------------------------------
 # Osx builds, build binaries on native_dyn and native_static. On anyother things, build only the libraries
-    | macos   | native_dyn         | d      | d        | dB        | B           |               |                      |
+    | macos     | native_dyn         | d      | d        | dB        | B           |               |                         | macos-x86_64-dyn       |
-    | macos   | native_static      |        |          | BP        | BP          |               | macos-x86_64         |
+    | macos     | native_static      |        |          | BP        | BP          |               | macos-x86_64            |                        |
-    | macos   | native_mixed       | BP     | BP       |           |             |               | macos-x86_64         |
+    | macos     | native_mixed       | BP     | BP       |           |             |               | macos-x86_64            |                        |
-    | macos   | iOS_arm64          | dB     | B        |           |             |               |                      |
+    | macos     | iOS_arm64          | dB     | dB       |           |             |               |                         | ios-arm64-dyn          |
-    | macos   | iOS_x86_64         | dB     | B        |           |             |               |                      |
+    | macos     | iOSSimulator_x86_64| dB     | dB       |           |             |               |                         | ios-x86_64-dyn         |
-    | macos   | macOS_arm64_static |        |          | BP        | BP          |               | macos-arm64          |
+    | macos     | iOSSimulator_arm64 | B      | B        |           |             |               |                         |                        |
-    | macos   | macOS_arm64_mixed  | BP     | BP       |           |             |               | macos-arm64          |
+    | macos     | macOS_arm64_static |        |          | BP        | BP          |               | macos-arm64             |                        |
-    | macos   | macOS_x86_64       | B      | B        |           |             |               |                      |
+    | macos     | macOS_arm64_mixed  | dBP    | dBP      | d         |             |               | macos-arm64             | macos-aarch64-dyn      |
-    ----------------------------------------------------------------------------------------------
+    | macos     | macOS_x86_64       | B      | B        |           |             |               |                         |                        |
-    |         | flatpak            |        |          |           |             | BP            |                      |
+    | macos     | apple_all_static   |        | BP       |           |             |               | xcframework             |                        |
-    |         | native_static      | d      | d        | dBPSD     | dBPSD       |               | linux-x86_64         |
+    ----------------------------------------------------------------------------------------------------------------------------------------------
-    |         | native_mixed       | BPS    | BPS      |           |             |               | linux-x86_64         |
+    | focal     | flatpak            |        |          |           |             | BP            |                         |                        |
-    |         | native_dyn         | d      | d        | dB        | dB          | dBPS          |                      |
+    | focal     | native_static      | d      | d        | dBPSD     | dBPSD       |               | linux-x86_64            | linux-x86_64-static    |
+    | focal     | native_mixed       | BPS    | BPS      |           |             |               | linux-x86_64            |                        |
+    | focal     | native_dyn         | d      | d        | dB        | dB          |               |                         | linux-x86_64-dyn       |
+    | jammy     | native_dyn         |        |          |           |             | dBPS          |                         | linux-x86_64-dyn       |
     # libzim CI is building alpine_dyn but not us
-    |         | android_arm        | dBP    | dBP      |           |             |               | android-arm          |
+    | focal     | android_arm        | dBP    | dBP      |           |             |               | android-arm             | android-arm            |
-    |         | android_arm64      | dBP    | dBP      |           |             |               | android-arm64        |
+    | focal     | android_arm64      | dBP    | dBP      |           |             |               | android-arm64           | android-arm64          |
-    |         | android_x86        | BP     | BP       |           |             |               | android-x86          |
+    | focal     | android_x86        | BP     | BP       |           |             |               | android-x86             |                        |
-    |         | android_x86_64     | BP     | BP       |           |             |               | android-x86_64       |
+    | focal     | android_x86_64     | BP     | BP       |           |             |               | android-x86_64          |                        |
-    |         | armv6_static       |        |          | BP        | BP          |               | linux-armv6          |
+    | focal     | armv6_static       |        |          | BP        | BP          |               | linux-armv6             |                        |
-    |         | armv6_mixed        | BP     |          |           |             |               | linux-armv6          |
+    | focal     | armv6_mixed        | BP     |          |           |             |               | linux-armv6             |                        |
-    |         | armv6_dyn          |        |          | B         | B           |               |                      |
+    | focal     | armv6_dyn          |        |          | B         | B           |               |                         |                        |
-    |         | armv8_static       |        |          | BP        | BP          |               | linux-armv8          |
+    | focal     | armv8_static       |        |          | BP        | BP          |               | linux-armv8             |                        |
-    |         | armv8_mixed        | BP     |          |           |             |               | linux-armv8          |
+    | focal     | armv8_mixed        | BP     |          |           |             |               | linux-armv8             |                        |
-    |         | armv8_dyn          |        |          | B         | B           |               |                      |
+    | focal     | armv8_dyn          |        |          | B         | B           |               |                         |                        |
-    |         | aarch64_static     |        |          | BP        | BP          |               | linux-aarch64        |
+    | focal     | aarch64_static     |        |          | BP        | BP          |               | linux-aarch64           |                        |
-    |         | aarch64_mixed      | BP     |          |           |             |               | linux-aarch64        |
+    | focal     | aarch64_mixed      | BP     |          |           |             |               | linux-aarch64           |                        |
-    |         | aarch64_dyn        | d      |          | B         | B           |               |                      |
+    | focal     | aarch64_dyn        | d      |          | B         | B           |               |                         | linux-aarch64-dyn      |
-    |         | aarch64_musl_static|        |          | BP        | BP          |               | linux-aarch64-musl   |
+    | focal     | aarch64_musl_static|        |          | BP        | BP          |               | linux-aarch64-musl      |                        |
-    |         | aarch64_musl_mixed | BP     |          |           |             |               | linux-aarch64-musl   |
+    | focal     | aarch64_musl_mixed | BP     |          |           |             |               | linux-aarch64-musl      |                        |
-    |         | aarch64_musl_dyn   | d      |          | B         | B           |               |                      |
+    | focal     | aarch64_musl_dyn   | d      |          | B         | B           |               |                         | linux-aarch64-musl-dyn |
-    |         | x86-64_musl_static |        |          | BP        | BP          |               | linux-x86_64-musl    |
+    | focal     | x86-64_musl_static |        |          | BP        | BP          |               | linux-x86_64-musl       |                        |
-    |         | x86-64_musl_mixed  | BP     |          |           |             |               | linux-x86_64-musl    |
+    | focal     | x86-64_musl_mixed  | BP     |          |           |             |               | linux-x86_64-musl       |                        |
-    |         | win32_static       | d      | dB       | dBP       | dBP         |               | win-i686             |
+    | focal     | i586_static        |        |          | BP        | BP          |               | linux-i586              |                        |
-    |         | win32_dyn          | d      | dB       | dB        | dB          |               |                      |
+    | focal     | i586_dyn           |        |          | B         | B           |               |                         |                        |
-    |         | i586_static        |        |          | BP        | BP          |               | linux-i586           |
+    | focal     | wasm               | dBP    |          |           |             |               | wasm-emscripten         | wasm                   |
-    |         | i586_dyn           |        |          | B         | B           |               |                      |
-    |         | wasm               | dBP    |          |           |             |               | wasm-emscripten      |
 """
 
 
@@ -90,10 +96,10 @@ def selector_match(selector, value):
 
 class Context(NamedTuple):
     OS_NAME: str
-    PLATFORM_TARGET: str
+    COMPILE_CONFIG: str
 
     def match(self, row):
-        for key in ["OS_NAME", "PLATFORM_TARGET"]:
+        for key in ["OS_NAME", "COMPILE_CONFIG"]:
             context_value = getattr(self, key)
             selector = row[key]
             if not selector_match(selector, context_value):
@@ -107,10 +113,11 @@ SOURCE_PUBLISH = "S"
 DEPS = "d"
 DOCKER = "D"
 
-def select_build_targets(criteria):
-    from common import PLATFORM_TARGET, OS_NAME
 
-    context = Context(PLATFORM_TARGET=PLATFORM_TARGET, OS_NAME=OS_NAME)
+def select_build_targets(criteria):
+    from common import COMPILE_CONFIG, OS_NAME
+
+    context = Context(COMPILE_CONFIG=COMPILE_CONFIG, OS_NAME=OS_NAME)
 
     reader = csv.DictReader(strip_array(BUILD_DEF), dialect=TableDialect())
     for row in reader:
@@ -129,17 +136,26 @@ def select_build_targets(criteria):
             print(build_order)
             return build_order
 
-    raise "No definition match with current context."
+    raise ValueError("No definition match with current context.")
 
-def get_platform_name():
-    from common import PLATFORM_TARGET, OS_NAME
 
-    context = Context(PLATFORM_TARGET=PLATFORM_TARGET, OS_NAME=OS_NAME)
+def get_column_value(column_name):
+    from common import COMPILE_CONFIG, OS_NAME
+
+    context = Context(COMPILE_CONFIG=COMPILE_CONFIG, OS_NAME=OS_NAME)
 
     reader = csv.DictReader(strip_array(BUILD_DEF), dialect=TableDialect())
     for row in reader:
         if context.match(row):
-            name = row["platform_name"]
+            name = row[column_name]
             return name or None
 
-    raise "No definition match with current context."
+    raise ValueError("No definition match with current context.")
+
+
+def get_platform_name():
+    return get_column_value("platform_name")
+
+
+def get_dependency_archive_name():
+    return get_column_value("dependency_name")

.github/scripts/build_projects.py

@@ -8,16 +8,16 @@ from common import (
     fix_macos_rpath,
     upload_archive,
     OS_NAME,
-    PLATFORM_TARGET,
+    COMPILE_CONFIG,
     DEV_BRANCH,
 )
 
 for target in select_build_targets(BUILD):
-    run_kiwix_build(target, platform=PLATFORM_TARGET)
+    run_kiwix_build(target, config=COMPILE_CONFIG)
     if target == "kiwix-desktop":
         archive = create_desktop_image(make_release=False)
     else:
-        if PLATFORM_TARGET == "native_mixed" and OS_NAME == "macos":
+        if COMPILE_CONFIG == "native_mixed" and OS_NAME == "macos":
             fix_macos_rpath(target)
         archive = make_archive(target, make_release=False)
     if archive and DEV_BRANCH:

.github/scripts/build_release_nightly.py

@@ -13,7 +13,7 @@ from common import (
     fix_macos_rpath,
     BASE_DIR,
     OS_NAME,
-    PLATFORM_TARGET,
+    COMPILE_CONFIG,
     MAKE_RELEASE,
     notarize_macos_build,
 )
@@ -21,23 +21,22 @@ from common import (
 from build_definition import select_build_targets, BUILD, PUBLISH, SOURCE_PUBLISH
 
 
+def release_filter(project):
+    return release_versions.get(project) is not None
 
 
 # Filter what to build if we are doing a release.
+TARGETS = select_build_targets(PUBLISH)
+
 if MAKE_RELEASE:
-    TARGETS = select_build_targets(PUBLISH)
-    def release_filter(project):
-        return release_versions.get(project) is not None
     TARGETS = tuple(filter(release_filter, TARGETS))
-else:
-    TARGETS = select_build_targets(BUILD)
 
 for target in TARGETS:
-    run_kiwix_build(target, platform=PLATFORM_TARGET, make_release=MAKE_RELEASE)
+    run_kiwix_build(target, config=COMPILE_CONFIG, make_release=MAKE_RELEASE)
     if target == "kiwix-desktop":
         archive = create_desktop_image(make_release=MAKE_RELEASE)
     else:
-        if OS_NAME == "macos" and PLATFORM_TARGET.endswith("_mixed"):
+        if OS_NAME == "macos" and COMPILE_CONFIG.endswith("_mixed"):
             fix_macos_rpath(target)
             notarize_macos_build(target)
         archive = make_archive(target, make_release=MAKE_RELEASE)
@@ -57,13 +56,11 @@ if MAKE_RELEASE:
         if target not in source_published_targets:
             continue
         run_kiwix_build(
-            target, platform=PLATFORM_TARGET, make_release=MAKE_RELEASE, make_dist=True
+            target, config=COMPILE_CONFIG, make_release=MAKE_RELEASE, make_dist=True
         )
         full_target_name = "{}-{}".format(target, main_project_versions[target])
         if target == "kiwix-desktop":
-            archive = (
+            archive = BASE_DIR / full_target_name / "{}.tar.gz".format(full_target_name)
-                BASE_DIR / full_target_name / "{}.tar.gz".format(full_target_name)
-            )
         else:
             archive = (
                 BASE_DIR
@@ -74,5 +71,5 @@ if MAKE_RELEASE:
         upload_archive(archive, target, make_release=MAKE_RELEASE)
 
     # Publish flathub
-    if PLATFORM_TARGET == "flatpak" and "kiwix-desktop" in TARGETS:
+    if COMPILE_CONFIG == "flatpak" and "kiwix-desktop" in TARGETS:
         update_flathub_git()

.github/scripts/common.py

@@ -1,17 +1,19 @@
 import os
 from os import environ as _environ
-from pathlib import Path
+from pathlib import Path, PurePosixPath
 from datetime import date
 import tarfile
 import zipfile
 import subprocess
 import re
 import shutil
+import platform
 
 import requests
 
-from build_definition import get_platform_name
+from build_definition import get_platform_name, get_dependency_archive_name
 
+from kiwixbuild.dependencies.apple_xcframework import AppleXCFramework
 from kiwixbuild.versions import (
     main_project_versions,
     release_versions,
@@ -19,22 +21,40 @@ from kiwixbuild.versions import (
 )
 
 
+def get_build_dir(config) -> Path:
+    command = ["kiwix-build"]
+    command.extend(["--config", config])
+    command.append("--get-build-dir")
+    command.append("--use-target-arch-name")
+    return Path(
+        subprocess.run(command, cwd=str(HOME), check=True, stdout=subprocess.PIPE)
+        .stdout.strip()
+        .decode("utf8")
+    )
 
-PLATFORM_TARGET = _environ["PLATFORM_TARGET"]
+
+COMPILE_CONFIG = _environ["COMPILE_CONFIG"]
 OS_NAME = _environ["OS_NAME"]
 HOME = Path(os.path.expanduser("~"))
 
-BASE_DIR = HOME / "BUILD_{}".format(PLATFORM_TARGET)
+BASE_DIR = get_build_dir(COMPILE_CONFIG)
 SOURCE_DIR = HOME / "SOURCE"
 ARCHIVE_DIR = HOME / "ARCHIVE"
 TOOLCHAIN_DIR = BASE_DIR / "TOOLCHAINS"
 INSTALL_DIR = BASE_DIR / "INSTALL"
-TMP_DIR = Path("/tmp")
+default_tmp_dir = os.getenv("TEMP") if platform.system() == "Windows" else "/tmp"
-KBUILD_SOURCE_DIR = HOME / "kiwix-build"
+TMP_DIR = Path(os.getenv("TMP_DIR", default_tmp_dir))
+if platform.system() == "Windows":
+    KBUILD_SOURCE_DIR = Path(_environ["GITHUB_WORKSPACE"])
+    BIN_EXT = ".exe"
+else:
+    KBUILD_SOURCE_DIR = HOME / "kiwix-build"
+    BIN_EXT = ""
+
 
 _ref = _environ.get("GITHUB_REF", "").split("/")[-1]
 MAKE_RELEASE = re.fullmatch(r"r_[0-9]+", _ref) is not None
-MAKE_RELEASE = MAKE_RELEASE and (_environ.get('GITHUB_EVENT_NAME') != 'schedule')
+MAKE_RELEASE = MAKE_RELEASE and (_environ.get("GITHUB_EVENT_NAME") != "schedule")
 
 if not MAKE_RELEASE and _ref != "main":
     DEV_BRANCH = _ref
@@ -44,19 +64,25 @@ else:
 FLATPAK_HTTP_GIT_REMOTE = "https://github.com/flathub/org.kiwix.desktop.git"
 FLATPAK_GIT_REMOTE = "git@github.com:flathub/org.kiwix.desktop.git"
 
-BIN_EXT = ".exe" if PLATFORM_TARGET.startswith("win32_") else ""
 
 def major_version(version: str) -> str:
     return version.split(".")[0]
 
+
+# Depending of base distribution, libraries are in "lib64" (redhat base) or "lib/<arch>" (debian base).
+# On top of that, when cross-compiling, libraries are always put in `lib/<arch>`.
+# As we use this as glob regex to select which files to add to archive, this is not a problem to have both.
+def lib_prefix(file):
+    yield "lib64/" + file
+    yield "lib/*/" + file
+
+
 # We have build everything. Now create archives for public deployement.
 EXPORT_FILES = {
     "kiwix-tools": (
         INSTALL_DIR / "bin",
-        [
+        [f + BIN_EXT for f in ("kiwix-manage", "kiwix-search", "kiwix-serve")]
-            f + BIN_EXT
+        + ["icu*.dll"],
-            for f in ("kiwix-manage", "kiwix-search", "kiwix-serve")
-        ],
     ),
     "zim-tools": (
         INSTALL_DIR / "bin",
@@ -71,37 +97,52 @@ EXPORT_FILES = {
                 "zimpatch",
                 "zimsplit",
                 "zimwriterfs",
-                "zimrecreate"
+                "zimrecreate",
             )
-        ],
+        ]
+        + ["icu*.dll"],
     ),
     "libzim": (
         INSTALL_DIR,
         (
+            ## Linux
             # We need to package all dependencies (`*.a`) on wasm
-            "lib/*/libzim.a" if PLATFORM_TARGET != "wasm" else "lib/*.a",
+            *lib_prefix("libzim.a" if COMPILE_CONFIG != "wasm" else "*.a"),
-            "lib/*/libzim.so",
+            *lib_prefix("libzim.so"),
-            "lib/*/libzim.so.{version}".format(
+            *lib_prefix(
-                version=main_project_versions["libzim"]
+                "libzim.so.{version}".format(version=main_project_versions["libzim"])
             ),
-            "lib/*/libzim.so.{version}".format(
+            *lib_prefix(
-                version=major_version(main_project_versions["libzim"])
+                "libzim.so.{version}".format(
+                    version=major_version(main_project_versions["libzim"])
+                )
             ),
+            ## MacOS
             "lib/libzim.{}.dylib".format(
                 major_version(main_project_versions["libzim"])
             ),
             "lib/libzim.dylib",
             "lib/*/libzim.pc",
+            ## Windows
+            "bin/zim-{version}.dll".format(
+                version=major_version(main_project_versions["libzim"])
+            ),
+            "bin/icu*.dll",
+            "bin/zim-{version}.pdb".format(
+                version=major_version(main_project_versions["libzim"])
+            ),
+            "lib/zim.lib",
+            ## Includes and others
             "include/zim/**/*.h",
             "share/icu/{}/icudt{}l.dat".format(
-                base_deps_versions["icu4c"],
+                base_deps_versions["icu4c"], major_version(base_deps_versions["icu4c"])
-                major_version(base_deps_versions["icu4c"])
+            ),
-            )
         ),
     ),
     "libkiwix": (
         INSTALL_DIR,
         (
+            "lib/CoreKiwix.xcframework/",
             "lib/*/libkiwix.so",
             "lib/*/libkiwix.so.{version}".format(
                 version=main_project_versions["libkiwix"]
@@ -112,19 +153,27 @@ EXPORT_FILES = {
             "lib/libkiwix.{}.dylib".format(
                 major_version(main_project_versions["libkiwix"])
             ),
+            "bin/kiwix-{version}.dll".format(
+                version=major_version(main_project_versions["libkiwix"])
+            ),
+            "bin/icu*.dll",
+            "bin/kiwix-{version}.pdb".format(
+                version=major_version(main_project_versions["libkiwix"])
+            ),
+            "lib/kiwix.lib",
             "lib/libkiwix.dylib",
             "lib/*/libkiwix.pc",
             "include/kiwix/**/*.h",
             "share/icu/{}/icudt{}l.dat".format(
-                base_deps_versions["icu4c"],
+                base_deps_versions["icu4c"], major_version(base_deps_versions["icu4c"])
-                major_version(base_deps_versions["icu4c"])
+            ),
-            )
         ),
     ),
 }
 
 DATE = date.today().isoformat()
 
+
 def print_message(message, *args, **kwargs):
     message = message.format(*args, **kwargs)
     message = "{0} {1} {0}".format("-" * 3, message)
@@ -134,23 +183,26 @@ def print_message(message, *args, **kwargs):
 MANIFEST_TEMPLATE = """{archive_name}
 ***************************
 
-Dependencies archive for {target} on platform {platform}
+Dependencies archive for {target} using config {config}
 Generated at {date}
 """
 
 
-def write_manifest(manifest_file, archive_name, target, platform):
+def write_manifest(manifest_file, archive_name, target, config):
     with manifest_file.open(mode="w") as f:
         f.write(
             MANIFEST_TEMPLATE.format(
-                archive_name=archive_name, target=target, platform=platform, date=DATE,
+                archive_name=archive_name,
+                target=target,
+                config=config,
+                date=DATE,
             )
         )
 
 
 def run_kiwix_build(
     target,
-    platform,
+    config,
     build_deps_only=False,
     target_only=False,
     make_release=False,
@@ -161,7 +213,8 @@ def run_kiwix_build(
     command.append("--hide-progress")
     command.append("--fast-clone")
     command.append("--assume-packages-installed")
-    command.extend(["--target-platform", platform])
+    command.append("--use-target-arch-name")
+    command.extend(["--config", config])
     if build_deps_only:
         command.append("--build-deps-only")
     if target_only:
@@ -183,54 +236,123 @@ def run_kiwix_build(
     print_message("Build ended")
 
 
-def upload(file_to_upload, host, dest_path):
+try:
-    if not file_to_upload.exists():
+    import paramiko
-        print_message("No {} to upload!", file_to_upload)
-        return
 
-    if ":" in host:
+    def upload(file_to_upload, host, dest_path):
-        host, port = host.split(":", 1)
+        if not file_to_upload.exists():
-    else:
+            print_message("No {} to upload!", file_to_upload)
-        port = "22"
+            return
 
-    # sending SFTP mkdir command to the sftp interactive mode and not batch (-b) mode
+        if ":" in host:
-    # as the latter would exit on any mkdir error while it is most likely
+            host, port = host.split(":", 1)
-    # the first parts of the destination is already present and thus can't be created
+        else:
-    sftp_commands = "\n".join(
+            port = "22"
-        [
+        if "@" in host:
-            f"mkdir {part}"
+            user, host = host.split("@", 1)
-            for part in list(reversed(Path(dest_path).parents)) + [dest_path]
+        else:
+            user = None
+
+        from contextlib import contextmanager
+
+        @contextmanager
+        def get_client():
+            client = paramiko.client.SSHClient()
+            client.set_missing_host_key_policy(paramiko.client.WarningPolicy)
+            print_message(f"Connect to {host}:{port}")
+            client.connect(
+                host,
+                port=port,
+                username=user,
+                key_filename=_environ.get("SSH_KEY"),
+                look_for_keys=False,
+                compress=True,
+            )
+            try:
+                yield client
+            finally:
+                client.close()
+
+        @contextmanager
+        def get_sftp():
+            with get_client() as client:
+                sftp = client.open_sftp()
+                try:
+                    yield sftp
+                finally:
+                    sftp.close()
+
+        dest_path = PurePosixPath(dest_path)
+        remote_file = dest_path.joinpath(file_to_upload.name)
+
+        with get_sftp() as sftp:
+            for part in list(reversed(dest_path.parents)) + [dest_path]:
+                part = str(part)
+                try:
+                    sftp.stat(part)
+                except FileNotFoundError:
+                    sftp.mkdir(part)
+
+            print_message(f"Sending archive {file_to_upload} to {remote_file}")
+            sftp.put(str(file_to_upload), str(remote_file), confirm=True)
+
+except ModuleNotFoundError:
+    # On old system (bionic) paramiko is really complex to install
+    # Keep the old implementaion on sush system.
+
+    def upload(file_to_upload, host, dest_path):
+        if not file_to_upload.exists():
+            print_message("No {} to upload!", file_to_upload)
+            return
+
+        if ":" in host:
+            host, port = host.split(":", 1)
+        else:
+            port = "22"
+
+        # Using SFTP to create the directory hierarchy because we can not
+        # use SSH (no shell for this user); and then scp to upload the file.
+        #
+        # Sending SFTP mkdir command to the SFTP interactive mode and not batch (-b) mode
+        # as the latter would exit on any mkdir error while it is most likely
+        # the first parts of the destination is already present and thus can't be created
+        sftp_commands = "\n".join(
+            [
+                f"mkdir {part}"
+                for part in list(reversed(Path(dest_path).parents)) + [dest_path]
+            ]
+        )
+        command = [
+            "sftp",
+            "-c",
+            "aes128-ctr",
+            "-i",
+            _environ.get("SSH_KEY"),
+            "-P",
+            port,
+            "-o",
+            "StrictHostKeyChecking=no",
+            host,
         ]
-    )
+        print_message("Creating dest path {}", dest_path)
-    command = [
+        subprocess.run(command, input=sftp_commands.encode("utf-8"), check=True)
-        "sftp",
-        "-i",
-        _environ.get("SSH_KEY"),
-        "-P",
-        port,
-        "-o",
-        "StrictHostKeyChecking=no",
-        host,
-    ]
-    print_message("Creating dest path {}", dest_path)
-    subprocess.run(command, input=sftp_commands.encode("utf-8"), check=True)
 
-    command = [
+        command = [
-        "scp",
+            "scp",
-        "-c",
+            "-c",
-        "aes128-ctr",
+            "aes128-ctr",
-        "-rp",
+            "-rp",
-        "-P",
+            "-P",
-        port,
+            port,
-        "-i",
+            "-i",
-        _environ.get("SSH_KEY"),
+            _environ.get("SSH_KEY"),
-        "-o",
+            "-o",
-        "StrictHostKeyChecking=no",
+            "StrictHostKeyChecking=no",
-        str(file_to_upload),
+            str(file_to_upload),
-        "{}:{}".format(host, dest_path),
+            "{}:{}".format(host, dest_path),
-    ]
+        ]
-    print_message("Sending archive with command {}", command)
+        print_message("Sending archive with command {}", command)
-    subprocess.check_call(command)
+        subprocess.check_call(command)
 
 
 def upload_archive(archive, project, make_release, dev_branch=None):
@@ -238,7 +360,7 @@ def upload_archive(archive, project, make_release, dev_branch=None):
         print_message("No archive {} to upload!", archive)
         return
 
-    if project.startswith("kiwix-") or project in ['libkiwix']:
+    if project.startswith("kiwix-") or project in ["libkiwix"]:
         host = "ci@master.download.kiwix.org:30022"
         dest_path = "/data/download/"
     else:
@@ -262,29 +384,37 @@ def upload_archive(archive, project, make_release, dev_branch=None):
 
 # This remove "share/doc" and "share/man" from the thing to copy in the deps archive
 def filter_install_dir(path):
-    for dir in path.glob('*'):
+    for dir in path.glob("*"):
-        if dir.name not in ['share']:
+        if dir.name not in ["share"]:
             yield dir
         else:
-            for sub_dir in dir.glob('*'):
+            for sub_dir in dir.glob("*"):
-                if sub_dir.name not in ['doc', 'man']:
+                if sub_dir.name not in ["doc", "man"]:
                     yield sub_dir
 
-# Full: True if we are creating a full archive to be used as cache by kiwix-build (base_deps2_{os}_{platform}_{base_deps_version}.tar.xz)
+
-# Full: False if we are creating a archive to be used as pre-cached dependencies for project's CI (deps2_{os}_{platform}_{target}.tar.xz)
+# Full: True if we are creating a full archive to be used as cache by kiwix-build (base_deps_{os}_{config}_{base_deps_version}.tar.gz)
+# Full: False if we are creating a archive to be used as pre-cached dependencies for project's CI (deps_{config}_{target}.tar.gz)
 def make_deps_archive(target=None, name=None, full=False):
-    archive_name = name or "deps2_{}_{}_{}.tar.xz".format(
+    archive_name = name or "deps_{}_{}.tar.gz".format(
-        OS_NAME, PLATFORM_TARGET, target
+        get_dependency_archive_name(), target
     )
     print_message("Create archive {}.", archive_name)
     files_to_archive = list(filter_install_dir(INSTALL_DIR))
     files_to_archive += HOME.glob("BUILD_*/LOGS")
-    if PLATFORM_TARGET.endswith("_mixed"):
+    if COMPILE_CONFIG == "apple_all_static":
-        static_platform = PLATFORM_TARGET.replace("_mixed", "_static")
+        for subconfig in AppleXCFramework.subConfigNames:
-        files_to_archive += filter_install_dir(HOME / ("BUILD_" + static_platform) / "INSTALL")
+            base_dir = get_build_dir(subconfig)
-    if PLATFORM_TARGET.startswith("android_"):
+            files_to_archive += filter_install_dir(base_dir / "INSTALL")
+            if (base_dir / "meson_cross_file.txt").exists():
+                files_to_archive.append(base_dir / "meson_cross_file.txt")
+
+    if COMPILE_CONFIG.endswith("_mixed"):
+        static_config = COMPILE_CONFIG.replace("_mixed", "_static")
+        files_to_archive += filter_install_dir(get_build_dir(static_config) / "INSTALL")
+    if COMPILE_CONFIG.startswith("android_"):
         files_to_archive += filter_install_dir(HOME / "BUILD_neutral" / "INSTALL")
-        base_dir = HOME / "BUILD_{}".format(PLATFORM_TARGET)
+        base_dir = get_build_dir(COMPILE_CONFIG)
         if (base_dir / "meson_cross_file.txt").exists():
             files_to_archive.append(base_dir / "meson_cross_file.txt")
     # Copy any toolchain
@@ -294,7 +424,7 @@ def make_deps_archive(target=None, name=None, full=False):
         files_to_archive.append(BASE_DIR / "meson_cross_file.txt")
 
     manifest_file = BASE_DIR / "manifest.txt"
-    write_manifest(manifest_file, archive_name, target, PLATFORM_TARGET)
+    write_manifest(manifest_file, archive_name, target, COMPILE_CONFIG)
     files_to_archive.append(manifest_file)
 
     relative_path = HOME
@@ -302,18 +432,20 @@ def make_deps_archive(target=None, name=None, full=False):
         files_to_archive += ARCHIVE_DIR.glob(".*_ok")
         files_to_archive += BASE_DIR.glob("*/.*_ok")
         # Add also static build for mixed target
-        if PLATFORM_TARGET.endswith("_mixed"):
+        if COMPILE_CONFIG.endswith("_mixed"):
-            static_platform = PLATFORM_TARGET.replace("_mixed", "_static")
+            static_config = COMPILE_CONFIG.replace("_mixed", "_static")
-            files_to_archive += (HOME / ("BUILD_" + static_platform)).glob("*/.*_ok")
+            files_to_archive += get_build_dir(static_config).glob("*/.*_ok")
         # Native dyn and static is needed for potential cross compilation that use native tools (icu)
-        files_to_archive += (HOME / "BUILD_native_dyn").glob("*/.*_ok")
+        files_to_archive += get_build_dir("native_dyn").glob("*/.*_ok")
-        files_to_archive += (HOME / "BUILD_native_static").glob("*/.*_ok")
+        files_to_archive += get_build_dir("native_static").glob("*/.*_ok")
-        files_to_archive += HOME.glob("BUILD_android*/**/.*_ok")
+        files_to_archive += HOME.glob("BUILD_*android*/**/.*_ok")
+        files_to_archive += HOME.glob("BUILD_*apple-macos*/**/.*_ok")
+        files_to_archive += HOME.glob("BUILD_*apple-ios*/**/.*_ok")
         files_to_archive += SOURCE_DIR.glob("*/.*_ok")
         files_to_archive += SOURCE_DIR.glob("zim-testing-suite-*/*")
 
     archive_file = TMP_DIR / archive_name
-    with tarfile.open(str(archive_file), "w:xz") as tar:
+    with tarfile.open(str(archive_file), "w:gz") as tar:
         for name in set(files_to_archive):
             print(".{}".format(name), flush=True)
             tar.add(str(name), arcname=str(name.relative_to(relative_path)))
@@ -329,6 +461,25 @@ def get_postfix(project):
     return postfix
 
 
+def sign_binary(path):
+    # We assume here that signtool and certificate are properly configured.
+    # Env var `SIGNTOOL_THUMBPRINT` must contain thumbprint of the certificate to use.
+    command = [
+        os.getenv("SIGNTOOL_PATH", "signtool.exe"),
+        "sign",
+        "/fd",
+        "sha256",
+        "/tr",
+        "http://ts.ssl.com",
+        "/td",
+        "sha256",
+        "/sha1",
+        os.environ["SIGNTOOL_THUMBPRINT"],
+        str(path),
+    ]
+    subprocess.run(command, check=True)
+
+
 def make_archive(project, make_release):
     platform_name = get_platform_name()
     if not platform_name:
@@ -350,7 +501,13 @@ def make_archive(project, make_release):
     files_to_archive = []
     for export_file in export_files:
         files_to_archive.extend(base_dir.glob(export_file))
-    if platform_name == "win-i686":
+
+    if make_release and platform.system() == "Windows":
+        for file in files_to_archive:
+            if str(file).endswith(".exe"):
+                sign_binary(file)
+
+    if platform_name == "win-i686" or platform.system() == "Windows":
         open_archive = lambda a: zipfile.ZipFile(
             str(a), "w", compression=zipfile.ZIP_DEFLATED
         )
@@ -380,15 +537,31 @@ def create_desktop_image(make_release):
         postfix = DATE
         src_dir = SOURCE_DIR / "kiwix-desktop"
 
-    if PLATFORM_TARGET == "flatpak":
+    if COMPILE_CONFIG == "flatpak":
         build_path = BASE_DIR / "org.kiwix.desktop.flatpak"
         app_name = "org.kiwix.desktop.{}.flatpak".format(postfix)
         print_message("archive is {}", build_path)
+    elif platform.system() == "Windows":
+        archive_basename = "kiwix-desktop_windows_x64_{}".format(postfix)
+        working_dir = INSTALL_DIR / archive_basename
+        build_path = Path(str(working_dir) + ".zip")
+        app_name = build_path.name
+        command = [
+            "python",
+            KBUILD_SOURCE_DIR / "scripts" / "package_kiwix-desktop_windows.py",
+            str(INSTALL_DIR),
+            str(working_dir),
+            str(build_path),
+        ]
+        if make_release:
+            command += ["-s"]
+        print_message("Package archive of kiwix-desktop")
+        subprocess.check_call(command, cwd=str(HOME))
     else:
         build_path = HOME / "Kiwix-{}-x86_64.AppImage".format(postfix)
         app_name = "kiwix-desktop_x86_64_{}.appimage".format(postfix)
         command = [
-            "kiwix-build/scripts/create_kiwix-desktop_appImage.sh",
+            KBUILD_SOURCE_DIR / "scripts" / "create_kiwix-desktop_appImage.sh",
             str(INSTALL_DIR),
             str(src_dir),
             str(HOME / "AppDir"),
@@ -422,7 +595,7 @@ def update_flathub_git():
     call(command)
     shutil.copy(str(BASE_DIR / "org.kiwix.desktop.json"), str(git_repo_dir))
     patch_dir = KBUILD_SOURCE_DIR / "kiwixbuild" / "patches"
-    for dep in ["pugixml"]:
+    for dep in ["pugixml", "libmicrohttpd"]:
         for f in patch_dir.glob("{}_*.patch".format(dep)):
             shutil.copy(str(f), str(git_repo_dir / "patches"))
     command = ["git", "add", "-A", "."]
@@ -448,7 +621,6 @@ def update_flathub_git():
 
 
 def fix_macos_rpath(project):
-
     base_dir, export_files = EXPORT_FILES[project]
     for file in filter(lambda f: f.endswith(".dylib"), export_files):
         lib = base_dir / file
@@ -467,22 +639,35 @@ def trigger_workflow(repo, workflow="docker.yml", ref="main", inputs=None):
     ref: branch or tag name
     inputs: dict of inputs to pass to the workflow"""
     print_message(
-        "triggering workflow `{workflow}` on {repo}@{ref} "
+        "triggering workflow `{workflow}` on {repo}@{ref} " "with inputs={inputs}",
-        "with inputs={inputs}", workflow=workflow, repo=repo, ref=ref, inputs=inputs)
+        workflow=workflow,
+        repo=repo,
+        ref=ref,
+        inputs=inputs,
+    )
 
     url = "{base_url}/repos/{repo}/actions/workflows/{workflow}/dispatches".format(
         base_url=os.getenv("GITHUB_API_URL", "https://api.github.com"),
-        repo=repo, workflow=workflow)
+        repo=repo,
+        workflow=workflow,
+    )
 
-    resp = requests.post(url, headers={
+    resp = requests.post(
+        url,
+        headers={
             "Content-Type": "application/json",
-            "Authorization": "token {token}".format(
+            "Authorization": "token {token}".format(token=os.getenv("GITHUB_PAT", "")),
-                token=os.getenv('GITHUB_PAT', '')),
             "Accept": "application/vnd.github.v3+json",
-        }, json={"ref": ref, "inputs": inputs}, timeout=5)
+        },
+        json={"ref": ref, "inputs": inputs},
+        timeout=5,
+    )
     if resp.status_code != 204:
-        raise ValueError("Unexpected HTTP {code}: {reason}".format(
+        raise ValueError(
-                         code=resp.status_code, reason=resp.reason))
+            "Unexpected HTTP {code}: {reason}".format(
+                code=resp.status_code, reason=resp.reason
+            )
+        )
 
 
 def trigger_docker_publish(target):
@@ -490,13 +675,14 @@ def trigger_docker_publish(target):
         return
 
     version = get_postfix(target)
-    repo = {
+    repo = {"zim-tools": "openzim/zim-tools", "kiwix-tools": "kiwix/kiwix-tools"}.get(
-        "zim-tools": "openzim/zim-tools",
+        target
-        "kiwix-tools": "kiwix/kiwix-tools"}.get(target)
+    )
 
     try:
-        trigger_workflow(repo, workflow="docker.yml", ref="main",
+        trigger_workflow(
-                         inputs={"version": version})
+            repo, workflow="docker.yml", ref="main", inputs={"version": version}
+        )
         print_message("triggered docker workflow on {repo}", repo=repo)
     except Exception as exc:
         print_message("Error triggering workflow: {exc}", exc=exc)
@@ -504,39 +690,52 @@ def trigger_docker_publish(target):
 
 
 def notarize_macos_build(project):
-    """ sign and notarize files for macOS
+    """sign and notarize files for macOS
 
-        Expects the following environment:
+    Expects the following environment:
-        - `SIGNING_IDENTITY` environ with Certificate name/identity
+    - `SIGNING_IDENTITY` environ with Certificate name/identity
-        - `KEYCHAIN` environ with path to the keychain storing credentials
+    - `KEYCHAIN` environ with path to the keychain storing credentials
-        - `KEYCHAIN_PROFILE` environ with name of the profile in that keychain
+    - `KEYCHAIN_PROFILE` environ with name of the profile in that keychain
-        - `KEYCHAIN_PASSWORD` environ with password to unlock the keychain
+    - `KEYCHAIN_PASSWORD` environ with password to unlock the keychain
-        """
+    """
     if project != "libzim":
         return
 
     # currently only supports libzim use case: sign every dylib
     base_dir, export_files = EXPORT_FILES[project]
-    filepaths = [base_dir.joinpath(file)
+    filepaths = [
-                 for file in filter(lambda f: f.endswith(".dylib"), export_files)
+        base_dir.joinpath(file)
-                 if not base_dir.joinpath(file).is_symlink()]
+        for file in filter(lambda f: f.endswith(".dylib"), export_files)
+        if not base_dir.joinpath(file).is_symlink()
+    ]
 
     if not filepaths:
         return
 
     for filepath in filepaths:
-        subprocess.check_call(["/usr/bin/codesign", "--force", "--sign",
+        subprocess.check_call(
-                               os.getenv("SIGNING_IDENTITY", "no-signing-ident"),
+            [
-                               "--keychain",
+                "/usr/bin/codesign",
-                               os.getenv("KEYCHAIN", "no-keychain-path"),
+                "--force",
-                               str(filepath), "--deep", "--timestamp"], env=os.environ)
+                "--sign",
+                os.getenv("SIGNING_IDENTITY", "no-signing-ident"),
+                "--keychain",
+                os.getenv("KEYCHAIN", "no-keychain-path"),
+                str(filepath),
+                "--deep",
+                "--timestamp",
+            ],
+            env=os.environ,
+        )
 
     # create a zip of the dylibs and upload for notarization
     zip_name = "{}.zip".format(project)
     subprocess.check_call(
         ["/usr/bin/ditto", "-c", "-k", "--keepParent"]
-        + [str(f) for f in filepaths] + [zip_name],
+        + [str(f) for f in filepaths]
-        env=os.environ)
+        + [zip_name],
+        env=os.environ,
+    )
 
     # make sure keychain is unlocked
     subprocess.check_call(

.github/scripts/compile_all_deps.py

@@ -6,13 +6,13 @@ from common import (
     run_kiwix_build,
     make_deps_archive,
     upload,
-    PLATFORM_TARGET,
+    COMPILE_CONFIG,
     DEV_BRANCH,
 )
 from build_definition import select_build_targets, DEPS
 
 for target in select_build_targets(DEPS):
-    run_kiwix_build(target, platform=PLATFORM_TARGET, build_deps_only=True)
+    run_kiwix_build(target, config=COMPILE_CONFIG, build_deps_only=True)
     archive_file = make_deps_archive(target=target)
     if DEV_BRANCH:
         destination = "/data/tmp/ci/dev_preview/" + DEV_BRANCH

.github/scripts/ensure_base_deps.py

@@ -13,10 +13,12 @@ from common import (
     upload,
     make_deps_archive,
     HOME,
-    PLATFORM_TARGET,
+    COMPILE_CONFIG,
     OS_NAME,
+    MAKE_RELEASE,
 )
 
+
 def download_base_archive(base_name):
     url = "http://tmp.kiwix.org/ci/{}".format(base_name)
     file_path = str(HOME / base_name)
@@ -30,29 +32,41 @@ def download_base_archive(base_name):
             file.write(batch)
     return file_path
 
-ARCHIVE_NAME_TEMPLATE = "base_deps2_{os}_{platform}_{version}.tar.xz"
 
-if PLATFORM_TARGET == 'flatpak':
+def get_archive_name():
-    base_dep_archive_name = "base_deps2_flatpak.tar.xz"
+    ARCHIVE_NAME_TEMPLATE = "base_deps_{os}_{config}_{version}.tar.gz"
-else:
+
-    base_dep_archive_name = ARCHIVE_NAME_TEMPLATE.format(
+    if COMPILE_CONFIG == "flatpak":
+        return "base_deps_flatpak.tar.gz"
+
+    return ARCHIVE_NAME_TEMPLATE.format(
         os=OS_NAME,
-        platform=PLATFORM_TARGET,
+        config=COMPILE_CONFIG,
         version=base_deps_meta_version,
     )
 
-print_message("Getting archive {}", base_dep_archive_name)
+
-try:
+def main():
-    local_filename = download_base_archive(base_dep_archive_name)
+    base_dep_archive_name = get_archive_name()
-    with tarfile.open(local_filename) as f:
+    print_message("Getting archive {}", base_dep_archive_name)
-        f.extractall(str(HOME))
+    try:
-    os.remove(str(local_filename))
+        local_filename = download_base_archive(base_dep_archive_name)
-except URLError:
+        with tarfile.open(local_filename) as f:
-    if PLATFORM_TARGET == "flatpak":
+            f.extractall(str(HOME))
-        print_message("Cannot get archive. Move on")
+        os.remove(str(local_filename))
+    except URLError:
+        if COMPILE_CONFIG == "flatpak":
+            print_message("Cannot get archive. Move on")
+        else:
+            print_message("Cannot get archive. Build dependencies")
+            run_kiwix_build("alldependencies", config=COMPILE_CONFIG)
+            archive_file = make_deps_archive(name=base_dep_archive_name, full=True)
+            upload(archive_file, "ci@tmp.kiwix.org:30022", "/data/tmp/ci")
+            os.remove(str(archive_file))
+
+
+if __name__ == "__main__":
+    if MAKE_RELEASE:
+        print_message("We are building release. Don't download deps archive.")
     else:
-        print_message("Cannot get archive. Build dependencies")
+        main()
-        run_kiwix_build("alldependencies", platform=PLATFORM_TARGET)
-        archive_file = make_deps_archive(name=base_dep_archive_name, full=True)
-        upload(archive_file, "ci@tmp.kiwix.org:30022", "/data/tmp/ci")
-        os.remove(str(archive_file))

.github/scripts/upload_failure_logs.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+
+import tarfile
+from pathlib import Path
+from common import upload, OS_NAME, COMPILE_CONFIG, HOME
+
+ARCHIVE_NAME = Path(f"fail_log_{OS_NAME}_{COMPILE_CONFIG}.tar.gz")
+
+
+files_to_archive = []
+files_to_archive += HOME.glob("BUILD_*")
+files_to_archive += [HOME / "SOURCE", HOME / "LOGS", HOME / "TOOLCHAINS"]
+
+with tarfile.open(ARCHIVE_NAME, "w:xz") as tar:
+    for name in set(files_to_archive):
+        tar.add(str(name))
+
+upload(ARCHIVE_NAME, "ci@tmp.kiwix.org:30022", "/data/tmp/ci")

.github/scripts/upload_failure_logs.sh

@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-cd $HOME
-
-ARCHIVE_NAME=fail_log_${OS_NAME}_${PLATFORM_TARGET}.tar.gz
-tar -czf ${ARCHIVE_NAME} $HOME/BUILD_* $HOME/SOURCE $HOME/LOGS $HOME/TOOLCHAINS
-
-echo "Uploading archive $ARCHIVE_NAME"
-
-scp -c aes128-ctr -P 30022 -p -i ${SSH_KEY} \
-  -o PasswordAuthentication=no \
-  -o StrictHostKeyChecking=no \
-  $ARCHIVE_NAME \
-  ci@tmp.kiwix.org:/data/tmp/ci

.github/stale.yml

@@ -1,15 +0,0 @@
-daysUntilClose: false
-staleLabel: stale
-
-issues:
-  daysUntilStale: 60
-  markComment: >
-    This issue has been automatically marked as stale because it has not had
-    recent activity. It will be now be reviewed manually. Thank you
-    for your contributions.
-pulls:
-   daysUntilStale: 7
-   markComment: >
-     This pull request has been automatically marked as stale because it has not had
-     recent activity. It will be now be reviewed manually. Thank you
-     for your contributions.

.github/workflows/cd.yml

@@ -1,18 +1,114 @@
-name: Release&Nigthly
+name: CD
 
 on:
  push:
    tags:
      - r_[0-9]+
  schedule:
-   - cron: '0 1 * * *'
+   - cron: '0 3 * * *'
 
 jobs:
+  Windows:
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - native_mixed
+          - native_dyn
+          - native_static
+    runs-on: windows-2022
+    env:
+      OS_NAME: windows
+      COMPILE_CONFIG: ${{matrix.config}}
+      HOME: 'C:\\Users\\runneradmin'
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+    - name: Setup python 3.12
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+    - name: Install packages
+      run: |
+        choco.exe install pkgconfiglite ninja
+    - name: Install python modules
+      shell: bash
+      run: |
+        pip3 install meson pytest requests distro paramiko
+        pip3 install --no-deps $GITHUB_WORKSPACE
+    - name: Install QT
+      if: ${{ matrix.config == 'native_mixed' }}
+      uses: jurplel/install-qt-action@v4
+      with:
+        version: 5.15.2
+        modules: "qtwebengine"
+        setup-python: false
+      env:
+        AQT_CONFIG: ${{ github.workspace }}/.github/configs/aqt.ini
+    - name: Setup MSVC compiler
+      uses: bus1/cabuild/action/msdevshell@v1
+      with:
+        architecture: x64
+    - name: secret
+      shell: bash
+      run: |
+        echo "${{secrets.ssh_key}}" > $SSH_KEY
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+    - name: Install and configure eSigner CKA and Windows SDK
+      if: github.event_name == 'push'
+      env:
+        ESIGNER_URL: https://github.com/SSLcom/eSignerCKA/releases/download/v1.0.7/SSL.COM-eSigner-CKA_1.0.7.zip
+      run: |
+          Set-StrictMode -Version 'Latest'
+
+          # Download and Unzip eSignerCKA Setup
+          Invoke-WebRequest -OutFile eSigner_CKA_Setup.zip "$env:ESIGNER_URL"
+          Expand-Archive -Force eSigner_CKA_Setup.zip
+          Remove-Item eSigner_CKA_Setup.zip
+          Move-Item -Destination “eSigner_CKA_Installer.exe” -Path “eSigner_CKA_*\*.exe”
+
+          # Install eSignerCKA
+          New-Item -ItemType Directory -Force -Path "C:\esigner"
+          ./eSigner_CKA_Installer.exe /CURRENTUSER /VERYSILENT /SUPPRESSMSGBOXES /DIR=”C:\esigner” /TYPE=automatic | Out-Null
+          Remove-Item "eSigner_CKA_Installer.exe"
+
+          # Configure the CKA with SSL.com credentials
+          C:\esigner\eSignerCKATool.exe config -mode product -user "${{ secrets.ESIGNER_USERNAME }}" -pass "${{ secrets.ESIGNER_PASSWORD }}" -totp "${{ secrets.ESIGNER_TOTP_SECRET }}" -key "C:\esigner\master.key" -r
+          C:\esigner\eSignerCKATool.exe unload
+          C:\esigner\eSignerCKATool.exe load
+
+          # Find certificate
+          $CodeSigningCert = Get-ChildItem Cert:\CurrentUser\My -CodeSigningCert | Select-Object -First 1
+          echo Certificate: $CodeSigningCert
+
+          # Extract thumbprint and subject name
+          $Thumbprint = $CodeSigningCert.Thumbprint
+          echo "SIGNTOOL_THUMBPRINT=$Thumbprint" >> $env:GITHUB_ENV
+
+    - name: Ensure base deps
+      run: |
+        python .github\\scripts\\ensure_base_deps.py
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+    - name: Build Release
+      run: |
+        python .github\\scripts\\build_release_nightly.py
+      env:
+        SIGNTOOL_PATH: "C:/Program Files (x86)/Windows Kits/10/bin/10.0.22621.0/x86/signtool.exe"
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+    - name: Upload failure logs
+      if: failure()
+      run: |
+        python .github\\scripts\\upload_failure_logs.py
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+
   Linux:
     strategy:
       fail-fast: false
       matrix:
-        target:
+        config:
           - native_static
           - native_mixed
           - native_dyn
@@ -34,18 +130,18 @@ jobs:
           - android_x86_64
         image_variant: ['focal']
         include:
-          - target: native_mixed
+          - config: native_mixed
-            image_variant: bionic
+            image_variant: manylinux
-          - target: aarch64_mixed
+          - config: aarch64_mixed
-            image_variant: bionic
+            image_variant: manylinux
-          - target: win32_static
+          - config: native_dyn
-            image_variant: f35
+            image_variant: jammy
     env:
       HOME: /home/runner
       SSH_KEY: /tmp/id_rsa
     runs-on: ubuntu-22.04
     container:
-      image: "ghcr.io/kiwix/kiwix-build_ci_${{matrix.image_variant}}:2023-10-30"
+      image: "ghcr.io/kiwix/kiwix-build_ci_${{matrix.image_variant}}:2024-11-30"
       options: "--device /dev/fuse --privileged"
     steps:
     - name: Checkout code
@@ -69,21 +165,19 @@ jobs:
         cd $HOME
         kiwix-build/.github/scripts/ensure_base_deps.py
       env:
-        PLATFORM_TARGET: ${{matrix.target}}
+        COMPILE_CONFIG: ${{matrix.config}}
     - name: Build release
       shell: bash
       run: |
         cd $HOME
         kiwix-build/.github/scripts/build_release_nightly.py
       env:
-        PLATFORM_TARGET: ${{matrix.target}}
+        COMPILE_CONFIG: ${{matrix.config}}
-        BINTRAY_USER: kiwix
-        BINTRAY_PASS: ${{secrets.bintray_pass}}
     - name: Upload failure logs
       if: failure()
-      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh
+      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py
       env:
-        PLATFORM_TARGET: ${{matrix.target}}
+        COMPILE_CONFIG: ${{matrix.config}}
 
   Flatpak:
     strategy:
@@ -91,7 +185,7 @@ jobs:
     env:
       HOME: /home/runner
       SSH_KEY: /tmp/id_rsa
-      PLATFORM_TARGET: flatpak
+      COMPILE_CONFIG: flatpak
       OS_NAME: focal
     runs-on: ubuntu-22.04
     steps:
@@ -126,18 +220,19 @@ jobs:
         kiwix-build/.github/scripts/build_release_nightly.py
     - name: Upload failure logs
       if: failure()
-      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh
+      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py
 
   Macos:
     strategy:
       fail-fast: false
       matrix:
-        target:
+        config:
           - native_dyn
           - native_static
           - native_mixed
           - macOS_arm64_static
           - macOS_arm64_mixed
+          - apple_all_static
     runs-on: macos-13
     env:
       SSH_KEY: /tmp/id_rsa
@@ -152,14 +247,14 @@ jobs:
       # https://github.com/actions/runner-images/blob/main/images/macos/macos-13-Readme.md#xcode
       run: sudo xcode-select -s /Applications/Xcode_15.0.1.app
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
-    - name: Setup python 3.8
+    - name: Setup python 3.10
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v5
       with:
-        python-version: '3.8'
+        python-version: '3.10'
     - name: Install packages
       run:
-        brew install pkg-config ninja automake autoconf
+        brew install automake # ninja
     - name: Install python modules
       run: |
         pip3 install meson pytest requests distro
@@ -194,30 +289,30 @@ jobs:
         cd $HOME
         $GITHUB_WORKSPACE/.github/scripts/ensure_base_deps.py
       env:
-        PLATFORM_TARGET: ${{matrix.target}}
+        COMPILE_CONFIG: ${{matrix.config}}
     - name: Build release
       shell: bash
       run: |
         cd $HOME
         $GITHUB_WORKSPACE/.github/scripts/build_release_nightly.py
       env:
-        PLATFORM_TARGET: ${{matrix.target}}
+        COMPILE_CONFIG: ${{matrix.config}}
     - name: Upload failure logs
       if: failure()
-      run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.sh
+      run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.py
       env:
-        PLATFORM_TARGET: ${{matrix.target}}
+        COMPILE_CONFIG: ${{matrix.config}}
 
 
   Trigger_Docker:
     needs: [Linux]
     runs-on: ubuntu-22.04
     env:
-      PLATFORM_TARGET: LINUX_DOCKER_TRIGGER
+      COMPILE_CONFIG: native_static
-      OS_NAME: LINUX_DOCKER_TRIGGER
+      OS_NAME: focal
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     - name: Install python modules
       shell: bash
       run: |

.github/workflows/ci.yml

@@ -6,11 +6,80 @@ on:
    - cron: '0 1 * * *'
 
 jobs:
+  Windows:
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - native_mixed
+          - native_dyn
+          - native_static
+    runs-on: windows-2022
+    env:
+      OS_NAME: windows
+      COMPILE_CONFIG: ${{matrix.config}}
+      HOME: 'C:\\Users\\runneradmin'
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+    - name: Setup python 3.12
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+    - name: Install packages
+      run: |
+        choco.exe install pkgconfiglite ninja
+    - name: Install python modules
+      shell: bash
+      run: |
+        pip3 install meson pytest requests distro paramiko
+        pip3 install --no-deps $GITHUB_WORKSPACE
+    - name: Install QT
+      if: ${{ matrix.config == 'native_mixed' }}
+      uses: jurplel/install-qt-action@v4
+      with:
+        version: 6.4.3
+        modules: "qtwebengine qtwebchannel qtpositioning"
+        setup-python: false
+      env:
+        AQT_CONFIG: ${{ github.workspace }}/.github/configs/aqt.ini
+    - name: Setup MSVC compiler
+      uses: bus1/cabuild/action/msdevshell@v1
+      with:
+        architecture: x64
+    - name: secret
+      shell: bash
+      run: |
+        echo "${{secrets.ssh_key}}" > $SSH_KEY
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+    - name: Ensure base deps
+      run: |
+        python .github\\scripts\\ensure_base_deps.py
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+    - name: Compile all deps
+      run: |
+        python .github\\scripts\\compile_all_deps.py
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+    - name: Build projects
+      run: |
+        python .github\\scripts\\build_projects.py
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+    - name: Upload failure logs
+      if: failure()
+      run: |
+        python .github\\scripts\\upload_failure_logs.py
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+
   Linux:
     strategy:
       fail-fast: false
       matrix:
-        target:
+        config:
           - native_static
           - native_dyn
           - native_mixed
@@ -37,20 +106,19 @@ jobs:
           - android_x86_64
         image_variant: ['focal']
         include:
-          - target: native_mixed
+          - config: native_mixed
-            image_variant: bionic
+            image_variant: manylinux
-          - target: aarch64_mixed
+          - config: aarch64_mixed
-            image_variant: bionic
+            image_variant: manylinux
-          - target: win32_static
+          - config: native_dyn
-            image_variant: f35
+            image_variant: jammy
-          - target: win32_dyn
-            image_variant: f35
     env:
       HOME: /home/runner
       SSH_KEY: /tmp/id_rsa
+      OS_NAME: ${{matrix.image_variant}}
     runs-on: ubuntu-22.04
     container:
-      image: "ghcr.io/kiwix/kiwix-build_ci_${{matrix.image_variant}}:2023-10-30"
+      image: "ghcr.io/kiwix/kiwix-build_ci_${{matrix.image_variant}}:2024-11-30"
       options: "--device /dev/fuse --privileged"
     steps:
     - name: Checkout code
@@ -63,6 +131,10 @@ jobs:
         pip3 install --user --no-deps .
       env:
         REP: ${{github.repository}}
+    - name: Install paramiko
+      if: ${{matrix.image_variant != 'bionic' }}
+      shell: bash
+      run: pip3 install --user paramiko
     - name: secret
       shell: bash
       run: |
@@ -74,26 +146,26 @@ jobs:
         cd $HOME
         kiwix-build/.github/scripts/ensure_base_deps.py
       env:
-        PLATFORM_TARGET: ${{matrix.target}}
+        COMPILE_CONFIG: ${{matrix.config}}
     - name: Compile all deps
       shell: bash
       run: |
         cd $HOME
         kiwix-build/.github/scripts/compile_all_deps.py
       env:
-        PLATFORM_TARGET: ${{matrix.target}}
+        COMPILE_CONFIG: ${{matrix.config}}
     - name: Build projects
       shell: bash
       run: |
         cd $HOME
         kiwix-build/.github/scripts/build_projects.py
       env:
-        PLATFORM_TARGET: ${{matrix.target}}
+        COMPILE_CONFIG: ${{matrix.config}}
     - name: Upload failure logs
       if: failure()
-      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh
+      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py
       env:
-        PLATFORM_TARGET: ${{matrix.target}}
+        COMPILE_CONFIG: ${{matrix.config}}
 
   Flatpak:
     strategy:
@@ -101,7 +173,7 @@ jobs:
     env:
       HOME: /home/runner
       SSH_KEY: /tmp/id_rsa
-      PLATFORM_TARGET: flatpak
+      COMPILE_CONFIG: flatpak
       OS_NAME: focal
     runs-on: ubuntu-22.04
     steps:
@@ -112,6 +184,7 @@ jobs:
         git clone https://github.com/${REP}
         cd ./${REP##*/}
         git checkout --force ${GITHUB_SHA}
+        pip3 install --user paramiko
         pip3 install --user --no-deps .
       env:
         REP: ${{github.repository}}
@@ -136,21 +209,23 @@ jobs:
         kiwix-build/.github/scripts/build_projects.py
     - name: Upload failure logs
       if: failure()
-      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh
+      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py
 
   Macos:
     strategy:
       fail-fast: false
       matrix:
-        target:
+        config:
           - native_dyn
           - native_static
           - native_mixed
           - iOS_arm64
-          - iOS_x86_64
+          - iOSSimulator_x86_64
+          - iOSSimulator_arm64
           - macOS_arm64_static
           - macOS_arm64_mixed
           - macOS_x86_64
+          - apple_all_static
     runs-on: macos-13
     env:
       SSH_KEY: /tmp/id_rsa
@@ -160,17 +235,17 @@ jobs:
       # https://github.com/actions/runner-images/blob/main/images/macos/macos-13-Readme.md#xcode
       run: sudo xcode-select -s /Applications/Xcode_15.0.1.app
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
-    - name: Setup python 3.8
+    - name: Setup Python 3.10
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v5
       with:
-        python-version: '3.8'
+        python-version: '3.10'
     - name: Install packages
       run:
-        brew install pkg-config ninja automake autoconf
+        brew install automake # ninja
     - name: Install python modules
       run: |
-        pip3 install meson pytest requests distro
+        pip3 install meson pytest requests distro paramiko
         pip3 install --no-deps $GITHUB_WORKSPACE
     - name: secret
       shell: bash
@@ -183,23 +258,23 @@ jobs:
         cd $HOME
         $GITHUB_WORKSPACE/.github/scripts/ensure_base_deps.py
       env:
-        PLATFORM_TARGET: ${{matrix.target}}
+        COMPILE_CONFIG: ${{matrix.config}}
     - name: Compile all deps
       shell: bash
       run: |
         cd $HOME
         $GITHUB_WORKSPACE/.github/scripts/compile_all_deps.py
       env:
-        PLATFORM_TARGET: ${{matrix.target}}
+        COMPILE_CONFIG: ${{matrix.config}}
     - name: Build projects
       shell: bash
       run: |
         cd $HOME
         $GITHUB_WORKSPACE/.github/scripts/build_projects.py
       env:
-        PLATFORM_TARGET: ${{matrix.target}}
+        COMPILE_CONFIG: ${{matrix.config}}
     - name: Upload failure logs
       if: failure()
-      run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.sh
+      run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.py
       env:
-        PLATFORM_TARGET: ${{matrix.target}}
+        COMPILE_CONFIG: ${{matrix.config}}

README.md

@@ -12,7 +12,7 @@ Kiwix Build audience is:
 * Kiwix developer team for its own CI/CD
 
 [![CI Build Status](https://github.com/kiwix/kiwix-build/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/kiwix/kiwix-build/actions/workflows/ci.yml?query=branch%3Amain)
-[![CD Build Status](https://github.com/kiwix/kiwix-build/actions/workflows/releaseNigthly.yml/badge.svg?branch=main)](https://github.com/kiwix/kiwix-build/actions/workflows/releaseNigthly.yml?query=branch%3Amain)
+[![CD Build Status](https://github.com/kiwix/kiwix-build/actions/workflows/cd.yml/badge.svg?branch=main)](https://github.com/kiwix/kiwix-build/actions/workflows/cd.yml?query=branch%3Amain)
 [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
 
 Prerequisites
@@ -78,19 +78,17 @@ invalid choice: 'not-existing-target' (choose from 'alldependencies', 'android-n
 ...
 ```
 
-#### Target platform
+#### Config
 
-If no target platform is specified, the default will be `native_dyn`.
+If no config is specified, the default will be `native_dyn`.
 
-You can select another target platform using the option
+You can select another config using the option
-`--target-platform`. For now, there is ten different supported
+`--config`. For now, there is ten different supported
 platforms:
 
 - native_dyn
 - native_mixed
 - native_static
-- win32_dyn
-- win32_static
 - android
 - android_arm
 - android_arm64
@@ -98,42 +96,31 @@ platforms:
 - android_x86_64
 - flatpak
 
-So, if you want to compile `kiwix-tools` for win32 using static linkage:
+All `native_*` config means using the native compiler without any cross-compilation option.
-```bash
+Other may simply use cross-compilation or may download a specific toolchain to use.
-kiwix-build --target-platform win32_dyn
-```
 
 Android
 -------
 
 `kiwix-android` (https://github.com/kiwix/kiwix-android) depends of
 the `libkiwix` project.
-It uses a special `.aar` file that represent (and embed) the libkiwix for
-all supported android arch. This is a kind of fat archive we have for MacOs.
-
-The `.aar` file is build using the `libkiwix-app` project.
-`libkiwix-app` itself is architecture independent (it is just a packaging of
-other archives) but it use `libkiwix` who is architecture dependent.
 
 When building `libkiwix`, you should directly use the
 target-platform `android_<arch>`:
 ```bash
-kiwix-build libkiwix --target-platform android_arm
+kiwix-build libkiwix --config android_arm
 ```
 
-But, `libkiwix-app` is mainly multi arch.
+You may directly use the special config `android` which will build different android architectures
-To compile `libkiwix-app`, you must use the `android` platform:
 ```bash
-kiwix-build --target-platform android libkiwix-app
+kiwix-build --config android libkiwix
-kiwix-build libkiwix-app # because `android` platform is the default for `libkiwix-app`
 ```
 
-By default, when using platform `android`, `libkiwix` will be build for
+By default, it will build for all android architecture,
-all architectures. This can be changed by using the option `--android-arch`:
+you can limit this with option `--android-arch`:
 ```bash
-kiwix-build libkiwix-app # aar with all architectures
+kiwix-build libkiwix --config android --android-arch arm # aar with arm architecture
-kiwix-build libkiwix-app --android-arch arm # aar with arm architecture
+kiwix-build libkiwix --config android --android-arch arm --android-arch arm64 # aan with arm and arm64 architectures
-kiwix-build libkiwix-app --android-arch arm --android-arch arm64 # aan with arm and arm64 architectures
 ```
 
 To build `kiwix-android` itself, you should see the documentation of `kiwix-android`.
@@ -148,13 +135,13 @@ To do so, you should directly use the target-platfrom `ios_multi`.
 As for `android`, `kiwix-build` will build the library several times
 (once for each platform) and then create the fat library.
 ```bash
-kiwix-build --target-platform iOS_multi libkiwix
+kiwix-build --config iOS_multi libkiwix
 ```
 
 You can specify the supported architectures with the option `--ios-arch`:
 ```bash
-kiwix-build --target-platform iOS_multi libkiwix # all architetures
+kiwix-build --config iOS_multi libkiwix # all architetures
-kiwix-build --target-platform iOS_multi --ios-arch arm --ios-arch arm64 # arm and arm64 arch only
+kiwix-build --config iOS_multi --ios-arch arm --ios-arch arm64 # arm and arm64 arch only
 ```
 
 Outputs
@@ -163,9 +150,9 @@ Outputs
 Kiwix-build.py will create several directories:
 - `ARCHIVES`: All the downloaded archives go there.
 - `SOURCES`: All the sources (extracted from archives and patched) go there.
-- `BUILD_<target_platform>`: All the build files go there.
+- `BUILD_<config>`: All the build files go there.
-- `BUILD_<target_platform>/INSTALL`: The installed files go there.
+- `BUILD_<config>/INSTALL`: The installed files go there.
-- `BUILD_<target_platform>/LOGS`: The logs files of the build.
+- `BUILD_<config>/LOGS`: The logs files of the build.
 
 If you want to install all those directories elsewhere, you can pass the
 `--working-dir` option to `kiwix-build`:

actions/dl_deps_archive/.gitignore

@@ -0,0 +1 @@
+node_modules

actions/dl_deps_archive/README.md

@@ -0,0 +1,55 @@
+# Download dependencies archive action
+
+This action download dependencies archive made by kiwix-build.
+It is intended to be used in projects using theses dependencies
+
+## Inputs
+
+### `base_url`
+
+The base url where we can download the archive.
+This input is provided for greater customization but you probably shouldn't set it.
+
+### `os_name`
+
+The os "name" on which the compilation is done.
+By default this use the `OS_NAME` env var, which is set in the docker file.
+
+
+### `target_platform`
+
+**Required** The targeted platform. Must be provided. Values are kind of :
+- native_dyn
+- android_arm
+- ...
+
+### `project`
+
+The name of the project being compiled.
+By default, the name of the repository.
+
+
+### `branch`
+
+The name of the "branch" to try to download (`/dev_preview/<branch>`).
+By default, the current branch on which the action is run.
+
+### `extract_dir`
+
+Where to extract the dependencies archive. By default it is `$HOME`
+
+
+## Example usage
+
+```yaml
+uses: kiwix/kiwix-build/actions/dl_deps_archive@main
+with:
+  target_platform: ${{ matrix.target_platform }}
+```
+
+```
+uses: kiwix/kiwix-build/actions/dl_deps_archive@main
+with:
+  target_platform: native_mixed
+  os_name: windows
+```

actions/dl_deps_archive/action.yml

@@ -0,0 +1,25 @@
+name: 'DL deps archive'
+description: 'Download the dependencies archive of the project'
+inputs:
+  base_url:  # id of input
+    description: 'What is the base url to download the archive'
+    required: false
+    default: 'https://tmp.kiwix.org/ci'
+  os_name:
+    description: "On which os are we running. [Default to env var `OS_NAME`]"
+    required: false
+  target_platform:
+    description: "What platform are we targetting"
+    required: true
+  project:
+    description: "What project are we building"
+    required: false
+  branch:
+    description: "On which branch are we ?"
+    required: false
+  extract_dir:
+    description: "Where to extract our dependencies. [Default to env var `HOME`]"
+    required: false
+runs:
+  using: 'node20'
+  main: 'dist/index.js'

actions/dl_deps_archive/index.js

@@ -0,0 +1,59 @@
+const tc = require("@actions/tool-cache");
+const core = require("@actions/core");
+const path = require("path");
+const os = require("os");
+
+function getInput(name, dflt) {
+  const val = process.env[`INPUT_${name.replace(/ /g, "_").toUpperCase()}`];
+  if (!val) {
+    return dflt;
+  }
+  return val;
+}
+
+function addLocalPath(inputPath) {
+  process.env["PATH"] = `${inputPath}${path.delimiter}${process.env["PATH"]}`;
+}
+
+async function run() {
+  try {
+    const base_url = core.getInput("base_url");
+    const target = core.getInput("target_platform");
+    const project = getInput(
+      "project",
+      process.env["GITHUB_REPOSITORY"].split("/")[1],
+    );
+    const branch = getInput(
+      "branch",
+      process.env["GITHUB_HEAD_REF"] || process.env["GITHUB_REF_NAME"],
+    );
+    const extract_dir = getInput(
+      "extract_dir",
+      process.env["HOME"] || process.env["GITHUB_WORKSPACE"],
+    );
+
+    let archivePath;
+    try {
+      const archive_url = `${base_url}/dev_preview/${branch}/deps_${target}_${project}.tar.gz`;
+      process.stdout.write("Downloading " + archive_url + "\n");
+      archivePath = await tc.downloadTool(archive_url);
+    } catch (error) {
+      const archive_url = `${base_url}/deps_${target}_${project}.tar.gz`;
+      process.stdout.write("Downloading " + archive_url + "\n");
+      archivePath = await tc.downloadTool(archive_url);
+    }
+
+    process.stdout.write("Extracting " + archivePath + " to " + extract_dir);
+    const archive_dir = await tc.extractTar(archivePath, extract_dir);
+    process.stdout.write("Extracted to " + archive_dir);
+  } catch (error) {
+    core.setFailed(error.message);
+  }
+}
+
+if (os.platform() === "win32") {
+  addLocalPath("C:\\Program Files\\Git\\usr\\bin");
+}
+core.setCommandEcho(true);
+
+run();

actions/dl_deps_archive/package-lock.json

@@ -0,0 +1,144 @@
+{
+  "name": "dl_deps_archive",
+  "version": "1.0.0",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "dl_deps_archive",
+      "version": "1.0.0",
+      "license": "ISC",
+      "dependencies": {
+        "@actions/core": "^1.10.1",
+        "@actions/tool-cache": "^2.0.1",
+        "path": "^0.12.7"
+      }
+    },
+    "node_modules/@actions/core": {
+      "version": "1.10.1",
+      "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.10.1.tgz",
+      "integrity": "sha512-3lBR9EDAY+iYIpTnTIXmWcNbX3T2kCkAEQGIQx4NVQ0575nk2k3GRZDTPQG+vVtS2izSLmINlxXf0uLtnrTP+g==",
+      "dependencies": {
+        "@actions/http-client": "^2.0.1",
+        "uuid": "^8.3.2"
+      }
+    },
+    "node_modules/@actions/exec": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@actions/exec/-/exec-1.1.1.tgz",
+      "integrity": "sha512-+sCcHHbVdk93a0XT19ECtO/gIXoxvdsgQLzb2fE2/5sIZmWQuluYyjPQtrtTHdU1YzTZ7bAPN4sITq2xi1679w==",
+      "dependencies": {
+        "@actions/io": "^1.0.1"
+      }
+    },
+    "node_modules/@actions/http-client": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.2.0.tgz",
+      "integrity": "sha512-q+epW0trjVUUHboliPb4UF9g2msf+w61b32tAkFEwL/IwP0DQWgbCMM0Hbe3e3WXSKz5VcUXbzJQgy8Hkra/Lg==",
+      "dependencies": {
+        "tunnel": "^0.0.6",
+        "undici": "^5.25.4"
+      }
+    },
+    "node_modules/@actions/io": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/@actions/io/-/io-1.1.3.tgz",
+      "integrity": "sha512-wi9JjgKLYS7U/z8PPbco+PvTb/nRWjeoFlJ1Qer83k/3C5PHQi28hiVdeE2kHXmIL99mQFawx8qt/JPjZilJ8Q=="
+    },
+    "node_modules/@actions/tool-cache": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/@actions/tool-cache/-/tool-cache-2.0.1.tgz",
+      "integrity": "sha512-iPU+mNwrbA8jodY8eyo/0S/QqCKDajiR8OxWTnSk/SnYg0sj8Hp4QcUEVC1YFpHWXtrfbQrE13Jz4k4HXJQKcA==",
+      "dependencies": {
+        "@actions/core": "^1.2.6",
+        "@actions/exec": "^1.0.0",
+        "@actions/http-client": "^2.0.1",
+        "@actions/io": "^1.1.1",
+        "semver": "^6.1.0",
+        "uuid": "^3.3.2"
+      }
+    },
+    "node_modules/@actions/tool-cache/node_modules/uuid": {
+      "version": "3.4.0",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
+      "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==",
+      "deprecated": "Please upgrade  to version 7 or higher.  Older versions may use Math.random() in certain circumstances, which is known to be problematic.  See https://v8.dev/blog/math-random for details.",
+      "bin": {
+        "uuid": "bin/uuid"
+      }
+    },
+    "node_modules/@fastify/busboy": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.0.tgz",
+      "integrity": "sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==",
+      "engines": {
+        "node": ">=14"
+      }
+    },
+    "node_modules/inherits": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
+      "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw=="
+    },
+    "node_modules/path": {
+      "version": "0.12.7",
+      "resolved": "https://registry.npmjs.org/path/-/path-0.12.7.tgz",
+      "integrity": "sha512-aXXC6s+1w7otVF9UletFkFcDsJeO7lSZBPUQhtb5O0xJe8LtYhj/GxldoL09bBj9+ZmE2hNoHqQSFMN5fikh4Q==",
+      "dependencies": {
+        "process": "^0.11.1",
+        "util": "^0.10.3"
+      }
+    },
+    "node_modules/process": {
+      "version": "0.11.10",
+      "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz",
+      "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==",
+      "engines": {
+        "node": ">= 0.6.0"
+      }
+    },
+    "node_modules/semver": {
+      "version": "6.3.1",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
+      "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
+      "bin": {
+        "semver": "bin/semver.js"
+      }
+    },
+    "node_modules/tunnel": {
+      "version": "0.0.6",
+      "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz",
+      "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==",
+      "engines": {
+        "node": ">=0.6.11 <=0.7.0 || >=0.7.3"
+      }
+    },
+    "node_modules/undici": {
+      "version": "5.28.4",
+      "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.4.tgz",
+      "integrity": "sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g==",
+      "dependencies": {
+        "@fastify/busboy": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=14.0"
+      }
+    },
+    "node_modules/util": {
+      "version": "0.10.4",
+      "resolved": "https://registry.npmjs.org/util/-/util-0.10.4.tgz",
+      "integrity": "sha512-0Pm9hTQ3se5ll1XihRic3FDIku70C+iHUdT/W926rSgHV5QgXsYbKZN8MSC3tJtSkhuROzvsQjAaFENRXr+19A==",
+      "dependencies": {
+        "inherits": "2.0.3"
+      }
+    },
+    "node_modules/uuid": {
+      "version": "8.3.2",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
+      "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
+      "bin": {
+        "uuid": "dist/bin/uuid"
+      }
+    }
+  }
+}

actions/dl_deps_archive/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "dl_deps_archive",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@actions/core": "^1.10.1",
+    "@actions/tool-cache": "^2.0.1",
+    "path": "^0.12.7"
+  }
+}

appveyor.yml

@@ -1,69 +0,0 @@
-image: Visual Studio 2019
-version: '0.1.{build}'
-environment:
-  EXTRA_DIR: 'C:\extra'
-  MINGW64_EXTRA_DIR: '/c/extra'
-  PATH: 'C:\\Python36\\Scripts;%PATH%;%EXTRA_DIR%\bin;C:\\Program Files (x86)\\Windows Kits\\10\\bin\\%platform%'
-  PKG_CONFIG_PATH: '%EXTRA_DIR%\lib\pkgconfig'
-  MSYS2_PATH_TYPE: 'inherit'
-  MSYS2_ARG_CONV_EXCL: '-Tp'
-  MINGW64_RUN: 'C:\msys64\usr\bin\bash.exe --login -c'
-  MSYSTEM: 'MINGW64'
-  KIWIX_DESKTOP_VERSION: '2.3.1-3'
-  KIWIX_DESKTOP_RELEASE: 0
-
-
-init:
-  - cmd: |
-     echo %TEST_VAR%
-     REM Set VC 2019
-     "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars64.bat"
-     mkdir C:\extra\bin
-
-install:
-  - cmd: |
-     cd C:\projects
-     REM Install meson
-     C:\Python36\Scripts\pip install meson
-
-     REM Install ninja
-     curl -o ninja-win.zip -L https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-win.zip
-     7z e ninja-win.zip -o%EXTRA_DIR%\bin
-
-     REM Install pkg-config-lite
-     curl --insecure -o pkg-config-lite-0.28-1.zip https://netix.dl.sourceforge.net/project/pkgconfiglite/0.28-1/pkg-config-lite-0.28-1_bin-win32.zip
-     7z e pkg-config-lite-0.28-1.zip -o%EXTRA_DIR%\bin pkg-config-lite-0.28-1/bin/pkg-config.exe
-     cd kiwix-build
-
-     REM Install openssh (scp) for deployement
-     choco install openssh openssl.light
-
-  - cmd: openssl enc -d -aes-256-cbc -md sha256 -pass pass:%ENC_PASSWD% -in appveyor\nightlybot_id_key.enc -out appveyor\nightlybot_id_key
-
-
-
-build_script:
-  - IF "%USE_CACHE%" EQU "1" appveyor\setup_from_cache.cmd
-  - IF "%USE_CACHE%" NEQ "1" appveyor\install_zlib.cmd
-  - IF "%USE_CACHE%" NEQ "1" appveyor\install_lzma.cmd
-  - IF "%USE_CACHE%" NEQ "1" appveyor\install_zstd.cmd
-  - IF "%USE_CACHE%" NEQ "1" appveyor\install_libcurl.cmd
-  - IF "%USE_CACHE%" NEQ "1" appveyor\install_pthread.cmd
-  - IF "%USE_CACHE%" NEQ "1" appveyor\install_dirent.cmd
-  - IF "%USE_CACHE%" NEQ "1" appveyor\install_xapian.cmd
-  - IF "%USE_CACHE%" NEQ "1" appveyor\install_icu.cmd
-  - IF "%USE_CACHE%" NEQ "1" appveyor\install_pugixml.cmd
-  - IF "%USE_CACHE%" NEQ "1" appveyor\install_aria2.cmd
-  - IF "%USE_CACHE%" NEQ "1" appveyor\install_mustache.cmd
-  - IF "%USE_CACHE%" NEQ "1" appveyor\install_libmicrohttpd.cmd
-  - IF "%USE_CACHE%" NEQ "1" %MINGW64_RUN% "cd /c/projects/kiwix-build && appveyor/create_cache.sh"
-  - appveyor\install_libzim.cmd
-  - appveyor\install_libkiwix.cmd
-  - appveyor\install_kiwix-desktop.cmd
-
-before_deploy:
-  - cmd: openssl enc -d -pbkdf2 -aes-256-cbc -md sha256 -pass pass:%win_certificate_secret% -in appveyor\kiwix.pfx.enc -out appveyor\kiwix.pfx
-  - cmd: '%MINGW64_RUN% "cd /c/projects/kiwix-build && appveyor/package_kiwix-desktop.sh"'
-
-deploy_script:
-  - cmd: '%MINGW64_RUN% "cd /c/projects/kiwix-build && appveyor/deploy.sh"'

appveyor/build_xapian.sh

@@ -1,16 +0,0 @@
-
-XAPIAN_SOURCE=$(pwd)/..
-
-$XAPIAN_SOURCE/configure \
-  CC="cl -nologo" \
-  CXX="$XAPIAN_SOURCE/compile cl -nologo" \
-  CXXFLAGS="-EHsc -MD" AR=lib \
-  CPPFLAGS="-I${MINGW64_EXTRA_DIR}/include" \
-  LDFLAGS="-L${MINGW64_EXTRA_DIR}/lib" \
-  --disable-backend-remote \
-  --disable-documentation \
-  --prefix=${MINGW64_EXTRA_DIR}
-
-make -j2
-
-make install

appveyor/create_cache.sh

@@ -1,11 +0,0 @@
-
-set -e
-
-ARCHIVE_NAME="deps_windows_windows.zip"
-SSH_KEY=C:\\projects\\kiwix-build\\appveyor\\nightlybot_id_key
-
-7z a -tzip $ARCHIVE_NAME $MINGW64_EXTRA_DIR
-scp -P 30022 -vrp -i ${SSH_KEY} -o StrictHostKeyChecking=no \
-  ${ARCHIVE_NAME} \
-  ci@tmp.kiwix.org:/data/tmp/ci
-

appveyor/deploy.sh

@@ -1,21 +0,0 @@
-set -e
-
-NIGHTLY_DATE=$(date +%Y-%m-%d)
-NIGHTLY_KIWIX_ARCHIVES_DIR=/c/projects/NIGHTLY_KIWIX_ARCHIVES/${NIGHTLY_DATE}
-RELEASE_KIWIX_ARCHIVES_DIR=/c/projects/RELEASE_KIWIX_ARCHIVES
-SSH_KEY=C:\\projects\\kiwix-build\\appveyor\\nightlybot_id_key
-
-if [[ "$APPVEYOR_SCHEDULED_BUILD" = "True" ]]
-then
-  scp -P 30022 -vrp -i ${SSH_KEY} -o StrictHostKeyChecking=no \
-    ${NIGHTLY_KIWIX_ARCHIVES_DIR} \
-    ci@master.download.kiwix.org:/data/download/nightly
-fi
-
-if [[ "$APPVEYOR_REPO_TAG" = "true" ]]
-then
-  RELEASE_ARCHIVES=$(find $RELEASE_KIWIX_ARCHIVES_DIR -type f)
-  scp -P 30022 -vrp -i ${SSH_KEY} -o StrictHostKeyChecking=no \
-      ${RELEASE_ARCHIVES} \
-      ci@master.download.kiwix.org:/data/download/release/kiwix-desktop
-fi

appveyor/fix_desktop_makefile.py

@@ -1,29 +0,0 @@
-import re, sys, os
-
-makefile = sys.argv[-1]
-print("Patching '{}'".format(makefile))
-
-with open(makefile, 'r') as f:
-    lines = f.readlines()
-
-with open(makefile, 'w') as f:
-    for line in lines:
-#        if "/SUBSYSTEM:WINDOWS" in line:
-#            line = line.replace("/SUBSYSTEM:WINDOWS", "/SUBSYSTEM:CONSOLE")
-#            f.write(line)
-#            continue
-        if not line.startswith('LIBS  '):
-            f.write(line)
-            continue
-        print("-- INPUT : {}".format(line))
-        for lib in ('kiwix', 'zim', 'pugixml', 'z', 'zstd'):
-            line = line.replace('{}.lib'.format(lib), 'lib{}.a'.format(lib))
-        for lib in ('lzma', 'curl'):
-            line = line.replace('lib{}.lib'.format(lib), 'lib{}.a'.format(lib))
-        line = line.strip()
-#        line += " pthreadVC2.lib"
-#        line += " icuin.lib icudt.lib icuuc.lib"
-        line += " Rpcrt4.lib Ws2_32.lib winmm.lib Shlwapi.lib"
-        line += os.linesep
-        print("++ OUTPUT : {}".format(line))
-        f.write(line)

appveyor/install_aria2.cmd

@@ -1,4 +0,0 @@
-REM ========================================================
-REM Install aria2
-curl -fsSL -O https://github.com/aria2/aria2/releases/download/release-1.36.0/aria2-1.36.0-win-64bit-build1.zip || exit /b 1
-7z e aria2-1.36.0-win-64bit-build1.zip -o%EXTRA_DIR% aria2-1.36.0-win-64bit-build1\aria2c.exe || exit /b 1

appveyor/install_dirent.cmd

@@ -1,3 +0,0 @@
-REM ========================================================
-REM Install dirent.h
-curl -fsSL -o%EXTRA_DIR%\include\dirent.h https://raw.githubusercontent.com/tronkko/dirent/master/include/dirent.h || exit /b 1

appveyor/install_git_ninja_pkgconfig.cmd

@@ -1,14 +0,0 @@
-REM Set VC 2017
-"C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat"
-mkdir C:\extra\bin
-cd C:\projects
-C:\Python36\Scripts\pip install meson || exit /b 1
-
-REM Set ninja
-curl -fsSL -o ninja-win.zip https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-win.zip || exit /b 1
-7z e ninja-win.zip -o%EXTRA_DIR%\bin || exit /b 1
-
-REM Set pkg-config-lit
-curl --insecure -fsSL -o pkg-config-lite-0.28-1.zip https://netix.dl.sourceforge.net/project/pkgconfiglite/0.28-1/pkg-config-lite-0.28-1_bin-win32.zip || exit /b 1
-7z e pkg-config-lite-0.28-1.zip -o%EXTRA_DIR%\bin pkg-config-lite-0.28-1/bin/pkg-config.exe || exit /b 1
-cd kiwix-build

appveyor/install_icu.cmd

@@ -1,7 +0,0 @@
-REM ========================================================
-REM Install icu
-curl -SL -O http://public.kymeria.fr/KIWIX/windows/icu4c-62_1-Win64-MSVC2017.zip || exit /b 1
-7z x icu4c-62_1-Win64-MSVC2017.zip -o%EXTRA_DIR% -r include || exit /b 1
-7z e icu4c-62_1-Win64-MSVC2017.zip -o%EXTRA_DIR%\lib lib64\* || exit /b 1
-7z e icu4c-62_1-Win64-MSVC2017.zip -o%EXTRA_DIR%\bin bin64\*.dll || exit /b 1
-curl -fsSL -o%PKG_CONFIG_PATH%\icu-i18n.pc http://public.kymeria.fr/KIWIX/windows/icu-i18n.pc || exit /b 1

appveyor/install_kiwix-desktop.cmd

@@ -1,22 +0,0 @@
-REM ========================================================
-REM Install kiwix-desktop
-git clone https://github.com/kiwix/kiwix-desktop || exit /b 1
-cd kiwix-desktop
-echo "Running qmake"
-SET _WITH_CONSOLE=1
-IF %KIWIX_DESKTOP_RELEASE% EQU 1 (
-  IF %APPVEYOR_REPO_TAG% == true (
-    SET _WITH_CONSOLE=0
-  )
-)
-IF %_WITH_CONSOLE% EQU 1 (
-    C:\Qt\5.15\msvc2019_64\bin\qmake.exe "CONFIG+=static console" || exit /b 1
-) else (
-    C:\Qt\5.15\msvc2019_64\bin\qmake.exe "CONFIG+=static" || exit /b 1
-)
-
-echo "Running fix_desktop"
-C:\Python36\Python ..\appveyor\fix_desktop_makefile.py Makefile.Release || exit /b 1
-echo "Running nmake"
-nmake || exit /b 1
-cd ..

appveyor/install_libcurl.cmd

@@ -1,14 +0,0 @@
-REM ========================================================
-REM Install libcurl
-curl -fsSL -O https://curl.haxx.se/download/curl-7.61.1.zip || exit /b 1
-7z x curl-7.61.1.zip || exit /b 1
-cd curl-7.61.1\winbuild
-nmake /f Makefile.vc mode=static MACHINE=x64 DEBUG=no VC=15 ENABLE_IDN=no || exit /b 1
-mkdir %EXTRA_DIR%\include\curl
-copy ..\builds\libcurl-vc15-x64-release-static-ipv6-sspi-winssl\include\curl\*.h %EXTRA_DIR%\include\curl
-copy ..\builds\libcurl-vc15-x64-release-static-ipv6-sspi-winssl\lib\libcurl_a.lib %EXTRA_DIR%\lib
-move %EXTRA_DIR%\lib\libcurl_a.lib %EXTRA_DIR%\lib\libcurl.a
-dir %EXTRA_DIR%\include\curl
-dir %EXTRA_DIR%\lib
-curl -fsSl -o%PKG_CONFIG_PATH%\libcurl.pc http://public.kymeria.fr/KIWIX/windows/libcurl.pc || exit /b 1
-cd ..\..

appveyor/install_libkiwix.cmd

@@ -1,10 +0,0 @@
-REM ========================================================
-REM Install libkiwix
-git clone https://github.com/kiwix/libkiwix.git || exit /b 1
-cd libkiwix
-set CPPFLAGS="-I%EXTRA_DIR%/include"
-meson . build --prefix %EXTRA_DIR% --default-library static --buildtype release || exit /b 1
-cd build
-ninja || exit /b 1
-ninja install || exit /b 1
-cd ..\..

appveyor/install_libmicrohttpd.cmd

@@ -1,11 +0,0 @@
-REM ========================================================
-REM Install libmicrohttpd
-curl -fsSL -O https://ftpmirror.gnu.org/libmicrohttpd/libmicrohttpd-latest-w32-bin.zip || exit /b 1
-7z e libmicrohttpd-latest-w32-bin.zip -o%EXTRA_DIR%/include libmicrohttpd-*-w32-bin/x86_64/VS2019/Release-static/microhttpd.h || exit /b 1
-7z e libmicrohttpd-latest-w32-bin.zip -o%EXTRA_DIR%/lib libmicrohttpd-*-w32-bin/x86_64/VS2019/Release-static/libmicrohttpd.lib || exit /b 1
-7z e libmicrohttpd-latest-w32-bin.zip -o%EXTRA_DIR%/lib/pkgconfig libmicrohttpd-*-w32-bin/x86_64/MinGW/static/mingw64/lib/pkgconfig/* || exit /b 1
-
-rename %EXTRA_DIR%\lib\libmicrohttpd.lib microhttpd.lib
-dir %EXTRA_DIR%
-dir %EXTRA_DIR%\lib
-dir %EXTRA_DIR%\lib\pkgconfig

appveyor/install_libzim.cmd

@@ -1,9 +0,0 @@
-REM ========================================================
-REM Install libzim
-git clone https://github.com/openzim/libzim.git || exit /b 1
-cd libzim
-meson . build --prefix %EXTRA_DIR% --default-library static --buildtype release || exit /b 1
-cd build
-ninja || exit /b 1
-ninja install || exit /b 1
-cd ..\..

appveyor/install_lzma.cmd

@@ -1,6 +0,0 @@
-REM ========================================================
-REM Install lzma
-curl -fsSL -O https://tukaani.org/xz/xz-5.2.4-windows.zip || exit /b 1
-7z x xz-5.2.4-windows.zip -o%EXTRA_DIR% -r include || exit /b 1
-7z e xz-5.2.4-windows.zip -o%EXTRA_DIR%\lib bin_x86-64\liblzma.a || exit /b 1
-curl -fsSL -o%PKG_CONFIG_PATH%\liblzma.pc http://public.kymeria.fr/KIWIX/windows/liblzma.pc || exit /b 1

appveyor/install_mustache.cmd

@@ -1,3 +0,0 @@
-REM ========================================================
-REM Install mustache.hpp
-curl -fsSL -o%EXTRA_DIR%\include\mustache.hpp https://raw.githubusercontent.com/kainjow/mustache/v3.2.1/mustache.hpp || exit /b 1

appveyor/install_pthread.cmd

@@ -1,14 +0,0 @@
-REM ========================================================
-REM Install pthread
-curl -fsSL -O ftp://sourceware.org/pub/pthreads-win32/pthreads-w32-2-9-1-release.zip || exit /b 1
-7z x pthreads-w32-2-9-1-release.zip -r pthreads.2 || exit /b 1
-cd pthreads.2
-REM Patch is pthread_timespec.patch
-curl -fsSL -O http://public.kymeria.fr/KIWIX/windows/pthread.h || exit /b 1
-nmake clean VC-inlined || exit /b 1
-copy pthread.h %EXTRA_DIR%\include
-copy sched.h %EXTRA_DIR%\include
-copy pthreadVC2.lib %EXTRA_DIR%\lib
-copy pthreadVC2.dll %EXTRA_DIR%\bin
-curl -fsSL -o%PKG_CONFIG_PATH%\libpthreadVC2.pc http://public.kymeria.fr/KIWIX/windows/libpthreadVC2.pc || exit /b 1
-cd ..

appveyor/install_pugixml.cmd

@@ -1,10 +0,0 @@
-REM ========================================================
-REM Install pugixml
-curl -fsSL -O http://public.kymeria.fr/KIWIX/windows/pugixml-1.2-meson.zip || exit /b 1
-7z x pugixml-1.2-meson.zip -o. || exit /b 1
-cd pugixml-1.2-meson
-meson . build --prefix %EXTRA_DIR% --default-library static --buildtype release || exit /b 1
-cd build
-ninja || exit /b 1
-ninja install || exit /b 1
-cd ..\..

appveyor/install_xapian.cmd

@@ -1,9 +0,0 @@
-REM ========================================================
-REM Install xapian
-curl -fsSL -O http://mirror.download.kiwix.org/dev/kiwix-build/xapian-core-1.4.18.zip || exit /b 1
-7z x xapian-core-1.4.18.zip || exit /b 1
-cd xapian-core-1.4.18
-mkdir build
-cd build
-%MINGW64_RUN% "cd /c/Projects/kiwix-build/xapian-core-1.4.18/build && /c/Projects/kiwix-build/appveyor/build_xapian.sh" > build_xapian.log || exit /b 1
-cd ..\..

appveyor/install_zlib.cmd

@@ -1,10 +0,0 @@
-REM ========================================================
-REM Install zlib
-curl -fsSL -O http://mirror.download.kiwix.org/dev/kiwix-build/zlib-1.2.12.meson.zip || exit /b 1
-7z x zlib-1.2.12.meson.zip || exit /b 1
-cd zlib-1.2.12
-meson . build --prefix %EXTRA_DIR% --default-library static --buildtype release || exit /b 1
-cd build
-ninja || exit /b 1
-ninja install || exit /b 1
-cd ..\..

appveyor/install_zstd.cmd

@@ -1,10 +0,0 @@
-REM ========================================================
-REM Install zstd
-curl -fsSL -o zstd-v1.5.2.zip https://github.com/facebook/zstd/archive/refs/tags/v1.5.2.zip || exit /b 1
-7z x zstd-v1.5.2.zip || exit /b 1
-cd zstd-1.5.2/build/meson
-meson . builddir --prefix %EXTRA_DIR% --default-library static --buildtype release -Dbin_programs=false -Dbin_contrib=false || exit /b 1
-cd builddir
-ninja || exit /b 1
-ninja install || exit /b 1
-cd ..\..\..\..

appveyor/package_kiwix-desktop.sh

@@ -1,35 +0,0 @@
-
-set -e
-
-if [[ "$APPVEYOR_REPO_TAG" = "false" ]]
-then
-  NIGHTLY_DATE=$(date +%Y-%m-%d)
-  KIWIX_ARCHIVES_DIR=/c/projects/NIGHTLY_KIWIX_ARCHIVES/${NIGHTLY_DATE}
-  KIWIX_DIR=kiwix-desktop_windows_x64_$NIGHTLY_DATE
-else
-  if [ $KIWIX_DESKTOP_RELEASE  -eq 1 ]
-  then
-    KIWIX_ARCHIVES_DIR=/c/projects/RELEASE_KIWIX_ARCHIVES
-    KIWIX_DIR=kiwix-desktop_windows_x64_${KIWIX_DESKTOP_VERSION}
-  fi
-fi
-
-if [[ "$KIWIX_DIR" ]]
-then
-  KIWIX_ARCH_NAME=${KIWIX_DIR}.zip
-
-  mkdir $KIWIX_DIR
-  mkdir -p KIWIX_ARCHIVES_DIR
-
-  cp /c/projects/kiwix-build/kiwix-desktop/Release/kiwix-desktop.exe $KIWIX_DIR
-/c/Qt/5.15/msvc2019_64/bin/windeployqt --compiler-runtime $KIWIX_DIR
-
-  cp $MINGW64_EXTRA_DIR/aria2c.exe $KIWIX_DIR
-  cp $MINGW64_EXTRA_DIR/bin/*.dll $KIWIX_DIR
-  cp /c/OpenSSL-v111-Win64/bin/libcrypto-1_1-x64.dll $KIWIX_DIR
-  cp /c/OpenSSL-v111-Win64/bin/libssl-1_1-x64.dll $KIWIX_DIR
-
-  signtool.exe sign -f appveyor/kiwix.pfx -p $win_certificate_password -t http://timestamp.digicert.com -d "Kiwix-desktop application" -fd SHA256 $KIWIX_DIR/kiwix-desktop.exe
-
-  7z a -tzip $KIWIX_ARCHIVES_DIR/$KIWIX_ARCH_NAME $KIWIX_DIR
-fi

appveyor/setup_from_cache.cmd

@@ -1,4 +0,0 @@
-REM ========================================================
-REM Setup from cache
-curl -fsSL -O http://tmp.kiwix.org/ci/deps_windows_windows.zip || exit /b 1
-7z x deps_windows_windows.zip -oc: -aoa || exit /b 1

kiwixbuild/__init__.py

@@ -4,81 +4,163 @@ import os, sys
 import argparse
 
 from .dependencies import Dependency
-from .platforms import PlatformInfo
+from .configs import ConfigInfo
 from .builder import Builder
 from .flatpak_builder import FlatpakBuilder
 from . import _global
 
+
 def parse_args():
     parser = argparse.ArgumentParser()
-    parser.add_argument('target', default='kiwix-tools', nargs='?', metavar='TARGET',
+    parser.add_argument(
-                        choices=Dependency.all_deps.keys())
+        "target",
-    parser.add_argument('--working-dir', default=".",
+        default="kiwix-tools",
-                        help=("Directory where kiwix-build puts all its files "
+        nargs="?",
-                              "(source, archive and build)\n"
+        metavar="TARGET",
-                              "working-dir can be absolute path or a relative (to cwd) one."))
+        choices=Dependency.all_deps.keys(),
-    parser.add_argument('--build-dir', default=".",
+    )
-                        help=("Directory where kiwix-build puts all build files.\n"
+    parser.add_argument(
-                              "build-dir can be absolute path or a relative (to working-dir) one."))
+        "--working-dir",
-    parser.add_argument('--libprefix', default=None)
+        default=".",
-    parser.add_argument('--target-platform', choices=PlatformInfo.all_platforms)
+        help=(
-    parser.add_argument('--verbose', '-v', action="store_true",
+            "Directory where kiwix-build puts all its files "
-                        help=("Print all logs on stdout instead of in specific"
+            "(source, archive and build)\n"
-                              " log files per commands"))
+            "working-dir can be absolute path or a relative (to cwd) one."
-    parser.add_argument('--hide-progress', action='store_false', dest='show_progress',
+        ),
-                        help="Hide intermediate progress information.")
+    )
-    parser.add_argument('--skip-source-prepare', action='store_true',
+    parser.add_argument(
-                        help="Skip the source download part")
+        "--build-dir",
-    parser.add_argument('--build-deps-only', action='store_true',
+        default=".",
-                        help="Build only the dependencies of the specified target.")
+        help=(
-    parser.add_argument('--build-nodeps', action='store_true',
+            "Directory where kiwix-build puts all build files.\n"
-                        help="Build only the target, not its dependencies.")
+            "build-dir can be absolute path or a relative (to working-dir) one."
-    parser.add_argument('--make-dist', action='store_true',
+        ),
-                        help="Build distrubution (dist) source archive")
+    )
-    parser.add_argument('--make-release', action='store_true',
+    parser.add_argument("--libprefix", default=None)
-                        help="Build a release version")
+    parser.add_argument(
-    subgroup = parser.add_argument_group('advanced')
+        "--config", choices=ConfigInfo.all_configs, default="native_dyn"
-    subgroup.add_argument('--no-cert-check', action='store_true',
+    )
-                          help="Skip SSL certificate verification during download")
+    parser.add_argument(
-    subgroup.add_argument('--clean-at-end', action='store_true',
+        "--verbose",
-                          help="Clean all intermediate files after the (successfull) build")
+        "-v",
-    subgroup.add_argument('--dont-install-packages', action='store_true',
+        action="store_true",
-                          help="Do not try to install packages before compiling")
+        help=(
-    subgroup.add_argument('--assume-packages-installed', action='store_true',
+            "Print all logs on stdout instead of in specific" " log files per commands"
-                          help="Assume the package to install to be aleady installed")
+        ),
-    subgroup.add_argument('--android-arch', action='append',
+    )
-                          help=("Specify the architecture to build for android application/libraries.\n"
+    parser.add_argument(
-                                "Can be specified several times to build for several architectures.\n"
+        "--hide-progress",
-                                "If not specified, all architectures will be build."))
+        action="store_false",
-    subgroup.add_argument('--ios-arch', action='append',
+        dest="show_progress",
-                          help=("Specify the architecture to build for ios application/libraries.\n"
+        help="Hide intermediate progress information.",
-                                "Can be specified several times to build for several architectures.\n"
+    )
-                                "If not specified, all architectures will be build."))
+    parser.add_argument(
-    subgroup.add_argument('--fast-clone', action='store_true',
+        "--skip-source-prepare",
-                          help=("Do not clone the whole repository.\n"
+        action="store_true",
-                                "This is useful for one shot build but it is not recommended if you want "
+        help="Skip the source download part",
-                                "to develop with the cloned sources."))
+    )
+    parser.add_argument(
+        "--build-deps-only",
+        action="store_true",
+        help="Build only the dependencies of the specified target.",
+    )
+    parser.add_argument(
+        "--build-nodeps",
+        action="store_true",
+        help="Build only the target, not its dependencies.",
+    )
+    parser.add_argument(
+        "--make-dist",
+        action="store_true",
+        help="Build distrubution (dist) source archive",
+    )
+    parser.add_argument(
+        "--make-release", action="store_true", help="Build a release version"
+    )
+    subgroup = parser.add_argument_group("advanced")
+    subgroup.add_argument(
+        "--no-cert-check",
+        action="store_true",
+        help="Skip SSL certificate verification during download",
+    )
+    subgroup.add_argument(
+        "--clean-at-end",
+        action="store_true",
+        help="Clean all intermediate files after the (successfull) build",
+    )
+    subgroup.add_argument(
+        "--dont-install-packages",
+        action="store_true",
+        help="Do not try to install packages before compiling",
+    )
+    subgroup.add_argument(
+        "--assume-packages-installed",
+        action="store_true",
+        help="Assume the package to install to be aleady installed",
+    )
+    subgroup.add_argument(
+        "--android-arch",
+        action="append",
+        help=(
+            "Specify the architecture to build for android application/libraries.\n"
+            "Can be specified several times to build for several architectures.\n"
+            "If not specified, all architectures will be build."
+        ),
+    )
+    subgroup.add_argument(
+        "--ios-arch",
+        action="append",
+        help=(
+            "Specify the architecture to build for ios application/libraries.\n"
+            "Can be specified several times to build for several architectures.\n"
+            "If not specified, all architectures will be build."
+        ),
+    )
+    subgroup.add_argument(
+        "--fast-clone",
+        action="store_true",
+        help=(
+            "Do not clone the whole repository.\n"
+            "This is useful for one shot build but it is not recommended if you want "
+            "to develop with the cloned sources."
+        ),
+    )
+    subgroup.add_argument(
+        "--use-target-arch-name",
+        action="store_true",
+        help=(
+            "Name the build directory using the arch name instead of the config name.\n"
+            "Different configs may create binary for the same arch so this option is "
+            "not recommended when working with several config on the same computer.\n"
+            "However, when generating dependencies for other it is better to have a "
+            "directory named using the target instead of the used config.\n"
+            "Intended to be used in CI only."
+        ),
+    )
+    subgroup.add_argument(
+        "--get-build-dir", action="store_true", help="Print the output directory."
+    )
     options = parser.parse_args()
 
     if not options.android_arch:
-        options.android_arch = ['arm', 'arm64', 'x86', 'x86_64']
+        options.android_arch = ["arm", "arm64", "x86", "x86_64"]
     if not options.ios_arch:
-        options.ios_arch = ['arm64', 'x86_64']
+        options.ios_arch = ["arm64", "x86_64"]
-
-    if not options.target_platform:
-        options.target_platform = 'native_dyn'
 
     return options
 
+
 def main():
     options = parse_args()
     options.working_dir = os.path.abspath(options.working_dir)
     _global.set_options(options)
-    neutralEnv = buildenv.PlatformNeutralEnv()
+    neutralEnv = buildenv.NeutralEnv(options.get_build_dir)
     _global.set_neutralEnv(neutralEnv)
-    if options.target_platform == 'flatpak':
+    if options.config == "flatpak":
         builder = FlatpakBuilder()
     else:
         builder = Builder()
-    builder.run()
+    if options.get_build_dir:
-
+        print(ConfigInfo.get_config(options.config).buildEnv.build_dir)
+    else:
+        builder.run()

kiwixbuild/_global.py

@@ -5,23 +5,29 @@ _neutralEnv = None
 _options = None
 _target_steps = _OrderedDict()
 
+
 def set_neutralEnv(env):
     global _neutralEnv
     _neutralEnv = env
 
+
 def neutralEnv(what):
     return getattr(_neutralEnv, what)
 
+
 def set_options(options):
     global _options
     _options = options
 
+
 def option(what):
     return getattr(_options, what)
 
+
 def add_target_step(key, what):
     _target_steps[key] = what
 
+
 def get_target_step(key, default_context=None):
     if isinstance(key, tuple):
         context, target = key
@@ -29,24 +35,6 @@ def get_target_step(key, default_context=None):
         context, target = default_context, key
     return _target_steps[(context, target)]
 
+
 def target_steps():
     return _target_steps
-
-
-def backend():
-    global _backend
-    if _backend is not None:
-        return _backend
-
-    _platform = platform.system()
-    if _platform == 'Windows':
-        print('ERROR: kiwix-build is not intented to run on Windows platform.\n'
-              'There is no backend for Windows, so we can\'t launch any commands.')
-        sys.exit(0)
-    if _platform == 'Linux':
-        _platform, _, _ = platform.linux_distribution()
-        _platform = _platform.lower()
-        _backend = backends.Linux()
-
-    return _backend
-

kiwixbuild/buildenv.py

@@ -1,93 +1,96 @@
-
 import os, sys, shutil
 import subprocess
 import platform
 import distro
 
-from .utils import pj, download_remote, Defaultdict
+from .utils import pj, download_remote, escape_path
 from ._global import neutralEnv, option
 
 
-class PlatformNeutralEnv:
+class NeutralEnv:
-    def __init__(self):
+    def __init__(self, dummy_run):
-        self.working_dir = option('working_dir')
+        self.working_dir = option("working_dir")
         self.source_dir = pj(self.working_dir, "SOURCE")
         self.archive_dir = pj(self.working_dir, "ARCHIVE")
         self.toolchain_dir = pj(self.working_dir, "TOOLCHAINS")
-        self.log_dir = pj(self.working_dir, 'LOGS')
+        self.log_dir = pj(self.working_dir, "LOGS")
-        for d in (self.source_dir,
+        for d in (self.source_dir, self.archive_dir, self.toolchain_dir, self.log_dir):
-                  self.archive_dir,
-                  self.toolchain_dir,
-                  self.log_dir):
             os.makedirs(d, exist_ok=True)
         self.detect_platform()
-        self.ninja_command = self._detect_ninja()
+        if dummy_run:
-        if not self.ninja_command:
+            # If this is for a dummy run, we will not run anything.
-            sys.exit("ERROR: ninja command not found.")
+            # To check for command (and so, don't enforce their presence)
-        self.meson_command = self._detect_meson()
+            return
-        if not self.meson_command:
+        self.ninja_command = self._detect_command(
-            sys.exit("ERROR: meson command not found.")
+            "ninja", default=[["ninja"], ["ninja-build"]]
-        self.qmake_command = self._detect_qmake()
+        )
-        if not self.qmake_command:
+        self.meson_command = self._detect_command(
-            print("WARNING: qmake command not found.", file=sys.stderr)
+            "meson", default=[["meson.py"], ["meson"]]
-        self.mesontest_command = "{} test".format(self.meson_command)
+        )
+        self.mesontest_command = [*self.meson_command, "test"]
+        self.patch_command = self._detect_command("patch")
+        self.git_command = self._detect_command("git")
+        if platform.system() == "Windows":
+            self.make_command = self._detect_command("nmake", options=["/?", "/NOLOGO"])
+        else:
+            self.make_command = self._detect_command("make")
+        self.cmake_command = self._detect_command("cmake")
+        self.qmake_command = self._detect_command(
+            "qmake", required=False, default=[["qmake"], ["qmake-qt5"]]
+        )
 
     def detect_platform(self):
         _platform = platform.system()
         self.distname = _platform
-        if _platform == 'Windows':
+        if _platform == "Linux":
-            print('ERROR: kiwix-build is not intented to run on Windows platform.\n'
-                  'It should probably not work, but well, you still can have a try.')
-            cont = input('Do you want to continue ? [y/N]')
-            if cont.lower() != 'y':
-                sys.exit(0)
-        if _platform == 'Linux':
             self.distname = distro.id()
-            if self.distname == 'ubuntu':
+            if self.distname == "ubuntu":
-                self.distname = 'debian'
+                self.distname = "debian"
 
     def download(self, what, where=None):
         where = where or self.archive_dir
         download_remote(what, where)
 
-    def _detect_binary(self, *bin_variants):
+    def _detect_command(self, name, default=None, options=["--version"], required=True):
-        for n in bin_variants:
+        if default is None:
+            default = [[name]]
+        env_key = "KBUILD_{}_COMMAND".format(name.upper())
+        if env_key in os.environ:
+            default = [os.environ[env_key].split()] + default
+        for command in default:
             try:
-                retcode = subprocess.check_call([n, '--version'],
+                retcode = subprocess.check_call(
-                                                stdout=subprocess.DEVNULL)
+                    command + options, stdout=subprocess.DEVNULL
-            except (FileNotFoundError, PermissionError):
+                )
+            except (FileNotFoundError, PermissionError, OSError):
                 # Doesn't exist in PATH or isn't executable
                 continue
             if retcode == 0:
-                return n
+                return command
-
+        else:
-
+            if required:
-    def _detect_ninja(self):
+                sys.exit("ERROR: {} command not found".format(name))
-        return self._detect_binary('ninja', 'ninja-build')
+            else:
-
+                print("WARNING: {} command not found".format(name), file=sys.stderr)
-    def _detect_meson(self):
+                return ["{}_NOT_FOUND".format(name.upper())]
-        return self._detect_binary('meson.py', 'meson')
-
-    def _detect_qmake(self):
-        return self._detect_binary('qmake-qt5', 'qmake')
 
 
 class BuildEnv:
-    def __init__(self, platformInfo):
+    def __init__(self, configInfo):
-        build_dir = "BUILD_{}".format(platformInfo.name)
+        self.configInfo = configInfo
-        self.platformInfo = platformInfo
+        self.base_build_dir = pj(option("working_dir"), option("build_dir"))
-        self.base_build_dir = pj(option('working_dir'), option('build_dir'))
+        build_dir = (
+            configInfo.arch_name if option("use_target_arch_name") else configInfo.name
+        )
+        build_dir = f"BUILD_{build_dir}"
         self.build_dir = pj(self.base_build_dir, build_dir)
         self.install_dir = pj(self.build_dir, "INSTALL")
         self.toolchain_dir = pj(self.build_dir, "TOOLCHAINS")
-        self.log_dir = pj(self.build_dir, 'LOGS')
+        self.log_dir = pj(self.build_dir, "LOGS")
-        for d in (self.build_dir,
+        for d in (self.build_dir, self.install_dir, self.toolchain_dir, self.log_dir):
-                  self.install_dir,
-                  self.toolchain_dir,
-                  self.log_dir):
             os.makedirs(d, exist_ok=True)
 
-        self.libprefix = option('libprefix') or self._detect_libdir()
+        self.libprefix = option("libprefix") or self._detect_libdir()
 
     def clean_intermediate_directories(self):
         for subdir in os.listdir(self.build_dir):
@@ -100,69 +103,81 @@ class BuildEnv:
                 os.remove(subpath)
 
     def _is_debianlike(self):
-        return os.path.isfile('/etc/debian_version')
+        return os.path.isfile("/etc/debian_version")
 
     def _detect_libdir(self):
-        if self.platformInfo.libdir is not None:
+        if self.configInfo.libdir is not None:
-            return self.platformInfo.libdir
+            return self.configInfo.libdir
         if self._is_debianlike():
             try:
-                pc = subprocess.Popen(['dpkg-architecture', '-qDEB_HOST_MULTIARCH'],
+                pc = subprocess.Popen(
-                                      stdout=subprocess.PIPE,
+                    ["dpkg-architecture", "-qDEB_HOST_MULTIARCH"],
-                                      stderr=subprocess.DEVNULL)
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.DEVNULL,
+                )
                 (stdo, _) = pc.communicate()
                 if pc.returncode == 0:
                     archpath = stdo.decode().strip()
-                    return 'lib/' + archpath
+                    return "lib/" + archpath
             except Exception:
                 pass
-        if os.path.isdir('/usr/lib64') and not os.path.islink('/usr/lib64'):
+        if os.path.isdir("/usr/lib64") and not os.path.islink("/usr/lib64"):
-            return 'lib64'
+            return "lib64"
-        return 'lib'
+        return "lib"
 
     def get_env(self, *, cross_comp_flags, cross_compilers, cross_path):
-        env = self.platformInfo.get_env()
+        env = self.configInfo.get_env()
-        pkgconfig_path = pj(self.install_dir, self.libprefix, 'pkgconfig')
+        pkgconfig_path = pj(self.install_dir, self.libprefix, "pkgconfig")
-        env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path])
+        env["PKG_CONFIG_PATH"].append(pkgconfig_path)
 
-        env['PATH'] = ':'.join([pj(self.install_dir, 'bin'), env['PATH']])
+        env["PATH"].insert(0, pj(self.install_dir, "bin"))
 
-        env['LD_LIBRARY_PATH'] = ':'.join([env['LD_LIBRARY_PATH'],
+        env["LD_LIBRARY_PATH"].extend(
-                                          pj(self.install_dir, 'lib'),
+            [
-                                          pj(self.install_dir, self.libprefix)
+                pj(self.install_dir, "lib"),
-                                          ])
+                pj(self.install_dir, self.libprefix),
+            ]
+        )
 
-        env['QMAKE_CXXFLAGS'] = " ".join(['-I'+pj(self.install_dir, 'include'), env['QMAKE_CXXFLAGS']])
+        env["QMAKE_CXXFLAGS"] = " ".join(
-        env['CPPFLAGS'] = " ".join(['-I'+pj(self.install_dir, 'include'), env['CPPFLAGS']])
+            [escape_path("-I" + pj(self.install_dir, "include")), env["QMAKE_CXXFLAGS"]]
-        env['QMAKE_LFLAGS'] = " ".join(['-L'+pj(self.install_dir, 'lib'),
+        )
-                                   '-L'+pj(self.install_dir, self.libprefix),
+        env["CPPFLAGS"] = " ".join(
-                                   env['QMAKE_LFLAGS']])
+            [escape_path("-I" + pj(self.install_dir, "include")), env["CPPFLAGS"]]
-        env['LDFLAGS'] = " ".join(['-L'+pj(self.install_dir, 'lib'),
+        )
-                                   '-L'+pj(self.install_dir, self.libprefix),
+        env["QMAKE_LFLAGS"] = " ".join(
-                                   env['LDFLAGS']])
+            [
+                escape_path("-L" + pj(self.install_dir, "lib")),
+                escape_path("-L" + pj(self.install_dir, self.libprefix)),
+                env["QMAKE_LFLAGS"],
+            ]
+        )
+        env["LDFLAGS"] = " ".join(
+            [
+                escape_path("-L" + pj(self.install_dir, "lib")),
+                escape_path("-L" + pj(self.install_dir, self.libprefix)),
+                env["LDFLAGS"],
+            ]
+        )
 
         if cross_comp_flags:
-            self.platformInfo.set_comp_flags(env)
+            self.configInfo.set_comp_flags(env)
         if cross_compilers:
-            self.platformInfo.set_compiler(env)
+            self.configInfo.set_compiler(env)
         if cross_path:
-            env['PATH'] = ':'.join(self.platformInfo.get_bin_dir() + [env['PATH']])
+            env["PATH"][0:0] = self.configInfo.get_bin_dir()
         return env
 
-
     @property
     def configure_wrapper(self):
-        wrapper = getattr(self.platformInfo, "configure_wrapper", "")
+        try:
-        if wrapper:
+            yield self.configInfo.configure_wrapper
-            return "{} ".format(wrapper)
+        except AttributeError:
-        else:
+            pass
-            return ""
 
     @property
     def make_wrapper(self):
-        wrapper = getattr(self.platformInfo, "make_wrapper", "")
+        try:
-        if wrapper:
+            yield self.configInfo.make_wrapper
-            return "{} ".format(wrapper)
+        except AttributeError:
-        else:
+            pass
-            return ""
-

kiwixbuild/builder.py

@@ -1,30 +1,36 @@
-
 import sys
 from collections import OrderedDict
 from .buildenv import *
 
-from .platforms import PlatformInfo
+from .configs import ConfigInfo
 from .utils import remove_duplicates, StopBuild, colorize
 from .dependencies import Dependency
 from .packages import PACKAGE_NAME_MAPPERS
 from ._global import (
-    neutralEnv, option,
+    neutralEnv,
-    add_target_step, get_target_step, target_steps,
+    option,
-    backend)
+    add_target_step,
+    get_target_step,
+    target_steps,
+)
 from . import _global
 
+
 class Builder:
     def __init__(self):
         self._targets = {}
-        PlatformInfo.get_platform('neutral', self._targets)
+        ConfigInfo.get_config("neutral", self._targets)
 
-        target_platform = option('target_platform')
+        config_name = option("config")
-        platform = PlatformInfo.get_platform(target_platform, self._targets)
+        config = ConfigInfo.get_config(config_name, self._targets)
-        if neutralEnv('distname') not in platform.compatible_hosts:
+        if neutralEnv("distname") not in config.compatible_hosts:
-            print((colorize('ERROR')+': The target platform {} cannot be build on host {}.\n'
+            print(
-                   'Select another target platform or change your host system.'
+                (
-                  ).format(platform.name, neutralEnv('distname')))
+                    colorize("ERROR") + ": The config {} cannot be build on host {}.\n"
-        self.targetDefs = platform.add_targets(option('target'), self._targets)
+                    "Select another config or change your host system."
+                ).format(config.name, neutralEnv("distname"))
+            )
+        self.targetDefs = config.add_targets(option("target"), self._targets)
 
     def finalize_target_steps(self):
         steps = []
@@ -32,36 +38,36 @@ class Builder:
             steps += self.order_steps(targetDef)
         steps = list(remove_duplicates(steps))
 
-        if option('build_nodeps'):
+        if option("build_nodeps"):
-            # add all platform steps
+            # add all config steps
             for dep in steps:
                 stepClass = Dependency.all_deps[dep[1]]
                 if stepClass.dont_skip:
                     add_target_step(dep, self._targets[dep])
 
-            src_targetDef = ('source', targetDef[1])
+            src_targetDef = ("source", targetDef[1])
             add_target_step(src_targetDef, self._targets[src_targetDef])
             add_target_step(targetDef, self._targets[targetDef])
         else:
             for dep in steps:
-                if option('build_deps_only') and dep[1] == targetDef[1]:
+                if option("build_deps_only") and dep[1] == targetDef[1]:
                     continue
                 add_target_step(dep, self._targets[dep])
         self.instanciate_steps()
 
     def order_steps(self, targetDef):
-        for pltName in PlatformInfo.all_running_platforms:
+        for cfgName in ConfigInfo.all_running_configs:
-            plt = PlatformInfo.all_platforms[pltName]
+            cfg = ConfigInfo.all_configs[cfgName]
-            for tlcName in plt.toolchain_names:
+            for tlcName in cfg.toolchain_names:
                 tlc = Dependency.all_deps[tlcName]
-                yield('source', tlcName)
+                yield ("source", tlcName)
-                yield('neutral' if tlc.neutral else pltName, tlcName)
+                yield ("neutral" if tlc.neutral else cfgName, tlcName)
-        _targets =dict(self._targets)
+        _targets = dict(self._targets)
         yield from self.order_dependencies(targetDef, _targets)
 
     def order_dependencies(self, targetDef, targets):
-        targetPlatformName, targetName = targetDef
+        targetConfigName, targetName = targetDef
-        if targetPlatformName == 'source':
+        if targetConfigName == "source":
             # Do not try to order sources, they will be added as dep by the
             # build step two lines later.
             return
@@ -70,43 +76,45 @@ class Builder:
         except KeyError:
             return
 
-        targetPlatform = PlatformInfo.get_platform(targetPlatformName)
+        targetConfig = ConfigInfo.get_config(targetConfigName)
-        for dep in target.get_dependencies(targetPlatform, True):
+        for dep in target.get_dependencies(targetConfig, True):
-            depPlatform, depName = targetPlatform.get_fully_qualified_dep(dep)
+            depConfig, depName = targetConfig.get_fully_qualified_dep(dep)
-            if (depPlatform, depName) in targets:
+            if (depConfig, depName) in targets:
-                yield from self.order_dependencies((depPlatform, depName), targets)
+                yield from self.order_dependencies((depConfig, depName), targets)
-        yield ('source', targetName)
+        yield ("source", targetName)
         yield targetDef
 
     def instanciate_steps(self):
         for stepDef in list(target_steps()):
-            stepPlatform, stepName = stepDef
+            stepConfig, stepName = stepDef
             stepClass = Dependency.all_deps[stepName]
-            if stepPlatform == 'source':
+            if stepConfig == "source":
                 source = get_target_step(stepDef)(stepClass)
                 add_target_step(stepDef, source)
             else:
-                source = get_target_step(stepName, 'source')
+                source = get_target_step(stepName, "source")
-                env = PlatformInfo.get_platform(stepPlatform).buildEnv
+                env = ConfigInfo.get_config(stepConfig).buildEnv
                 builder = get_target_step(stepDef)(stepClass, source, env)
                 add_target_step(stepDef, builder)
 
     def prepare_sources(self):
-        if option('skip_source_prepare'):
+        if option("skip_source_prepare"):
             print(colorize("SKIP"))
             return
 
-        sourceDefs = remove_duplicates(tDef for tDef in target_steps() if tDef[0]=='source')
+        sourceDefs = remove_duplicates(
+            tDef for tDef in target_steps() if tDef[0] == "source"
+        )
         for sourceDef in sourceDefs:
             print("prepare sources {} :".format(sourceDef[1]))
             source = get_target_step(sourceDef)
             source.prepare()
 
     def build(self):
-        builderDefs = (tDef for tDef in target_steps() if tDef[0] != 'source')
+        builderDefs = (tDef for tDef in target_steps() if tDef[0] != "source")
         for builderDef in builderDefs:
             builder = get_target_step(builderDef)
-            if option('make_dist') and builderDef[1] == option('target'):
+            if option("make_dist") and builderDef[1] == option("target"):
                 print("make dist {} ({}):".format(builder.name, builderDef[0]))
                 builder.make_dist()
                 continue
@@ -116,19 +124,19 @@ class Builder:
 
     def _get_packages(self):
         packages_list = []
-        for platform in PlatformInfo.all_running_platforms.values():
+        for config in ConfigInfo.all_running_configs.values():
-            mapper_name = "{host}_{target}".format(
+            mapper_name = "{host}_{config}".format(
-                host=neutralEnv('distname'),
+                host=neutralEnv("distname"), config=config
-                target=platform)
+            )
             package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {})
-            packages_list += package_name_mapper.get('COMMON', [])
+            packages_list += package_name_mapper.get("COMMON", [])
 
         to_drop = []
         for builderDef in self._targets:
-            platformName, builderName = builderDef
+            configName, builderName = builderDef
-            mapper_name = "{host}_{target}".format(
+            mapper_name = "{host}_{config}".format(
-                host=neutralEnv('distname'),
+                host=neutralEnv("distname"), config=configName
-                target=platformName)
+            )
             package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {})
             packages = package_name_mapper.get(builderName)
             if packages:
@@ -144,20 +152,20 @@ class Builder:
         packages_to_have = self._get_packages()
         packages_to_have = remove_duplicates(packages_to_have)
 
-        if option('assume_packages_installed'):
+        if option("assume_packages_installed"):
             print(colorize("SKIP") + ", Assume package installed")
             return
 
-        distname = neutralEnv('distname')
+        distname = neutralEnv("distname")
-        if distname in ('fedora', 'redhat', 'centos'):
+        if distname in ("fedora", "redhat", "centos"):
-            package_installer = 'sudo dnf install {}'
+            package_installer = "sudo dnf install {}"
-            package_checker = 'rpm -q --quiet {}'
+            package_checker = "rpm -q --quiet {}"
-        elif distname in ('debian', 'Ubuntu'):
+        elif distname in ("debian", "Ubuntu"):
-            package_installer = 'sudo apt-get install {}'
+            package_installer = "sudo apt-get install {}"
             package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1'
-        elif distname == 'Darwin':
+        elif distname == "Darwin":
-            package_installer = 'brew install {}'
+            package_installer = "brew install {}"
-            package_checker = 'brew ls --version {} > /dev/null'
+            package_checker = "brew ls --version {} > /dev/null"
 
         packages_to_install = []
         for package in packages_to_have:
@@ -176,31 +184,30 @@ class Builder:
             print(command)
             subprocess.check_call(command, shell=True)
         else:
-            print(colorize("SKIP")+ ", No package to install.")
+            print(colorize("SKIP") + ", No package to install.")
 
     def run(self):
         try:
             print("[INSTALL PACKAGES]")
-            if option('dont_install_packages'):
+            if option("dont_install_packages"):
                 print(colorize("SKIP"))
             else:
                 self.install_packages()
             self.finalize_target_steps()
-            print("[SETUP PLATFORMS]")
+            print("[SETUP TOOLCHAINS]")
-            for platform in PlatformInfo.all_running_platforms.values():
+            for config in ConfigInfo.all_running_configs.values():
-                platform.finalize_setup()
+                config.finalize_setup()
             print("[PREPARE]")
             self.prepare_sources()
             print("[BUILD]")
             self.build()
             # No error, clean intermediate file at end of build if needed.
             print("[CLEAN]")
-            if option('clean_at_end'):
+            if option("clean_at_end"):
-                for platform in PlatformInfo.all_running_platforms.values():
+                for config in ConfigInfo.all_running_configs.values():
-                    platform.clean_intermediate_directories()
+                    config.clean_intermediate_directories()
             else:
                 print(colorize("SKIP"))
         except StopBuild as e:
             print(e)
             sys.exit("Stopping build due to errors")
-

kiwixbuild/configs/__init__.py

@@ -0,0 +1,3 @@
+from .base import *
+
+from . import android, armhf, musl, flatpak, i586, ios, native, neutral, wasm

kiwixbuild/configs/android.py

@@ -0,0 +1,169 @@
+from .base import ConfigInfo, MetaConfigInfo
+from kiwixbuild.utils import pj
+from kiwixbuild._global import get_target_step, option
+
+
+class AndroidConfigInfo(ConfigInfo):
+    build = "android"
+    static = True
+    toolchain_names = ["android-ndk"]
+    compatible_hosts = ["fedora", "debian"]
+
+    def __str__(self):
+        return "android"
+
+    @property
+    def libdir(self):
+        return "lib/{}".format(self.arch_full)
+
+    @property
+    def binaries_name(self):
+        arch_full = self.arch_full
+        return {
+            "CC": "{}-{}".format(arch_full, "clang"),
+            "CXX": "{}-{}".format(arch_full, "clang++"),
+            "AR": "{}-{}".format(arch_full, "ar"),
+            "STRIP": "{}-{}".format(arch_full, "strip"),
+            "RANLIB": "{}-{}".format(arch_full, "ranlib"),
+            "LD": "{}-{}".format(arch_full, "ld"),
+        }
+
+    def binaries(self):
+        install_path = self.install_path
+        binaries = {
+            k: pj(install_path, "bin", v) for k, v in self.binaries_name.items()
+        }
+        binaries["PKGCONFIG"] = "pkg-config"
+        return binaries
+
+    @property
+    def ndk_builder(self):
+        return get_target_step("android-ndk", self.name)
+
+    @property
+    def install_path(self):
+        return self.ndk_builder.install_path
+
+    def get_cross_config(self):
+        extra_libs = [
+            "-llog",
+            "-Wl,--exclude-libs,libgcc.a",
+            "-Wl,--exclude-libs,libunwind.a",
+        ]
+        extra_cflags = [
+            "-I{}".format(include_dir) for include_dir in self.get_include_dirs()
+        ]
+        if hasattr(self, "march"):
+            extra_libs.append("-march={}".format(self.march))
+            extra_cflags.append("-march={}".format(self.march))
+        return {
+            "exe_wrapper_def": "",
+            "install_path": self.install_path,
+            "binaries": self.binaries(),
+            "root_path": pj(self.install_path, "sysroot"),
+            "extra_libs": extra_libs,
+            "extra_cflags": extra_cflags,
+            "host_machine": {
+                "system": "Android",
+                "lsystem": "android",
+                "cpu_family": self.arch,
+                "cpu": self.cpu,
+                "endian": "little",
+                "abi": self.abi,
+            },
+        }
+
+    def get_env(self):
+        env = super().get_env()
+        root_path = pj(self.install_path, "sysroot")
+        env["PKG_CONFIG_LIBDIR"] = pj(root_path, "lib", "pkgconfig")
+        env["NDK_DEBUG"] = "0"
+        return env
+
+    def get_bin_dir(self):
+        return [pj(self.install_path, "bin")]
+
+    def set_comp_flags(self, env):
+        super().set_comp_flags(env)
+        root_path = pj(self.install_path, "sysroot")
+        march = "-march={}".format(self.march) if hasattr(self, "march") else ""
+        env["CFLAGS"] = (
+            "-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} {} ".format(
+                root_path, march
+            )
+            + env["CFLAGS"]
+        )
+        env["CXXFLAGS"] = (
+            "-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} {} ".format(
+                root_path, march
+            )
+            + env["CXXFLAGS"]
+        )
+        env["LDFLAGS"] = "--sysroot={} {} ".format(root_path, march) + env["LDFLAGS"]
+
+    def set_compiler(self, env):
+        binaries = self.binaries()
+        for k, v in binaries.items():
+            env[k] = v
+
+    @property
+    def configure_options(self):
+        yield "--host={}".format(self.arch_full)
+
+    def finalize_setup(self):
+        super().finalize_setup()
+        self.buildEnv.cmake_crossfile = self._gen_crossfile(
+            "cmake_android_cross_file.txt", "cmake_cross_file.txt"
+        )
+        self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
+
+
+class AndroidArm(AndroidConfigInfo):
+    name = "android_arm"
+    arch = cpu = "arm"
+    arch_full = "arm-linux-androideabi"
+    abi = "armeabi-v7a"
+    march = "armv7-a"
+
+
+class AndroidArm64(AndroidConfigInfo):
+    name = "android_arm64"
+    arch = "arm64"
+    arch_full = "aarch64-linux-android"
+    cpu = "aarch64"
+    abi = "arm64-v8a"
+
+
+class AndroidX86(AndroidConfigInfo):
+    name = "android_x86"
+    arch = abi = "x86"
+    arch_full = "i686-linux-android"
+    cpu = "i686"
+
+
+class AndroidX8664(AndroidConfigInfo):
+    name = "android_x86_64"
+    arch = cpu = abi = "x86_64"
+    arch_full = "x86_64-linux-android"
+
+
+class Android(MetaConfigInfo):
+    name = "android"
+    compatible_hosts = ["fedora", "debian"]
+
+    @property
+    def arch_name(self):
+        return "multi-linux-android"
+
+    @property
+    def subConfigNames(self):
+        return ["android_{}".format(arch) for arch in option("android_arch")]
+
+    def add_targets(self, targetName, targets):
+        return super().add_targets(targetName, targets)
+
+    def __str__(self):
+        return self.name
+
+    def set_comp_flags(self, env):
+        pass

kiwixbuild/configs/armhf.py

@@ -0,0 +1,183 @@
+from .base import ConfigInfo, MixedMixin
+
+from kiwixbuild.utils import pj
+from kiwixbuild._global import get_target_step
+
+
+# Base config for arm
+class ArmConfigInfo(ConfigInfo):
+    compatible_hosts = ["fedora", "debian", "almalinux"]
+
+    def get_cross_config(self):
+        return {
+            "binaries": self.binaries,
+            "exe_wrapper_def": "",
+            "root_path": self.root_path,
+            "extra_libs": [],
+            "extra_cflags": [
+                "-I{}".format(include_dir) for include_dir in self.get_include_dirs()
+            ],
+            "host_machine": {
+                "system": "linux",
+                "lsystem": "linux",
+                "cpu_family": self.cpu_family,
+                "cpu": self.cpu,
+                "endian": "little",
+                "abi": "",
+            },
+        }
+
+    @property
+    def libdir(self):
+        return "lib/{}".format(self.arch_full)
+
+    @property
+    def toolchain(self):
+        return get_target_step(self.build, "neutral")
+
+    @property
+    def root_path(self):
+        return self.toolchain.build_path
+
+    @property
+    def binaries(self):
+        binaries = (
+            (k, "{}-{}".format(self.arch_full, v))
+            for k, v in (
+                ("CC", "gcc"),
+                ("CXX", "g++"),
+                ("AR", "ar"),
+                ("STRIP", "strip"),
+                ("WINDRES", "windres"),
+                ("RANLIB", "ranlib"),
+                ("LD", "ld"),
+                ("LDSHARED", "g++ -shared"),
+            )
+        )
+        binaries = {k: pj(self.root_path, "bin", v) for k, v in binaries}
+        binaries["PKGCONFIG"] = "pkg-config"
+        return binaries
+
+    @property
+    def exe_wrapper_def(self):
+        try:
+            which("qemu-arm")
+        except subprocess.CalledProcessError:
+            return ""
+        else:
+            return "exe_wrapper = 'qemu-arm'"
+
+    @property
+    def configure_options(self):
+        yield "--host={}".format(self.arch_full)
+
+    def get_bin_dir(self):
+        return [pj(self.root_path, "bin")]
+
+    def get_env(self):
+        env = super().get_env()
+        env["LD_LIBRARY_PATH"][0:0] = [
+            pj(self.root_path, self.arch_full, "lib64"),
+            pj(self.root_path, "lib"),
+        ]
+        env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig")
+        env["QEMU_LD_PREFIX"] = pj(self.root_path, self.arch_full, "libc")
+        env["QEMU_SET_ENV"] = "LD_LIBRARY_PATH={}".format(
+            ":".join(
+                [pj(self.root_path, self.arch_full, "lib"), str(env["LD_LIBRARY_PATH"])]
+            )
+        )
+        return env
+
+    def set_comp_flags(self, env):
+        super().set_comp_flags(env)
+        env["CFLAGS"] = (
+            " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
+            + env["CFLAGS"]
+        )
+        env["CXXFLAGS"] = (
+            " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
+            + env["CXXFLAGS"]
+        )
+
+    def set_compiler(self, env):
+        for k, v in self.binaries.items():
+            env[k] = v
+
+    def finalize_setup(self):
+        super().finalize_setup()
+        self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt")
+        self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
+
+
+class Armv6(ArmConfigInfo):
+    build = "armv6"
+    arch_full = "armv6-rpi-linux-gnueabihf"
+    toolchain_names = ["armv6"]
+    cpu_family = "arm"
+    cpu = "armv6"
+
+
+class Armv6Dyn(Armv6):
+    name = "armv6_dyn"
+    static = False
+
+
+class Armv6Static(Armv6):
+    name = "armv6_static"
+    static = True
+
+
+class Armv6Mixed(MixedMixin("armv6_static"), Armv6):
+    name = "armv6_mixed"
+    static = False
+
+
+class Armv8(ArmConfigInfo):
+    build = "armv8"
+    arch_full = "armv8-rpi3-linux-gnueabihf"
+    toolchain_names = ["armv8"]
+    cpu_family = "arm"
+    cpu = "armv8"
+
+
+class Armv8Dyn(Armv8):
+    name = "armv8_dyn"
+    static = False
+
+
+class Armv8Static(Armv8):
+    name = "armv8_static"
+    static = True
+
+
+class Armv8Mixed(MixedMixin("armv8_static"), Armv8):
+    name = "armv8_mixed"
+    static = False
+
+
+class Aarch64(ArmConfigInfo):
+    build = "aarch64"
+    arch_full = "aarch64-linux-gnu"
+    toolchain_names = ["aarch64"]
+    cpu_family = "aarch64"
+    cpu = "aarch64"
+
+    @property
+    def root_path(self):
+        return self.toolchain.build_path
+
+
+class Aarch64Dyn(Aarch64):
+    name = "aarch64_dyn"
+    static = False
+
+
+class Aarch64Static(Aarch64):
+    name = "aarch64_static"
+    static = True
+
+
+class Aarch64Mixed(MixedMixin("aarch64_static"), Aarch64):
+    name = "aarch64_mixed"
+    static = False

kiwixbuild/configs/base.py

@@ -1,4 +1,3 @@
-
 import os, sys
 import subprocess
 
@@ -8,107 +7,105 @@ from kiwixbuild.buildenv import BuildEnv
 from kiwixbuild._global import neutralEnv, option, target_steps
 
 _SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
-TEMPLATES_DIR = pj(os.path.dirname(_SCRIPT_DIR), 'templates')
+TEMPLATES_DIR = pj(os.path.dirname(_SCRIPT_DIR), "templates")
 
-class _MetaPlatform(type):
+
+class _MetaConfig(type):
     def __new__(cls, name, bases, dct):
         _class = type.__new__(cls, name, bases, dct)
-        if name not in ('PlatformInfo', 'MetaPlatformInfo') and 'name' in dct:
+        if name not in ("ConfigInfo", "MetaConfigInfo") and "name" in dct:
-            dep_name = dct['name']
+            dep_name = dct["name"]
-            PlatformInfo.all_platforms[dep_name] = _class
+            ConfigInfo.all_configs[dep_name] = _class
         return _class
 
 
-class PlatformInfo(metaclass=_MetaPlatform):
+class ConfigInfo(metaclass=_MetaConfig):
-    all_platforms = {}
+    all_configs = {}
-    all_running_platforms = {}
+    all_running_configs = {}
     toolchain_names = []
-    configure_option = ""
+    configure_options = []
     mixed = False
     libdir = None
 
+    @property
+    def arch_name(self):
+        return self.arch_full
+
     @classmethod
-    def get_platform(cls, name, targets=None):
+    def get_config(cls, name, targets=None):
-        if name not in cls.all_running_platforms:
+        if name not in cls.all_running_configs:
             if targets is None:
                 print("Should not got there.")
-                print(cls.all_running_platforms)
+                print(cls.all_running_configs)
                 raise KeyError(name)
-            cls.all_running_platforms[name] = cls.all_platforms[name](targets)
+            cls.all_running_configs[name] = cls.all_configs[name](targets)
-        return cls.all_running_platforms[name]
+        return cls.all_running_configs[name]
 
     def __init__(self, targets):
-        self.all_running_platforms[self.name] = self
+        self.all_running_configs[self.name] = self
         self.buildEnv = BuildEnv(self)
         self.setup_toolchains(targets)
 
     def __str__(self):
-        return "{}_{}".format(self.build, 'static' if self.static else 'dyn')
+        return "{}_{}".format(self.build, "static" if self.static else "dyn")
 
     def setup_toolchains(self, targets):
         for tlc_name in self.toolchain_names:
             ToolchainClass = Dependency.all_deps[tlc_name]
-            targets[('source', tlc_name)] = ToolchainClass.Source
+            targets[("source", tlc_name)] = ToolchainClass.Source
-            plt_name = 'neutral' if ToolchainClass.neutral else self.name
+            cfg_name = "neutral" if ToolchainClass.neutral else self.name
-            targets[(plt_name, tlc_name)] = ToolchainClass.Builder
+            targets[(cfg_name, tlc_name)] = ToolchainClass.Builder
 
     def add_targets(self, targetName, targets):
         if (self.name, targetName) in targets:
             return []
         targetClass = Dependency.all_deps[targetName]
-        targets[('source', targetName)] = targetClass.Source
+        targets[("source", targetName)] = targetClass.Source
         targets[(self.name, targetName)] = targetClass.Builder
         for dep in targetClass.Builder.get_dependencies(self, False):
             if isinstance(dep, tuple):
-                depPlatformName, depName = dep
+                depConfigName, depName = dep
             else:
-                depPlatformName, depName = self.name, dep
+                depConfigName, depName = self.name, dep
-            depPlatform = self.get_platform(depPlatformName, targets)
+            depConfig = self.get_config(depConfigName, targets)
-            depPlatform.add_targets(depName, targets)
+            depConfig.add_targets(depName, targets)
         return [(self.name, targetName)]
 
     def get_fully_qualified_dep(self, dep):
-         if isinstance(dep, tuple):
+        if isinstance(dep, tuple):
             return dep
-         else:
+        else:
             return self.name, dep
 
-
     def get_cross_config(self):
         return {}
 
     def get_include_dirs(self):
-        return [pj(self.buildEnv.install_dir, 'include')]
+        return [pj(self.buildEnv.install_dir, "include")]
 
     def get_env(self):
         return DefaultEnv()
 
-
     def get_bin_dir(self):
         return []
 
-
     def set_compiler(self, env):
         pass
 
-
     def set_comp_flags(self, env):
         if self.static:
-            env['CFLAGS'] = env['CFLAGS'] + ' -fPIC'
+            env["CFLAGS"] = env["CFLAGS"] + " -fPIC"
-            env['CXXFLAGS'] = env['CXXFLAGS'] + ' -fPIC'
+            env["CXXFLAGS"] = env["CXXFLAGS"] + " -fPIC"
-
 
     def _gen_crossfile(self, name, outname=None):
         if outname is None:
             outname = name
         crossfile = pj(self.buildEnv.build_dir, outname)
         template_file = pj(TEMPLATES_DIR, name)
-        with open(template_file, 'r') as f:
+        with open(template_file, "r") as f:
             template = f.read()
-        content = template.format(
+        content = template.format(**self.get_cross_config())
-            **self.get_cross_config()
+        with open(crossfile, "w") as outfile:
-        )
-        with open(crossfile, 'w') as outfile:
             outfile.write(content)
         return crossfile
 
@@ -121,56 +118,62 @@ class PlatformInfo(metaclass=_MetaPlatform):
         self.buildEnv.clean_intermediate_directories()
 
 
-
+class MetaConfigInfo(ConfigInfo):
-class MetaPlatformInfo(PlatformInfo):
+    subConfigNames = []
-    subPlatformNames = []
 
     def add_targets(self, targetName, targets):
         targetDefs = []
-        for platformName in self.subPlatformNames:
+        for configName in self.subConfigNames:
-            platform = self.get_platform(platformName, targets)
+            config = self.get_config(configName, targets)
-            targetDefs += platform.add_targets(targetName, targets)
+            targetDefs += config.add_targets(targetName, targets)
         return targetDefs
 
 
-
 def MixedMixin(static_name):
     class MixedMixinClass:
         mixed = True
         static = False
 
         def add_targets(self, targetName, targets):
-            print(targetName)
+            if option("target") == targetName:
-            if option('target') == targetName:
                 return super().add_targets(targetName, targets)
             else:
-                static_platform = self.get_platform(static_name, targets)
+                static_config = self.get_config(static_name, targets)
-                return static_platform.add_targets(targetName, targets)
+                return static_config.add_targets(targetName, targets)
 
         def get_fully_qualified_dep(self, dep):
             if isinstance(dep, tuple):
                 return dep
-            if option('target') == dep:
+            if option("target") == dep:
                 return self.name, dep
             return static_name, dep
 
         @property
         def static_buildEnv(self):
-            static_platform = self.get_platform(static_name)
+            static_config = self.get_config(static_name)
-            return static_platform.buildEnv
+            return static_config.buildEnv
 
         def get_include_dirs(self):
             return [
-                pj(self.buildEnv.install_dir, 'include'),
+                pj(self.buildEnv.install_dir, "include"),
-                pj(self.static_buildEnv.install_dir, 'include')
+                pj(self.static_buildEnv.install_dir, "include"),
             ]
 
         def get_env(self):
             env = super().get_env()
-            env['PATH'] = ':'.join([pj(self.static_buildEnv.install_dir, 'bin')] + [env['PATH']])
+            env["PATH"].insert(0, pj(self.static_buildEnv.install_dir, "bin"))
-            pkgconfig_path = pj(self.static_buildEnv.install_dir, self.static_buildEnv.libprefix, 'pkgconfig')
+            pkgconfig_path = pj(
-            env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path])
+                self.static_buildEnv.install_dir,
-            env['CPPFLAGS'] = " ".join(['-I'+pj(self.static_buildEnv.install_dir, 'include'), env['CPPFLAGS']])
+                self.static_buildEnv.libprefix,
+                "pkgconfig",
+            )
+            env["PKG_CONFIG_PATH"].append(pkgconfig_path)
+            env["CPPFLAGS"] = " ".join(
+                [
+                   "-I" + pj(self.static_buildEnv.install_dir, "include"),
+                    env["CPPFLAGS"],
+                ]
+            )
             return env
 
     return MixedMixinClass

kiwixbuild/configs/flatpak.py

@@ -0,0 +1,19 @@
+from .base import ConfigInfo
+from kiwixbuild._global import option, neutralEnv
+
+
+class FlatpakConfigInfo(ConfigInfo):
+    name = "flatpak"
+    arch_name = "flatpak"
+    build = "flatpak"
+    static = ""
+    toolchain_names = ["org.kde", "io.qt.qtwebengine"]
+    compatible_hosts = ["debian", "fedora"]
+
+    def __str__(self):
+        return "flatpak"
+
+    def get_env(self):
+        env = super().get_env()
+        env["FLATPAK_USER_DIR"] = self.buildEnv.build_dir
+        return env

kiwixbuild/configs/i586.py

@@ -0,0 +1,79 @@
+import os
+
+from .base import ConfigInfo
+from kiwixbuild.utils import which, pj
+
+
+class I586ConfigInfo(ConfigInfo):
+    build = "i586"
+    arch_full = "i586-linux-gnu"
+    compatible_hosts = ["fedora", "debian"]
+
+    def get_cross_config(self):
+        return {
+            "binaries": self.binaries,
+            "exe_wrapper_def": "",
+            "extra_libs": ["-m32", "-march=i586", "-mno-sse"],
+            "extra_cflags": [
+                "-m32",
+                "-march=i586",
+                "-mno-sse",
+                *(
+                    "-I{}".format(include_dir)
+                    for include_dir in self.get_include_dirs()
+                ),
+            ],
+            "host_machine": {
+                "system": "linux",
+                "lsystem": "linux",
+                "cpu_family": "x86",
+                "cpu": "i586",
+                "endian": "little",
+                "abi": "",
+            },
+        }
+
+    @property
+    def configure_options(self):
+        yield f"--host={self.arch_full}"
+
+    @property
+    def binaries(self):
+        return {
+            k: which(v)
+            for k, v in (
+                ("CC", os.environ.get("CC", "gcc")),
+                ("CXX", os.environ.get("CXX", "g++")),
+                ("AR", "ar"),
+                ("STRIP", "strip"),
+                ("RANLIB", "ranlib"),
+                ("LD", "ld"),
+                ("PKGCONFIG", "pkg-config"),
+            )
+        }
+
+    def set_comp_flags(self, env):
+        super().set_comp_flags(env)
+        env["CFLAGS"] = "-m32 -march=i586 -mno-sse " + env["CFLAGS"]
+        env["CXXFLAGS"] = "-m32 -march=i586 -mno-sse " + env["CXXFLAGS"]
+        env["LDFLAGS"] = "-m32 -march=i586 -mno-sse " + env["LDFLAGS"]
+
+    def get_bin_dir(self):
+        return []
+
+    def finalize_setup(self):
+        super().finalize_setup()
+        self.buildEnv.cmake_crossfile = self._gen_crossfile(
+            "cmake_i586_cross_file.txt", "cmake_cross_file.txt"
+        )
+        self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
+
+
+class I586Dyn(I586ConfigInfo):
+    name = "i586_dyn"
+    static = False
+
+
+class I586Static(I586ConfigInfo):
+    name = "i586_static"
+    static = True

kiwixbuild/configs/ios.py

@@ -0,0 +1,249 @@
+import subprocess
+
+from kiwixbuild._global import option
+from kiwixbuild.utils import pj, xrun_find
+from .base import ConfigInfo, MetaConfigInfo, MixedMixin
+from kiwixbuild.dependencies.apple_xcframework import AppleXCFramework
+
+
+MIN_MACOS_VERSION = "12.0"
+
+
+class AppleConfigInfo(ConfigInfo):
+    build = "iOS"
+    static = True
+    compatible_hosts = ["Darwin"]
+    arch = None
+    host = None
+    target = None
+    sdk_name = None
+    min_iphoneos_version = None
+    min_macos_version = None
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._root_path = None
+
+    @property
+    def arch_name(self):
+        return self.target
+
+    @property
+    def root_path(self):
+        if self._root_path is None:
+            command = "xcrun --sdk {} --show-sdk-path".format(self.sdk_name)
+            self._root_path = subprocess.check_output(command, shell=True)[:-1].decode()
+        return self._root_path
+
+    def __str__(self):
+        return "iOS"
+
+    def finalize_setup(self):
+        super().finalize_setup()
+        self.buildEnv.cmake_crossfile = self._gen_crossfile(
+            "cmake_ios_cross_file.txt", "cmake_cross_file.txt"
+        )
+        self.buildEnv.meson_crossfile = self._gen_crossfile(
+            "meson_ios_cross_file.txt", "meson_cross_file.txt"
+        )
+
+    def get_cross_config(self):
+        config = {
+            "root_path": self.root_path,
+            "binaries": self.binaries,
+            "exe_wrapper_def": "",
+            "extra_libs": [
+                "-isysroot",
+                self.root_path,
+                "-arch",
+                self.arch,
+                "-target",
+                self.target,
+            ],
+            "extra_cflags": [
+                "-isysroot",
+                self.root_path,
+                "-arch",
+                self.arch,
+                "-target",
+                self.target,
+                *(
+                    "-I{}".format(include_dir)
+                    for include_dir in self.get_include_dirs()
+                ),
+            ],
+            "host_machine": {
+                "system": "Darwin",
+                "lsystem": "darwin",
+                "cpu_family": self.arch,
+                "cpu": self.cpu,
+                "endian": "",
+                "abi": "",
+            },
+        }
+        if self.min_iphoneos_version:
+            config["extra_libs"].append(
+                "-miphoneos-version-min={}".format(self.min_iphoneos_version)
+            )
+            config["extra_cflags"].append(
+                "-miphoneos-version-min={}".format(self.min_iphoneos_version)
+            )
+        if self.min_macos_version:
+            config["extra_libs"].append(
+                "-mmacosx-version-min={}".format(self.min_macos_version)
+            )
+            config["extra_cflags"].append(
+                "-mmacosx-version-min={}".format(self.min_macos_version)
+            )
+        return config
+
+    def get_env(self):
+        env = super().get_env()
+        cflags = [env["CFLAGS"]]
+        if self.min_iphoneos_version:
+            cflags.append("-miphoneos-version-min={}".format(self.min_iphoneos_version))
+        if self.min_macos_version:
+            cflags.append("-mmacosx-version-min={}".format(self.min_macos_version))
+        env["CFLAGS"] = " ".join(cflags)
+        return env
+
+    def set_comp_flags(self, env):
+        super().set_comp_flags(env)
+        cflags = [
+            "-isysroot {}".format(self.root_path),
+            "-arch {}".format(self.arch),
+            "-target {}".format(self.target),
+            env["CFLAGS"],
+        ]
+        if self.min_iphoneos_version:
+            cflags.append("-miphoneos-version-min={}".format(self.min_iphoneos_version))
+        env["CFLAGS"] = " ".join(cflags)
+        env["CXXFLAGS"] = " ".join(
+            [
+                env["CFLAGS"],
+                "-std=c++11",
+                env["CXXFLAGS"],
+            ]
+        )
+        env["LDFLAGS"] = " ".join(
+            [
+                " -arch {}".format(self.arch),
+                "-isysroot {}".format(self.root_path),
+            ]
+        )
+
+    def get_bin_dir(self):
+        return [pj(self.root_path, "bin")]
+
+    @property
+    def binaries(self):
+        return {
+            "CC": xrun_find("clang"),
+            "CXX": xrun_find("clang++"),
+            "AR": xrun_find("ar"),
+            "STRIP": xrun_find("strip"),
+            "RANLIB": xrun_find("ranlib"),
+            "LD": xrun_find("ld"),
+            "PKGCONFIG": "pkg-config",
+        }
+
+    @property
+    def configure_options(self):
+        yield f"--host={self.host}"
+
+
+class iOSArm64(AppleConfigInfo):
+    name = "iOS_arm64"
+    arch = cpu = "arm64"
+    host = "arm-apple-darwin"
+    target = "aarch64-apple-ios"
+    sdk_name = "iphoneos"
+    min_iphoneos_version = "15.0"
+
+
+class iOSx64Simulator(AppleConfigInfo):
+    name = "iOSSimulator_x86_64"
+    arch = cpu = "x86_64"
+    host = "x86_64-apple-darwin"
+    target = "x86-apple-ios-simulator"
+    sdk_name = "iphonesimulator"
+    min_iphoneos_version = "15.0"
+
+
+class iOSArm64Simulator(AppleConfigInfo):
+    name = "iOSSimulator_arm64"
+    arch = cpu = "arm64"
+    host = "arm-apple-darwin"
+    target = "aarch64-apple-ios-simulator"
+    sdk_name = "iphonesimulator"
+    min_iphoneos_version = "15.0"
+
+
+class macOSArm64(AppleConfigInfo):
+    name = "macOS_arm64_static"
+    arch = cpu = "arm64"
+    host = "aarch64-apple-darwin"
+    target = "arm64-apple-macos"
+    sdk_name = "macosx"
+    min_iphoneos_version = None
+    min_macos_version = MIN_MACOS_VERSION
+
+
+class macOSArm64Mixed(MixedMixin("macOS_arm64_static"), AppleConfigInfo):
+    name = "macOS_arm64_mixed"
+    arch = cpu = "arm64"
+    host = "aarch64-apple-darwin"
+    target = "arm64-apple-macos"
+    sdk_name = "macosx"
+    min_iphoneos_version = None
+    min_macos_version = MIN_MACOS_VERSION
+
+
+class macOSx64(AppleConfigInfo):
+    name = "macOS_x86_64"
+    arch = cpu = "x86_64"
+    host = "x86_64-apple-darwin"
+    target = "x86_64-apple-macos"
+    sdk_name = "macosx"
+    min_iphoneos_version = None
+    min_macos_version = MIN_MACOS_VERSION
+
+
+class IOS(MetaConfigInfo):
+    name = "iOS_multi"
+    compatible_hosts = ["Darwin"]
+
+    @property
+    def arch_name(self):
+        return self.name
+
+    @property
+    def subConfigNames(self):
+        return ["iOS_{}".format(arch) for arch in option("ios_arch")]
+
+    def add_targets(self, targetName, targets):
+        super().add_targets(targetName, targets)
+        return ConfigInfo.add_targets(self, "_ios_fat_lib", targets)
+
+    def __str__(self):
+        return self.name
+
+
+class AppleStaticAll(MetaConfigInfo):
+    name = "apple_all_static"
+    compatible_hosts = ["Darwin"]
+
+    @property
+    def arch_name(self):
+        return self.name
+
+    @property
+    def subConfigNames(self):
+        return AppleXCFramework.subConfigNames
+
+    def add_targets(self, targetName, targets):
+        super().add_targets(targetName, targets)
+        return ConfigInfo.add_targets(self, "apple_xcframework", targets)
+
+    def __str__(self):
+        return self.name

kiwixbuild/configs/musl.py

@@ -0,0 +1,157 @@
+from .base import ConfigInfo, MixedMixin
+
+from kiwixbuild.utils import pj
+from kiwixbuild._global import get_target_step
+
+
+class MuslConfigInfo(ConfigInfo):
+    compatible_hosts = ["fedora", "debian"]
+
+    def get_cross_config(self):
+        return {
+            "binaries": self.binaries,
+            "exe_wrapper_def": "",
+            "root_path": self.root_path,
+            "extra_libs": [],
+            "extra_cflags": [
+                "-I{}".format(include_dir) for include_dir in self.get_include_dirs()
+            ],
+            "host_machine": {
+                "system": "linux",
+                "lsystem": "linux",
+                "cpu_family": self.cpu_family,
+                "cpu": self.cpu,
+                "endian": "little",
+                "abi": "",
+            },
+        }
+
+    @property
+    def toolchain(self):
+        return get_target_step(self.build, "neutral")
+
+    @property
+    def root_path(self):
+        return self.toolchain.build_path
+
+    @property
+    def binaries(self):
+        binaries = (
+            (k, "{}-{}".format(self.arch_full, v))
+            for k, v in (
+                ("CC", "gcc"),
+                ("CXX", "g++"),
+                ("AR", "ar"),
+                ("STRIP", "strip"),
+                ("WINDRES", "windres"),
+                ("RANLIB", "ranlib"),
+                ("LD", "ld"),
+                ("LDSHARED", "g++ -shared"),
+            )
+        )
+        binaries = {k: pj(self.root_path, "bin", v) for k, v in binaries}
+        binaries["PKGCONFIG"] = "pkg-config"
+        return binaries
+
+    @property
+    def exe_wrapper_def(self):
+        try:
+            which(self.qemu)
+        except subprocess.CalledProcessError:
+            return ""
+        except AttributeError:
+            return ""
+        else:
+            return f"exe_wrapper = '{self.qemu}'"
+
+    @property
+    def configure_options(self):
+        return [f"--host={self.arch_full}"]
+
+    def get_bin_dir(self):
+        return [pj(self.root_path, "bin")]
+
+    def get_env(self):
+        env = super().get_env()
+        env["LD_LIBRARY_PATH"][0:0] = [
+            pj(self.root_path, self.arch_full, "lib64"),
+            pj(self.root_path, "lib"),
+        ]
+        env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig")
+        env["QEMU_LD_PREFIX"] = pj(self.root_path, self.arch_full, "libc")
+        env["QEMU_SET_ENV"] = "LD_LIBRARY_PATH={}".format(
+            ":".join(
+                [pj(self.root_path, self.arch_full, "lib"), str(env["LD_LIBRARY_PATH"])]
+            )
+        )
+        return env
+
+    def set_comp_flags(self, env):
+        super().set_comp_flags(env)
+        env["LD_LIBRARY_PATH"].insert(0, pj(self.root_path, self.arch_full, "lib"))
+        env["CFLAGS"] = (
+            " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
+            + env["CFLAGS"]
+        )
+        env["CXXFLAGS"] = (
+            " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
+            + env["CXXFLAGS"]
+        )
+
+    def set_compiler(self, env):
+        for k, v in self.binaries.items():
+            env[k] = v
+
+    def finalize_setup(self):
+        super().finalize_setup()
+        self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt")
+        self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
+
+
+class Aarch64MuslConfigInfo(MuslConfigInfo):
+    build = "aarch64_musl"
+    arch_full = "aarch64-linux-musl"
+    toolchain_names = ["aarch64_musl"]
+    libdir = "lib/aarch64-linux-musl"
+    cpu_family = "arm"
+    cpu = "armhf"
+    qemu = "qemu-arm"
+
+
+class Aarch64MuslDyn(Aarch64MuslConfigInfo):
+    name = "aarch64_musl_dyn"
+    static = False
+
+
+class Aarch64MuslStatic(Aarch64MuslConfigInfo):
+    name = "aarch64_musl_static"
+    static = True
+
+
+class Aarch64MuslMixed(MixedMixin("aarch64_musl_static"), Aarch64MuslConfigInfo):
+    name = "aarch64_musl_mixed"
+    static = False
+
+
+class X86_64MuslConfigInfo(MuslConfigInfo):
+    build = "x86-64_musl"
+    arch_full = "x86_64-linux-musl"
+    toolchain_names = ["x86-64_musl"]
+    libdir = "lib/x86_64-linux-musl"
+    cpu_family = "x86_64"
+    cpu = "x86_64"
+
+
+class X86_64MuslDyn(X86_64MuslConfigInfo):
+    name = "x86-64_musl_dyn"
+    static = False
+
+
+class X86_64MuslStatic(X86_64MuslConfigInfo):
+    name = "x86-64_musl_static"
+    static = True
+
+
+class x86_64MuslMixed(MixedMixin("x86-64_musl_static"), X86_64MuslConfigInfo):
+    name = "x86-64_musl_mixed"
+    static = False

kiwixbuild/configs/native.py

@@ -0,0 +1,44 @@
+from .base import ConfigInfo, MixedMixin
+
+from kiwixbuild.utils import pj
+from kiwixbuild._global import option, neutralEnv
+from kiwixbuild.configs.ios import MIN_MACOS_VERSION
+import sysconfig
+import platform
+import sys
+
+
+class NativeConfigInfo(ConfigInfo):
+    build = "native"
+
+    def get_env(self):
+        env = super().get_env()
+        if neutralEnv("distname") == "fedora":
+            env["QT_SELECT"] = "5-64"
+        if neutralEnv("distname") == "Darwin":
+            env["CFLAGS"] += f"-mmacosx-version-min={MIN_MACOS_VERSION}"
+        return env
+
+    @property
+    def arch_name(self):
+        if sys.platform == "darwin":
+            return f"{platform.machine()}-apple-darwin"
+        return sysconfig.get_platform()
+
+
+class NativeDyn(NativeConfigInfo):
+    name = "native_dyn"
+    static = False
+    compatible_hosts = ["fedora", "debian", "Darwin", "almalinux", "Windows"]
+
+
+class NativeStatic(NativeConfigInfo):
+    name = "native_static"
+    static = True
+    compatible_hosts = ["fedora", "debian", "Darwin", "almalinux", "Windows"]
+
+
+class NativeMixed(MixedMixin("native_static"), NativeConfigInfo):
+    name = "native_mixed"
+    static = False
+    compatible_hosts = ["fedora", "debian", "Darwin", "almalinux", "Windows"]

kiwixbuild/configs/neutral.py

@@ -0,0 +1,11 @@
+from .base import ConfigInfo
+
+
+class NeutralConfigInfo(ConfigInfo):
+    name = "neutral"
+    arch_name = "neutral"
+    static = ""
+    compatible_hosts = ["fedora", "debian", "Darwin"]
+
+    def __str__(self):
+        return "neutral"

kiwixbuild/configs/wasm.py

@@ -0,0 +1,113 @@
+from .base import ConfigInfo
+
+from kiwixbuild.utils import pj
+from kiwixbuild._global import get_target_step
+
+
+class WasmConfigInfo(ConfigInfo):
+    name = "wasm"
+    static = True
+    build = "wasm"
+    arch_full = "wasm64-emscripten"
+    libdir = "lib"
+    # arch_full = 'wasm64-linux'
+    toolchain_names = ["emsdk"]
+    compatible_hosts = ["fedora", "debian"]
+    exe_wrapper_def = ""
+
+    def get_cross_config(self):
+        return {
+            "binaries": self.binaries,
+            "exe_wrapper_def": "",
+            "root_path": self.root_path,
+            "extra_libs": [],
+            "extra_cflags": [],
+            "host_machine": {
+                "system": "emscripten",
+                "lsystem": "emscripten",
+                "cpu_family": "wasm64",
+                "cpu": "wasm64",
+                "endian": "little",
+                "abi": "",
+            },
+        }
+
+    @property
+    def wasm_sdk(self):
+        return get_target_step("emsdk", self.name)
+
+    @property
+    def install_path(self):
+        return self.wasm_sdk.install_path
+
+    @property
+    def root_path(self):
+        return self.install_path
+
+    @property
+    def binaries(self):
+        binaries = (
+            ("CC", "emcc"),
+            ("CXX", "em++"),
+            ("AR", "emar"),
+            ("STRIP", "emstrip"),
+            ("WINDRES", "windres"),
+            ("RANLIB", "emranlib"),
+            ("LD", "wasm-ld"),
+        )
+        binaries = {
+            k: pj(self.install_path, "upstream", "emscripten", v) for k, v in binaries
+        }
+        binaries["PKGCONFIG"] = "pkg-config"
+        return binaries
+
+    @property
+    def configure_options(self):
+        # return ""
+        return [f"--host={self.arch_full}"]
+
+    @property
+    def configure_wrapper(self):
+        return "emconfigure"
+
+    @property
+    def make_wrapper(self):
+        return "emmake"
+
+    def get_bin_dir(self):
+        return [pj(self.install_path, "bin")]
+
+    def get_env(self):
+        env = super().get_env()
+        env["PATH"].extend(
+            [
+                self.install_path,
+                pj(self.install_path, "upstream", "emscripten"),
+                pj(self.install_path, "node", "14.18.2_64bit", "bin"),
+            ]
+        )
+        env["EMSDK"] = self.install_path
+        env["EMSDK_NODE"] = pj(
+            self.install_path, "node", "14.18.2_64bit", "bin", "node"
+        )
+        return env
+
+    def set_comp_flags(self, env):
+        super().set_comp_flags(env)
+        env["CFLAGS"] = (
+            " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
+            + env["CFLAGS"]
+        )
+        env["CXXFLAGS"] = (
+            " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
+            + env["CXXFLAGS"]
+        )
+
+    def set_compiler(self, env):
+        for k, v in self.binaries.items():
+            env[k] = v
+
+    def finalize_setup(self):
+        super().finalize_setup()
+        self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt")
+        self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")

kiwixbuild/configs/win64.py

@@ -0,0 +1,103 @@
+import subprocess
+
+from .base import ConfigInfo
+from kiwixbuild.utils import which, pj
+from kiwixbuild._global import neutralEnv
+
+
+class Win64ConfigInfo(ConfigInfo):
+    extra_libs = [
+        "-lmingw32",
+        "-lwinmm",
+        "-lws2_32",
+        "-lshlwapi",
+        "-lrpcrt4",
+        "-lmsvcr100",
+        "-liphlpapi",
+        "-lshell32",
+        "-lkernel32",
+    ]
+    build = "win64"
+    compatible_hosts = ["fedora", "debian"]
+    arch_full = "x86_64-w64-mingw32"
+
+    def get_cross_config(self):
+        return {
+            "exe_wrapper_def": self.exe_wrapper_def,
+            "binaries": self.binaries,
+            "root_path": self.root_path,
+            "extra_libs": self.extra_libs,
+            "extra_cflags": ["-DWIN32"],
+            "host_machine": {
+                "system": "Windows",
+                "lsystem": "windows",
+                "cpu_family": "x86_64",
+                "cpu": "x86_64",
+                "endian": "little",
+                "abi": "",
+            },
+        }
+
+    def finalize_setup(self):
+        super().finalize_setup()
+        self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt")
+        self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
+
+    @property
+    def root_path(self):
+        root_paths = {
+            "fedora": "/usr/x86_64-w64-mingw32/sys-root/mingw",
+            "debian": "/usr/x86_64-w64-mingw32",
+        }
+        return root_paths[neutralEnv("distname")]
+
+    @property
+    def binaries(self):
+        return {
+            k: which("{}-{}".format(self.arch_full, v))
+            for k, v in (
+                ("CC", "gcc"),
+                ("CXX", "g++"),
+                ("AR", "ar"),
+                ("STRIP", "strip"),
+                ("WINDRES", "windres"),
+                ("RANLIB", "ranlib"),
+                ("PKGCONFIG", "pkg-config"),
+            )
+        }
+
+    @property
+    def exe_wrapper_def(self):
+        try:
+            which("wine")
+        except subprocess.CalledProcessError:
+            return ""
+        else:
+            return "exe_wrapper = 'wine'"
+
+    @property
+    def configure_options(self):
+        return [f"--host={self.arch_full}"]
+
+    def set_compiler(self, env):
+        for k, v in self.binaries.items():
+            env[k] = v
+
+    def get_bin_dir(self):
+        return [pj(self.root_path, "bin")]
+
+    def get_env(self):
+        env = super().get_env()
+        env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig")
+        env["LIBS"] = " ".join(self.extra_libs) + " " + env["LIBS"]
+        return env
+
+
+class Win64Dyn(Win64ConfigInfo):
+    name = "win64_dyn"
+    static = False
+
+
+class Win64Static(Win64ConfigInfo):
+    name = "win64_static"
+    static = True

kiwixbuild/dependencies/__init__.py

@@ -1,7 +1,7 @@
-
 from .base import *
 from . import (
     all_dependencies,
+    boostregex,
     tc_android_ndk,
     aria2,
     tc_armhf,
@@ -28,5 +28,5 @@ from . import (
     zim_tools,
     zim_testing_suite,
     zlib,
-    zstd
+    zstd,
 )

kiwixbuild/dependencies/all_dependencies.py

@@ -1,30 +1,61 @@
-from .base import (
+from os import environ
-    Dependency,
+
-    NoopSource,
+from .base import Dependency, NoopSource, NoopBuilder
-    NoopBuilder)
 
 from kiwixbuild._global import neutralEnv
 
+
 class AllBaseDependencies(Dependency):
     name = "alldependencies"
 
     Source = NoopSource
+
     class Builder(NoopBuilder):
         @classmethod
-        def get_dependencies(cls, platformInfo, allDeps):
+        def get_dependencies(cls, configInfo, allDeps):
-            if platformInfo.build == "wasm":
+            if configInfo.build == "wasm" or environ.get("OS_NAME") == "manylinux":
-                return ['zlib', 'lzma', 'zstd', 'icu4c', 'xapian-core']
+                return ["zlib", "lzma", "zstd", "icu4c", "xapian-core"]
 
-            base_deps = ['zlib', 'lzma', 'zstd', 'xapian-core', 'pugixml', 'libcurl', 'icu4c', 'mustache', 'libmicrohttpd', 'zim-testing-suite']
+            if neutralEnv("distname") == "Windows":
-            # Add specific dependencies depending of the platform
+                base_deps = [
-            if platformInfo.build not in ('android', 'iOS'):
+                    "zlib",
-                # For zimtools
+                    "zstd",
-                base_deps += ['docoptcpp']
+                    "xapian-core",
-                if platformInfo.build != 'win32':
+                    "zim-testing-suite",
-                    # zimwriterfs
+                    "icu4c",
-                    base_deps += ['libmagic', 'gumbo']
+                    "boostregex",
-                if platformInfo.build == 'native' and neutralEnv('distname') != 'Darwin':
+                    "docoptcpp",
-                    # We compile kiwix-desktop only on native and not on `Darwin`
+                ]
-                    # So we need aria2 only there
+
-                    base_deps += ['aria2']
+                if not configInfo.name.endswith("_dyn"):
+                    base_deps += [
+                        "pugixml",
+                        "libcurl",
+                        "mustache",
+                        "libmicrohttpd",
+                    ]
+            else:
+                base_deps = [
+                    "zlib",
+                    "lzma",
+                    "zstd",
+                    "xapian-core",
+                    "pugixml",
+                    "libcurl",
+                    "icu4c",
+                    "mustache",
+                    "libmicrohttpd",
+                    "zim-testing-suite",
+                ]
+                # Add specific dependencies depending of the config
+                if configInfo.build not in ("android", "iOS"):
+                    # For zimtools
+                    base_deps += ["docoptcpp", "libmagic", "gumbo"]
+                    if (
+                        configInfo.build == "native"
+                        and neutralEnv("distname") != "Darwin"
+                    ):
+                        # We compile kiwix-desktop only on native and not on `Darwin`
+                        # So we need aria2 only there
+                        base_deps += ["aria2"]
             return base_deps

kiwixbuild/dependencies/apple_xcframework.py

@@ -0,0 +1,126 @@
+import os
+import shutil
+from pathlib import Path
+
+from kiwixbuild.configs import ConfigInfo
+from kiwixbuild.utils import pj, run_command
+from .base import Dependency, NoopSource, Builder as BaseBuilder
+
+
+class AppleXCFramework(Dependency):
+    name = "apple_xcframework"
+    subConfigNames = [
+        "macOS_x86_64",
+        "macOS_arm64_static",
+        "iOS_arm64",
+        "iOSSimulator_x86_64",
+        "iOSSimulator_arm64",
+    ]
+    Source = NoopSource
+
+    class Builder(BaseBuilder):
+        @property
+        def all_subconfigs(self):
+            return self.buildEnv.configInfo.subConfigNames
+
+        @property
+        def macos_subconfigs(self):
+            return [
+                target for target in self.all_subconfigs if target.startswith("macOS")
+            ]
+
+        @property
+        def iossimulator_subconfigs(self):
+            return [
+                target
+                for target in self.all_subconfigs
+                if target.startswith("iOSSimulator")
+            ]
+
+        @property
+        def ios_subconfigs(self):
+            return [
+                target for target in self.all_subconfigs if target.startswith("iOS_")
+            ]
+
+        @classmethod
+        def get_dependencies(cls, configInfo, alldeps):
+            return [(target, "libkiwix") for target in AppleXCFramework.subConfigNames]
+
+        @property
+        def final_path(self):
+            return pj(self.buildEnv.install_dir, "lib", "CoreKiwix.xcframework")
+
+        def _remove_if_exists(self, context):
+            if not os.path.exists(self.final_path):
+                return
+
+            shutil.rmtree(self.final_path)
+
+        def _merge_libs(self, context):
+            """create merged.a in all targets to bundle all static archives"""
+            xcf_libs = []
+            for target in self.all_subconfigs:
+                static_ars = []
+
+                cfg = ConfigInfo.get_config(target)
+                lib_dir = pj(cfg.buildEnv.install_dir, "lib")
+                static_ars = [str(f) for f in Path(lib_dir).glob("*.a")]
+
+                # create merged.a from all *.a in install_dir/lib
+                command = ["libtool", "-static", "-o", "merged.a", *static_ars]
+                run_command(command, lib_dir, context)
+
+                # will be included in xcframework
+                if target in self.ios_subconfigs:
+                    xcf_libs.append(pj(lib_dir, "merged.a"))
+
+            return xcf_libs
+
+        def make_fat_with(self, configs, folder_name, context):
+            """create fat merged.a in {folder_name} install/lib with {configs}"""
+            libs = []
+            for target in configs:
+                cfg = ConfigInfo.get_config(target)
+                libs.append(pj(cfg.buildEnv.install_dir, "lib", "merged.a"))
+
+            fat_dir = pj(self.buildEnv.build_dir, folder_name)
+            os.makedirs(fat_dir, exist_ok=True)
+
+            output_merged = pj(fat_dir, "merged.a")
+            command = ["lipo", "-create", "-output", output_merged, *libs]
+            run_command(command, self.buildEnv.build_dir, context)
+
+            return [output_merged]
+
+        def _build_xcframework(self, xcf_libs, context):
+            # create xcframework
+            ref_conf = ConfigInfo.get_config(self.macos_subconfigs[0])
+            command = ["xcodebuild", "-create-xcframework"]
+            for lib in xcf_libs:
+                command += [
+                    "-library",
+                    lib,
+                    "-headers",
+                    pj(ref_conf.buildEnv.install_dir, "include"),
+                ]
+            command += ["-output", self.final_path]
+            run_command(command, self.buildEnv.build_dir, context)
+
+        def build(self):
+            xcf_libs = []
+            self.command("remove_if_exists", self._remove_if_exists)
+            xcf_libs += self.command("merge_libs", self._merge_libs)
+            xcf_libs += self.command(
+                "make_macos_fat",
+                self.make_fat_with,
+                self.macos_subconfigs,
+                "macOS_fat",
+            )
+            xcf_libs += self.command(
+                "make_simulator_fat",
+                self.make_fat_with,
+                self.iossimulator_subconfigs,
+                "iOS-simulator_fat",
+            )
+            self.command("build_xcframework", self._build_xcframework, xcf_libs)

kiwixbuild/dependencies/aria2.py

@@ -1,24 +1,53 @@
-from .base import (
+from .base import Dependency, ReleaseDownload, MakeBuilder, NoopBuilder
-    Dependency,
-    ReleaseDownload,
-    MakeBuilder
-)
 
-from kiwixbuild.utils import Remotefile, run_command
+from kiwixbuild.utils import Remotefile, run_command, pj
+import platform
+from shutil import copy2
+
+# Important: in case of aria2c update,
+# 'scripts/create_kiwix-desktop_appImage.sh' should not be forgotten!
 
 class Aria2(Dependency):
     name = "aria2"
 
-    class Source(ReleaseDownload):
+    if platform.system() == "Windows":
-        archive = Remotefile('aria2-1.36.0.tar.xz',
-                             '58d1e7608c12404f0229a3d9a4953d0d00c18040504498b483305bcb3de907a5',
-                             'https://github.com/aria2/aria2/releases/download/release-1.36.0/aria2-1.36.0.tar.xz')
 
-        def _post_prepare_script(self, context):
+        class Source(ReleaseDownload):
-            context.try_skip(self.extract_path)
+            archive = Remotefile(
-            command = "autoreconf -i"
+                "aria2-1.37.0-win-64bit-build1.zip",
-            run_command(command, self.extract_path, context)
+                "67d015301eef0b612191212d564c5bb0a14b5b9c4796b76454276a4d28d9b288",
+                "https://dev.kiwix.org/kiwix-desktop/aria2-1.37.0-win-64bit-build1.zip",
+            )
 
-    class Builder(MakeBuilder):
+        class Builder(NoopBuilder):
-        dependencies = ['zlib']
+            def build(self):
-        configure_option = "--disable-libaria2 --disable-websocket --without-sqlite3"
+                self.command("copy_binary", self._copy_binary)
+
+            def _copy_binary(self, context):
+                context.try_skip(self.build_path)
+                copy2(
+                    pj(self.source_path, "aria2c.exe"),
+                    pj(self.buildEnv.install_dir, "bin"),
+                )
+
+    else:
+
+        class Source(ReleaseDownload):
+            archive = Remotefile(
+                "aria2-1.37.0.tar.xz",
+                "60a420ad7085eb616cb6e2bdf0a7206d68ff3d37fb5a956dc44242eb2f79b66b",
+                "https://dev.kiwix.org/kiwix-desktop/aria2-1.37.0.tar.xz",
+            )
+
+            def _post_prepare_script(self, context):
+                context.try_skip(self.extract_path)
+                command = ["autoreconf", "-i"]
+                run_command(command, self.extract_path, context)
+
+        class Builder(MakeBuilder):
+            dependencies = ["zlib"]
+            configure_options = [
+                "--disable-libaria2",
+                "--disable-websocket",
+                "--without-sqlite3",
+            ]

kiwixbuild/dependencies/base.py

@@ -2,8 +2,19 @@ import subprocess
 import os
 import shutil
 import time
+import platform
 
-from kiwixbuild.utils import pj, Context, SkipCommand, WarningMessage, extract_archive, Defaultdict, StopBuild, run_command, colorize, copy_tree
+from kiwixbuild.utils import (
+    pj,
+    Context,
+    SkipCommand,
+    WarningMessage,
+    extract_archive,
+    StopBuild,
+    run_command,
+    colorize,
+    copy_tree,
+)
 from kiwixbuild.versions import main_project_versions, base_deps_versions
 from kiwixbuild._global import neutralEnv, option, get_target_step
 
@@ -13,8 +24,8 @@ SCRIPT_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 class _MetaDependency(type):
     def __new__(cls, name, bases, dct):
         _class = type.__new__(cls, name, bases, dct)
-        if name != 'Dependency':
+        if name != "Dependency":
-            dep_name = dct['name']
+            dep_name = dct["name"]
             Dependency.all_deps[dep_name] = _class
         return _class
 
@@ -29,7 +40,7 @@ class Dependency(metaclass=_MetaDependency):
     def version(cls):
         if cls.name in base_deps_versions:
             return base_deps_versions[cls.name]
-        elif option('make_release'):
+        elif option("make_release"):
             return main_project_versions.get(cls.name, None)
         return None
 
@@ -42,8 +53,9 @@ class Dependency(metaclass=_MetaDependency):
 
 class Source:
     """Base Class to the real preparator
-       A source preparator must install source in the self.source_dir attribute
+    A source preparator must install source in the self.source_dir attribute
-       inside the neutralEnv.source_dir."""
+    inside the neutralEnv.source_dir."""
+
     def __init__(self, target):
         self.target = target
 
@@ -61,22 +73,22 @@ class Source:
 
     @property
     def source_path(self):
-        return pj(neutralEnv('source_dir'), self.source_dir)
+        return pj(neutralEnv("source_dir"), self.source_dir)
 
     @property
     def _log_dir(self):
-        return neutralEnv('log_dir')
+        return neutralEnv("log_dir")
 
     def _patch(self, context):
         context.try_skip(self.source_path)
         for p in self.patches:
-            patch_file_path = pj(SCRIPT_DIR, 'patches', p)
+            patch_file_path = pj(SCRIPT_DIR, "patches", p)
-            patch_command = "patch -p1 -i {patch}".format(patch=patch_file_path)
+            patch_command = [*neutralEnv("patch_command"), "-p1", "-i", patch_file_path]
             run_command(patch_command, self.source_path, context)
 
     def command(self, name, function, *args):
         print("  {} {} : ".format(name, self.name), end="", flush=True)
-        log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
+        log = pj(self._log_dir, "cmd_{}_{}.log".format(name, self.name))
         context = Context(name, log, True)
         try:
             start_time = time.time()
@@ -92,7 +104,7 @@ class Source:
         except subprocess.CalledProcessError:
             print(colorize("ERROR"))
             try:
-                with open(log, 'r') as f:
+                with open(log, "r") as f:
                     print(f.read())
             except:
                 pass
@@ -112,34 +124,45 @@ class ReleaseDownload(Source):
 
     @property
     def archives(self):
-        return (self.archive, )
+        return (self.archive,)
 
     @property
     def extract_path(self):
-        return pj(neutralEnv('source_dir'), self.source_dir)
+        return pj(neutralEnv("source_dir"), self.source_dir)
 
     def _download(self, context):
-        context.try_skip(neutralEnv('archive_dir'), self.full_name)
+        context.try_skip(neutralEnv("archive_dir"), self.full_name)
-        for archive in self.archives:
+        archive_iter = iter(self.archives)
-            neutralEnv('download')(archive)
+        archive = next(archive_iter, None)
+        while archive:
+            try:
+                neutralEnv("download")(archive)
+            except SkipCommand as e:
+                archive = next(archive_iter, None)
+                if not archive:
+                    raise e
+                continue
+            archive = next(archive_iter, None)
 
     def _extract(self, context):
         context.try_skip(self.extract_path)
         if os.path.exists(self.extract_path):
             shutil.rmtree(self.extract_path)
         for archive in self.archives:
-            extract_archive(pj(neutralEnv('archive_dir'), archive.name),
+            extract_archive(
-                            neutralEnv('source_dir'),
+                pj(neutralEnv("archive_dir"), archive.name),
-                            topdir=self.archive_top_dir,
+                neutralEnv("source_dir"),
-                            name=self.source_dir)
+                topdir=self.archive_top_dir,
+                name=self.source_dir,
+            )
 
     def prepare(self):
-        self.command('download', self._download)
+        self.command("download", self._download)
-        self.command('extract', self._extract)
+        self.command("extract", self._extract)
-        if hasattr(self, 'patches'):
+        if hasattr(self, "patches"):
-            self.command('patch', self._patch)
+            self.command("patch", self._patch)
-        if hasattr(self, '_post_prepare_script'):
+        if hasattr(self, "_post_prepare_script"):
-            self.command('post_prepare_script', self._post_prepare_script)
+            self.command("post_prepare_script", self._post_prepare_script)
 
 
 class GitClone(Source):
@@ -152,70 +175,66 @@ class GitClone(Source):
 
     @property
     def source_dir(self):
-        if option('make_release'):
+        if option("make_release"):
             return "{}_release".format(self.git_dir)
         else:
             return self.git_dir
 
     @property
     def git_path(self):
-        return pj(neutralEnv('source_dir'), self.source_dir)
+        return pj(neutralEnv("source_dir"), self.source_dir)
 
     @property
     def git_ref(self):
-        if option('make_release'):
+        if option("make_release"):
             return self.release_git_ref
         else:
             return self.base_git_ref
 
     def _git_init(self, context):
-        if option('fast_clone') and self.force_full_clone == False:
+        if option("fast_clone") and self.force_full_clone == False:
-            command = "git clone --depth=1 --branch {} {} {}".format(
+            command = [
-                self.git_ref, self.git_remote, self.source_dir)
+                *neutralEnv("git_command"),
-            run_command(command, neutralEnv('source_dir'), context)
+                "clone",
+                "--depth=1",
+                "--branch",
+                self.git_ref,
+                self.git_remote,
+                self.source_dir,
+            ]
+            run_command(command, neutralEnv("source_dir"), context)
         else:
-            command = "git clone {} {}".format(self.git_remote, self.source_dir)
+            command = [
-            run_command(command, neutralEnv('source_dir'), context)
+                *neutralEnv("git_command"),
-            command = "git checkout {}".format(self.git_ref)
+                "clone",
+                self.git_remote,
+                self.source_dir,
+            ]
+            run_command(command, neutralEnv("source_dir"), context)
+            command = [*neutralEnv("git_command"), "checkout", self.git_ref]
             run_command(command, self.git_path, context)
 
     def _git_update(self, context):
-        command = "git fetch origin {}".format(self.git_ref)
+        command = [*neutralEnv("git_command"), "fetch", "origin", self.git_ref]
         run_command(command, self.git_path, context)
         try:
-            command = "git merge --ff-only origin/{}".format(self.git_ref)
+            command = [
+                *neutralEnv("git_command"),
+                "merge",
+                "--ff-only",
+                f"origin/{self.git_ref}",
+            ]
             run_command(command, self.git_path, context)
         except subprocess.CalledProcessError:
             raise WarningMessage("Cannot update, please check log for information")
 
     def prepare(self):
         if not os.path.exists(self.git_path):
-            self.command('gitinit', self._git_init)
+            self.command("gitinit", self._git_init)
         else:
-            self.command('gitupdate', self._git_update)
+            self.command("gitupdate", self._git_update)
-        if hasattr(self, '_post_prepare_script'):
+        if hasattr(self, "_post_prepare_script"):
-            self.command('post_prepare_script', self._post_prepare_script)
+            self.command("post_prepare_script", self._post_prepare_script)
-
-
-class SvnClone(Source):
-    @property
-    def source_dir(self):
-        return self.svn_dir
-
-    @property
-    def svn_path(self):
-        return pj(neutralEnv('source_dir'), self.svn_dir)
-
-    def _svn_export(self, context):
-        if os.path.exists(self.svn_path):
-            raise SkipCommand()
-        command = "svn export {} {}".format(self.svn_remote, self.svn_dir)
-        run_command(command, neutralEnv('source_dir'), context)
-
-    def prepare(self):
-        self.command('svnexport', self._svn_export)
-        if hasattr(self, 'patches'):
-            self.command('patch', self._patch)
 
 
 class Builder:
@@ -228,7 +247,7 @@ class Builder:
         self.buildEnv = buildEnv
 
     @classmethod
-    def get_dependencies(cls, platformInfo, allDeps):
+    def get_dependencies(cls, configInfo, allDeps):
         return cls.dependencies
 
     @property
@@ -252,7 +271,7 @@ class Builder:
 
     def command(self, name, function, *args):
         print("  {} {} : ".format(name, self.name), end="", flush=True)
-        log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
+        log = pj(self._log_dir, "cmd_{}_{}.log".format(name, self.name))
         context = Context(name, log, self.target.force_native_build)
         if self.target.force_build:
             context.no_skip = True
@@ -270,7 +289,7 @@ class Builder:
         except subprocess.CalledProcessError:
             print(colorize("ERROR"))
             try:
-                with open(log, 'r') as f:
+                with open(log, "r") as f:
                     print(f.read())
             except:
                 pass
@@ -280,39 +299,43 @@ class Builder:
             raise
 
     def build(self):
-        if hasattr(self, '_pre_build_script'):
+        if hasattr(self, "_pre_build_script"):
-            self.command('pre_build_script', self._pre_build_script)
+            self.command("pre_build_script", self._pre_build_script)
-        self.command('configure', self._configure)
+        self.command("configure", self._configure)
-        if hasattr(self, '_post_configure_script'):
+        if hasattr(self, "_post_configure_script"):
-            self.command('post_configure_script', self._post_configure_script)
+            self.command("post_configure_script", self._post_configure_script)
-        self.command('compile', self._compile)
+        self.command("compile", self._compile)
-        if hasattr(self, '_test'):
+        if hasattr(self, "_test"):
-            self.command('test', self._test)
+            self.command("test", self._test)
-        self.command('install', self._install)
+        self.command("install", self._install)
-        if hasattr(self, '_post_build_script'):
+        if hasattr(self, "_post_build_script"):
-            self.command('post_build_script', self._post_build_script)
+            self.command("post_build_script", self._post_build_script)
 
     def make_dist(self):
-        if hasattr(self, '_pre_build_script'):
+        if hasattr(self, "_pre_build_script"):
-            self.command('pre_build_script', self._pre_build_script)
+            self.command("pre_build_script", self._pre_build_script)
-        self.command('configure', self._configure)
+        self.command("configure", self._configure)
-        self.command('make_dist', self._make_dist)
+        self.command("make_dist", self._make_dist)
 
     def set_flatpak_buildsystem(self, module):
-        if getattr(self, 'flatpak_buildsystem', None):
+        if getattr(self, "flatpak_buildsystem", None):
-            module['buildsystem'] = self.flatpak_buildsystem
+            module["buildsystem"] = self.flatpak_buildsystem
-        if getattr(self, 'subsource_dir', None):
+        if getattr(self, "subsource_dir", None):
-            module['subdir'] = self.subsource_dir
+            module["subdir"] = self.subsource_dir
-        if getattr(self, 'flatpack_build_options', None):
+        if getattr(self, "flatpack_build_options", None):
-            module['build-options'] = self.flatpack_build_options
+            module["build-options"] = self.flatpack_build_options
-        if getattr(self, 'configure_option', ''):
+        if getattr(self, "configure_options", ""):
-            module['config-opts'] = self.configure_option.split(' ')
+            module["config-opts"] = list(self.configure_options)
 
     def get_env(self, *, cross_comp_flags, cross_compilers, cross_path):
-        env = self.buildEnv.get_env(cross_comp_flags=cross_comp_flags, cross_compilers=cross_compilers, cross_path=cross_path)
+        env = self.buildEnv.get_env(
-        for dep in self.get_dependencies(self.buildEnv.platformInfo, False):
+            cross_comp_flags=cross_comp_flags,
+            cross_compilers=cross_compilers,
+            cross_path=cross_path,
+        )
+        for dep in self.get_dependencies(self.buildEnv.configInfo, False):
             try:
-                builder = get_target_step(dep, self.buildEnv.platformInfo.name)
+                builder = get_target_step(dep, self.buildEnv.configInfo.name)
                 builder.set_env(env)
             except KeyError:
                 # Some target may be missing (installed by a package, ...)
@@ -339,7 +362,7 @@ class TcCopyBuilder(Builder):
         return pj(self.buildEnv.toolchain_dir, self.target.full_name())
 
     def build(self):
-        self.command('copy', self._copy)
+        self.command("copy", self._copy)
 
     def _copy(self, context):
         context.try_skip(self.build_path)
@@ -354,233 +377,239 @@ class TcCopyBuilder(Builder):
 
 
 class MakeBuilder(Builder):
-    configure_option_template = "{dep_options} {static_option} {env_option} --prefix {install_dir} --libdir {libdir}"
+    configure_options = []
-    configure_option = ""
+    dynamic_configure_options = ["--enable-shared", "--disable-static"]
-    dynamic_configure_option = "--enable-shared --disable-static"
+    static_configure_options = ["--enable-static", "--disable-shared"]
-    static_configure_option = "--enable-static --disable-shared"
+    make_options = ["-j4"]
-    make_option = ""
+    install_options = []
-    install_option = ""
     configure_script = "configure"
     configure_env = {
-        '_format_CFLAGS' : '{env[CFLAGS]} -O3',
+        "_format_CFLAGS": "{env[CFLAGS]} -O3",
-        '_format_CXXFLAGS': '{env[CXXFLAGS]} -O3'
+        "_format_CXXFLAGS": "{env[CXXFLAGS]} -O3",
     }
-    make_target = ""
+    make_targets = []
     flatpak_buildsystem = None
 
     @property
-    def make_install_target(self):
+    def make_install_targets(self):
-        if self.buildEnv.platformInfo.build in ('iOS', "wasm"):
+        if self.buildEnv.configInfo.build in ("iOS", "wasm"):
-            return 'install'
+            yield "install"
-        return 'install-strip'
+        else:
+            yield "install-strip"
 
     @property
-    def all_configure_option(self):
+    def all_configure_options(self):
-        option = self.configure_option_template.format(
+        yield from self.configure_options
-            dep_options=self.configure_option,
+        if self.buildEnv.configInfo.static:
-            static_option=self.static_configure_option if self.buildEnv.platformInfo.static else self.dynamic_configure_option,
+            yield from self.static_configure_options
-            env_option=self.buildEnv.platformInfo.configure_option if not self.target.force_native_build else "",
+        else:
-            install_dir=self.buildEnv.install_dir,
+            yield from self.dynamic_configure_options
-            libdir=pj(self.buildEnv.install_dir, self.buildEnv.libprefix)
+        if not self.target.force_native_build:
-            )
+            yield from self.buildEnv.configInfo.configure_options
-        return option
+        yield from ("--prefix", self.buildEnv.install_dir)
+        yield from ("--libdir", pj(self.buildEnv.install_dir, self.buildEnv.libprefix))
 
     def set_configure_env(self, env):
         dep_conf_env = self.configure_env
         if not dep_conf_env:
             return
         for k, v in dep_conf_env.items():
-            if k.startswith('_format_'):
+            if k.startswith("_format_"):
                 v = v.format(buildEnv=self.buildEnv, env=env)
                 env[k[8:]] = v
             else:
                 env[k] = v
 
-
     def _configure(self, context):
         context.try_skip(self.build_path)
-        command = "{configure_wrapper}{configure_script} {configure_option}"
+        command = [
-        command = command.format(
+            *self.buildEnv.configure_wrapper,
-            configure_wrapper=self.buildEnv.configure_wrapper,
+            pj(self.source_path, self.configure_script),
-            configure_script=pj(self.source_path, self.configure_script),
+            *self.all_configure_options,
-            configure_option=self.all_configure_option
+        ]
-        )
         env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
         self.set_configure_env(env)
         run_command(command, self.build_path, context, env=env)
 
     def _compile(self, context):
         context.try_skip(self.build_path)
-        command = "{make_wrapper}make -j4 {make_target} {make_option}".format(
+        command = [
-            make_wrapper=self.buildEnv.make_wrapper,
+            *self.buildEnv.make_wrapper,
-            make_target=self.make_target,
+            *neutralEnv("make_command"),
-            make_option=self.make_option
+            *self.make_targets,
-        )
+            *self.make_options,
+        ]
         env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
         run_command(command, self.build_path, context, env=env)
 
     def _install(self, context):
         context.try_skip(self.build_path)
-        command = "{make_wrapper}make {make_install_target} {make_option}".format(
+        command = [
-            make_wrapper=self.buildEnv.make_wrapper,
+            *self.buildEnv.make_wrapper,
-            make_install_target=self.make_install_target,
+            *neutralEnv("make_command"),
-            make_option=self.make_option
+            *self.make_install_targets,
-        )
+            *self.make_options,
+        ]
         env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
         run_command(command, self.build_path, context, env=env)
 
     def _make_dist(self, context):
         context.try_skip(self.build_path)
-        command = "{make_wrapper}make dist".format(
+        command = [*self.buildEnv.make_wrapper, *neutralEnv("make_command"), "dist"]
-            make_wrapper=self.buildEnv.make_wrapper
-        )
         env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
         run_command(command, self.build_path, context, env=env)
 
 
 class CMakeBuilder(MakeBuilder):
-    flatpak_buildsystem = 'cmake'
+    flatpak_buildsystem = "cmake"
 
     def _configure(self, context):
         context.try_skip(self.build_path)
-        cross_option = ""
+        cross_options = []
         if not self.target.force_native_build and self.buildEnv.cmake_crossfile:
-            cross_option = "-DCMAKE_TOOLCHAIN_FILE={}".format(self.buildEnv.cmake_crossfile)
+            cross_options += [f"-DCMAKE_TOOLCHAIN_FILE={self.buildEnv.cmake_crossfile}"]
-        command = ("cmake {configure_option}"
+        command = [
-                   " -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON"
+            *neutralEnv("cmake_command"),
-                   " -DCMAKE_INSTALL_PREFIX={install_dir}"
+            *self.configure_options,
-                   " -DCMAKE_INSTALL_LIBDIR={libdir}"
+            "-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON",
-                   " {source_path}"
+            f"-DCMAKE_INSTALL_PREFIX={self.buildEnv.install_dir}",
-                   " {cross_option}")
+            f"-DCMAKE_INSTALL_LIBDIR={self.buildEnv.libprefix}",
-        command = command.format(
+            self.source_path,
-            configure_option=self.configure_option,
+            *cross_options,
-            install_dir=self.buildEnv.install_dir,
+        ]
-            libdir=self.buildEnv.libprefix,
+        env = self.get_env(
-            source_path=self.source_path,
+            cross_comp_flags=True, cross_compilers=False, cross_path=True
-            cross_option=cross_option
         )
-        env = self.get_env(cross_comp_flags=True, cross_compilers=False, cross_path=True)
         self.set_configure_env(env)
         run_command(command, self.build_path, context, env=env)
 
     def set_flatpak_buildsystem(self, module):
-        super().set_flatpak_buildsystem( module)
+        super().set_flatpak_buildsystem(module)
-        module['buildir'] = True
+        module["buildir"] = True
 
 
 class QMakeBuilder(MakeBuilder):
-    qmake_target = ""
+    qmake_targets = []
-    flatpak_buildsystem = 'qmake'
+    flatpak_buildsystem = "qmake"
 
     @property
-    def env_option(self):
+    def make_options(self):
-        options = ""
+        if platform.system() == "Windows":
-        if 'QMAKE_CC' in os.environ:
+            return []
-            options += 'QMAKE_CC={} '.format(os.environ['QMAKE_CC'])
+        return super().make_options
-        if 'QMAKE_CXX' in os.environ:
+
-            options += 'QMAKE_CXX={} '.format(os.environ['QMAKE_CXX'])
+    @property
-        return options
+    def env_options(self):
+        if "QMAKE_CC" in os.environ:
+            yield f"QMAKE_CC={os.environ['QMAKE_CC']}"
+        if "QMAKE_CXX" in os.environ:
+            yield f"QMAKE_CXX={os.environ['QMAKE_CXX']}"
 
     def _configure(self, context):
         context.try_skip(self.build_path)
-        cross_option = ""
+        command = [
-        command = ("{command} {configure_option}"
+            *neutralEnv("qmake_command"),
-                   " {env_option}"
+            *self.configure_options,
-                   " {source_path}"
+            *self.env_options,
-                   " {cross_option}")
+            self.source_path,
-        command = command.format(
+        ]
-            command = neutralEnv('qmake_command'),
+        env = self.get_env(
-            configure_option=self.configure_option,
+            cross_comp_flags=True, cross_compilers=False, cross_path=True
-            env_option=self.env_option,
-            source_path=self.source_path,
-            cross_option=cross_option
         )
-        env = self.get_env(cross_comp_flags=True, cross_compilers=False, cross_path=True)
         self.set_configure_env(env)
         run_command(command, self.build_path, context, env=env)
 
     def _make_dist(self, context):
-        command = "git archive -o {build_dir}/{name}.tar.gz --prefix={name}/ HEAD"
+        command = [
-        command = command.format(
+            *neutralEnv("git_command"),
-            build_dir = self.build_path,
+            "archive",
-            name = self.target.full_name()
+            "-o",
-        )
+            f"{self.build_path}/{self.target.full_name()}.tar.gz",
+            f"--prefix={self.target.full_name()}/",
+            "HEAD",
+        ]
         run_command(command, self.source_path, context)
 
 
-
 class MesonBuilder(Builder):
-    configure_option = ""
+    configure_options = []
-    test_option = ""
+    test_options = []
-    flatpak_buildsystem = 'meson'
+    flatpak_buildsystem = "meson"
 
     @property
     def build_type(self):
-        return 'release' if option('make_release') else 'debug'
+        if platform.system() == "Windows":
+            return "release"
+
+        return "release" if option("make_release") else "debug"
 
     @property
-    def strip_option(self):
+    def strip_options(self):
-        return '--strip' if option('make_release') else ''
+        if option("make_release"):
+            yield "--strip"
 
     @property
     def library_type(self):
-        return 'static' if self.buildEnv.platformInfo.static else 'shared'
+        return "static" if self.buildEnv.configInfo.static else "shared"
 
     def _configure(self, context):
-        context.no_skip = False
         context.try_skip(self.build_path)
         if os.path.exists(self.build_path):
             shutil.rmtree(self.build_path)
         os.makedirs(self.build_path)
-        configure_option = self.configure_option.format(buildEnv=self.buildEnv)
+        cross_options = []
-        cross_option = ""
         if not self.target.force_native_build and self.buildEnv.meson_crossfile:
-            cross_option = "--cross-file {}".format(
+            cross_options += ["--cross-file", self.buildEnv.meson_crossfile]
-                self.buildEnv.meson_crossfile)
+        command = [
-        command = ("{command} . {build_path}"
+            *neutralEnv("meson_command"),
-                   " --buildtype={build_type} {strip_option}"
+            "setup",
-                   " --default-library={library_type}"
+            ".",
-                   " {configure_option}"
+            self.build_path,
-                   " --prefix={buildEnv.install_dir}"
+            f"--buildtype={self.build_type}",
-                   " --libdir={buildEnv.libprefix}"
+            *self.strip_options,
-                   " {cross_option}")
+            f"--default-library={self.library_type}",
-        command = command.format(
+            *self.configure_options,
-            command=neutralEnv('meson_command'),
+            f"--prefix={self.buildEnv.install_dir}",
-            build_type=self.build_type,
+            f"--libdir={self.buildEnv.libprefix}",
-            strip_option=self.strip_option,
+            *cross_options,
-            library_type=self.library_type,
+        ]
-            configure_option=configure_option,
+        env = self.get_env(
-            build_path=self.build_path,
+            cross_comp_flags=False, cross_compilers=False, cross_path=True
-            buildEnv=self.buildEnv,
-            cross_option=cross_option
         )
-        env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True)
         run_command(command, self.source_path, context, env=env)
 
     def _compile(self, context):
         context.try_skip(self.build_path)
-        command = "{} -v".format(neutralEnv('ninja_command'))
+        command = [*neutralEnv("ninja_command"), "-v"]
-        env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True)
+        env = self.get_env(
+            cross_comp_flags=False, cross_compilers=False, cross_path=True
+        )
         run_command(command, self.build_path, context, env=env)
 
     def _test(self, context):
         context.try_skip(self.build_path)
-        if ( self.buildEnv.platformInfo.build == 'android'
+        if self.buildEnv.configInfo.build == "android" or (
-             or (self.buildEnv.platformInfo.build != 'native'
+            self.buildEnv.configInfo.build != "native"
-                 and not self.buildEnv.platformInfo.static)
+            and not self.buildEnv.configInfo.static
-           ):
+        ):
             raise SkipCommand()
-        command = "{} --verbose {}".format(neutralEnv('mesontest_command'), self.test_option)
+        command = [*neutralEnv("mesontest_command"), "--verbose", *self.test_options]
-        env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True)
+        env = self.get_env(
+            cross_comp_flags=False, cross_compilers=False, cross_path=True
+        )
         run_command(command, self.build_path, context, env=env)
 
     def _install(self, context):
         context.try_skip(self.build_path)
-        command = "{} -v install".format(neutralEnv('ninja_command'))
+        command = [*neutralEnv("ninja_command"), "-v", "install"]
-        env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True)
+        env = self.get_env(
+            cross_comp_flags=False, cross_compilers=False, cross_path=True
+        )
         run_command(command, self.build_path, context, env=env)
 
     def _make_dist(self, context):
-        command = "{} -v dist".format(neutralEnv('ninja_command'))
+        command = [*neutralEnv("ninja_command"), "-v", "dist"]
-        env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True)
+        env = self.get_env(
+            cross_comp_flags=False, cross_compilers=False, cross_path=True
+        )
         run_command(command, self.build_path, context, env=env)

kiwixbuild/dependencies/boostregex.py

@@ -0,0 +1,27 @@
+from .base import Dependency, ReleaseDownload, Builder as BaseBuilder
+
+from kiwixbuild.utils import Remotefile, pj
+from shutil import copytree
+
+
+class BoostRegex(Dependency):
+    name = "boostregex"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile(
+            "regex-boost-1.86.0.zip",
+            "",
+            "https://codeload.github.com/boostorg/regex/zip/refs/tags/boost-1.86.0",
+        )
+
+    class Builder(BaseBuilder):
+        def build(self):
+            self.command("copy_headers", self._copy_headers)
+
+        def _copy_headers(self, context):
+            context.try_skip(self.build_path)
+            copytree(
+                pj(self.source_path, "include", "boost"),
+                pj(self.buildEnv.install_dir, "include", "boost"),
+                dirs_exist_ok=True,
+            )

kiwixbuild/dependencies/docoptcpp.py

@@ -1,21 +1,34 @@
-from .base import (
+from .base import Dependency, ReleaseDownload, MesonBuilder
-    Dependency,
-    GitClone,
-    CMakeBuilder)
 
 from kiwixbuild.utils import Remotefile
-
+from kiwixbuild._global import neutralEnv
-
 
 class docoptcpp(Dependency):
-    name = 'docoptcpp'
+    name = "docoptcpp"
 
-    class Source(GitClone):
+    class Source(ReleaseDownload):
-        git_remote = "https://github.com/docopt/docopt.cpp.git"
+        name = "docoptcpp"
-        git_dir = "docopt.cpp"
+        src_archive = Remotefile(
-        force_full_clone = True
+            "v0.6.3.tar.gz",
-        git_ref = "3dd23e3280f213bacefdf5fcb04857bf52e90917"
+            "28af5a0c482c6d508d22b14d588a3b0bd9ff97135f99c2814a5aa3cbff1d6632",
+            "https://github.com/docopt/docopt.cpp/archive/v0.6.3.tar.gz",
+        )
 
-    class Builder(CMakeBuilder):
+        meson_archive = Remotefile(
-        make_install_target = 'install'
+            "docopt_0.6.3-3_patch.zip",
+            "1f641187f9d3f35b0a5ebd2011876ef8e9b04b69b7b163095dd7dfa16219ad01",
+            "https://wrapdb.mesonbuild.com/v2/docopt_0.6.3-3/get_patch",
+        )
 
+        archives = [src_archive, meson_archive]
+        patches = [
+            "docopt_meson_install_pkgconfig.patch",
+            "docopt_meson_use_boostregex.patch",
+        ]
+
+    class Builder(MesonBuilder):
+        @classmethod
+        def get_dependencies(cls, configInfo, allDeps):
+            if neutralEnv("distname") == "Windows":
+                return ["boostregex"]
+            return []

kiwixbuild/dependencies/gumbo.py

@@ -1,8 +1,4 @@
-from .base import (
+from .base import Dependency, ReleaseDownload, MakeBuilder
-    Dependency,
-    ReleaseDownload,
-    MakeBuilder
-)
 
 from kiwixbuild.utils import Remotefile, run_command
 
@@ -11,13 +7,15 @@ class Gumbo(Dependency):
     name = "gumbo"
 
     class Source(ReleaseDownload):
-        archive = Remotefile('gumbo-0.10.1.tar.gz',
+        archive = Remotefile(
-                             '28463053d44a5dfbc4b77bcf49c8cee119338ffa636cc17fc3378421d714efad',
+            "gumbo-0.10.1.tar.gz",
-                             'https://github.com/google/gumbo-parser/archive/v0.10.1.tar.gz')
+            "28463053d44a5dfbc4b77bcf49c8cee119338ffa636cc17fc3378421d714efad",
+            "https://github.com/google/gumbo-parser/archive/v0.10.1.tar.gz",
+        )
 
         def _post_prepare_script(self, context):
             context.try_skip(self.extract_path)
-            command = "./autogen.sh"
+            command = ["./autogen.sh"]
             run_command(command, self.extract_path, context)
 
     Builder = MakeBuilder

kiwixbuild/dependencies/icu4c.py

@@ -1,90 +1,191 @@
 from .base import (
     Dependency,
     ReleaseDownload,
-    MakeBuilder
+    MakeBuilder,
+    Builder as BaseBuilder,
 )
 
 from kiwixbuild.utils import pj, SkipCommand, Remotefile, extract_archive
 from kiwixbuild._global import get_target_step, neutralEnv
 import os, shutil
 import fileinput
+import platform
 
-class Icu(Dependency):
+if platform.system() == "Windows":
-    name = "icu4c"
 
-    class Source(ReleaseDownload):
+    class Icu(Dependency):
-        archive_src = Remotefile('icu4c-73_2-src.tgz',
+        name = "icu4c"
-                                 '818a80712ed3caacd9b652305e01afc7fa167e6f2e94996da44b90c2ab604ce1',
-                                 'https://github.com/unicode-org/icu/releases/download/release-73-2/icu4c-73_2-src.tgz')
-        archive_data = Remotefile('icu4c-73_2-data.zip',
-                                  'ca1ee076163b438461e484421a7679fc33a64cd0a54f9d4b401893fa1eb42701',
-                                  'https://github.com/unicode-org/icu/releases/download/release-73-2/icu4c-73_2-data.zip')
 
-
+        class Source(ReleaseDownload):
-        archives = [archive_src, archive_data]
+            archive = Remotefile(
-
+                "icu4c-74_1-Win64-MSVC2022.zip",
-        def _extract(self, context):
+                "",
-            context.try_skip(self.extract_path)
+                "https://github.com/unicode-org/icu/releases/download/release-74-1/icu4c-74_1-Win64-MSVC2022.zip",
-            if os.path.exists(self.extract_path):
-                shutil.rmtree(self.extract_path)
-            extract_archive(
-                pj(neutralEnv('archive_dir'), self.archive_src.name),
-                neutralEnv('source_dir'),
-                topdir=None,
-                name=self.source_dir)
-            shutil.rmtree(pj(neutralEnv('source_dir'), self.source_dir, 'source', 'data'))
-            extract_archive(
-                pj(neutralEnv('archive_dir'), self.archive_data.name),
-                pj(neutralEnv('source_dir'), self.source_dir, 'source'),
-                topdir='data',
-                name='data'
             )
 
-        patches = [
+        class Builder(BaseBuilder):
-                   "icu4c_fix_static_lib_name_mingw.patch",
+            def build(self):
-        #           "icu4c_android_elf64_st_info.patch",
+                self.command("copy_headers", self._copy_headers)
-        #           "icu4c_custom_data.patch",
+                self.command("copy_bins", self._copy_bin)
-        #           "icu4c_noxlocale.patch",
+                self.command("generate_pkg_config", self._generate_pkg_config)
-                   "icu4c_rpath.patch",
-        #           "icu4c_build_config.patch",
-                   "icu4c_wasm.patch"
-                  ]
 
+            def _copy_headers(self, context):
+                context.try_skip(self.build_path)
+                shutil.copytree(
+                    pj(self.source_path, "include", "unicode"),
+                    pj(self.buildEnv.install_dir, "include", "unicode"),
+                )
 
-    class Builder(MakeBuilder):
+            def _copy_bin(self, context):
-        subsource_dir = "source"
+                context.try_skip(self.build_path)
-        make_install_target = "install"
+                shutil.copytree(
+                    pj(self.source_path, "lib64"),
+                    pj(self.buildEnv.install_dir, "lib"),
+                    dirs_exist_ok=True,
+                )
 
-        @classmethod
+                def ignore_non_dll(path, names):
-        def get_dependencies(cls, platformInfo, allDeps):
+                    return [n for n in names if not n.endswith(".dll")]
-            plt = 'native_static' if platformInfo.static else 'native_dyn'
-            return [(plt, 'icu4c')]
 
-        @property
+                shutil.copytree(
-        def configure_option(self):
+                    pj(self.source_path, "bin64"),
-            options = ("--disable-samples --disable-tests --disable-extras "
+                    pj(self.buildEnv.install_dir, "bin"),
-                       "--disable-dyload --enable-rpath "
+                    ignore=ignore_non_dll,
-                       "--disable-icuio --disable-layoutex")
+                    dirs_exist_ok=True,
-            platformInfo = self.buildEnv.platformInfo
+                )
-            if platformInfo.build != 'native':
-                icu_native_builder = get_target_step(
-                    'icu4c',
-                    'native_static' if platformInfo.static else 'native_dyn')
-                options += " --with-cross-build={} --disable-tools".format(
-                    icu_native_builder.build_path)
-            if platformInfo.build in ('android', 'wasm'):
-                options += " --with-data-packaging=archive"
-            return options
 
-        def set_env(self, env):
+            def _generate_pkg_config(self, context):
-            env['ICU_DATA_FILTER_FILE'] = pj(os.path.dirname(os.path.realpath(__file__)), "icu4c_data_filter.json")
+                context.try_skip(self.build_path)
 
-        def _post_configure_script(self, context):
+                pkg_config_template = """ Copyright (C) 2016 and later: Unicode, Inc. and others.
-            if self.buildEnv.platformInfo.build != "wasm":
+# License & terms of use: http://www.unicode.org/copyright.html
-                context.skip()
+# Copyright (C) 2010-2013, International Business Machines Corporation. All Rights Reserved.
-            context.try_skip(self.build_path)
+
-            for line in fileinput.input(pj(self.build_path, 'Makefile'), inplace=True):
+# CFLAGS contains only anything end users should set
-                if line == "#DATASUBDIR = data\n":
+CFLAGS = 
-                    print("DATASUBDIR = data")
+# CXXFLAGS contains only anything end users should set
-                else:
+CXXFLAGS =  -std=c++11
-                    print(line, end="")
+# DEFS only contains those UCONFIG_CPPFLAGS which are not auto-set by platform.h
+DEFS = 
+prefix = {prefix}
+exec_prefix = ${{prefix}}
+libdir = ${{exec_prefix}}/lib
+includedir = ${{prefix}}/include
+baselibs = -lpthread -lm 
+UNICODE_VERSION=15.0
+ICUPREFIX=icu
+ICULIBSUFFIX=
+LIBICU=lib${{ICUPREFIX}}
+pkglibdir=${{libdir}}/icu${{ICULIBSUFFIX}}/73.1
+ICUDATA_NAME = icudt73l
+ICUDESC=International Components for Unicode
+
+Version: 73.1
+Cflags: -I${{includedir}}
+Description: International Components for Unicode: Internationalization library
+Name: icu-i18n
+Libs: -L${{libdir}} -licuin -licuuc -licudt"""
+
+                pkg_config_content = pkg_config_template.format(
+                    prefix=self.buildEnv.install_dir
+                )
+
+                os.makedirs(
+                    pj(self.buildEnv.install_dir, "lib", "pkgconfig"), exist_ok=True
+                )
+                with open(
+                    pj(self.buildEnv.install_dir, "lib", "pkgconfig", "icu-i18n.pc"),
+                    mode="w",
+                ) as f:
+                    f.write(pkg_config_content)
+
+else:
+
+    class Icu(Dependency):
+        name = "icu4c"
+
+        class Source(ReleaseDownload):
+            archive_src = Remotefile(
+                "icu4c-73_2-src.tgz",
+                "818a80712ed3caacd9b652305e01afc7fa167e6f2e94996da44b90c2ab604ce1",
+                "https://github.com/unicode-org/icu/releases/download/release-73-2/icu4c-73_2-src.tgz",
+            )
+            archive_data = Remotefile(
+                "icu4c-73_2-data.zip",
+                "ca1ee076163b438461e484421a7679fc33a64cd0a54f9d4b401893fa1eb42701",
+                "https://github.com/unicode-org/icu/releases/download/release-73-2/icu4c-73_2-data.zip",
+            )
+
+            archives = [archive_src, archive_data]
+
+            def _extract(self, context):
+                context.try_skip(self.extract_path)
+                if os.path.exists(self.extract_path):
+                    shutil.rmtree(self.extract_path)
+                extract_archive(
+                    pj(neutralEnv("archive_dir"), self.archive_src.name),
+                    neutralEnv("source_dir"),
+                    topdir=None,
+                    name=self.source_dir,
+                )
+                shutil.rmtree(
+                    pj(neutralEnv("source_dir"), self.source_dir, "source", "data")
+                )
+                extract_archive(
+                    pj(neutralEnv("archive_dir"), self.archive_data.name),
+                    pj(neutralEnv("source_dir"), self.source_dir, "source"),
+                    topdir="data",
+                    name="data",
+                )
+
+            patches = [
+                "icu4c_fix_static_lib_name_mingw.patch",
+                "icu4c_rpath.patch",
+                "icu4c_wasm.patch",
+            ]
+
+        class Builder(MakeBuilder):
+            subsource_dir = "source"
+            make_install_targets = ["install"]
+
+            @classmethod
+            def get_dependencies(cls, configInfo, allDeps):
+                plt = "native_static" if configInfo.static else "native_dyn"
+                return [(plt, "icu4c")]
+
+            @property
+            def configure_options(self):
+                yield "--disable-samples"
+                yield "--disable-tests"
+                yield "--disable-extras"
+                yield "--disable-dyload"
+                yield "--enable-rpath"
+                yield "--disable-icuio"
+                yield "--disable-layoutex"
+                configInfo = self.buildEnv.configInfo
+                if configInfo.build != "native":
+                    icu_native_builder = get_target_step(
+                        "icu4c", "native_static" if configInfo.static else "native_dyn"
+                    )
+                    yield f"--with-cross-build={icu_native_builder.build_path}"
+                    yield "--disable-tools"
+                if configInfo.build in ("android", "wasm"):
+                    yield "--with-data-packaging=archive"
+
+            def set_env(self, env):
+                env["ICU_DATA_FILTER_FILE"] = pj(
+                    os.path.dirname(os.path.realpath(__file__)),
+                    "icu4c_data_filter.json",
+                )
+
+            def _post_configure_script(self, context):
+                if self.buildEnv.configInfo.build != "wasm":
+                    context.skip()
+                context.try_skip(self.build_path)
+                for line in fileinput.input(
+                    pj(self.build_path, "Makefile"), inplace=True
+                ):
+                    if line == "#DATASUBDIR = data\n":
+                        print("DATASUBDIR = data")
+                    else:
+                        print(line, end="")

kiwixbuild/dependencies/ios_fat_lib.py

@@ -1,13 +1,9 @@
 import os
 
-from kiwixbuild.platforms import PlatformInfo
+from kiwixbuild.configs import ConfigInfo
 from kiwixbuild.utils import pj, copy_tree, run_command
 from kiwixbuild._global import option
-from .base import (
+from .base import Dependency, NoopSource, Builder as BaseBuilder
-    Dependency,
-    NoopSource,
-    Builder as BaseBuilder)
-
 
 
 class IOSFatLib(Dependency):
@@ -16,37 +12,39 @@ class IOSFatLib(Dependency):
     Source = NoopSource
 
     class Builder(BaseBuilder):
-
         @classmethod
         def get_dependencies(self, platfomInfo, alldeps):
-            base_target = option('target')
+            base_target = option("target")
-            return [('iOS_{}'.format(arch), base_target) for arch in option('ios_arch')]
+            return [("iOS_{}".format(arch), base_target) for arch in option("ios_arch")]
 
         def _copy_headers(self, context):
-            plt = PlatformInfo.get_platform('iOS_{}'.format(option('ios_arch')[0]))
+            plt = ConfigInfo.get_config("iOS_{}".format(option("ios_arch")[0]))
-            include_src = pj(plt.buildEnv.install_dir, 'include')
+            include_src = pj(plt.buildEnv.install_dir, "include")
-            include_dst = pj(self.buildEnv.install_dir, 'include')
+            include_dst = pj(self.buildEnv.install_dir, "include")
             copy_tree(include_src, include_dst)
 
         def _merge_libs(self, context):
             lib_dirs = []
-            for arch in option('ios_arch'):
+            for arch in option("ios_arch"):
-                plt = PlatformInfo.get_platform('iOS_{}'.format(arch))
+                plt = ConfigInfo.get_config("iOS_{}".format(arch))
-                lib_dirs.append(pj(plt.buildEnv.install_dir, 'lib'))
+                lib_dirs.append(pj(plt.buildEnv.install_dir, "lib"))
             libs = []
             for f in os.listdir(lib_dirs[0]):
                 if os.path.islink(pj(lib_dirs[0], f)):
                     continue
-                if f.endswith('.a') or f.endswith('.dylib'):
+                if f.endswith(".a") or f.endswith(".dylib"):
                     libs.append(f)
-            os.makedirs(pj(self.buildEnv.install_dir, 'lib'), exist_ok=True)
+            os.makedirs(pj(self.buildEnv.install_dir, "lib"), exist_ok=True)
-            command_tmp = "lipo -create {input} -output {output}"
             for l in libs:
-                command = command_tmp.format(
+                command = [
-                    input=" ".join(pj(d, l) for d in lib_dirs),
+                    "lipo",
-                    output=pj(self.buildEnv.install_dir, 'lib', l))
+                    "-create",
+                    *[pj(d, l) for d in lib_dirs],
+                    "-output",
+                    pj(self.buildEnv.install_dir, "lib", l),
+                ]
                 run_command(command, self.buildEnv.install_dir, context)
 
         def build(self):
-            self.command('copy_headers', self._copy_headers)
+            self.command("copy_headers", self._copy_headers)
-            self.command('merge_libs', self._merge_libs)
+            self.command("merge_libs", self._merge_libs)

kiwixbuild/dependencies/kiwix_desktop.py

@@ -1,7 +1,7 @@
-from .base import (
+from kiwixbuild._global import option
-    Dependency,
+from .base import Dependency, GitClone, QMakeBuilder
-    GitClone,
+import platform
-    QMakeBuilder)
+
 
 class KiwixDesktop(Dependency):
     name = "kiwix-desktop"
@@ -13,25 +13,27 @@ class KiwixDesktop(Dependency):
 
     class Builder(QMakeBuilder):
         dependencies = ["qt", "qtwebengine", "libkiwix", "aria2"]
-        make_install_target = 'install'
         configure_env = None
 
-        flatpack_build_options = {
+        flatpack_build_options = {"env": {"QMAKEPATH": "/app"}}
-            "env": [
-                "QMAKEPATH=/app/lib"
-            ]
-        }
 
         @property
-        def configure_option(self):
+        def make_targets(self):
-            if self.buildEnv.platformInfo.name == 'flatpak':
+            if platform.system() == "Windows":
-                options = [
+                yield "release-all"
-                    'QMAKE_INCDIR+=/app/include/QtWebEngine',
-                    'QMAKE_INCDIR+=/app/include/QtWebEngineCore',
-                    'QMAKE_INCDIR+=/app/include/QtWebEngineWidgets'
-                ]
             else:
-                options = ["PREFIX={}".format(self.buildEnv.install_dir)]
+                yield from super().make_targets
-                if self.buildEnv.platformInfo.static:
+
-                    options.append('"CONFIG+=static"')
+        @property
-            return " ".join(options)
+        def make_install_targets(self):
+            if platform.system() == "Windows":
+                yield "release-install"
+            else:
+                yield "install"
+
+        @property
+        def configure_options(self):
+            if self.buildEnv.configInfo.name != "flatpak":
+                yield f"PREFIX={self.buildEnv.install_dir}"
+            if self.buildEnv.configInfo.static:
+                yield "CONFIG+=static"

kiwixbuild/dependencies/kiwix_tools.py

@@ -1,7 +1,5 @@
-from .base import (
+from .base import Dependency, GitClone, MesonBuilder
-    Dependency,
+
-    GitClone,
-    MesonBuilder)
 
 class KiwixTools(Dependency):
     name = "kiwix-tools"
@@ -12,10 +10,9 @@ class KiwixTools(Dependency):
         git_dir = "kiwix-tools"
 
     class Builder(MesonBuilder):
-        dependencies = ["libkiwix"]
+        dependencies = ["libkiwix", "docoptcpp"]
 
         @property
-        def configure_option(self):
+        def configure_options(self):
-            if self.buildEnv.platformInfo.static:
+            if self.buildEnv.configInfo.static:
-                return "-Dstatic-linkage=true"
+                yield "-Dstatic-linkage=true"
-            return ""

kiwixbuild/dependencies/libcurl.py

@@ -3,28 +3,59 @@ import os
 from .base import (
     Dependency,
     ReleaseDownload,
-    MakeBuilder,
+    MesonBuilder,
 )
 
 from kiwixbuild.utils import Remotefile, pj, Defaultdict, SkipCommand, run_command
 from kiwixbuild._global import get_target_step
 
+
 class LibCurl(Dependency):
     name = "libcurl"
 
     class Source(ReleaseDownload):
         name = "libcurl"
-        archive = Remotefile('curl-7.67.0.tar.xz',
+        src_archive = Remotefile(
-                             'f5d2e7320379338c3952dcc7566a140abb49edb575f9f99272455785c40e536c',
+            "curl-8.4.0.tar.xz",
-                             'https://curl.haxx.se/download/curl-7.67.0.tar.xz')
+            "16c62a9c4af0f703d28bda6d7bbf37ba47055ad3414d70dec63e2e6336f2a82d",
+            "https://curl.se/download/curl-8.4.0.tar.xz",
+        )
+        meson_archive = Remotefile(
+            "curl_8.4.0-2_patch.zip",
+            "bbb6ae75225c36ef9bb336cface729794c7c070c623a003fff40bd416042ff6e",
+            "https://dev.kiwix.org/libkiwix/curl_8.4.0-2_patch.zip",
+        )
+        archives = [src_archive, meson_archive]
 
-    class Builder(MakeBuilder):
+    class Builder(MesonBuilder):
-        dependencies = ['zlib']
+        dependencies = ["zlib"]
-        configure_option = " ".join(
+        configure_options = [
-            ["--without-{}".format(p)
+            f"-D{p}=disabled"
-                for p in ('libssh2', 'ssl', 'libmetalink', 'librtmp',
+            for p in (
-                          'nghttp2', 'libidn2', 'brotli')] +
+                "psl",
-            ["--disable-{}".format(p)
+                "kerberos-auth",
-                for p in ('ftp', 'file', 'ldap', 'ldaps', 'rtsp', 'dict',
+                "gss-api",
-                          'telnet', 'tftp', 'pop3', 'imap', 'smb', 'smtp',
+                "ssh",
-                          'gopher', 'manual')])
+                "rtmp",
+                "http2",
+                "idn",
+                "brotli",
+                "ftp",
+                "file",
+                "ldap",
+                "ldaps",
+                "rtsp",
+                "dict",
+                "telnet",
+                "tftp",
+                "pop3",
+                "imap",
+                "smb",
+                "smtp",
+                "gopher",
+                "tool",
+            )
+        ]
+
+        def _test(self, context):
+            context.skip("No Test")

kiwixbuild/dependencies/libkiwix.py

@@ -1,12 +1,10 @@
 import shutil, os
 
-from .base import (
+from .base import Dependency, GitClone, MesonBuilder
-    Dependency,
-    GitClone,
-    MesonBuilder)
 from kiwixbuild.utils import pj, copy_tree
 from kiwixbuild._global import option, get_target_step, neutralEnv
 
+
 class Libkiwix(Dependency):
     name = "libkiwix"
     force_build = True
@@ -16,22 +14,39 @@ class Libkiwix(Dependency):
         git_dir = "libkiwix"
 
     class Builder(MesonBuilder):
-        dependencies = ["pugixml", "libzim", "zlib", "lzma", "libcurl", "libmicrohttpd", "icu4c", "mustache", "xapian-core"]
+        dependencies = [
-        strip_option = ''
+            "pugixml",
+            "libzim",
+            "zlib",
+            "libcurl",
+            "libmicrohttpd",
+            "icu4c",
+            "mustache",
+            "xapian-core",
+        ]
+        strip_options = []
 
         @property
-        def configure_option(self):
+        def build_type(self):
-            platformInfo = self.buildEnv.platformInfo
+            if self.buildEnv.configInfo.build == "android":
-            if platformInfo.build == 'android':
+                return "debug"
-                return '-Dstatic-linkage=true -Dwerror=false'
+            return super().build_type
-            if platformInfo.name == 'flatpak':
+
-                return '--wrap-mode=nodownload'
+        @property
-            if platformInfo.mixed and option('target') == 'libkiwix':
+        def configure_options(self):
-                return "-Dstatic-linkage=true"
+            configInfo = self.buildEnv.configInfo
-            return ''
+            if configInfo.build == "android":
+                yield "-Dstatic-linkage=true"
+                yield "-Dwerror=false"
+            if configInfo.build == "iOS":
+                yield "-Db_bitcode=true"
+            if configInfo.name == "flatpak":
+                yield "--wrap-mode=nodownload"
+            if configInfo.mixed and option("target") == "libkiwix":
+                yield "-Dstatic-linkage=true"
 
         @property
         def library_type(self):
-            if self.buildEnv.platformInfo.build == 'android':
+            if self.buildEnv.configInfo.build == "android":
-                return 'shared'
+                return "shared"
             return super().library_type

kiwixbuild/dependencies/libmagic.py

@@ -6,44 +6,45 @@ from .base import (
     MakeBuilder,
 )
 
-from kiwixbuild.utils import Remotefile, pj, Defaultdict, SkipCommand, run_command
+from kiwixbuild.utils import Remotefile, pj, SkipCommand, run_command
 from kiwixbuild._global import get_target_step
 
+
 class LibMagic(Dependency):
     name = "libmagic"
 
     class Source(ReleaseDownload):
         name = "libmagic"
         source_dir = "libmagic"
-        archive_top_dir = 'file-5.44'
+        archive_top_dir = "file-5.44"
-        archive = Remotefile('file-5.44.tar.gz',
+        archive = Remotefile(
-                             '3751c7fba8dbc831cb8d7cc8aff21035459b8ce5155ef8b0880a27d028475f3b')
+            "file-5.44.tar.gz",
+            "3751c7fba8dbc831cb8d7cc8aff21035459b8ce5155ef8b0880a27d028475f3b",
+        )
 
     class Builder(MakeBuilder):
         @property
-        def configure_option(self):
+        def configure_options(self):
-            return ("--disable-bzlib "
+            yield "--disable-bzlib"
-                    "--disable-xzlib "
+            yield "--disable-xzlib"
-                    "--disable-zstdlib "
+            yield "--disable-zstdlib"
-                    "--disable-lzlib "
+            yield "--disable-lzlib"
-                  )
 
         @classmethod
-        def get_dependencies(cls, platformInfo, allDeps):
+        def get_dependencies(cls, configInfo, allDeps):
-            if platformInfo.build != 'native':
+            if configInfo.build != "native":
-                return [('native_static', 'libmagic')]
+                return [("native_static", "libmagic")]
             return []
 
         def _compile(self, context):
-            platformInfo = self.buildEnv.platformInfo
+            configInfo = self.buildEnv.configInfo
-            if platformInfo.build == 'native':
+            if configInfo.build == "native":
                 return super()._compile(context)
             context.try_skip(self.build_path)
-            command = "make -j4 {make_target} {make_option}".format(
+            command = ["make", "-j4", *self.make_targets, *self.make_options]
-                make_target=self.make_target,
+            env = self.buildEnv.get_env(
-                make_option=self.make_option
+                cross_comp_flags=True, cross_compilers=True, cross_path=True
             )
-            env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
+            libmagic_native_builder = get_target_step("libmagic", "native_static")
-            libmagic_native_builder = get_target_step('libmagic', 'native_static')
+            env["PATH"].insert(0, pj(libmagic_native_builder.build_path, "src"))
-            env['PATH'] = ':'.join([pj(libmagic_native_builder.build_path, 'src'), env['PATH']])
             run_command(command, self.build_path, context, env=env)

kiwixbuild/dependencies/libmicrohttpd.py

@@ -1,17 +1,33 @@
-from .base import (
+from .base import Dependency, ReleaseDownload, MesonBuilder
-    Dependency,
-    ReleaseDownload,
-    MakeBuilder)
 
 from kiwixbuild.utils import Remotefile
 
+
 class MicroHttpd(Dependency):
     name = "libmicrohttpd"
 
     class Source(ReleaseDownload):
-        archive = Remotefile('libmicrohttpd-0.9.76.tar.gz',
+        src_archive = Remotefile(
-                             'f0b1547b5a42a6c0f724e8e1c1cb5ce9c4c35fb495e7d780b9930d35011ceb4c',
+            "libmicrohttpd-0.9.76.tar.gz",
-                             'https://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.76.tar.gz')
+            "f0b1547b5a42a6c0f724e8e1c1cb5ce9c4c35fb495e7d780b9930d35011ceb4c",
+            "https://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.76.tar.gz",
+        )
+        meson_archive = Remotefile(
+            "libmicrohttpd_0.9.16-3_patch.zip",
+            "0954c094a0d4cfe0dd799d8df8a04face6669f7b4d51a7386a9c3e2d37b9c3b3",
+            "https://wrapdb.mesonbuild.com/v2/libmicrohttpd_0.9.76-3/get_patch",
+        )
+        archives = [src_archive, meson_archive]
+        patches = [
+            "libmicrohttpd_meson_pkgconfig.patch",
+            "libmicrohttpd_meson_timeval_tvsec_size.patch",
+            "libmicrohttpd_meson_winet6.patch",
+        ]
 
-    class Builder(MakeBuilder):
+    class Builder(MesonBuilder):
-        configure_option = "--disable-https --without-libgcrypt --without-libcurl --disable-doc --disable-examples"
+        configure_options = [
+            "-Dgnutls=disabled",
+            "-Dgcrypt=disabled",
+            "-Dcurl=disabled",
+            "-Dexpat=disabled",
+        ]

kiwixbuild/dependencies/libzim.py

@@ -1,8 +1,6 @@
-from .base import (
+from .base import Dependency, GitClone, MesonBuilder
-    Dependency,
+from kiwixbuild._global import option, get_target_step, neutralEnv
-    GitClone,
+
-    MesonBuilder)
-from kiwixbuild._global import option, get_target_step
 
 class Libzim(Dependency):
     name = "libzim"
@@ -13,38 +11,47 @@ class Libzim(Dependency):
         git_dir = "libzim"
 
     class Builder(MesonBuilder):
-        test_option = "-t 8"
+        test_options = ["-t", "8"]
-        strip_option = ''
+        strip_options = []
+
+        @property
+        def build_type(self):
+            if self.buildEnv.configInfo.build == "android":
+                return "debug"
+            return super().build_type
 
         @classmethod
-        def get_dependencies(cls, platformInfo, allDeps):
+        def get_dependencies(cls, configInfo, allDeps):
-            deps = ['lzma', 'zstd', 'xapian-core', 'icu4c']
+            if neutralEnv("distname") == "Windows":
-            if platformInfo.name not in ('flatpak', 'wasm'):
+                return ["zstd", "xapian-core", "icu4c", "zim-testing-suite"]
-                deps.append('zim-testing-suite')
+            deps = ["lzma", "zstd", "xapian-core", "icu4c"]
+            if configInfo.name not in ("flatpak", "wasm"):
+                deps.append("zim-testing-suite")
             return deps
 
         @property
-        def configure_option(self):
+        def configure_options(self):
-            platformInfo = self.buildEnv.platformInfo
+            configInfo = self.buildEnv.configInfo
-            config_options = []
+            if neutralEnv("distname") == "Windows":
-            if platformInfo.build == 'android':
+                yield "-Dwith_xapian_fuller=false"
-                config_options.append("-DUSE_BUFFER_HEADER=false")
+                yield "-Dwerror=false"
-                config_options.append("-Dstatic-linkage=true")
+            if configInfo.build == "android":
-            if platformInfo.mixed and option('target') == 'libzim':
+                yield "-DUSE_BUFFER_HEADER=false"
-                config_options.append("-Dstatic-linkage=true")
+                yield "-Dstatic-linkage=true"
-            if platformInfo.name == "flatpak":
+            if configInfo.mixed and option("target") == "libzim":
-                config_options.append("--wrap-mode=nodownload")
+                yield "-Dstatic-linkage=true"
-                config_options.append("-Dtest_data_dir=none")
+            if configInfo.name == "flatpak":
-            if platformInfo.name == "wasm":
+                yield "--wrap-mode=nodownload"
-                config_options.append("-Dexamples=false")
+                yield "-Dtest_data_dir=none"
-                config_options.append("-DUSE_MMAP=false")
+            if configInfo.name == "wasm":
-            if platformInfo.name not in ("flatpak", "wasm"):
+                yield "-Dexamples=false"
-                zim_testing_suite = get_target_step('zim-testing-suite', 'source')
+                yield "-DUSE_MMAP=false"
-                config_options.append('-Dtest_data_dir={}'.format(zim_testing_suite.source_path))
+            if configInfo.name not in ("flatpak", "wasm"):
-            return " ".join(config_options)
+                zim_testing_suite = get_target_step("zim-testing-suite", "source")
+                yield "-Dtest_data_dir={}".format(zim_testing_suite.source_path)
 
         @property
         def library_type(self):
-            if self.buildEnv.platformInfo.build == 'android':
+            if self.buildEnv.configInfo.build == "android":
-                return 'shared'
+                return "shared"
             return super().library_type

kiwixbuild/dependencies/lzma.py

@@ -1,28 +1,26 @@
 from .base import (
     Dependency,
     ReleaseDownload,
-    MakeBuilder)
+    MesonBuilder)
 
 from kiwixbuild.utils import Remotefile
 
+
 class lzma(Dependency):
     name = 'lzma'
 
     class Source(ReleaseDownload):
-        archive = Remotefile('xz-5.2.6.tar.gz',
+        src_archive = Remotefile(
-                             'a2105abee17bcd2ebd15ced31b4f5eda6e17efd6b10f921a01cda4a44c91b3a0',
+            "xz-5.2.6.tar.gz",
-                             'https://altushost-swe.dl.sourceforge.net/project/lzmautils/xz-5.2.6.tar.gz'
+            "a2105abee17bcd2ebd15ced31b4f5eda6e17efd6b10f921a01cda4a44c91b3a0",
-                            )
+            "https://altushost-swe.dl.sourceforge.net/project/lzmautils/xz-5.2.6.tar.gz",
+        )
+        meson_patch = Remotefile(
+            "liblzma_5.2.6-3_patch.zip",
+            "1c71536d364e1a3ce6bea61266576f89cc5cce4d3b9e11f3494417dafa29780b",
+            "https://wrapdb.mesonbuild.com/v2/liblzma_5.2.6-3/get_patch",
+        )
+        archives = [src_archive, meson_patch]
+        patches = ['lzma_meson_install.patch']
 
-    class Builder(MakeBuilder):
+    Builder = MesonBuilder
-        @property
-        def configure_option(self):
-            return ("--disable-xz "
-                    "--disable-xzdec "
-                    "--disable-lzmadec "
-                    "--disable-lzmainfo "
-                    "--disable-lzma-links "
-                    "--disable-scripts "
-                    "--disable-doc "
-#                    "--disable-symbol-versions "
-                  )

kiwixbuild/dependencies/mustache.py

@@ -1,28 +1,30 @@
-from .base import (
+from .base import Dependency, ReleaseDownload, Builder as BaseBuilder
-    Dependency,
-    ReleaseDownload,
-    Builder as BaseBuilder)
 
 from kiwixbuild.utils import Remotefile, pj
 from shutil import copy2
 
+
 class Mustache(Dependency):
     name = "mustache"
 
     class Source(ReleaseDownload):
-        archive = Remotefile('Mustache-4.1.tar.gz',
+        archive = Remotefile(
-                             'acd66359feb4318b421f9574cfc5a511133a77d916d0b13c7caa3783c0bfe167',
+            "Mustache-4.1.tar.gz",
-                             'https://github.com/kainjow/Mustache/archive/v4.1.tar.gz')
+            "acd66359feb4318b421f9574cfc5a511133a77d916d0b13c7caa3783c0bfe167",
+            "https://github.com/kainjow/Mustache/archive/v4.1.tar.gz",
+        )
 
     class Builder(BaseBuilder):
         def build(self):
-            self.command('copy_header', self._copy_header)
+            self.command("copy_header", self._copy_header)
 
         def _copy_header(self, context):
             context.try_skip(self.build_path)
-            copy2(pj(self.source_path, 'mustache.hpp'),
+            copy2(
-                  pj(self.buildEnv.install_dir, 'include'))
+                pj(self.source_path, "mustache.hpp"),
+                pj(self.buildEnv.install_dir, "include"),
+            )
 
         def set_flatpak_buildsystem(self, module):
-            module['buildsystem'] = 'simple'
+            module["buildsystem"] = "simple"
-            module['build-commands'] = ['cp mustache.hpp /app/include']
+            module["build-commands"] = ["cp mustache.hpp /app/include"]

kiwixbuild/dependencies/pugixml.py

@@ -1,19 +1,18 @@
-from .base import (
+from .base import Dependency, ReleaseDownload, MesonBuilder
-    Dependency,
-    ReleaseDownload,
-    MesonBuilder)
 
 from kiwixbuild.utils import Remotefile
 
+
 class Pugixml(Dependency):
     name = "pugixml"
 
     class Source(ReleaseDownload):
-        archive = Remotefile('pugixml-1.2.tar.gz',
+        archive = Remotefile(
-                             '0f422dad86da0a2e56a37fb2a88376aae6e931f22cc8b956978460c9db06136b')
+            "pugixml-1.2.tar.gz",
+            "0f422dad86da0a2e56a37fb2a88376aae6e931f22cc8b956978460c9db06136b",
+        )
         patches = ["pugixml_meson.patch"]
         flatpak_dest = "src"
 
     class Builder(MesonBuilder):
-        build_type = 'release'
+        strip_options = []
-        strip_option = ''

kiwixbuild/dependencies/qt.py

@@ -1,70 +1,25 @@
 import shutil
 
-from .base import (
+from .base import Dependency, NoopBuilder, NoopSource
-    Dependency,
-    ReleaseDownload,
-    MakeBuilder,
-    QMakeBuilder)
 
-from kiwixbuild.utils import Remotefile, pj, SkipCommand
+from kiwixbuild.utils import SkipCommand, colorize
 
 
 class Qt(Dependency):
-    name = 'qt'
+    name = "qt"
 
-    class Source(ReleaseDownload):
+    Source = NoopSource
-        name = "qt"
-        source_dir = "qt-5.10.1"
-        archive = Remotefile('qt-everywhere-src-5.10.1.tar.xz',
-                             '',
-                             'http://ftp.oregonstate.edu/.1/blfs/conglomeration/qt5/qt-everywhere-src-5.10.1.tar.xz')
 
-    class Builder(MakeBuilder):
+    class Builder(NoopBuilder):
-        dependencies = ['icu4c', 'zlib']
+        def build(self):
-        configure_option_template = "{dep_options} {static_option} {env_option} -prefix {install_dir} -libdir {libdir}"
+            error_msg = f"""WARNING: kiwix-build cannot build {self.name} for you.
-        dynamic_configure_option = "-shared"
+You must install it yourself using official Qt installer or your distribution system."""
-        static_configure_option = "-static"
+            print(colorize(error_msg, "WARNING"))
 
-        @property
-        def configure_option(self):
-            skip_modules = [
-                'qt3d',
-                'qtcanvas3d',
-                'qtcharts',
-                'qtconnectivity',
-                'qtdatavis3d',
-            #    'qtdeclarative',
-                'qtdoc',
-                'qtgamepad',
-                'qtgraphicaleffects',
-                'qtlocation',
-                'qtmultimedia',
-                'qtnetworkauth',
-                'qtpurchasing',
-            #    'qtquickcontrols',
-                'qtquickcontrols2',
-                'qtremoteobjects',
-                'qtscript',
-                'qtscxml',
-                'qtsensors',
-                'qtserialbus',
-                'qtserialport',
-                'qtspeech',
-                'qtvirtualkeyboard',
-                'qtwayland',
-                'qtwebglplugin',
-                'qtwebsockets',
-#                'qtwebview',
-]
-            skip_modules = " ".join("-skip {}".format(m) for m in skip_modules)
-            options = "-recheck -opensource -confirm-license -ccache -make libs {}".format(skip_modules)
-            return options
 
 class QtWebEngine(Dependency):
     name = "qtwebengine"
 
-    Source = Qt.Source
+    Source = NoopSource
 
-    class Builder(QMakeBuilder):
+    Builder = Qt.Builder
-        dependencies = ['qt']
-        subsource_dir = "qtwebengine"

kiwixbuild/dependencies/tc_android_ndk.py

@@ -5,22 +5,28 @@ from kiwixbuild.utils import Remotefile, add_execution_right, run_command
 
 pj = os.path.join
 
+
 class android_ndk(Dependency):
     dont_skip = True
     neutral = False
-    name = 'android-ndk'
+    name = "android-ndk"
-    gccver = '4.9.x'
+    gccver = "4.9.x"
-    api = '24'
+    api = "24"
 
     class Source(ReleaseDownload):
-        archive = Remotefile('android-ndk-r21e-linux-x86_64.zip',
+        archive = Remotefile(
-                             'ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e',
+            "android-ndk-r21e-linux-x86_64.zip",
-                             'https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip')
+            "ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e",
+            "https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip",
+        )
 
         @property
         def source_dir(self):
             return self.target.full_name()
 
+        patches = [
+            "android-ndk-r21e-linux-x86_64-python3.12+.patch",
+        ]
 
     class Builder(Builder):
         @property
@@ -32,37 +38,46 @@ class android_ndk(Dependency):
             return self.target.api
 
         @property
-        def platform(self):
+        def config(self):
-            return 'android-'+self.api
+            return "android-" + self.api
 
         @property
         def arch(self):
-            return self.buildEnv.platformInfo.arch
+            return self.buildEnv.configInfo.arch
 
         @property
         def arch_full(self):
-            return self.buildEnv.platformInfo.arch_full
+            return self.buildEnv.configInfo.arch_full
 
-        def _build_platform(self, context):
+        def _build_toolchain(self, context):
             context.try_skip(self.build_path)
-            script = pj(self.source_path, 'build/tools/make_standalone_toolchain.py')
+            script = pj(self.source_path, "build/tools/make_standalone_toolchain.py")
             add_execution_right(script)
-            command = '{script} --arch={arch} --api={api} --install-dir={install_dir} --force'
+            command = [
-            command = command.format(
+                script,
-                script=script,
+                f"--arch={self.arch}",
-                arch=self.arch,
+                f"--api={self.api}",
-                api=self.api,
+                f"--install-dir={self.install_path}",
-                install_dir=self.install_path
+                "--force",
+            ]
+            env = self.buildEnv.get_env(
+                cross_comp_flags=False, cross_compilers=False, cross_path=False
             )
-            env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False)
             run_command(command, self.build_path, context, env=env)
 
         def _fix_permission_right(self, context):
             context.try_skip(self.build_path)
-            bin_dirs = [pj(self.install_path, 'bin'),
+            bin_dirs = [
-                        pj(self.install_path, self.arch_full, 'bin'),
+                pj(self.install_path, "bin"),
-                        pj(self.install_path, 'libexec', 'gcc', self.arch_full, self.target.gccver)
+                pj(self.install_path, self.arch_full, "bin"),
-                       ]
+                pj(
+                    self.install_path,
+                    "libexec",
+                    "gcc",
+                    self.arch_full,
+                    self.target.gccver,
+                ),
+            ]
             for root, dirs, files in os.walk(self.install_path):
                 if not root in bin_dirs:
                     continue
@@ -74,6 +89,5 @@ class android_ndk(Dependency):
                     add_execution_right(file_path)
 
         def build(self):
-            self.command('build_platform', self._build_platform)
+            self.command("build_toolchain", self._build_toolchain)
-            self.command('fix_permission_right', self._fix_permission_right)
+            self.command("fix_permission_right", self._fix_permission_right)
-

kiwixbuild/dependencies/tc_armhf.py

@@ -4,21 +4,22 @@ from kiwixbuild.utils import Remotefile
 # The arm toolchains
 # This is based on toolchains published here : https://github.com/tttapa/docker-arm-cross-toolchain
 
-base_url = "https://github.com/tttapa/docker-arm-cross-toolchain/releases/download/0.1.0/"
+base_url = (
+    "https://github.com/tttapa/docker-arm-cross-toolchain/releases/download/0.1.0/"
+)
 
 
-aarch_base_url = 'https://master.dl.sourceforge.net/project/raspberry-pi-cross-compilers/Bonus%20Raspberry%20Pi%20GCC%2064-Bit%20Toolchains/Raspberry%20Pi%20GCC%2064-Bit%20Cross-Compiler%20Toolchains/Stretch/GCC%206.3.0/'
-
 class armv6_toolchain(Dependency):
     dont_skip = True
     neutral = True
-    name = 'armv6'
+    name = "armv6"
 
     class Source(ReleaseDownload):
-        archive = Remotefile('x-tools-armv6-rpi-linux-gnueabihf.tar.xz',
+        archive = Remotefile(
-                             '4c371c4c5b55ebd1f3d7dd26b14703632d9ba47423f901bcd9303d83ad444434',
+            "x-tools-armv6-rpi-linux-gnueabihf.tar.xz",
-                             base_url + 'x-tools-armv6-rpi-linux-gnueabihf.tar.xz')
+            "4c371c4c5b55ebd1f3d7dd26b14703632d9ba47423f901bcd9303d83ad444434",
-
+            base_url + "x-tools-armv6-rpi-linux-gnueabihf.tar.xz",
+        )
 
     class Builder(TcCopyBuilder):
         src_subdir = "armv6-rpi-linux-gnueabihf"
@@ -27,24 +28,28 @@ class armv6_toolchain(Dependency):
 class armv8_toolchain(Dependency):
     dont_skip = True
     neutral = True
-    name = 'armv8'
+    name = "armv8"
 
     class Source(ReleaseDownload):
-        archive = Remotefile('x-tools-armv8-rpi-linux-gnueabihf.tar.xz',
+        archive = Remotefile(
-                             'cc28f5c3f6a3e7d9985f98779c4e72224b4eb5a7e4dc2bcdefd90cb241fb94a5',
+            "x-tools-armv8-rpi-linux-gnueabihf.tar.xz",
-                             base_url + 'x-tools-armv8-rpi3-linux-gnueabihf.tar.xz')
+            "cc28f5c3f6a3e7d9985f98779c4e72224b4eb5a7e4dc2bcdefd90cb241fb94a5",
+            base_url + "x-tools-armv8-rpi3-linux-gnueabihf.tar.xz",
+        )
 
     class Builder(TcCopyBuilder):
         src_subdir = "armv8-rpi3-linux-gnueabihf"
 
+
 class aarch64_toolchain(Dependency):
     dont_skip = True
     neutral = True
     name = "aarch64"
 
     class Source(ReleaseDownload):
-        archive = Remotefile('cross-gcc-6.3.0-pi_64.tar.gz',
+        archive = Remotefile(
-                             '1b048bb8886ad63d21797cd9129fc37b9ea0dfaac7e3c36f888aa16fbec1d320',
+            "cross-gcc-6.3.0-pi_64.tar.gz",
-                             aarch_base_url + 'cross-gcc-6.3.0-pi_64.tar.gz')
+            "1b048bb8886ad63d21797cd9129fc37b9ea0dfaac7e3c36f888aa16fbec1d320",
+        )
 
     Builder = TcCopyBuilder

kiwixbuild/dependencies/tc_emsdk.py

@@ -5,21 +5,23 @@ from kiwixbuild.utils import Remotefile, run_command, copy_tree
 
 pj = os.path.join
 
+
 class emsdk(Dependency):
     dont_skip = True
     neutral = False
-    name = 'emsdk'
+    name = "emsdk"
 
     class Source(ReleaseDownload):
-        archive = Remotefile('emsdk-3.1.24.tar.gz',
+        archive = Remotefile(
-                             '1aa5365ccb2147701cc9d1e59a5a49577c1d6aea55da7c450df2d5ffa48b8a58',
+            "emsdk-3.1.41.tar.gz",
-                             'https://codeload.github.com/emscripten-core/emsdk/tar.gz/refs/tags/3.1.24')
+            "147a2d72df34227bdb4ffedc587a8cb674a42269c40458f3f69ae37e8966cdc6",
+            "https://codeload.github.com/emscripten-core/emsdk/tar.gz/refs/tags/3.1.41",
+        )
 
         @property
         def source_dir(self):
             return self.target.full_name()
 
-
     class Builder(Builder):
         @property
         def install_path(self):
@@ -31,17 +33,15 @@ class emsdk(Dependency):
 
         def _install(self, context):
             context.try_skip(self.build_path)
-            command = "./emsdk install 3.1.24"
+            command = ["./emsdk", "install", "3.1.24"]
             run_command(command, self.install_path, context)
 
         def _activate(self, context):
             context.try_skip(self.build_path)
-            command = "./emsdk activate 3.1.24"
+            command = ["./emsdk", "activate", "3.1.24"]
             run_command(command, self.install_path, context)
 
-
         def build(self):
-            self.command('copy_source', self._copy_source)
+            self.command("copy_source", self._copy_source)
-            self.command('install', self._install)
+            self.command("install", self._install)
-            self.command('activate', self._activate)
+            self.command("activate", self._activate)
-

kiwixbuild/dependencies/tc_flatpak.py

@@ -5,64 +5,85 @@ from kiwixbuild.utils import Remotefile, add_execution_right, run_command
 
 pj = os.path.join
 
+
 class org_kde(Dependency):
     neutral = False
-    name = 'org.kde'
+    name = "org.kde"
 
     Source = NoopSource
 
     class Builder(Builder):
         def _setup_remote(self, context):
-            command = "flatpak --user remote-add --if-not-exists {remote_name} {remote_url}"
+            command = [
-            command = command.format(
+                "flatpak",
-                remote_name = 'flathub',
+                "--user",
-                remote_url = 'https://flathub.org/repo/flathub.flatpakrepo'
+                "remote-add",
+                "--if-not-exists",
+                "flathub",
+                "https://flathub.org/repo/flathub.flatpakrepo",
+            ]
+            env = self.buildEnv.get_env(
+                cross_comp_flags=False, cross_compilers=False, cross_path=False
             )
-            env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False)
             run_command(command, self.buildEnv.build_dir, context, env=env)
 
         def _install_sdk(self, context):
-            command = "flatpak --user install --noninteractive --verbose -y {remote_name} {name}.Sdk//{version} {name}.Platform//{version}"
+            command = [
-            command = command.format(
+                "flatpak",
-                remote_name = 'flathub',
+                "--user",
-                name = self.target.name,
+                "install",
-                version = self.target.version()
+                "--noninteractive",
+                "--verbose",
+                "-y",
+                "flathub",
+                f"{self.target.name}.Sdk//{self.target.version()}",
+                f"{self.target.name}.Platform//{self.target.version()}",
+            ]
+            env = self.buildEnv.get_env(
+                cross_comp_flags=False, cross_compilers=False, cross_path=False
             )
-            env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False)
             run_command(command, self.buildEnv.build_dir, context, env=env)
 
         def build(self):
-            self.command('setup_remote', self._setup_remote)
+            self.command("setup_remote", self._setup_remote)
-            self.command('install_sdk', self._install_sdk)
+            self.command("install_sdk", self._install_sdk)
 
 
 class io_qt_qtwebengine(Dependency):
     neutral = False
-    name = 'io.qt.qtwebengine'
+    name = "io.qt.qtwebengine"
 
     Source = NoopSource
 
     class Builder(Builder):
         def _setup_remote(self, context):
-            command = "flatpak --user remote-add --if-not-exists {remote_name} {remote_url}"
+            command = [
-            command = command.format(
+                "flatpak",
-                remote_name = 'flathub',
+                "--user",
-                remote_url = 'https://flathub.org/repo/flathub.flatpakrepo'
+                "remote-add",
+                "--if-not-exists",
+                "flathub",
+                "https://flathub.org/repo/flathub.flatpakrepo",
+            ]
+            env = self.buildEnv.get_env(
+                cross_comp_flags=False, cross_compilers=False, cross_path=False
             )
-            env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False)
             run_command(command, self.buildEnv.build_dir, context, env=env)
 
         def _install_sdk(self, context):
-            command = "flatpak --user install -y {remote_name} {name}.BaseApp//{version}"
+            command = [
-            command = command.format(
+                "flatpak",
-                remote_name = 'flathub',
+                "--user",
-                name = self.target.name,
+                "install",
-                version = self.target.version()
+                "-y",
+                "flathub",
+                f"{self.target.name}.BaseApp//{self.target.version()}",
+            ]
+            env = self.buildEnv.get_env(
+                cross_comp_flags=False, cross_compilers=False, cross_path=False
             )
-            env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False)
             run_command(command, self.buildEnv.build_dir, context, env=env)
 
         def build(self):
-            self.command('setup_remote', self._setup_remote)
+            self.command("setup_remote", self._setup_remote)
-            self.command('install_sdk', self._install_sdk)
+            self.command("install_sdk", self._install_sdk)
-

kiwixbuild/dependencies/tc_musl.py

@@ -1,15 +1,18 @@
 from .base import Dependency, ReleaseDownload, TcCopyBuilder
 from kiwixbuild.utils import Remotefile
 
+
 class aarch64_musl_toolchain(Dependency):
     dont_skip = True
     neutral = True
     name = "aarch64_musl"
 
     class Source(ReleaseDownload):
-        archive = Remotefile('aarch64-linux-musl-cross.tgz',
+        archive = Remotefile(
-                             '0f18a885b161815520bbb5757a4b4ab40d0898c29bebee58d0cddd6112e59cc6',
+            "aarch64-linux-musl-cross.tgz",
-                             'https://more.musl.cc/10/x86_64-linux-musl/aarch64-linux-musl-cross.tgz')
+            "0f18a885b161815520bbb5757a4b4ab40d0898c29bebee58d0cddd6112e59cc6",
+            "https://more.musl.cc/10/x86_64-linux-musl/aarch64-linux-musl-cross.tgz",
+        )
 
     Builder = TcCopyBuilder
 
@@ -20,8 +23,10 @@ class x86_64_musl_toolchain(Dependency):
     name = "x86-64_musl"
 
     class Source(ReleaseDownload):
-        archive = Remotefile('x86_64-linux-musl-cross.tgz',
+        archive = Remotefile(
-                             'a3d55de8105739fcfb8b10eaa72cdb5d779319726bacff24149388d7608d1ed8',
+            "x86_64-linux-musl-cross.tgz",
-                             'https://more.musl.cc/10/x86_64-linux-musl/x86_64-linux-musl-cross.tgz')
+            "a3d55de8105739fcfb8b10eaa72cdb5d779319726bacff24149388d7608d1ed8",
+            "https://more.musl.cc/10/x86_64-linux-musl/x86_64-linux-musl-cross.tgz",
+        )
 
     Builder = TcCopyBuilder

kiwixbuild/dependencies/uuid.py

@@ -1,25 +1,30 @@
-from .base import (
+from .base import Dependency, ReleaseDownload, MakeBuilder
-    Dependency,
-    ReleaseDownload,
-    MakeBuilder
-)
 
 from kiwixbuild.utils import Remotefile
 
 
 class UUID(Dependency):
-    name = 'uuid'
+    name = "uuid"
 
     class Source(ReleaseDownload):
-        archive = Remotefile('e2fsprogs-libs-1.43.4.tar.gz',
+        archive = Remotefile(
-                             'eed4516325768255c9745e7b82c9d7d0393abce302520a5b2cde693204b0e419',
+            "e2fsprogs-libs-1.43.4.tar.gz",
-                             'https://www.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v1.43.4/e2fsprogs-libs-1.43.4.tar.gz')
+            "eed4516325768255c9745e7b82c9d7d0393abce302520a5b2cde693204b0e419",
-        extract_dir = 'e2fsprogs-libs-1.43.4'
+            "https://www.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v1.43.4/e2fsprogs-libs-1.43.4.tar.gz",
+        )
+        extract_dir = "e2fsprogs-libs-1.43.4"
 
     class Builder(MakeBuilder):
-        configure_option = ("--enable-libuuid --disable-debugfs --disable-imager --disable-resizer --disable-defrag --enable-fsck"
+        configure_options = [
-                            " --disable-uuidd")
+            "--enable-libuuid",
-        configure_env = {'_format_CFLAGS': "{env.CFLAGS} -O3 -fPIC"}
+            "--disable-debugfs",
-        static_configure_option = dynamic_configure_option = ""
+            "--disable-imager",
-        make_target = 'libs'
+            "--disable-resizer",
-        make_install_target = 'install-libs'
+            "--disable-defrag",
+            "--enable-fsck",
+            "--disable-uuidd",
+        ]
+        configure_env = {"_format_CFLAGS": "{env.CFLAGS} -O3 -fPIC"}
+        static_configure_options = dynamic_configure_options = []
+        make_targets = ["libs"]
+        make_install_targets = ["install-libs"]

kiwixbuild/dependencies/xapian.py

@@ -1,30 +1,55 @@
-from .base import (
+from .base import Dependency, GitClone, ReleaseDownload, MakeBuilder, MesonBuilder
-    Dependency,
-    ReleaseDownload,
-    MakeBuilder
-)
 
 from kiwixbuild.utils import Remotefile
 from kiwixbuild._global import neutralEnv
 
+import platform
+
 
 class Xapian(Dependency):
     name = "xapian-core"
 
-    class Source(ReleaseDownload):
+    if platform.system() == "Windows":
-        archive = Remotefile('xapian-core-1.4.23.tar.xz',
-                             '30d3518172084f310dab86d262b512718a7f9a13635aaa1a188e61dc26b2288c')
 
-    class Builder(MakeBuilder):
+        class Source(GitClone):
-        configure_option = "--disable-sse --disable-backend-chert --disable-backend-remote --disable-documentation"
+            git_remote = "https://github.com/openzim/xapian-meson.git"
-        configure_env = {'_format_LDFLAGS': "{env.LDFLAGS} -L{buildEnv.install_dir}/{buildEnv.libprefix}",
+            git_dir = "xapian-core"
-                         '_format_CXXFLAGS': "{env.CXXFLAGS} -O3 -I{buildEnv.install_dir}/include"}
 
+        class Builder(MesonBuilder):
+            configure_options = [
+                "-Denable-sse=false",
+                "-Denable-backend-chert=false",
+                "-Denable-backend-remote=false",
+            ]
+            subsource_dir = "xapian-core"
 
-        @classmethod
+            @classmethod
-        def get_dependencies(cls, platformInfo, allDeps):
+            def get_dependencies(cls, configInfo, allDeps):
-            deps = ['zlib', 'lzma']
+                return ["zlib"]
-            if (platformInfo.build in ('win32', 'wasm')
+
-             or neutralEnv('distname') == 'Darwin'):
+    else:
-                return deps
+
-            return deps + ['uuid']
+        class Source(ReleaseDownload):
+            archive = Remotefile(
+                "xapian-core-1.4.23.tar.xz",
+                "30d3518172084f310dab86d262b512718a7f9a13635aaa1a188e61dc26b2288c",
+            )
+
+        class Builder(MakeBuilder):
+            configure_options = [
+                "--disable-sse",
+                "--disable-backend-chert",
+                "--disable-backend-remote",
+                "--disable-documentation",
+            ]
+            configure_env = {
+                "_format_LDFLAGS": "{env.LDFLAGS} -L{buildEnv.install_dir}/{buildEnv.libprefix}",
+                "_format_CXXFLAGS": "{env.CXXFLAGS} -I{buildEnv.install_dir}/include",
+            }
+
+            @classmethod
+            def get_dependencies(cls, configInfo, allDeps):
+                deps = ["zlib", "lzma"]
+                if configInfo.build == "wasm" or neutralEnv("distname") == "Darwin":
+                    return deps
+                return deps + ["uuid"]

kiwixbuild/dependencies/zim_testing_suite.py

@@ -1,8 +1,4 @@
-from .base import (
+from .base import Dependency, ReleaseDownload, NoopBuilder
-    Dependency,
-    ReleaseDownload,
-    NoopBuilder
-)
 
 from kiwixbuild.utils import Remotefile
 
@@ -12,8 +8,10 @@ class ZimTestingSuite(Dependency):
     dont_skip = True
 
     class Source(ReleaseDownload):
-        archive = Remotefile('zim-testing-suite-0.3.tar.gz',
+        archive = Remotefile(
-                             'cd7d1ccc48af3783af9156cb6bf3c18d9a3319a73fdeefe65f0b4cae402d3d66',
+            "zim-testing-suite-0.8.0.tar.gz",
-                             'https://github.com/openzim/zim-testing-suite/releases/download/v0.3/zim-testing-suite-0.3.tar.gz')
+            "28f51449a3f9aea02652ca21f32c5598fd610d6cec3810fa552bd0c0f7a2d5fc",
+            "https://github.com/openzim/zim-testing-suite/releases/download/0.8.0/zim-testing-suite-0.8.0.tar.gz",
+        )
 
     Builder = NoopBuilder

kiwixbuild/dependencies/zim_tools.py

@@ -1,7 +1,6 @@
-from .base import (
+from .base import Dependency, GitClone, MesonBuilder
-    Dependency,
+from kiwixbuild._global import neutralEnv
-    GitClone,
+
-    MesonBuilder)
 
 class ZimTools(Dependency):
     name = "zim-tools"
@@ -13,18 +12,16 @@ class ZimTools(Dependency):
 
     class Builder(MesonBuilder):
         @classmethod
-        def get_dependencies(cls, platformInfo, allDeps):
+        def get_dependencies(cls, configInfo, allDeps):
-            base_deps = ['libzim', 'docoptcpp', 'mustache']
+            base_deps = ["libzim", "docoptcpp", "mustache"]
-            if platformInfo.build != 'win32':
+            if neutralEnv("distname") != "Windows":
-                base_deps += ['libmagic', 'gumbo']
+                base_deps += ["libmagic", "gumbo"]
             return base_deps
 
         @property
-        def configure_option(self):
+        def configure_options(self):
-            base_option  = ""
+            # We don't build zimwriterfs on Windows, and so we don't have magic
-            # We don't build zimwriterfs on win32, and so we don't have magic
+            if neutralEnv("distname") != "Windows":
-            if self.buildEnv.platformInfo.build != 'win32':
+                yield f"-Dmagic-install-prefix={self.buildEnv.install_dir}"
-                base_option += " -Dmagic-install-prefix={buildEnv.install_dir}"
+            if self.buildEnv.configInfo.static:
-            if self.buildEnv.platformInfo.static:
+                yield "-Dstatic-linkage=true"
-                base_option += " -Dstatic-linkage=true"
-            return base_option

kiwixbuild/dependencies/zlib.py

@@ -3,51 +3,25 @@ import shutil
 from .base import (
     Dependency,
     ReleaseDownload,
-    MakeBuilder)
+    MesonBuilder)
 
 from kiwixbuild.utils import Remotefile, pj, SkipCommand
-
+from kiwixbuild._global import neutralEnv
 
 
 class zlib(Dependency):
-    name = 'zlib'
+    name = "zlib"
 
     class Source(ReleaseDownload):
-        archive = Remotefile('zlib-1.2.12.tar.gz',
+        src_archive = Remotefile(
-                             '91844808532e5ce316b3c010929493c0244f3d37593afd6de04f71821d5136d9')
+            "zlib-1.2.12.tar.gz",
-        patches = ['zlib_std_libname.patch']
+            "91844808532e5ce316b3c010929493c0244f3d37593afd6de04f71821d5136d9"
+        )
+        meson_patch = Remotefile(
+            "zlib_1.2.12-1_patch.zip",
+            "8ec8344f3fe7b06ad4be768fd416694bc56cb4545ce78b0f1c18b3e72b3ec936",
+            "https://wrapdb.mesonbuild.com/v2/zlib_1.2.12-1/get_patch")
+        archives = [src_archive, meson_patch]
+        #patches = ['zlib_std_libname.patch']
 
-    class Builder(MakeBuilder):
+    Builder = MesonBuilder
-        dynamic_configure_option = "--shared"
-        static_configure_option = "--static"
-        make_install_target = 'install'
-        configure_option_template = "{dep_options} {static_option} --prefix {install_dir} --libdir {libdir}"
-
-        def _pre_build_script(self, context):
-            context.try_skip(self.build_path)
-            shutil.copytree(self.source_path, self.build_path)
-
-
-        def _configure(self, context):
-            if self.buildEnv.platformInfo.build == 'win32':
-                raise SkipCommand()
-            return super()._configure(context)
-
-        @property
-        def make_option(self):
-            if self.buildEnv.platformInfo.build == 'win32':
-                return "--makefile win32/Makefile.gcc PREFIX={host}- SHARED_MODE={static} INCLUDE_PATH={include_path} LIBRARY_PATH={library_path} BINARY_PATH={binary_path}".format(
-                    host='i686-w64-mingw32',
-                    static="0" if self.buildEnv.platformInfo.static else "1",
-                    include_path=pj(self.buildEnv.install_dir, 'include'),
-                    library_path=pj(self.buildEnv.install_dir, self.buildEnv.libprefix),
-                    binary_path=pj(self.buildEnv.install_dir, 'bin'),
-                    )
-            return ""
-
-        @property
-        def make_target(self):
-            if self.buildEnv.platformInfo.static:
-                return "static"
-            else:
-                return "shared"

kiwixbuild/dependencies/zstd.py

@@ -1,22 +1,20 @@
-from .base import (
+from .base import Dependency, ReleaseDownload, MesonBuilder
-    Dependency,
-    ReleaseDownload,
-    MesonBuilder)
 
 from kiwixbuild.utils import Remotefile
 
 
-
 class zstd(Dependency):
-    name = 'zstd'
+    name = "zstd"
 
     class Source(ReleaseDownload):
-        archive = Remotefile('zstd-1.5.2.tar.gz',
+        archive = Remotefile(
-                             'f7de13462f7a82c29ab865820149e778cbfe01087b3a55b5332707abf9db4a6e',
+            "zstd-1.5.5.tar.gz",
-                             'https://github.com/facebook/zstd/archive/refs/tags/v1.5.2.tar.gz')
+            "98e9c3d949d1b924e28e01eccb7deed865eefebf25c2f21c702e5cd5b63b85e1",
+            "https://github.com/facebook/zstd/archive/refs/tags/v1.5.5.tar.gz",
+        )
 
     class Builder(MesonBuilder):
-        subsource_dir = 'build/meson'
+        subsource_dir = "build/meson"
-        build_type = 'release'
+        build_type = "release"
-        strip_option = ''
+        strip_options = []
-        configure_option = '-Dbin_programs=false -Dbin_contrib=false'
+        configure_options = ["-Dbin_programs=false", "-Dbin_contrib=false"]

kiwixbuild/flatpak_builder.py

@@ -1,17 +1,19 @@
-
 import sys
 from collections import OrderedDict
 from .buildenv import *
 
-from .platforms import PlatformInfo
+from .configs import ConfigInfo
 from .utils import remove_duplicates, run_command, StopBuild, Context
 from .dependencies import Dependency
 from .packages import PACKAGE_NAME_MAPPERS
 from .versions import base_deps_versions
 from ._global import (
-    neutralEnv, option,
+    neutralEnv,
-    add_target_step, get_target_step, target_steps,
+    option,
-    backend)
+    add_target_step,
+    get_target_step,
+    target_steps,
+)
 from . import _global
 from .dependencies.base import (
     Source,
@@ -22,7 +24,8 @@ from .dependencies.base import (
     CMakeBuilder,
     QMakeBuilder,
     MakeBuilder,
-    SCRIPT_DIR)
+    SCRIPT_DIR,
+)
 import json
 from shutil import copyfile
 from urllib.parse import urlparse
@@ -30,54 +33,61 @@ from urllib.request import urlopen
 import json
 
 MANIFEST = {
-    'app-id': 'org.kiwix.desktop',
+    "app-id": "org.kiwix.desktop",
-    'runtime': 'org.kde.Platform',
+    "runtime": "org.kde.Platform",
-    'runtime-version': base_deps_versions['org.kde'],
+    "runtime-version": base_deps_versions["org.kde"],
-    'base': 'io.qt.qtwebengine.BaseApp',
+    "base": "io.qt.qtwebengine.BaseApp",
-    'base-version': base_deps_versions['org.kde'], # keep BaseApp (qwebengine) in sync with org.kde
+    "base-version": base_deps_versions["io.qt.qtwebengine"],
-    'sdk': 'org.kde.Sdk',
+    "sdk": "org.kde.Sdk",
-    'command': 'kiwix-desktop',
+    "command": "kiwix-desktop",
-    'rename-icon': 'kiwix-desktop',
+    "rename-icon": "kiwix-desktop",
-    'finish-args': [
+    "finish-args": [
-        '--socket=wayland',
+        "--device=dri",
-        '--socket=x11',
+        "--env=QTWEBENGINEPROCESS_PATH=/app/bin/QtWebEngineProcess",
-        '--share=network',
+        "--socket=wayland",
-        '--share=ipc',
+        "--socket=fallback-x11",
-        '--device=dri',
+        "--socket=pulseaudio",
-        '--socket=pulseaudio',
+        "--share=network",
+        "--share=ipc",
+        "--filesystem=host:ro",
     ],
-    'cleanup': [
+    "cleanup": [
-        '/include',
+        "/include",
-        '/lib/pkgconfig',
+        "/lib/pkgconfig",
-        '/lib/cmake',
+        "/lib/cmake",
-        '/lib/*.la',
+        "/lib/*.la",
-        '/bin/curl',
+        "/bin/curl",
-        '/bin/copydatabase',
+        "/bin/copydatabase",
-        '/bin/kiwix-compile-resources',
+        "/bin/kiwix-compile-resources",
-        '/bin/kiwix-manage',
+        "/bin/kiwix-manage",
-        '/bin/kiwix-read',
+        "/bin/kiwix-read",
-        '/bin/kiwix-search',
+        "/bin/kiwix-search",
-        '/bin/quest',
+        "/bin/quest",
-        '/bin/simple*',
+        "/bin/simple*",
-        '/bin/xapian-*',
+        "/bin/xapian-*",
-        '/share/aclocal',
+        "/share/aclocal",
-        '/share/doc',
+        "/share/doc",
-        '/share/man'
+        "/share/man",
-    ]
+    ],
+    "cleanup-commands": ["/app/cleanup-BaseApp.sh"],
 }
 
-GET_REF_URL_API_TEMPLATE = 'https://api.github.com/repos{repo}/git/refs/tags/{ref}'
+GET_REF_URL_API_TEMPLATE = "https://api.github.com/repos{repo}/git/refs/tags/{ref}"
+
 
 class FlatpakBuilder:
     def __init__(self):
         self._targets = {}
-        PlatformInfo.get_platform('neutral', self._targets)
+        ConfigInfo.get_config("neutral", self._targets)
-        self.platform = PlatformInfo.get_platform('flatpak', self._targets)
+        self.config = ConfigInfo.get_config("flatpak", self._targets)
-        if neutralEnv('distname') not in self.platform.compatible_hosts:
+        if neutralEnv("distname") not in self.config.compatible_hosts:
-            print(('ERROR: The target platform {} cannot be build on host {}.\n'
+            print(
-                   'Select another target platform or change your host system.'
+                (
-                  ).format(self.platform.name, neutralEnv('distname')))
+                    "ERROR: The config {} cannot be build on host {}.\n"
-        self.targetDefs = self.platform.add_targets(option('target'), self._targets)
+                    "Select another config or change your host system."
+                ).format(self.config.name, neutralEnv("distname"))
+            )
+        self.targetDefs = self.config.add_targets(option("target"), self._targets)
 
     def finalize_target_steps(self):
         steps = []
@@ -85,14 +95,14 @@ class FlatpakBuilder:
             steps += self.order_steps(targetDef)
         steps = list(remove_duplicates(steps))
 
-        for pltName in PlatformInfo.all_running_platforms:
+        for cfgName in ConfigInfo.all_running_configs:
-            plt = PlatformInfo.all_platforms[pltName]
+            cfg = ConfigInfo.all_configs[cfgName]
-            for tlcName in plt.toolchain_names:
+            for tlcName in cfg.toolchain_names:
                 tlc = Dependency.all_deps[tlcName]
-                src_plt_step = ('source', tlcName)
+                src_cfg_step = ("source", tlcName)
-                add_target_step(src_plt_step, self._targets[src_plt_step])
+                add_target_step(src_cfg_step, self._targets[src_cfg_step])
-                blt_plt_step = ('neutral' if tlc.neutral else pltName, tlcName)
+                blt_cfg_step = ("neutral" if tlc.neutral else cfgName, tlcName)
-                add_target_step(blt_plt_step, self._targets[blt_plt_step])
+                add_target_step(blt_cfg_step, self._targets[blt_cfg_step])
 
         for dep in steps:
             add_target_step(dep, self._targets[dep])
@@ -103,8 +113,8 @@ class FlatpakBuilder:
         yield from self.order_dependencies(targetDef, _targets)
 
     def order_dependencies(self, targetDef, targets):
-        targetPlatformName, targetName = targetDef
+        targetConfigName, targetName = targetDef
-        if targetPlatformName == 'source':
+        if targetConfigName == "source":
             # Do not try to order sources, they will be added as dep by the
             # build step two lines later.
             return
@@ -113,27 +123,27 @@ class FlatpakBuilder:
         except KeyError:
             return
 
-        targetPlatform = PlatformInfo.get_platform(targetPlatformName)
+        targetConfig = ConfigInfo.get_config(targetConfigName)
-        for dep in target.get_dependencies(targetPlatform, True):
+        for dep in target.get_dependencies(targetConfig, True):
             if isinstance(dep, tuple):
-                depPlatform, depName = dep
+                depConfig, depName = dep
             else:
-                depPlatform, depName = targetPlatformName, dep
+                depConfig, depName = targetConfigName, dep
-            if (depPlatform, depName) in targets:
+            if (depConfig, depName) in targets:
-                yield from self.order_dependencies((depPlatform, depName), targets)
+                yield from self.order_dependencies((depConfig, depName), targets)
-        yield ('source', targetName)
+        yield ("source", targetName)
         yield targetDef
 
     def instanciate_steps(self):
         for stepDef in list(target_steps()):
-            stepPlatform, stepName = stepDef
+            stepConfig, stepName = stepDef
             stepClass = Dependency.all_deps[stepName]
-            if stepPlatform == 'source':
+            if stepConfig == "source":
                 source = get_target_step(stepDef)(stepClass)
                 add_target_step(stepDef, source)
             else:
-                source = get_target_step(stepName, 'source')
+                source = get_target_step(stepName, "source")
-                env = PlatformInfo.get_platform(stepPlatform).buildEnv
+                env = ConfigInfo.get_config(stepConfig).buildEnv
                 builder = get_target_step(stepDef)(stepClass, source, env)
                 add_target_step(stepDef, builder)
 
@@ -142,117 +152,132 @@ class FlatpakBuilder:
         modules = OrderedDict()
         for stepDef in steps:
             module = modules.setdefault(stepDef[1], {})
-            module['name'] = stepDef[1]
+            module["name"] = stepDef[1]
-            if stepDef[0] == 'source':
+            if stepDef[0] == "source":
                 source = get_target_step(stepDef)
-                if getattr(source, 'flatpak_no_autogen', False):
+                if getattr(source, "flatpak_no_autogen", False):
-                    module['no-autogen'] = True
+                    module["no-autogen"] = True
-                module_sources = module.setdefault('sources', [])
+                module_sources = module.setdefault("sources", [])
                 if isinstance(source, ReleaseDownload):
-                    src = {
+                    for archive in source.archives:
-                        'type': 'archive',
+                        src = {
-                        'sha256': source.archive.sha256,
+                            "type": "archive",
-                        'url': source.archive.url
+                            "dest-filename": archive.name,
-                    }
+                            "sha256": archive.sha256,
-                    if hasattr(source, 'flatpak_dest'):
+                            "url": archive.url,
-                        src['dest'] = source.flatpak_dest
+                        }
-                    module_sources.append(src)
+                        if hasattr(source, "flatpak_dest"):
+                            src["dest"] = source.flatpak_dest
+                        module_sources.append(src)
                 elif isinstance(source, GitClone):
                     src = {
-                        'type': 'git',
+                        "type": "git",
-                        'url': source.git_remote,
+                        "url": source.git_remote,
-                        'tag': source.git_ref
+                        "tag": source.git_ref,
                     }
                     try:
                         parsed = urlparse(source.git_remote)
-                        if parsed.hostname == 'github.com':
+                        if parsed.hostname == "github.com":
                             repo = parsed.path[:-4]
                             api_url = GET_REF_URL_API_TEMPLATE.format(
-                                repo = repo,
+                                repo=repo, ref=source.git_ref
-                                ref = source.git_ref)
+                            )
                             with urlopen(api_url) as r:
                                 ret = json.loads(r.read())
-                            src['commit'] = ret['object']['sha']
+                            src["commit"] = ret["object"]["sha"]
                     except:
                         pass
                     module_sources.append(src)
-                for p in getattr(source, 'patches', []):
+                for p in getattr(source, "patches", []):
-                    patch = {
+                    patch = {"type": "patch", "path": "patches/" + p}
-                        'type': 'patch',
-                        'path': 'patches/' + p
-                    }
                     module_sources.append(patch)
 
-                if hasattr(source, 'flatpak_command'):
+                if hasattr(source, "flatpak_command"):
-                    command = {
+                    command = {"type": "shell", "commands": [source.flatpak_command]}
-                        'type': 'shell',
-                        'commands': [source.flatpak_command]
-                    }
                     module_sources.append(command)
 
             else:
                 builder = get_target_step(stepDef)
                 builder.set_flatpak_buildsystem(module)
-                print(module['name'])
+                print(module["name"])
 
         manifest = MANIFEST.copy()
-        modules = [m for m in modules.values() if m.get('sources')]
+        modules = [m for m in modules.values() if m.get("sources")]
         for m in modules:
-            temp = m['sources']
+            temp = m["sources"]
-            del m['sources']
+            del m["sources"]
-            m['sources'] = temp
+            m["sources"] = temp
-        manifest['modules'] = modules
+        manifest["modules"] = modules
-        manifest_name = "{}.json".format(MANIFEST['app-id'])
+        manifest_name = "{}.json".format(MANIFEST["app-id"])
-        manifest_path = pj(self.platform.buildEnv.build_dir, manifest_name)
+        manifest_path = pj(self.config.buildEnv.build_dir, manifest_name)
-        with open(manifest_path, 'w') as f:
+        with open(manifest_path, "w") as f:
             f.write(json.dumps(manifest, indent=4))
 
     def copy_patches(self):
-        sourceDefs = (tDef for tDef in target_steps() if tDef[0] == 'source')
+        sourceDefs = (tDef for tDef in target_steps() if tDef[0] == "source")
         for sourceDef in sourceDefs:
             source = get_target_step(sourceDef)
-            if not hasattr(source, 'patches'):
+            if not hasattr(source, "patches"):
                 continue
             for p in source.patches:
-                path = pj(SCRIPT_DIR, 'patches', p)
+                path = pj(SCRIPT_DIR, "patches", p)
-                os.makedirs(pj(self.platform.buildEnv.build_dir, 'patches'), exist_ok=True)
+                os.makedirs(
-                dest = pj(self.platform.buildEnv.build_dir, 'patches', p)
+                    pj(self.config.buildEnv.build_dir, "patches"), exist_ok=True
+                )
+                dest = pj(self.config.buildEnv.build_dir, "patches", p)
                 copyfile(path, dest)
 
-
     def build(self):
-        log = pj(self.platform.buildEnv.log_dir, 'cmd_build_flatpak.log')
+        log = pj(self.config.buildEnv.log_dir, "cmd_build_flatpak.log")
-        context = Context('build', log, False)
+        context = Context("build", log, False)
-        command = "flatpak-builder --user --ccache --force-clean --keep-build-dirs --disable-rofiles-fuse --repo=repo builddir {id}.json"
+        command = [
-        command = command.format(id = MANIFEST['app-id'])
+            "flatpak-builder",
+            "--user",
+            "--ccache",
+            "--force-clean",
+            "--keep-build-dirs",
+            "--disable-rofiles-fuse",
+            "--repo=repo",
+            "builddir",
+            f"{MANIFEST['app-id']}.json",
+        ]
         try:
-            run_command(command, self.platform.buildEnv.build_dir, context, env=self.platform.get_env())
+            run_command(
+                command,
+                self.config.buildEnv.build_dir,
+                context,
+                env=self.config.get_env(),
+            )
             context._finalise()
         except subprocess.CalledProcessError:
-            with open(log, 'r') as f:
+            with open(log, "r") as f:
                 print(f.read())
             raise StopBuild()
 
     def bundle(self):
-        log = pj(self.platform.buildEnv.log_dir, 'cmd_bundle_flatpak.log')
+        log = pj(self.config.buildEnv.log_dir, "cmd_bundle_flatpak.log")
-        context = Context('bundle', log, False)
+        context = Context("bundle", log, False)
-        command = "flatpak build-bundle repo {id}.flatpak {id}"
+        app_id = MANIFEST["app-id"]
-        command = command.format(id = MANIFEST['app-id'])
+        command = ["flatpak", "build-bundle", "repo", f"{app_id}.flatpak", app_id]
         try:
-            run_command(command, self.platform.buildEnv.build_dir, context, env=self.platform.get_env())
+            run_command(
+                command,
+                self.config.buildEnv.build_dir,
+                context,
+                env=self.config.get_env(),
+            )
             context._finalise()
         except subprocess.CalledProcessError:
-            with open(log, 'r') as f:
+            with open(log, "r") as f:
                 print(f.read())
             raise StopBuild()
 
-
     def _get_packages(self):
-        package_name_mapper = PACKAGE_NAME_MAPPERS.get('flatpak', {})
+        package_name_mapper = PACKAGE_NAME_MAPPERS.get("flatpak", {})
 
         to_drop = []
         for builderDef in self._targets:
-            platformName, builderName = builderDef
+            configName, builderName = builderDef
             packages = package_name_mapper.get(builderName)
             if packages:
                 to_drop.append(builderDef)
@@ -266,16 +291,16 @@ class FlatpakBuilder:
             # dependencies we already have in the sdk.
             self._get_packages()
             self.finalize_target_steps()
-            print("[SETUP PLATFORMS]")
+            print("[SETUP TOOLCHAINS]")
-            for platform in PlatformInfo.all_running_platforms.values():
+            for config in ConfigInfo.all_running_configs.values():
-                platform.finalize_setup()
+                config.finalize_setup()
-            for pltName in PlatformInfo.all_running_platforms:
+            for cfgName in ConfigInfo.all_running_configs:
-                plt = PlatformInfo.all_platforms[pltName]
+                cfg = ConfigInfo.all_configs[cfgName]
-                for tlcName in plt.toolchain_names:
+                for tlcName in cfg.toolchain_names:
                     tlc = Dependency.all_deps[tlcName]
-                    builderDef = (pltName, tlcName)
+                    builderDef = (cfgName, tlcName)
                     builder = get_target_step(builderDef)
-                    print("build {} ({}):".format(builder.name, pltName[0]))
+                    print("build {} ({}):".format(builder.name, cfgName))
                     add_target_step(builderDef, builder)
                     builder.build()
             print("[GENERATE FLATPAK MANIFEST]")
@@ -287,11 +312,10 @@ class FlatpakBuilder:
             self.bundle()
             # No error, clean intermediate file at end of build if needed.
             print("[CLEAN]")
-            if option('clean_at_end'):
+            if option("clean_at_end"):
-                for platform in PlatformInfo.all_running_platforms.values():
+                for config in ConfigInfo.all_running_configs.values():
-                    platform.clean_intermediate_directories()
+                    config.clean_intermediate_directories()
             else:
                 print("SKIP")
         except StopBuild:
             sys.exit("Stopping build due to errors")
-

kiwixbuild/packages.py

@@ -1,107 +1,102 @@
-
+_fedora_common = [
-
+    "automake",
-_fedora_common = ['automake', 'libtool', 'cmake', 'git', 'subversion', 'ccache', 'pkgconf-pkg-config', 'gcc-c++', 'gettext-devel']
+    "libtool",
-_debian_common = ['automake', 'libtool', 'cmake', 'git', 'subversion', 'ccache', 'pkg-config', 'gcc', 'autopoint']
+    "cmake",
+    "git",
+    "subversion",
+    "ccache",
+    "pkgconf-pkg-config",
+    "gcc-c++",
+    "gettext-devel",
+]
+_debian_common = [
+    "automake",
+    "libtool",
+    "cmake",
+    "git",
+    "subversion",
+    "ccache",
+    "pkg-config",
+    "gcc",
+    "autopoint",
+]
 PACKAGE_NAME_MAPPERS = {
-    'flatpak': {
+    "flatpak": {
-        'zlib': True,
+        "zlib": True,
-        'lzma': True,
+        "lzma": True,
-        'icu4c': True,
+        "icu4c": True,
-        'qt': True,
+        "qt": True,
-        'qtwebengine': True,
+        "qtwebengine": True,
-        'uuid': True,
+        "uuid": True,
-        'libxml2': True,
+        "libxml2": True,
-        'libssl': True,
+        "libssl": True,
-        'libcurl': True,
+        "libcurl": True,
     },
-    'fedora_native_dyn': {
+    "fedora_native_dyn": {
-        'COMMON': _fedora_common,
+        "COMMON": _fedora_common,
-        'uuid': ['libuuid-devel'],
+        "uuid": ["libuuid-devel"],
-        'xapian-core': None, # Not the right version on fedora 25
+        "xapian-core": None,  # Not the right version on fedora 25
-        'pugixml': None, # ['pugixml-devel'] but package doesn't provide pkg-config file
+        "pugixml": None,  # ['pugixml-devel'] but package doesn't provide pkg-config file
-        'libmicrohttpd': ['libmicrohttpd-devel'],
+        "libmicrohttpd": ["libmicrohttpd-devel"],
-        'zlib': ['zlib-devel'],
+        "zlib": ["zlib-devel"],
-        'lzma': ['xz-devel'],
+        "lzma": ["xz-devel"],
-        'icu4c': None,
+        "icu4c": None,
-        'zimlib': None,
+        "zimlib": None,
-        'file' : ['file-devel'],
+        "file": ["file-devel"],
-        'gumbo' : ['gumbo-parser-devel'],
+        "gumbo": ["gumbo-parser-devel"],
-        'aria2': ['aria2'],
+        "aria2": ["aria2"],
-        'qt': ['qt5-qtbase-devel', 'qt5-qtsvg'],
+        "qt": ["qt5-qtbase-devel", "qt5-qtsvg"],
-        'qtwebengine': ['qt5-qtwebengine-devel']
+        "qtwebengine": ["qt5-qtwebengine-devel"],
     },
-    'fedora_native_static': {
+    "fedora_native_static": {
-        'COMMON': _fedora_common + ['glibc-static', 'libstdc++-static'],
+        "COMMON": _fedora_common + ["glibc-static", "libstdc++-static"],
-        'lzma': ['xz-devel', 'xz-static']
+        "lzma": ["xz-devel", "xz-static"],
         # Either there is no packages, or no static or too old
     },
-    'fedora_i586_dyn': {
+    "fedora_i586_dyn": {
-        'COMMON': _fedora_common + ['glibc-devel.i686', 'libstdc++-devel.i686'],
+        "COMMON": _fedora_common + ["glibc-devel.i686", "libstdc++-devel.i686"],
     },
-    'fedora_i586_static': {
+    "fedora_i586_static": {
-        'COMMON': _fedora_common + ['glibc-devel.i686'],
+        "COMMON": _fedora_common + ["glibc-devel.i686"],
     },
-    'fedora_win32_dyn': {
+    "fedora_armhf_static": {"COMMON": _fedora_common},
-        'COMMON': _fedora_common + ['mingw32-gcc-c++', 'mingw32-bzip2', 'mingw32-win-iconv', 'mingw32-winpthreads', 'wine'],
+    "fedora_armhf_dyn": {"COMMON": _fedora_common},
-        'zlib': ['mingw32-zlib'],
+    "fedora_android": {"COMMON": _fedora_common},
-        'lzma': ['mingw32-xz-libs'],
+    "debian_native_dyn": {
-        'libmicrohttpd': ['mingw32-libmicrohttpd'],
+        "COMMON": _debian_common + ["libbz2-dev", "libmagic-dev"],
+        "zlib": ["zlib1g-dev"],
+        "uuid": ["uuid-dev"],
+        "libmicrohttpd": ["libmicrohttpd-dev", "ccache"],
+        "qt": ["libqt5gui5", "qtbase5-dev", "qt5-default"],
+        "qtwebengine": ["qtwebengine5-dev"],
+        "aria2": ["aria2"],
     },
-    'fedora_win32_static': {
+    "debian_native_static": {
-        'COMMON': _fedora_common + ['mingw32-gcc-c++', 'mingw32-bzip2-static', 'mingw32-win-iconv-static', 'mingw32-winpthreads-static', 'wine'],
+        "COMMON": _debian_common + ["libbz2-dev", "libmagic-dev"],
-        'zlib': ['mingw32-zlib-static'],
-        'lzma': ['mingw32-xz-libs-static'],
-        'libmicrohttpd': None, # ['mingw32-libmicrohttpd-static'] packaging dependecy seems buggy, and some static lib are name libfoo.dll.a and
-                               # gcc cannot found them.
     },
-    'fedora_armhf_static': {
+    "debian_i586_dyn": {
-        'COMMON': _fedora_common
+        "COMMON": _debian_common
+        + ["libc6-dev-i386", "lib32stdc++6", "gcc-multilib", "g++-multilib"],
     },
-    'fedora_armhf_dyn': {
+    "debian_i586_static": {
-        'COMMON': _fedora_common
+        "COMMON": _debian_common
+        + ["libc6-dev-i386", "lib32stdc++6", "gcc-multilib", "g++-multilib"],
     },
-    'fedora_android': {
+    "debian_armhf_static": {
-        'COMMON': _fedora_common
+        "COMMON": _debian_common,
     },
-    'debian_native_dyn': {
+    "debian_armhf_dyn": {
-        'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'],
+        "COMMON": _debian_common,
-        'zlib': ['zlib1g-dev'],
-        'uuid': ['uuid-dev'],
-        'libmicrohttpd': ['libmicrohttpd-dev', 'ccache'],
-        'qt' : ['libqt5gui5', 'qtbase5-dev', 'qt5-default'],
-        'qtwebengine' : ['qtwebengine5-dev'],
-        'aria2': ['aria2'],
     },
-    'debian_native_static': {
+    "debian_android": {
-        'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'],
+        "COMMON": _debian_common,
     },
-    'debian_i586_dyn': {
+    "Darwin_native_dyn": {
-        'COMMON': _debian_common + ['libc6-dev-i386', 'lib32stdc++6', 'gcc-multilib', 'g++-multilib'],
+        "COMMON": ["autoconf", "automake", "libtool", "cmake", "pkg-config"],
+        "file": ["libmagic"],
     },
-    'debian_i586_static': {
+    "Darwin_iOS": {
-        'COMMON': _debian_common + ['libc6-dev-i386', 'lib32stdc++6', 'gcc-multilib', 'g++-multilib'],
+        "COMMON": ["autoconf", "automake", "libtool", "cmake", "pkg-config"],
-    },
+        "file": ["libmagic"],
-    'debian_win32_dyn': {
-        'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'],
-    },
-    'debian_win32_static': {
-        'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'],
-    },
-    'debian_armhf_static': {
-        'COMMON': _debian_common,
-    },
-    'debian_armhf_dyn': {
-        'COMMON': _debian_common,
-    },
-    'debian_android': {
-        'COMMON': _debian_common,
-    },
-    'Darwin_native_dyn': {
-        'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'],
-        'file': ['libmagic']
-    },
-    'Darwin_iOS': {
-        'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'],
-        'file': ['libmagic']
     },
 }

kiwixbuild/patches/android-ndk-r21e-linux-x86_64-python3.12+.patch

@@ -0,0 +1,33 @@
+diff -ur android-ndk-r21e/build/tools/make_standalone_toolchain.py android-ndk-r21e.patched/build/tools/make_standalone_toolchain.py
+--- android-ndk-r21e/build/tools/make_standalone_toolchain.py	2025-03-04 20:48:14.681288830 +0400
++++ android-ndk-r21e.patched/build/tools/make_standalone_toolchain.py	2025-03-05 12:10:47.252578915 +0400
+@@ -1,4 +1,4 @@
+-#!/usr/bin/env python
++#!/usr/bin/env python3
+ #
+ # Copyright (C) 2016 The Android Open Source Project
+ #
+@@ -21,7 +21,6 @@
+ """
+ import argparse
+ import atexit
+-from distutils.dir_util import copy_tree
+ import inspect
+ import logging
+ import os
+@@ -221,13 +220,13 @@
+ 
+ def create_toolchain(install_path, arch, api, toolchain_path, host_tag):
+     """Create a standalone toolchain."""
+-    copy_tree(toolchain_path, install_path)
++    shutil.copytree(toolchain_path, install_path)
+     triple = get_triple(arch)
+     make_clang_scripts(install_path, arch, api, host_tag == 'windows-x86_64')
+     replace_gcc_wrappers(install_path, triple, host_tag == 'windows-x86_64')
+ 
+     prebuilt_path = os.path.join(NDK_DIR, 'prebuilt', host_tag)
+-    copy_tree(prebuilt_path, install_path)
++    shutil.copytree(prebuilt_path, install_path, dirs_exist_ok=True)
+ 
+     gdbserver_path = os.path.join(
+         NDK_DIR, 'prebuilt', 'android-' + arch, 'gdbserver')

kiwixbuild/patches/docopt_meson_install_pkgconfig.patch

@@ -0,0 +1,31 @@
+diff '--color=auto' -ur docoptcpp-0.6.2/meson.build docoptcpp-0.6.2_patched/meson.build
+--- docoptcpp-0.6.2/meson.build	2024-08-26 14:28:47.553448529 +0200
++++ docoptcpp-0.6.2_patched/meson.build	2024-08-26 14:10:47.232603427 +0200
+@@ -10,11 +10,25 @@
+ 
+ # bug with missing dllexport. fixed in next version.
+ if cpp.get_argument_syntax() == 'msvc'
+-  doclib = static_library('docopt', 'docopt.cpp')
++  doclib = static_library('docopt', 'docopt.cpp', install: true)
+ else
+-  doclib = library('docopt', 'docopt.cpp')
++  doclib = library('docopt', 'docopt.cpp', install: true)
+ endif
+ 
+ executable('docopt_example', 'examples/naval_fate.cpp', link_with: doclib)
+ docopt_dep = declare_dependency(include_directories: include_directories('.'),
+     link_with: doclib)
++
++install_headers(
++  'docopt.h',
++  'docopt_value.h',
++  subdir: 'docopt'
++)
++
++pkg_mod = import('pkgconfig')
++pkg_mod.generate(
++  doclib,
++  version: meson.project_version(),
++  name: 'docopt',
++  filebase: 'docopt'
++)

kiwixbuild/patches/docopt_meson_use_boostregex.patch

@@ -0,0 +1,14 @@
+diff -ur docoptcpp-0.6.2/meson.build docoptcpp-0.6.2_boostregex/meson.build
+--- docoptcpp-0.6.2/meson.build	2024-08-28 17:22:46.256716100 +0200
++++ docoptcpp-0.6.2_boostregex/meson.build	2024-08-28 17:02:47.932681000 +0200
+@@ -8,6 +8,10 @@
+   add_project_arguments('-DDOCOPT_DLL', '-DDOCOPT_EXPORTS', language: 'cpp')
+ endif
+ 
++if cpp.get_id() =='msvc'
++  add_project_arguments('-DDOCTOPT_USE_BOOST_REGEX', '-DBOOST_REGEX_STANDALONE', language: 'cpp')
++endif
++
+ # bug with missing dllexport. fixed in next version.
+ if cpp.get_argument_syntax() == 'msvc'
+   doclib = static_library('docopt', 'docopt.cpp', install: true)

kiwixbuild/patches/libmicrohttpd_meson_pkgconfig.patch

@@ -0,0 +1,29 @@
+diff '--color=auto' -ur libmicrohttpd-0.9.76_orig/meson.build libmicrohttpd-0.9.76/meson.build
+--- libmicrohttpd-0.9.76_orig/meson.build	2024-08-18 14:55:29.372805433 +0200
++++ libmicrohttpd-0.9.76/meson.build	2024-08-18 17:19:44.087444728 +0200
+@@ -6,6 +6,8 @@
+   default_options: ['warning_level=1'],
+ )
+ 
++pkg = import('pkgconfig')
++
+ add_project_arguments('-D_GNU_SOURCE', language: 'c')
+ 
+ incdirs = include_directories('src/include')
+@@ -267,6 +269,16 @@
+   install: true,
+ )
+ 
++install_headers(
++  'src/include/microhttpd.h',
++)
++
++pkg.generate(
++  libmicrohttpd,
++  description: 'Libmicrohttpd',
++  name: 'libmicrohttpd',
++)
++
+ depinc = include_directories('.', 'src/include')
+ libmicrohttpd_dep = declare_dependency(
+   include_directories: depinc,