Merge pull request #819 from kiwix/release-zim-tools-3.6.0

Relase zim-tools 3.6.0

.github/FUNDING.yml

@@ -1,12 +0,0 @@
-# These are supported funding model platforms
-
-github: kiwix # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
-patreon: # Replace with a single Patreon username
-open_collective: # Replace with a single Open Collective username
-ko_fi: # Replace with a single Ko-fi username
-tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
-community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
-liberapay: # Replace with a single Liberapay username
-issuehunt: # Replace with a single IssueHunt username
-otechie: # Replace with a single Otechie username
-custom: # https://kiwix.org/support-us/

.github/configs/aqt.ini

@@ -0,0 +1,46 @@
+[DEFAULTS]
+
+[aqt]
+concurrency: 4
+baseurl: https://qt.mirror.constant.com/
+7zcmd: 7z
+print_stacktrace_on_error: False
+always_keep_archives: False
+archive_download_location: .
+min_module_size: 41
+
+[requests]
+connection_timeout: 3.5
+response_timeout: 30
+max_retries_on_connection_error: 5
+retry_backoff: 0.1
+max_retries_on_checksum_error: 5
+max_retries_to_retrieve_hash: 5
+hash_algorithm: sha256
+INSECURE_NOT_FOR_PRODUCTION_ignore_hash: False
+
+[mirrors]
+trusted_mirrors:
+    https://download.qt.io
+    https://qt.mirror.constant.com/
+    https://ftp.fau.de/qtproject/
+blacklist:
+    http://mirrors.ocf.berkeley.edu
+    http://mirrors.tuna.tsinghua.edu.cn
+    http://mirrors.geekpie.club
+fallbacks:
+    https://qtproject.mirror.liquidtelecom.com/
+    https://mirrors.aliyun.com/qt/
+    https://mirrors.ustc.edu.cn/qtproject/
+    https://ftp.jaist.ac.jp/pub/qtproject/
+    https://ftp.yz.yamagata-u.ac.jp/pub/qtproject/
+    https://qt-mirror.dannhauer.de/
+    https://ftp.fau.de/qtproject/
+    https://mirror.netcologne.de/qtproject/
+    https://mirrors.dotsrc.org/qtproject/
+    https://www.nic.funet.fi/pub/mirrors/download.qt-project.org/
+    https://master.qt.io/
+    https://mirrors.ukfast.co.uk/sites/qt.io/
+    https://ftp2.nluug.nl/languages/qt/
+    https://ftp1.nluug.nl/languages/qt/
+    https://qt.mirror.constant.com/

.github/scripts/build_definition.py

@@ -15,52 +15,56 @@ import csv, io, re
 # 'D' letter means we trigger the docker forkflow to build the docker image.
 # If a cell contains several letters, all are done.
 BUILD_DEF = """
-    | OS_NAME | COMPILE_CONFIG     | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | platform_name        |
+    | OS_NAME   | COMPILE_CONFIG     | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | platform_name           | dependency_name        |
-    =====================================================================================================================
+    ==============================================================================================================================================
-# Bionic is a special case as we need to compile libzim on old arch for python
+# manylinux is a special case as we need to compile libzim on old arch for python
-    | bionic  | native_mixed       | BP     |          |           |             |               | linux-x86_64-bionic  |
+    | manylinux | native_mixed       | BP     |          |           |             |               | linux-x86_64-manylinux  |                        |
-    | bionic  | aarch64_mixed      | BP     |          |           |             |               | linux-aarch64-bionic |
+    | manylinux | aarch64_mixed      | BP     |          |           |             |               | linux-aarch64-manylinux |                        |
-    --------------------------------------------------------------------------------------------------------------------
+    ----------------------------------------------------------------------------------------------------------------------------------------------
+# On Windows, we build only libzim for now. And only native_mixed as xapian doesn't compile as dll
+    | windows   | native_static      | Bd     | d        | BPd       | BPd         |               | win-x86_64              | win-x86_64-static      |
+    | windows   | native_dyn         | Bd     |          |           |             |               | win-x86_64              | win-x86_64-dyn         |
+    | windows   | native_mixed       | BPd    | d        |           |             | BPd           | win-x86_64              | win-x86_64-mixed       |
+    ----------------------------------------------------------------------------------------------------------------------------------------------
 # Osx builds, build binaries on native_dyn and native_static. On anyother things, build only the libraries
-    | macos   | native_dyn         | d      | d        | dB        | B           |               |                      |
+    | macos     | native_dyn         | d      | d        | dB        | B           |               |                         | macos-x86_64-dyn       |
-    | macos   | native_static      |        |          | BP        | BP          |               | macos-x86_64         |
+    | macos     | native_static      |        |          | BP        | BP          |               | macos-x86_64            |                        |
-    | macos   | native_mixed       | BP     | BP       |           |             |               | macos-x86_64         |
+    | macos     | native_mixed       | BP     | BP       |           |             |               | macos-x86_64            |                        |
-    | macos   | iOS_arm64          | dB     | dB       |           |             |               |                      |
+    | macos     | iOS_arm64          | dB     | dB       |           |             |               |                         | ios-arm64-dyn          |
-    | macos   | iOSSimulator_x86_64| dB     | dB       |           |             |               |                      |
+    | macos     | iOSSimulator_x86_64| dB     | dB       |           |             |               |                         | ios-x86_64-dyn         |
-    | macos   | iOSSimulator_arm64 | dB     | dB       |           |             |               |                      |
+    | macos     | iOSSimulator_arm64 | B      | B        |           |             |               |                         |                        |
-    | macos   | macOS_arm64_static |        |          | BP        | BP          |               | macos-arm64          |
+    | macos     | macOS_arm64_static |        |          | BP        | BP          |               | macos-arm64             |                        |
-    | macos   | macOS_arm64_mixed  | BP     | BP       |           |             |               | macos-arm64          |
+    | macos     | macOS_arm64_mixed  | dBP    | dBP      | d         |             |               | macos-arm64             | macos-aarch64-dyn      |
-    | macos   | macOS_x86_64       | B      | B        |           |             |               |                      |
+    | macos     | macOS_x86_64       | B      | B        |           |             |               |                         |                        |
-    | macos   | apple_all_static   |        | BP       |           |             |               | xcframework          |
+    | macos     | apple_all_static   |        | BP       |           |             |               | xcframework             |                        |
-    ----------------------------------------------------------------------------------------------
+    ----------------------------------------------------------------------------------------------------------------------------------------------
-    |         | flatpak            |        |          |           |             | BP            |                      |
+    | focal     | flatpak            |        |          |           |             | BP            |                         |                        |
-    |         | native_static      | d      | d        | dBPSD     | dBPSD       |               | linux-x86_64         |
+    | focal     | native_static      | d      | d        | dBPSD     | dBPSD       |               | linux-x86_64            | linux-x86_64-static    |
-    |         | native_mixed       | BPS    | BPS      |           |             |               | linux-x86_64         |
+    | focal     | native_mixed       | BPS    | BPS      |           |             |               | linux-x86_64            |                        |
-    |         | native_dyn         | d      | d        | dB        | dB          | dBPS          |                      |
+    | focal     | native_dyn         | d      | d        | dB        | dB          |               |                         | linux-x86_64-dyn       |
+    | jammy     | native_dyn         |        |          |           |             | dBPS          |                         | linux-x86_64-dyn       |
     # libzim CI is building alpine_dyn but not us
-    |         | android_arm        | dBP    | dBP      |           |             |               | android-arm          |
+    | focal     | android_arm        | dBP    | dBP      |           |             |               | android-arm             | android-arm            |
-    |         | android_arm64      | dBP    | dBP      |           |             |               | android-arm64        |
+    | focal     | android_arm64      | dBP    | dBP      |           |             |               | android-arm64           | android-arm64          |
-    |         | android_x86        | BP     | BP       |           |             |               | android-x86          |
+    | focal     | android_x86        | BP     | BP       |           |             |               | android-x86             |                        |
-    |         | android_x86_64     | BP     | BP       |           |             |               | android-x86_64       |
+    | focal     | android_x86_64     | BP     | BP       |           |             |               | android-x86_64          |                        |
-    |         | armv6_static       |        |          | BP        | BP          |               | linux-armv6          |
+    | focal     | armv6_static       |        |          | BP        | BP          |               | linux-armv6             |                        |
-    |         | armv6_mixed        | BP     |          |           |             |               | linux-armv6          |
+    | focal     | armv6_mixed        | BP     |          |           |             |               | linux-armv6             |                        |
-    |         | armv6_dyn          |        |          | B         | B           |               |                      |
+    | focal     | armv6_dyn          |        |          | B         | B           |               |                         |                        |
-    |         | armv8_static       |        |          | BP        | BP          |               | linux-armv8          |
+    | focal     | armv8_static       |        |          | BP        | BP          |               | linux-armv8             |                        |
-    |         | armv8_mixed        | BP     |          |           |             |               | linux-armv8          |
+    | focal     | armv8_mixed        | BP     |          |           |             |               | linux-armv8             |                        |
-    |         | armv8_dyn          |        |          | B         | B           |               |                      |
+    | focal     | armv8_dyn          |        |          | B         | B           |               |                         |                        |
-    |         | aarch64_static     |        |          | BP        | BP          |               | linux-aarch64        |
+    | focal     | aarch64_static     |        |          | BP        | BP          |               | linux-aarch64           |                        |
-    |         | aarch64_mixed      | BP     |          |           |             |               | linux-aarch64        |
+    | focal     | aarch64_mixed      | BP     |          |           |             |               | linux-aarch64           |                        |
-    |         | aarch64_dyn        | d      |          | B         | B           |               |                      |
+    | focal     | aarch64_dyn        | d      |          | B         | B           |               |                         | linux-aarch64-dyn      |
-    |         | aarch64_musl_static|        |          | BP        | BP          |               | linux-aarch64-musl   |
+    | focal     | aarch64_musl_static|        |          | BP        | BP          |               | linux-aarch64-musl      |                        |
-    |         | aarch64_musl_mixed | BP     |          |           |             |               | linux-aarch64-musl   |
+    | focal     | aarch64_musl_mixed | BP     |          |           |             |               | linux-aarch64-musl      |                        |
-    |         | aarch64_musl_dyn   | d      |          | B         | B           |               |                      |
+    | focal     | aarch64_musl_dyn   | d      |          | B         | B           |               |                         | linux-aarch64-musl-dyn |
-    |         | x86-64_musl_static |        |          | BP        | BP          |               | linux-x86_64-musl    |
+    | focal     | x86-64_musl_static |        |          | BP        | BP          |               | linux-x86_64-musl       |                        |
-    |         | x86-64_musl_mixed  | BP     |          |           |             |               | linux-x86_64-musl    |
+    | focal     | x86-64_musl_mixed  | BP     |          |           |             |               | linux-x86_64-musl       |                        |
-    |         | win32_static       | d      | dB       | dBP       | dBP         |               | win-i686             |
+    | focal     | i586_static        |        |          | BP        | BP          |               | linux-i586              |                        |
-    |         | win32_dyn          | d      | dB       | dB        | dB          |               |                      |
+    | focal     | i586_dyn           |        |          | B         | B           |               |                         |                        |
-    |         | i586_static        |        |          | BP        | BP          |               | linux-i586           |
+    | focal     | wasm               | dBP    |          |           |             |               | wasm-emscripten         | wasm                   |
-    |         | i586_dyn           |        |          | B         | B           |               |                      |
-    |         | wasm               | dBP    |          |           |             |               | wasm-emscripten      |
 """
 
 
@@ -135,7 +139,7 @@ def select_build_targets(criteria):
     raise ValueError("No definition match with current context.")
 
 
-def get_platform_name():
+def get_column_value(column_name):
     from common import COMPILE_CONFIG, OS_NAME
 
     context = Context(COMPILE_CONFIG=COMPILE_CONFIG, OS_NAME=OS_NAME)
@@ -143,7 +147,15 @@ def get_platform_name():
     reader = csv.DictReader(strip_array(BUILD_DEF), dialect=TableDialect())
     for row in reader:
         if context.match(row):
-            name = row["platform_name"]
+            name = row[column_name]
             return name or None
 
     raise ValueError("No definition match with current context.")
+
+
+def get_platform_name():
+    return get_column_value("platform_name")
+
+
+def get_dependency_archive_name():
+    return get_column_value("dependency_name")

.github/scripts/build_release_nightly.py

@@ -21,16 +21,15 @@ from common import (
 from build_definition import select_build_targets, BUILD, PUBLISH, SOURCE_PUBLISH
 
 
-# Filter what to build if we are doing a release.
-if MAKE_RELEASE:
-    TARGETS = select_build_targets(PUBLISH)
-
 def release_filter(project):
     return release_versions.get(project) is not None
 
+
+# Filter what to build if we are doing a release.
+TARGETS = select_build_targets(PUBLISH)
+
+if MAKE_RELEASE:
     TARGETS = tuple(filter(release_filter, TARGETS))
-else:
-    TARGETS = select_build_targets(BUILD)
 
 for target in TARGETS:
     run_kiwix_build(target, config=COMPILE_CONFIG, make_release=MAKE_RELEASE)

.github/scripts/common.py

@@ -1,16 +1,17 @@
 import os
 from os import environ as _environ
-from pathlib import Path
+from pathlib import Path, PurePosixPath
 from datetime import date
 import tarfile
 import zipfile
 import subprocess
 import re
 import shutil
+import platform
 
 import requests
 
-from build_definition import get_platform_name
+from build_definition import get_platform_name, get_dependency_archive_name
 
 from kiwixbuild.dependencies.apple_xcframework import AppleXCFramework
 from kiwixbuild.versions import (
@@ -20,17 +21,36 @@ from kiwixbuild.versions import (
 )
 
 
+def get_build_dir(config) -> Path:
+    command = ["kiwix-build"]
+    command.extend(["--config", config])
+    command.append("--get-build-dir")
+    command.append("--use-target-arch-name")
+    return Path(
+        subprocess.run(command, cwd=str(HOME), check=True, stdout=subprocess.PIPE)
+        .stdout.strip()
+        .decode("utf8")
+    )
+
+
 COMPILE_CONFIG = _environ["COMPILE_CONFIG"]
 OS_NAME = _environ["OS_NAME"]
 HOME = Path(os.path.expanduser("~"))
 
-BASE_DIR = HOME / "BUILD_{}".format(COMPILE_CONFIG)
+BASE_DIR = get_build_dir(COMPILE_CONFIG)
 SOURCE_DIR = HOME / "SOURCE"
 ARCHIVE_DIR = HOME / "ARCHIVE"
 TOOLCHAIN_DIR = BASE_DIR / "TOOLCHAINS"
 INSTALL_DIR = BASE_DIR / "INSTALL"
-TMP_DIR = Path(os.getenv("TMP_DIR", "/tmp"))
+default_tmp_dir = os.getenv("TEMP") if platform.system() == "Windows" else "/tmp"
+TMP_DIR = Path(os.getenv("TMP_DIR", default_tmp_dir))
+if platform.system() == "Windows":
+    KBUILD_SOURCE_DIR = Path(_environ["GITHUB_WORKSPACE"])
+    BIN_EXT = ".exe"
+else:
     KBUILD_SOURCE_DIR = HOME / "kiwix-build"
+    BIN_EXT = ""
+
 
 _ref = _environ.get("GITHUB_REF", "").split("/")[-1]
 MAKE_RELEASE = re.fullmatch(r"r_[0-9]+", _ref) is not None
@@ -44,18 +64,25 @@ else:
 FLATPAK_HTTP_GIT_REMOTE = "https://github.com/flathub/org.kiwix.desktop.git"
 FLATPAK_GIT_REMOTE = "git@github.com:flathub/org.kiwix.desktop.git"
 
-BIN_EXT = ".exe" if COMPILE_CONFIG.startswith("win32_") else ""
-
 
 def major_version(version: str) -> str:
     return version.split(".")[0]
 
 
+# Depending of base distribution, libraries are in "lib64" (redhat base) or "lib/<arch>" (debian base).
+# On top of that, when cross-compiling, libraries are always put in `lib/<arch>`.
+# As we use this as glob regex to select which files to add to archive, this is not a problem to have both.
+def lib_prefix(file):
+    yield "lib64/" + file
+    yield "lib/*/" + file
+
+
 # We have build everything. Now create archives for public deployement.
 EXPORT_FILES = {
     "kiwix-tools": (
         INSTALL_DIR / "bin",
-        [f + BIN_EXT for f in ("kiwix-manage", "kiwix-search", "kiwix-serve")],
+        [f + BIN_EXT for f in ("kiwix-manage", "kiwix-search", "kiwix-serve")]
+        + ["icu*.dll"],
     ),
     "zim-tools": (
         INSTALL_DIR / "bin",
@@ -72,23 +99,40 @@ EXPORT_FILES = {
                 "zimwriterfs",
                 "zimrecreate",
             )
-        ],
+        ]
+        + ["icu*.dll"],
     ),
     "libzim": (
         INSTALL_DIR,
         (
+            ## Linux
             # We need to package all dependencies (`*.a`) on wasm
-            "lib/*/libzim.a" if COMPILE_CONFIG != "wasm" else "lib/*.a",
+            *lib_prefix("libzim.a" if COMPILE_CONFIG != "wasm" else "*.a"),
-            "lib/*/libzim.so",
+            *lib_prefix("libzim.so"),
-            "lib/*/libzim.so.{version}".format(version=main_project_versions["libzim"]),
+            *lib_prefix(
-            "lib/*/libzim.so.{version}".format(
+                "libzim.so.{version}".format(version=main_project_versions["libzim"])
-                version=major_version(main_project_versions["libzim"])
             ),
+            *lib_prefix(
+                "libzim.so.{version}".format(
+                    version=major_version(main_project_versions["libzim"])
+                )
+            ),
+            ## MacOS
             "lib/libzim.{}.dylib".format(
                 major_version(main_project_versions["libzim"])
             ),
             "lib/libzim.dylib",
             "lib/*/libzim.pc",
+            ## Windows
+            "bin/zim-{version}.dll".format(
+                version=major_version(main_project_versions["libzim"])
+            ),
+            "bin/icu*.dll",
+            "bin/zim-{version}.pdb".format(
+                version=major_version(main_project_versions["libzim"])
+            ),
+            "lib/zim.lib",
+            ## Includes and others
             "include/zim/**/*.h",
             "share/icu/{}/icudt{}l.dat".format(
                 base_deps_versions["icu4c"], major_version(base_deps_versions["icu4c"])
@@ -109,6 +153,14 @@ EXPORT_FILES = {
             "lib/libkiwix.{}.dylib".format(
                 major_version(main_project_versions["libkiwix"])
             ),
+            "bin/kiwix-{version}.dll".format(
+                version=major_version(main_project_versions["libkiwix"])
+            ),
+            "bin/icu*.dll",
+            "bin/kiwix-{version}.pdb".format(
+                version=major_version(main_project_versions["libkiwix"])
+            ),
+            "lib/kiwix.lib",
             "lib/libkiwix.dylib",
             "lib/*/libkiwix.pc",
             "include/kiwix/**/*.h",
@@ -161,6 +213,7 @@ def run_kiwix_build(
     command.append("--hide-progress")
     command.append("--fast-clone")
     command.append("--assume-packages-installed")
+    command.append("--use-target-arch-name")
     command.extend(["--config", config])
     if build_deps_only:
         command.append("--build-deps-only")
@@ -183,6 +236,70 @@ def run_kiwix_build(
     print_message("Build ended")
 
 
+try:
+    import paramiko
+
+    def upload(file_to_upload, host, dest_path):
+        if not file_to_upload.exists():
+            print_message("No {} to upload!", file_to_upload)
+            return
+
+        if ":" in host:
+            host, port = host.split(":", 1)
+        else:
+            port = "22"
+        if "@" in host:
+            user, host = host.split("@", 1)
+        else:
+            user = None
+
+        from contextlib import contextmanager
+
+        @contextmanager
+        def get_client():
+            client = paramiko.client.SSHClient()
+            client.set_missing_host_key_policy(paramiko.client.WarningPolicy)
+            print_message(f"Connect to {host}:{port}")
+            client.connect(
+                host,
+                port=port,
+                username=user,
+                key_filename=_environ.get("SSH_KEY"),
+                look_for_keys=False,
+                compress=True,
+            )
+            try:
+                yield client
+            finally:
+                client.close()
+
+        @contextmanager
+        def get_sftp():
+            with get_client() as client:
+                sftp = client.open_sftp()
+                try:
+                    yield sftp
+                finally:
+                    sftp.close()
+
+        dest_path = PurePosixPath(dest_path)
+        remote_file = dest_path.joinpath(file_to_upload.name)
+
+        with get_sftp() as sftp:
+            for part in list(reversed(dest_path.parents)) + [dest_path]:
+                part = str(part)
+                try:
+                    sftp.stat(part)
+                except FileNotFoundError:
+                    sftp.mkdir(part)
+
+            print_message(f"Sending archive {file_to_upload} to {remote_file}")
+            sftp.put(str(file_to_upload), str(remote_file), confirm=True)
+
+except ModuleNotFoundError:
+    # On old system (bionic) paramiko is really complex to install
+    # Keep the old implementaion on sush system.
+
     def upload(file_to_upload, host, dest_path):
         if not file_to_upload.exists():
             print_message("No {} to upload!", file_to_upload)
@@ -193,7 +310,10 @@ def upload(file_to_upload, host, dest_path):
         else:
             port = "22"
 
-    # sending SFTP mkdir command to the sftp interactive mode and not batch (-b) mode
+        # Using SFTP to create the directory hierarchy because we can not
+        # use SSH (no shell for this user); and then scp to upload the file.
+        #
+        # Sending SFTP mkdir command to the SFTP interactive mode and not batch (-b) mode
         # as the latter would exit on any mkdir error while it is most likely
         # the first parts of the destination is already present and thus can't be created
         sftp_commands = "\n".join(
@@ -204,6 +324,8 @@ def upload(file_to_upload, host, dest_path):
         )
         command = [
             "sftp",
+            "-c",
+            "aes128-ctr",
             "-i",
             _environ.get("SSH_KEY"),
             "-P",
@@ -271,30 +393,28 @@ def filter_install_dir(path):
                     yield sub_dir
 
 
-# Full: True if we are creating a full archive to be used as cache by kiwix-build (base_deps2_{os}_{config}_{base_deps_version}.tar.xz)
+# Full: True if we are creating a full archive to be used as cache by kiwix-build (base_deps_{os}_{config}_{base_deps_version}.tar.gz)
-# Full: False if we are creating a archive to be used as pre-cached dependencies for project's CI (deps2_{os}_{config}_{target}.tar.xz)
+# Full: False if we are creating a archive to be used as pre-cached dependencies for project's CI (deps_{config}_{target}.tar.gz)
 def make_deps_archive(target=None, name=None, full=False):
-    archive_name = name or "deps2_{}_{}_{}.tar.xz".format(
+    archive_name = name or "deps_{}_{}.tar.gz".format(
-        OS_NAME, COMPILE_CONFIG, target
+        get_dependency_archive_name(), target
     )
     print_message("Create archive {}.", archive_name)
     files_to_archive = list(filter_install_dir(INSTALL_DIR))
     files_to_archive += HOME.glob("BUILD_*/LOGS")
     if COMPILE_CONFIG == "apple_all_static":
         for subconfig in AppleXCFramework.subConfigNames:
-            base_dir = HOME / "BUILD_{}".format(subconfig)
+            base_dir = get_build_dir(subconfig)
             files_to_archive += filter_install_dir(base_dir / "INSTALL")
             if (base_dir / "meson_cross_file.txt").exists():
                 files_to_archive.append(base_dir / "meson_cross_file.txt")
 
     if COMPILE_CONFIG.endswith("_mixed"):
         static_config = COMPILE_CONFIG.replace("_mixed", "_static")
-        files_to_archive += filter_install_dir(
+        files_to_archive += filter_install_dir(get_build_dir(static_config) / "INSTALL")
-            HOME / ("BUILD_" + static_config) / "INSTALL"
-        )
     if COMPILE_CONFIG.startswith("android_"):
         files_to_archive += filter_install_dir(HOME / "BUILD_neutral" / "INSTALL")
-        base_dir = HOME / "BUILD_{}".format(COMPILE_CONFIG)
+        base_dir = get_build_dir(COMPILE_CONFIG)
         if (base_dir / "meson_cross_file.txt").exists():
             files_to_archive.append(base_dir / "meson_cross_file.txt")
     # Copy any toolchain
@@ -314,18 +434,18 @@ def make_deps_archive(target=None, name=None, full=False):
         # Add also static build for mixed target
         if COMPILE_CONFIG.endswith("_mixed"):
             static_config = COMPILE_CONFIG.replace("_mixed", "_static")
-            files_to_archive += (HOME / ("BUILD_" + static_config)).glob("*/.*_ok")
+            files_to_archive += get_build_dir(static_config).glob("*/.*_ok")
         # Native dyn and static is needed for potential cross compilation that use native tools (icu)
-        files_to_archive += (HOME / "BUILD_native_dyn").glob("*/.*_ok")
+        files_to_archive += get_build_dir("native_dyn").glob("*/.*_ok")
-        files_to_archive += (HOME / "BUILD_native_static").glob("*/.*_ok")
+        files_to_archive += get_build_dir("native_static").glob("*/.*_ok")
-        files_to_archive += HOME.glob("BUILD_android*/**/.*_ok")
+        files_to_archive += HOME.glob("BUILD_*android*/**/.*_ok")
-        files_to_archive += HOME.glob("BUILD_macOS*/**/.*_ok")
+        files_to_archive += HOME.glob("BUILD_*apple-macos*/**/.*_ok")
-        files_to_archive += HOME.glob("BUILD_iOS*/**/.*_ok")
+        files_to_archive += HOME.glob("BUILD_*apple-ios*/**/.*_ok")
         files_to_archive += SOURCE_DIR.glob("*/.*_ok")
         files_to_archive += SOURCE_DIR.glob("zim-testing-suite-*/*")
 
     archive_file = TMP_DIR / archive_name
-    with tarfile.open(str(archive_file), "w:xz") as tar:
+    with tarfile.open(str(archive_file), "w:gz") as tar:
         for name in set(files_to_archive):
             print(".{}".format(name), flush=True)
             tar.add(str(name), arcname=str(name.relative_to(relative_path)))
@@ -341,6 +461,25 @@ def get_postfix(project):
     return postfix
 
 
+def sign_binary(path):
+    # We assume here that signtool and certificate are properly configured.
+    # Env var `SIGNTOOL_THUMBPRINT` must contain thumbprint of the certificate to use.
+    command = [
+        os.getenv("SIGNTOOL_PATH", "signtool.exe"),
+        "sign",
+        "/fd",
+        "sha256",
+        "/tr",
+        "http://ts.ssl.com",
+        "/td",
+        "sha256",
+        "/sha1",
+        os.environ["SIGNTOOL_THUMBPRINT"],
+        str(path),
+    ]
+    subprocess.run(command, check=True)
+
+
 def make_archive(project, make_release):
     platform_name = get_platform_name()
     if not platform_name:
@@ -362,7 +501,13 @@ def make_archive(project, make_release):
     files_to_archive = []
     for export_file in export_files:
         files_to_archive.extend(base_dir.glob(export_file))
-    if platform_name == "win-i686":
+
+    if make_release and platform.system() == "Windows":
+        for file in files_to_archive:
+            if str(file).endswith(".exe"):
+                sign_binary(file)
+
+    if platform_name == "win-i686" or platform.system() == "Windows":
         open_archive = lambda a: zipfile.ZipFile(
             str(a), "w", compression=zipfile.ZIP_DEFLATED
         )
@@ -396,11 +541,27 @@ def create_desktop_image(make_release):
         build_path = BASE_DIR / "org.kiwix.desktop.flatpak"
         app_name = "org.kiwix.desktop.{}.flatpak".format(postfix)
         print_message("archive is {}", build_path)
+    elif platform.system() == "Windows":
+        archive_basename = "kiwix-desktop_windows_x64_{}".format(postfix)
+        working_dir = INSTALL_DIR / archive_basename
+        build_path = Path(str(working_dir) + ".zip")
+        app_name = build_path.name
+        command = [
+            "python",
+            KBUILD_SOURCE_DIR / "scripts" / "package_kiwix-desktop_windows.py",
+            str(INSTALL_DIR),
+            str(working_dir),
+            str(build_path),
+        ]
+        if make_release:
+            command += ["-s"]
+        print_message("Package archive of kiwix-desktop")
+        subprocess.check_call(command, cwd=str(HOME))
     else:
         build_path = HOME / "Kiwix-{}-x86_64.AppImage".format(postfix)
         app_name = "kiwix-desktop_x86_64_{}.appimage".format(postfix)
         command = [
-            "kiwix-build/scripts/create_kiwix-desktop_appImage.sh",
+            KBUILD_SOURCE_DIR / "scripts" / "create_kiwix-desktop_appImage.sh",
             str(INSTALL_DIR),
             str(src_dir),
             str(HOME / "AppDir"),
@@ -434,7 +595,7 @@ def update_flathub_git():
     call(command)
     shutil.copy(str(BASE_DIR / "org.kiwix.desktop.json"), str(git_repo_dir))
     patch_dir = KBUILD_SOURCE_DIR / "kiwixbuild" / "patches"
-    for dep in ["pugixml"]:
+    for dep in ["pugixml", "libmicrohttpd"]:
         for f in patch_dir.glob("{}_*.patch".format(dep)):
             shutil.copy(str(f), str(git_repo_dir / "patches"))
     command = ["git", "add", "-A", "."]

.github/scripts/ensure_base_deps.py

@@ -15,6 +15,7 @@ from common import (
     HOME,
     COMPILE_CONFIG,
     OS_NAME,
+    MAKE_RELEASE,
 )
 
 
@@ -32,17 +33,21 @@ def download_base_archive(base_name):
     return file_path
 
 
-ARCHIVE_NAME_TEMPLATE = "base_deps2_{os}_{config}_{version}.tar.xz"
+def get_archive_name():
+    ARCHIVE_NAME_TEMPLATE = "base_deps_{os}_{config}_{version}.tar.gz"
 
     if COMPILE_CONFIG == "flatpak":
-    base_dep_archive_name = "base_deps2_flatpak.tar.xz"
+        return "base_deps_flatpak.tar.gz"
-else:
+
-    base_dep_archive_name = ARCHIVE_NAME_TEMPLATE.format(
+    return ARCHIVE_NAME_TEMPLATE.format(
         os=OS_NAME,
         config=COMPILE_CONFIG,
         version=base_deps_meta_version,
     )
 
+
+def main():
+    base_dep_archive_name = get_archive_name()
     print_message("Getting archive {}", base_dep_archive_name)
     try:
         local_filename = download_base_archive(base_dep_archive_name)
@@ -58,3 +63,10 @@ except URLError:
             archive_file = make_deps_archive(name=base_dep_archive_name, full=True)
             upload(archive_file, "ci@tmp.kiwix.org:30022", "/data/tmp/ci")
             os.remove(str(archive_file))
+
+
+if __name__ == "__main__":
+    if MAKE_RELEASE:
+        print_message("We are building release. Don't download deps archive.")
+    else:
+        main()

.github/scripts/upload_failure_logs.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+
+import tarfile
+from pathlib import Path
+from common import upload, OS_NAME, COMPILE_CONFIG, HOME
+
+ARCHIVE_NAME = Path(f"fail_log_{OS_NAME}_{COMPILE_CONFIG}.tar.gz")
+
+
+files_to_archive = []
+files_to_archive += HOME.glob("BUILD_*")
+files_to_archive += [HOME / "SOURCE", HOME / "LOGS", HOME / "TOOLCHAINS"]
+
+with tarfile.open(ARCHIVE_NAME, "w:xz") as tar:
+    for name in set(files_to_archive):
+        tar.add(str(name))
+
+upload(ARCHIVE_NAME, "ci@tmp.kiwix.org:30022", "/data/tmp/ci")

.github/scripts/upload_failure_logs.sh

@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-cd $HOME
-
-ARCHIVE_NAME=fail_log_${OS_NAME}_${COMPILE_CONFIG}.tar.gz
-tar -czf ${ARCHIVE_NAME} $HOME/BUILD_* $HOME/SOURCE $HOME/LOGS $HOME/TOOLCHAINS
-
-echo "Uploading archive $ARCHIVE_NAME"
-
-scp -c aes128-ctr -P 30022 -p -i ${SSH_KEY} \
-  -o PasswordAuthentication=no \
-  -o StrictHostKeyChecking=no \
-  $ARCHIVE_NAME \
-  ci@tmp.kiwix.org:/data/tmp/ci

.github/stale.yml

@@ -1,15 +0,0 @@
-daysUntilClose: false
-staleLabel: stale
-
-issues:
-  daysUntilStale: 60
-  markComment: >
-    This issue has been automatically marked as stale because it has not had
-    recent activity. It will be now be reviewed manually. Thank you
-    for your contributions.
-pulls:
-   daysUntilStale: 7
-   markComment: >
-     This pull request has been automatically marked as stale because it has not had
-     recent activity. It will be now be reviewed manually. Thank you
-     for your contributions.

.github/workflows/cd.yml

@@ -1,13 +1,109 @@
-name: Release&Nigthly
+name: CD
 
 on:
  push:
    tags:
      - r_[0-9]+
  schedule:
-   - cron: '0 1 * * *'
+   - cron: '0 3 * * *'
 
 jobs:
+  Windows:
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - native_mixed
+          - native_dyn
+          - native_static
+    runs-on: windows-2022
+    env:
+      OS_NAME: windows
+      COMPILE_CONFIG: ${{matrix.config}}
+      HOME: 'C:\\Users\\runneradmin'
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+    - name: Setup python 3.12
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+    - name: Install packages
+      run: |
+        choco.exe install pkgconfiglite ninja
+    - name: Install python modules
+      shell: bash
+      run: |
+        pip3 install meson pytest requests distro paramiko
+        pip3 install --no-deps $GITHUB_WORKSPACE
+    - name: Install QT
+      if: ${{ matrix.config == 'native_mixed' }}
+      uses: jurplel/install-qt-action@v4
+      with:
+        version: 5.15.2
+        modules: "qtwebengine"
+        setup-python: false
+      env:
+        AQT_CONFIG: ${{ github.workspace }}/.github/configs/aqt.ini
+    - name: Setup MSVC compiler
+      uses: bus1/cabuild/action/msdevshell@v1
+      with:
+        architecture: x64
+    - name: secret
+      shell: bash
+      run: |
+        echo "${{secrets.ssh_key}}" > $SSH_KEY
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+    - name: Install and configure eSigner CKA and Windows SDK
+      if: github.event_name == 'push'
+      env:
+        ESIGNER_URL: https://github.com/SSLcom/eSignerCKA/releases/download/v1.0.7/SSL.COM-eSigner-CKA_1.0.7.zip
+      run: |
+          Set-StrictMode -Version 'Latest'
+
+          # Download and Unzip eSignerCKA Setup
+          Invoke-WebRequest -OutFile eSigner_CKA_Setup.zip "$env:ESIGNER_URL"
+          Expand-Archive -Force eSigner_CKA_Setup.zip
+          Remove-Item eSigner_CKA_Setup.zip
+          Move-Item -Destination “eSigner_CKA_Installer.exe” -Path “eSigner_CKA_*\*.exe”
+
+          # Install eSignerCKA
+          New-Item -ItemType Directory -Force -Path "C:\esigner"
+          ./eSigner_CKA_Installer.exe /CURRENTUSER /VERYSILENT /SUPPRESSMSGBOXES /DIR=”C:\esigner” /TYPE=automatic | Out-Null
+          Remove-Item "eSigner_CKA_Installer.exe"
+
+          # Configure the CKA with SSL.com credentials
+          C:\esigner\eSignerCKATool.exe config -mode product -user "${{ secrets.ESIGNER_USERNAME }}" -pass "${{ secrets.ESIGNER_PASSWORD }}" -totp "${{ secrets.ESIGNER_TOTP_SECRET }}" -key "C:\esigner\master.key" -r
+          C:\esigner\eSignerCKATool.exe unload
+          C:\esigner\eSignerCKATool.exe load
+
+          # Find certificate
+          $CodeSigningCert = Get-ChildItem Cert:\CurrentUser\My -CodeSigningCert | Select-Object -First 1
+          echo Certificate: $CodeSigningCert
+
+          # Extract thumbprint and subject name
+          $Thumbprint = $CodeSigningCert.Thumbprint
+          echo "SIGNTOOL_THUMBPRINT=$Thumbprint" >> $env:GITHUB_ENV
+
+    - name: Ensure base deps
+      run: |
+        python .github\\scripts\\ensure_base_deps.py
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+    - name: Build Release
+      run: |
+        python .github\\scripts\\build_release_nightly.py
+      env:
+        SIGNTOOL_PATH: "C:/Program Files (x86)/Windows Kits/10/bin/10.0.22621.0/x86/signtool.exe"
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+    - name: Upload failure logs
+      if: failure()
+      run: |
+        python .github\\scripts\\upload_failure_logs.py
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+
   Linux:
     strategy:
       fail-fast: false
@@ -35,17 +131,17 @@ jobs:
         image_variant: ['focal']
         include:
           - config: native_mixed
-            image_variant: bionic
+            image_variant: manylinux
           - config: aarch64_mixed
-            image_variant: bionic
+            image_variant: manylinux
-          - config: win32_static
+          - config: native_dyn
-            image_variant: f35
+            image_variant: jammy
     env:
       HOME: /home/runner
       SSH_KEY: /tmp/id_rsa
     runs-on: ubuntu-22.04
     container:
-      image: "ghcr.io/kiwix/kiwix-build_ci_${{matrix.image_variant}}:2023-10-30"
+      image: "ghcr.io/kiwix/kiwix-build_ci_${{matrix.image_variant}}:2024-11-30"
       options: "--device /dev/fuse --privileged"
     steps:
     - name: Checkout code
@@ -77,11 +173,9 @@ jobs:
         kiwix-build/.github/scripts/build_release_nightly.py
       env:
         COMPILE_CONFIG: ${{matrix.config}}
-        BINTRAY_USER: kiwix
-        BINTRAY_PASS: ${{secrets.bintray_pass}}
     - name: Upload failure logs
       if: failure()
-      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh
+      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py
       env:
         COMPILE_CONFIG: ${{matrix.config}}
 
@@ -126,7 +220,7 @@ jobs:
         kiwix-build/.github/scripts/build_release_nightly.py
     - name: Upload failure logs
       if: failure()
-      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh
+      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py
 
   Macos:
     strategy:
@@ -153,14 +247,14 @@ jobs:
       # https://github.com/actions/runner-images/blob/main/images/macos/macos-13-Readme.md#xcode
       run: sudo xcode-select -s /Applications/Xcode_15.0.1.app
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
-    - name: Setup python 3.8
+    - name: Setup python 3.10
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v5
       with:
-        python-version: '3.8'
+        python-version: '3.10'
     - name: Install packages
       run:
-        brew install pkg-config ninja automake autoconf
+        brew install automake # ninja
     - name: Install python modules
       run: |
         pip3 install meson pytest requests distro
@@ -205,7 +299,7 @@ jobs:
         COMPILE_CONFIG: ${{matrix.config}}
     - name: Upload failure logs
       if: failure()
-      run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.sh
+      run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.py
       env:
         COMPILE_CONFIG: ${{matrix.config}}
 
@@ -215,10 +309,10 @@ jobs:
     runs-on: ubuntu-22.04
     env:
       COMPILE_CONFIG: native_static
-      OS_NAME: linux
+      OS_NAME: focal
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     - name: Install python modules
       shell: bash
       run: |

.github/workflows/ci.yml

@@ -6,6 +6,75 @@ on:
    - cron: '0 1 * * *'
 
 jobs:
+  Windows:
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - native_mixed
+          - native_dyn
+          - native_static
+    runs-on: windows-2022
+    env:
+      OS_NAME: windows
+      COMPILE_CONFIG: ${{matrix.config}}
+      HOME: 'C:\\Users\\runneradmin'
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+    - name: Setup python 3.12
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+    - name: Install packages
+      run: |
+        choco.exe install pkgconfiglite ninja
+    - name: Install python modules
+      shell: bash
+      run: |
+        pip3 install meson pytest requests distro paramiko
+        pip3 install --no-deps $GITHUB_WORKSPACE
+    - name: Install QT
+      if: ${{ matrix.config == 'native_mixed' }}
+      uses: jurplel/install-qt-action@v4
+      with:
+        version: 6.4.3
+        modules: "qtwebengine qtwebchannel qtpositioning"
+        setup-python: false
+      env:
+        AQT_CONFIG: ${{ github.workspace }}/.github/configs/aqt.ini
+    - name: Setup MSVC compiler
+      uses: bus1/cabuild/action/msdevshell@v1
+      with:
+        architecture: x64
+    - name: secret
+      shell: bash
+      run: |
+        echo "${{secrets.ssh_key}}" > $SSH_KEY
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+    - name: Ensure base deps
+      run: |
+        python .github\\scripts\\ensure_base_deps.py
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+    - name: Compile all deps
+      run: |
+        python .github\\scripts\\compile_all_deps.py
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+    - name: Build projects
+      run: |
+        python .github\\scripts\\build_projects.py
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+    - name: Upload failure logs
+      if: failure()
+      run: |
+        python .github\\scripts\\upload_failure_logs.py
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+
   Linux:
     strategy:
       fail-fast: false
@@ -38,19 +107,18 @@ jobs:
         image_variant: ['focal']
         include:
           - config: native_mixed
-            image_variant: bionic
+            image_variant: manylinux
           - config: aarch64_mixed
-            image_variant: bionic
+            image_variant: manylinux
-          - config: win32_static
+          - config: native_dyn
-            image_variant: f35
+            image_variant: jammy
-          - config: win32_dyn
-            image_variant: f35
     env:
       HOME: /home/runner
       SSH_KEY: /tmp/id_rsa
+      OS_NAME: ${{matrix.image_variant}}
     runs-on: ubuntu-22.04
     container:
-      image: "ghcr.io/kiwix/kiwix-build_ci_${{matrix.image_variant}}:2023-10-30"
+      image: "ghcr.io/kiwix/kiwix-build_ci_${{matrix.image_variant}}:2024-11-30"
       options: "--device /dev/fuse --privileged"
     steps:
     - name: Checkout code
@@ -63,6 +131,10 @@ jobs:
         pip3 install --user --no-deps .
       env:
         REP: ${{github.repository}}
+    - name: Install paramiko
+      if: ${{matrix.image_variant != 'bionic' }}
+      shell: bash
+      run: pip3 install --user paramiko
     - name: secret
       shell: bash
       run: |
@@ -91,7 +163,7 @@ jobs:
         COMPILE_CONFIG: ${{matrix.config}}
     - name: Upload failure logs
       if: failure()
-      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh
+      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py
       env:
         COMPILE_CONFIG: ${{matrix.config}}
 
@@ -112,6 +184,7 @@ jobs:
         git clone https://github.com/${REP}
         cd ./${REP##*/}
         git checkout --force ${GITHUB_SHA}
+        pip3 install --user paramiko
         pip3 install --user --no-deps .
       env:
         REP: ${{github.repository}}
@@ -136,7 +209,7 @@ jobs:
         kiwix-build/.github/scripts/build_projects.py
     - name: Upload failure logs
       if: failure()
-      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh
+      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py
 
   Macos:
     strategy:
@@ -162,17 +235,17 @@ jobs:
       # https://github.com/actions/runner-images/blob/main/images/macos/macos-13-Readme.md#xcode
       run: sudo xcode-select -s /Applications/Xcode_15.0.1.app
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
-    - name: Setup python 3.8
+    - name: Setup Python 3.10
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v5
       with:
-        python-version: '3.8'
+        python-version: '3.10'
     - name: Install packages
       run:
-        brew install pkg-config ninja automake autoconf
+        brew install automake # ninja
     - name: Install python modules
       run: |
-        pip3 install meson pytest requests distro
+        pip3 install meson pytest requests distro paramiko
         pip3 install --no-deps $GITHUB_WORKSPACE
     - name: secret
       shell: bash
@@ -202,6 +275,6 @@ jobs:
         COMPILE_CONFIG: ${{matrix.config}}
     - name: Upload failure logs
       if: failure()
-      run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.sh
+      run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.py
       env:
         COMPILE_CONFIG: ${{matrix.config}}

README.md

@@ -12,7 +12,7 @@ Kiwix Build audience is:
 * Kiwix developer team for its own CI/CD
 
 [![CI Build Status](https://github.com/kiwix/kiwix-build/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/kiwix/kiwix-build/actions/workflows/ci.yml?query=branch%3Amain)
-[![CD Build Status](https://github.com/kiwix/kiwix-build/actions/workflows/releaseNigthly.yml/badge.svg?branch=main)](https://github.com/kiwix/kiwix-build/actions/workflows/releaseNigthly.yml?query=branch%3Amain)
+[![CD Build Status](https://github.com/kiwix/kiwix-build/actions/workflows/cd.yml/badge.svg?branch=main)](https://github.com/kiwix/kiwix-build/actions/workflows/cd.yml?query=branch%3Amain)
 [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
 
 Prerequisites
@@ -89,8 +89,6 @@ platforms:
 - native_dyn
 - native_mixed
 - native_static
-- win32_dyn
-- win32_static
 - android
 - android_arm
 - android_arm64
@@ -101,11 +99,6 @@ platforms:
 All `native_*` config means using the native compiler without any cross-compilation option.
 Other may simply use cross-compilation or may download a specific toolchain to use.
 
-If you want to compile `kiwix-tools` for win32 using static linkage:
-```bash
-kiwix-build --config win32_dyn
-```
-
 Android
 -------
 

actions/dl_deps_archive/.gitignore

@@ -0,0 +1 @@
+node_modules

actions/dl_deps_archive/dist/index.js

@@ -30484,7 +30484,6 @@ function addLocalPath(inputPath) {
 async function run() {
   try {
     const base_url = core.getInput("base_url");
-    const os_name = getInput("os_name", process.env["OS_NAME"]);
     const target = core.getInput("target_platform");
     const project = getInput(
       "project",
@@ -30501,17 +30500,17 @@ async function run() {
 
     let archivePath;
     try {
-      const archive_url = `${base_url}/dev_preview/${branch}/deps2_${os_name}_${target}_${project}.tar.xz`;
+      const archive_url = `${base_url}/dev_preview/${branch}/deps_${target}_${project}.tar.gz`;
       process.stdout.write("Downloading " + archive_url + "\n");
       archivePath = await tc.downloadTool(archive_url);
     } catch (error) {
-      const archive_url = `${base_url}/deps2_${os_name}_${target}_${project}.tar.xz`;
+      const archive_url = `${base_url}/deps_${target}_${project}.tar.gz`;
       process.stdout.write("Downloading " + archive_url + "\n");
       archivePath = await tc.downloadTool(archive_url);
     }
 
     process.stdout.write("Extracting " + archivePath + " to " + extract_dir);
-    const archive_dir = await tc.extractTar(archivePath, extract_dir, "x");
+    const archive_dir = await tc.extractTar(archivePath, extract_dir);
     process.stdout.write("Extracted to " + archive_dir);
   } catch (error) {
     core.setFailed(error.message);

actions/dl_deps_archive/index.js

@@ -18,7 +18,6 @@ function addLocalPath(inputPath) {
 async function run() {
   try {
     const base_url = core.getInput("base_url");
-    const os_name = getInput("os_name", process.env["OS_NAME"]);
     const target = core.getInput("target_platform");
     const project = getInput(
       "project",
@@ -35,17 +34,17 @@ async function run() {
 
     let archivePath;
     try {
-      const archive_url = `${base_url}/dev_preview/${branch}/deps2_${os_name}_${target}_${project}.tar.xz`;
+      const archive_url = `${base_url}/dev_preview/${branch}/deps_${target}_${project}.tar.gz`;
       process.stdout.write("Downloading " + archive_url + "\n");
       archivePath = await tc.downloadTool(archive_url);
     } catch (error) {
-      const archive_url = `${base_url}/deps2_${os_name}_${target}_${project}.tar.xz`;
+      const archive_url = `${base_url}/deps_${target}_${project}.tar.gz`;
       process.stdout.write("Downloading " + archive_url + "\n");
       archivePath = await tc.downloadTool(archive_url);
     }
 
     process.stdout.write("Extracting " + archivePath + " to " + extract_dir);
-    const archive_dir = await tc.extractTar(archivePath, extract_dir, "x");
+    const archive_dir = await tc.extractTar(archivePath, extract_dir);
     process.stdout.write("Extracted to " + archive_dir);
   } catch (error) {
     core.setFailed(error.message);

actions/dl_deps_archive/package-lock.json

@@ -114,9 +114,9 @@
       }
     },
     "node_modules/undici": {
-      "version": "5.28.2",
+      "version": "5.28.4",
-      "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.2.tgz",
+      "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.4.tgz",
-      "integrity": "sha512-wh1pHJHnUeQV5Xa8/kyQhO7WFa8M34l026L5P/+2TYiakvGy5Rdc8jWZVyG7ieht/0WgJLEd3kcU5gKx+6GC8w==",
+      "integrity": "sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g==",
       "dependencies": {
         "@fastify/busboy": "^2.0.0"
       },

appveyor.yml

@@ -1,69 +0,0 @@
-image: Visual Studio 2019
-version: '0.1.{build}'
-environment:
-  EXTRA_DIR: 'C:\extra'
-  MINGW64_EXTRA_DIR: '/c/extra'
-  PATH: 'C:\\Python312\\Scripts;%PATH%;%EXTRA_DIR%\bin;C:\\Program Files (x86)\\Windows Kits\\10\\bin\\%platform%'
-  PKG_CONFIG_PATH: '%EXTRA_DIR%\lib\pkgconfig'
-  MSYS2_PATH_TYPE: 'inherit'
-  MSYS2_ARG_CONV_EXCL: '-Tp'
-  MINGW64_RUN: 'C:\msys64\usr\bin\bash.exe --login -c'
-  MSYSTEM: 'MINGW64'
-  KIWIX_DESKTOP_VERSION: '2.3.1-3'
-  KIWIX_DESKTOP_RELEASE: 0
-
-
-init:
-  - cmd: |
-     echo %TEST_VAR%
-     REM Set VC 2019
-     "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars64.bat"
-     mkdir C:\extra\bin
-
-install:
-  - cmd: |
-     cd C:\projects
-     REM Install meson
-     C:\Python312\Scripts\pip install meson
-
-     REM Install ninja
-     curl -o ninja-win.zip -L https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-win.zip
-     7z e ninja-win.zip -o%EXTRA_DIR%\bin
-
-     REM Install pkg-config-lite
-     curl --insecure -o pkg-config-lite-0.28-1.zip https://netix.dl.sourceforge.net/project/pkgconfiglite/0.28-1/pkg-config-lite-0.28-1_bin-win32.zip
-     7z e pkg-config-lite-0.28-1.zip -o%EXTRA_DIR%\bin pkg-config-lite-0.28-1/bin/pkg-config.exe
-     cd kiwix-build
-
-     REM Install openssh (scp) for deployement
-     choco install openssh openssl.light
-
-  - cmd: openssl enc -d -aes-256-cbc -md sha256 -pass pass:%ENC_PASSWD% -in appveyor\nightlybot_id_key.enc -out appveyor\nightlybot_id_key
-
-
-
-build_script:
-  - IF "%USE_CACHE%" EQU "1" appveyor\setup_from_cache.cmd
-  - IF "%USE_CACHE%" NEQ "1" appveyor\install_zlib.cmd
-  - IF "%USE_CACHE%" NEQ "1" appveyor\install_lzma.cmd
-  - IF "%USE_CACHE%" NEQ "1" appveyor\install_zstd.cmd
-  - IF "%USE_CACHE%" NEQ "1" appveyor\install_libcurl.cmd
-  - IF "%USE_CACHE%" NEQ "1" appveyor\install_pthread.cmd
-  - IF "%USE_CACHE%" NEQ "1" appveyor\install_dirent.cmd
-  - IF "%USE_CACHE%" NEQ "1" appveyor\install_xapian.cmd
-  - IF "%USE_CACHE%" NEQ "1" appveyor\install_icu.cmd
-  - IF "%USE_CACHE%" NEQ "1" appveyor\install_pugixml.cmd
-  - IF "%USE_CACHE%" NEQ "1" appveyor\install_aria2.cmd
-  - IF "%USE_CACHE%" NEQ "1" appveyor\install_mustache.cmd
-  - IF "%USE_CACHE%" NEQ "1" appveyor\install_libmicrohttpd.cmd
-  - IF "%USE_CACHE%" NEQ "1" %MINGW64_RUN% "cd /c/projects/kiwix-build && appveyor/create_cache.sh"
-  - appveyor\install_libzim.cmd
-  - appveyor\install_libkiwix.cmd
-  - appveyor\install_kiwix-desktop.cmd
-
-before_deploy:
-  - cmd: openssl enc -d -pbkdf2 -aes-256-cbc -md sha256 -pass pass:%win_certificate_secret% -in appveyor\kiwix.pfx.enc -out appveyor\kiwix.pfx
-  - cmd: '%MINGW64_RUN% "cd /c/projects/kiwix-build && appveyor/package_kiwix-desktop.sh"'
-
-deploy_script:
-  - cmd: '%MINGW64_RUN% "cd /c/projects/kiwix-build && appveyor/deploy.sh"'

appveyor/build_xapian.sh

@@ -1,16 +0,0 @@
-
-XAPIAN_SOURCE=$(pwd)/..
-
-$XAPIAN_SOURCE/configure \
-  CC="cl -nologo" \
-  CXX="$XAPIAN_SOURCE/compile cl -nologo" \
-  CXXFLAGS="-EHsc -MD" AR=lib \
-  CPPFLAGS="-I${MINGW64_EXTRA_DIR}/include" \
-  LDFLAGS="-L${MINGW64_EXTRA_DIR}/lib" \
-  --disable-backend-remote \
-  --disable-documentation \
-  --prefix=${MINGW64_EXTRA_DIR}
-
-make -j2
-
-make install

appveyor/create_cache.sh

@@ -1,11 +0,0 @@
-
-set -e
-
-ARCHIVE_NAME="deps_windows_windows.zip"
-SSH_KEY=C:\\projects\\kiwix-build\\appveyor\\nightlybot_id_key
-
-7z a -tzip $ARCHIVE_NAME $MINGW64_EXTRA_DIR
-scp -P 30022 -vrp -i ${SSH_KEY} -o StrictHostKeyChecking=no \
-  ${ARCHIVE_NAME} \
-  ci@tmp.kiwix.org:/data/tmp/ci
-

appveyor/deploy.sh

@@ -1,21 +0,0 @@
-set -e
-
-NIGHTLY_DATE=$(date +%Y-%m-%d)
-NIGHTLY_KIWIX_ARCHIVES_DIR=/c/projects/NIGHTLY_KIWIX_ARCHIVES/${NIGHTLY_DATE}
-RELEASE_KIWIX_ARCHIVES_DIR=/c/projects/RELEASE_KIWIX_ARCHIVES
-SSH_KEY=C:\\projects\\kiwix-build\\appveyor\\nightlybot_id_key
-
-if [[ "$APPVEYOR_SCHEDULED_BUILD" = "True" ]]
-then
-  scp -P 30022 -vrp -i ${SSH_KEY} -o StrictHostKeyChecking=no \
-    ${NIGHTLY_KIWIX_ARCHIVES_DIR} \
-    ci@master.download.kiwix.org:/data/download/nightly
-fi
-
-if [[ "$APPVEYOR_REPO_TAG" = "true" ]]
-then
-  RELEASE_ARCHIVES=$(find $RELEASE_KIWIX_ARCHIVES_DIR -type f)
-  scp -P 30022 -vrp -i ${SSH_KEY} -o StrictHostKeyChecking=no \
-      ${RELEASE_ARCHIVES} \
-      ci@master.download.kiwix.org:/data/download/release/kiwix-desktop
-fi

appveyor/fix_desktop_makefile.py

@@ -1,29 +0,0 @@
-import re, sys, os
-
-makefile = sys.argv[-1]
-print("Patching '{}'".format(makefile))
-
-with open(makefile, 'r') as f:
-    lines = f.readlines()
-
-with open(makefile, 'w') as f:
-    for line in lines:
-#        if "/SUBSYSTEM:WINDOWS" in line:
-#            line = line.replace("/SUBSYSTEM:WINDOWS", "/SUBSYSTEM:CONSOLE")
-#            f.write(line)
-#            continue
-        if not line.startswith('LIBS  '):
-            f.write(line)
-            continue
-        print("-- INPUT : {}".format(line))
-        for lib in ('kiwix', 'zim', 'pugixml', 'z', 'zstd'):
-            line = line.replace('{}.lib'.format(lib), 'lib{}.a'.format(lib))
-        for lib in ('lzma', 'curl'):
-            line = line.replace('lib{}.lib'.format(lib), 'lib{}.a'.format(lib))
-        line = line.strip()
-#        line += " pthreadVC2.lib"
-#        line += " icuin.lib icudt.lib icuuc.lib"
-        line += " Rpcrt4.lib Ws2_32.lib winmm.lib Shlwapi.lib"
-        line += os.linesep
-        print("++ OUTPUT : {}".format(line))
-        f.write(line)

appveyor/install_aria2.cmd

@@ -1,4 +0,0 @@
-REM ========================================================
-REM Install aria2
-curl -fsSL -O https://github.com/aria2/aria2/releases/download/release-1.36.0/aria2-1.36.0-win-64bit-build1.zip || exit /b 1
-7z e aria2-1.36.0-win-64bit-build1.zip -o%EXTRA_DIR% aria2-1.36.0-win-64bit-build1\aria2c.exe || exit /b 1

appveyor/install_dirent.cmd

@@ -1,3 +0,0 @@
-REM ========================================================
-REM Install dirent.h
-curl -fsSL -o%EXTRA_DIR%\include\dirent.h https://raw.githubusercontent.com/tronkko/dirent/master/include/dirent.h || exit /b 1

appveyor/install_git_ninja_pkgconfig.cmd

@@ -1,14 +0,0 @@
-REM Set VC 2017
-"C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat"
-mkdir C:\extra\bin
-cd C:\projects
-C:\Python36\Scripts\pip install meson || exit /b 1
-
-REM Set ninja
-curl -fsSL -o ninja-win.zip https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-win.zip || exit /b 1
-7z e ninja-win.zip -o%EXTRA_DIR%\bin || exit /b 1
-
-REM Set pkg-config-lit
-curl --insecure -fsSL -o pkg-config-lite-0.28-1.zip https://netix.dl.sourceforge.net/project/pkgconfiglite/0.28-1/pkg-config-lite-0.28-1_bin-win32.zip || exit /b 1
-7z e pkg-config-lite-0.28-1.zip -o%EXTRA_DIR%\bin pkg-config-lite-0.28-1/bin/pkg-config.exe || exit /b 1
-cd kiwix-build

appveyor/install_icu.cmd

@@ -1,7 +0,0 @@
-REM ========================================================
-REM Install icu
-curl -SL -O http://public.kymeria.fr/KIWIX/windows/icu4c-62_1-Win64-MSVC2017.zip || exit /b 1
-7z x icu4c-62_1-Win64-MSVC2017.zip -o%EXTRA_DIR% -r include || exit /b 1
-7z e icu4c-62_1-Win64-MSVC2017.zip -o%EXTRA_DIR%\lib lib64\* || exit /b 1
-7z e icu4c-62_1-Win64-MSVC2017.zip -o%EXTRA_DIR%\bin bin64\*.dll || exit /b 1
-curl -fsSL -o%PKG_CONFIG_PATH%\icu-i18n.pc http://public.kymeria.fr/KIWIX/windows/icu-i18n.pc || exit /b 1

appveyor/install_kiwix-desktop.cmd

@@ -1,22 +0,0 @@
-REM ========================================================
-REM Install kiwix-desktop
-git clone https://github.com/kiwix/kiwix-desktop || exit /b 1
-cd kiwix-desktop
-echo "Running qmake"
-SET _WITH_CONSOLE=1
-IF %KIWIX_DESKTOP_RELEASE% EQU 1 (
-  IF %APPVEYOR_REPO_TAG% == true (
-    SET _WITH_CONSOLE=0
-  )
-)
-IF %_WITH_CONSOLE% EQU 1 (
-    C:\Qt\5.15\msvc2019_64\bin\qmake.exe "CONFIG+=static console" || exit /b 1
-) else (
-    C:\Qt\5.15\msvc2019_64\bin\qmake.exe "CONFIG+=static" || exit /b 1
-)
-
-echo "Running fix_desktop"
-C:\Python312\Python ..\appveyor\fix_desktop_makefile.py Makefile.Release || exit /b 1
-echo "Running nmake"
-nmake || exit /b 1
-cd ..

appveyor/install_libcurl.cmd

@@ -1,14 +0,0 @@
-REM ========================================================
-REM Install libcurl
-curl -fsSL -O https://curl.haxx.se/download/curl-7.61.1.zip || exit /b 1
-7z x curl-7.61.1.zip || exit /b 1
-cd curl-7.61.1\winbuild
-nmake /f Makefile.vc mode=static MACHINE=x64 DEBUG=no VC=15 ENABLE_IDN=no || exit /b 1
-mkdir %EXTRA_DIR%\include\curl
-copy ..\builds\libcurl-vc15-x64-release-static-ipv6-sspi-winssl\include\curl\*.h %EXTRA_DIR%\include\curl
-copy ..\builds\libcurl-vc15-x64-release-static-ipv6-sspi-winssl\lib\libcurl_a.lib %EXTRA_DIR%\lib
-move %EXTRA_DIR%\lib\libcurl_a.lib %EXTRA_DIR%\lib\libcurl.a
-dir %EXTRA_DIR%\include\curl
-dir %EXTRA_DIR%\lib
-curl -fsSl -o%PKG_CONFIG_PATH%\libcurl.pc http://public.kymeria.fr/KIWIX/windows/libcurl.pc || exit /b 1
-cd ..\..

appveyor/install_libkiwix.cmd

@@ -1,10 +0,0 @@
-REM ========================================================
-REM Install libkiwix
-git clone https://github.com/kiwix/libkiwix.git || exit /b 1
-cd libkiwix
-set CPPFLAGS="-I%EXTRA_DIR%/include"
-meson . build --prefix %EXTRA_DIR% --default-library static --buildtype release || exit /b 1
-cd build
-ninja || exit /b 1
-ninja install || exit /b 1
-cd ..\..

appveyor/install_libmicrohttpd.cmd

@@ -1,11 +0,0 @@
-REM ========================================================
-REM Install libmicrohttpd
-curl -fsSL -O https://ftpmirror.gnu.org/libmicrohttpd/libmicrohttpd-latest-w32-bin.zip || exit /b 1
-7z e libmicrohttpd-latest-w32-bin.zip -o%EXTRA_DIR%/include libmicrohttpd-*-w32-bin/x86_64/VS2019/Release-static/microhttpd.h || exit /b 1
-7z e libmicrohttpd-latest-w32-bin.zip -o%EXTRA_DIR%/lib libmicrohttpd-*-w32-bin/x86_64/VS2019/Release-static/libmicrohttpd.lib || exit /b 1
-7z e libmicrohttpd-latest-w32-bin.zip -o%EXTRA_DIR%/lib/pkgconfig libmicrohttpd-*-w32-bin/x86_64/MinGW/static/mingw64/lib/pkgconfig/* || exit /b 1
-
-rename %EXTRA_DIR%\lib\libmicrohttpd.lib microhttpd.lib
-dir %EXTRA_DIR%
-dir %EXTRA_DIR%\lib
-dir %EXTRA_DIR%\lib\pkgconfig

appveyor/install_libzim.cmd

@@ -1,9 +0,0 @@
-REM ========================================================
-REM Install libzim
-git clone https://github.com/openzim/libzim.git || exit /b 1
-cd libzim
-meson . build --prefix %EXTRA_DIR% --default-library static --buildtype release || exit /b 1
-cd build
-ninja || exit /b 1
-ninja install || exit /b 1
-cd ..\..

appveyor/install_lzma.cmd

@@ -1,6 +0,0 @@
-REM ========================================================
-REM Install lzma
-curl -fsSL -O https://tukaani.org/xz/xz-5.2.4-windows.zip || exit /b 1
-7z x xz-5.2.4-windows.zip -o%EXTRA_DIR% -r include || exit /b 1
-7z e xz-5.2.4-windows.zip -o%EXTRA_DIR%\lib bin_x86-64\liblzma.a || exit /b 1
-curl -fsSL -o%PKG_CONFIG_PATH%\liblzma.pc http://public.kymeria.fr/KIWIX/windows/liblzma.pc || exit /b 1

appveyor/install_mustache.cmd

@@ -1,3 +0,0 @@
-REM ========================================================
-REM Install mustache.hpp
-curl -fsSL -o%EXTRA_DIR%\include\mustache.hpp https://raw.githubusercontent.com/kainjow/mustache/v3.2.1/mustache.hpp || exit /b 1

appveyor/install_pthread.cmd

@@ -1,14 +0,0 @@
-REM ========================================================
-REM Install pthread
-curl -fsSL -O ftp://sourceware.org/pub/pthreads-win32/pthreads-w32-2-9-1-release.zip || exit /b 1
-7z x pthreads-w32-2-9-1-release.zip -r pthreads.2 || exit /b 1
-cd pthreads.2
-REM Patch is pthread_timespec.patch
-curl -fsSL -O http://public.kymeria.fr/KIWIX/windows/pthread.h || exit /b 1
-nmake clean VC-inlined || exit /b 1
-copy pthread.h %EXTRA_DIR%\include
-copy sched.h %EXTRA_DIR%\include
-copy pthreadVC2.lib %EXTRA_DIR%\lib
-copy pthreadVC2.dll %EXTRA_DIR%\bin
-curl -fsSL -o%PKG_CONFIG_PATH%\libpthreadVC2.pc http://public.kymeria.fr/KIWIX/windows/libpthreadVC2.pc || exit /b 1
-cd ..

appveyor/install_pugixml.cmd

@@ -1,10 +0,0 @@
-REM ========================================================
-REM Install pugixml
-curl -fsSL -O http://public.kymeria.fr/KIWIX/windows/pugixml-1.2-meson.zip || exit /b 1
-7z x pugixml-1.2-meson.zip -o. || exit /b 1
-cd pugixml-1.2-meson
-meson . build --prefix %EXTRA_DIR% --default-library static --buildtype release || exit /b 1
-cd build
-ninja || exit /b 1
-ninja install || exit /b 1
-cd ..\..

appveyor/install_xapian.cmd

@@ -1,9 +0,0 @@
-REM ========================================================
-REM Install xapian
-curl -fsSL -O http://mirror.download.kiwix.org/dev/kiwix-build/xapian-core-1.4.18.zip || exit /b 1
-7z x xapian-core-1.4.18.zip || exit /b 1
-cd xapian-core-1.4.18
-mkdir build
-cd build
-%MINGW64_RUN% "cd /c/Projects/kiwix-build/xapian-core-1.4.18/build && /c/Projects/kiwix-build/appveyor/build_xapian.sh" > build_xapian.log || exit /b 1
-cd ..\..

appveyor/install_zlib.cmd

@@ -1,10 +0,0 @@
-REM ========================================================
-REM Install zlib
-curl -fsSL -O http://mirror.download.kiwix.org/dev/kiwix-build/zlib-1.2.12.meson.zip || exit /b 1
-7z x zlib-1.2.12.meson.zip || exit /b 1
-cd zlib-1.2.12
-meson . build --prefix %EXTRA_DIR% --default-library static --buildtype release || exit /b 1
-cd build
-ninja || exit /b 1
-ninja install || exit /b 1
-cd ..\..

appveyor/install_zstd.cmd

@@ -1,10 +0,0 @@
-REM ========================================================
-REM Install zstd
-curl -fsSL -o zstd-v1.5.2.zip https://github.com/facebook/zstd/archive/refs/tags/v1.5.2.zip || exit /b 1
-7z x zstd-v1.5.2.zip || exit /b 1
-cd zstd-1.5.2/build/meson
-meson . builddir --prefix %EXTRA_DIR% --default-library static --buildtype release -Dbin_programs=false -Dbin_contrib=false || exit /b 1
-cd builddir
-ninja || exit /b 1
-ninja install || exit /b 1
-cd ..\..\..\..

appveyor/package_kiwix-desktop.sh

@@ -1,35 +0,0 @@
-
-set -e
-
-if [[ "$APPVEYOR_REPO_TAG" = "false" ]]
-then
-  NIGHTLY_DATE=$(date +%Y-%m-%d)
-  KIWIX_ARCHIVES_DIR=/c/projects/NIGHTLY_KIWIX_ARCHIVES/${NIGHTLY_DATE}
-  KIWIX_DIR=kiwix-desktop_windows_x64_$NIGHTLY_DATE
-else
-  if [ $KIWIX_DESKTOP_RELEASE  -eq 1 ]
-  then
-    KIWIX_ARCHIVES_DIR=/c/projects/RELEASE_KIWIX_ARCHIVES
-    KIWIX_DIR=kiwix-desktop_windows_x64_${KIWIX_DESKTOP_VERSION}
-  fi
-fi
-
-if [[ "$KIWIX_DIR" ]]
-then
-  KIWIX_ARCH_NAME=${KIWIX_DIR}.zip
-
-  mkdir $KIWIX_DIR
-  mkdir -p KIWIX_ARCHIVES_DIR
-
-  cp /c/projects/kiwix-build/kiwix-desktop/Release/kiwix-desktop.exe $KIWIX_DIR
-/c/Qt/5.15/msvc2019_64/bin/windeployqt --compiler-runtime $KIWIX_DIR
-
-  cp $MINGW64_EXTRA_DIR/aria2c.exe $KIWIX_DIR
-  cp $MINGW64_EXTRA_DIR/bin/*.dll $KIWIX_DIR
-  cp /c/OpenSSL-v111-Win64/bin/libcrypto-1_1-x64.dll $KIWIX_DIR
-  cp /c/OpenSSL-v111-Win64/bin/libssl-1_1-x64.dll $KIWIX_DIR
-
-  signtool.exe sign -f appveyor/kiwix.pfx -p $win_certificate_password -t http://timestamp.digicert.com -d "Kiwix-desktop application" -fd SHA256 $KIWIX_DIR/kiwix-desktop.exe
-
-  7z a -tzip $KIWIX_ARCHIVES_DIR/$KIWIX_ARCH_NAME $KIWIX_DIR
-fi

appveyor/setup_from_cache.cmd

@@ -1,4 +0,0 @@
-REM ========================================================
-REM Setup from cache
-curl -fsSL -O http://tmp.kiwix.org/ci/deps_windows_windows.zip || exit /b 1
-7z x deps_windows_windows.zip -oc: -aoa || exit /b 1

kiwixbuild/__init__.py

@@ -125,6 +125,21 @@ def parse_args():
             "to develop with the cloned sources."
         ),
     )
+    subgroup.add_argument(
+        "--use-target-arch-name",
+        action="store_true",
+        help=(
+            "Name the build directory using the arch name instead of the config name.\n"
+            "Different configs may create binary for the same arch so this option is "
+            "not recommended when working with several config on the same computer.\n"
+            "However, when generating dependencies for other it is better to have a "
+            "directory named using the target instead of the used config.\n"
+            "Intended to be used in CI only."
+        ),
+    )
+    subgroup.add_argument(
+        "--get-build-dir", action="store_true", help="Print the output directory."
+    )
     options = parser.parse_args()
 
     if not options.android_arch:
@@ -139,10 +154,13 @@ def main():
     options = parse_args()
     options.working_dir = os.path.abspath(options.working_dir)
     _global.set_options(options)
-    neutralEnv = buildenv.NeutralEnv()
+    neutralEnv = buildenv.NeutralEnv(options.get_build_dir)
     _global.set_neutralEnv(neutralEnv)
     if options.config == "flatpak":
         builder = FlatpakBuilder()
     else:
         builder = Builder()
+    if options.get_build_dir:
+        print(ConfigInfo.get_config(options.config).buildEnv.build_dir)
+    else:
         builder.run()

kiwixbuild/buildenv.py

@@ -8,7 +8,7 @@ from ._global import neutralEnv, option
 
 
 class NeutralEnv:
-    def __init__(self):
+    def __init__(self, dummy_run):
         self.working_dir = option("working_dir")
         self.source_dir = pj(self.working_dir, "SOURCE")
         self.archive_dir = pj(self.working_dir, "ARCHIVE")
@@ -17,6 +17,10 @@ class NeutralEnv:
         for d in (self.source_dir, self.archive_dir, self.toolchain_dir, self.log_dir):
             os.makedirs(d, exist_ok=True)
         self.detect_platform()
+        if dummy_run:
+            # If this is for a dummy run, we will not run anything.
+            # To check for command (and so, don't enforce their presence)
+            return
         self.ninja_command = self._detect_command(
             "ninja", default=[["ninja"], ["ninja-build"]]
         )
@@ -26,21 +30,18 @@ class NeutralEnv:
         self.mesontest_command = [*self.meson_command, "test"]
         self.patch_command = self._detect_command("patch")
         self.git_command = self._detect_command("git")
+        if platform.system() == "Windows":
+            self.make_command = self._detect_command("nmake", options=["/?", "/NOLOGO"])
+        else:
             self.make_command = self._detect_command("make")
         self.cmake_command = self._detect_command("cmake")
-        self.qmake_command = self._detect_command("qmake", required=False)
+        self.qmake_command = self._detect_command(
+            "qmake", required=False, default=[["qmake"], ["qmake-qt5"]]
+        )
 
     def detect_platform(self):
         _platform = platform.system()
         self.distname = _platform
-        if _platform == "Windows":
-            print(
-                "ERROR: kiwix-build is not intented to run on Windows platform.\n"
-                "It should probably not work, but well, you still can have a try."
-            )
-            cont = input("Do you want to continue ? [y/N]")
-            if cont.lower() != "y":
-                sys.exit(0)
         if _platform == "Linux":
             self.distname = distro.id()
             if self.distname == "ubuntu":
@@ -70,15 +71,18 @@ class NeutralEnv:
             if required:
                 sys.exit("ERROR: {} command not found".format(name))
             else:
-                print("WARNING: {} command not found".format(name))
+                print("WARNING: {} command not found".format(name), file=sys.stderr)
                 return ["{}_NOT_FOUND".format(name.upper())]
 
 
 class BuildEnv:
     def __init__(self, configInfo):
-        build_dir = "BUILD_{}".format(configInfo.name)
         self.configInfo = configInfo
         self.base_build_dir = pj(option("working_dir"), option("build_dir"))
+        build_dir = (
+            configInfo.arch_name if option("use_target_arch_name") else configInfo.name
+        )
+        build_dir = f"BUILD_{build_dir}"
         self.build_dir = pj(self.base_build_dir, build_dir)
         self.install_dir = pj(self.build_dir, "INSTALL")
         self.toolchain_dir = pj(self.build_dir, "TOOLCHAINS")
@@ -124,13 +128,12 @@ class BuildEnv:
     def get_env(self, *, cross_comp_flags, cross_compilers, cross_path):
         env = self.configInfo.get_env()
         pkgconfig_path = pj(self.install_dir, self.libprefix, "pkgconfig")
-        env["PKG_CONFIG_PATH"] = ":".join([env["PKG_CONFIG_PATH"], pkgconfig_path])
+        env["PKG_CONFIG_PATH"].append(pkgconfig_path)
 
-        env["PATH"] = ":".join([escape_path(pj(self.install_dir, "bin")), env["PATH"]])
+        env["PATH"].insert(0, pj(self.install_dir, "bin"))
 
-        env["LD_LIBRARY_PATH"] = ":".join(
+        env["LD_LIBRARY_PATH"].extend(
             [
-                env["LD_LIBRARY_PATH"],
                 pj(self.install_dir, "lib"),
                 pj(self.install_dir, self.libprefix),
             ]
@@ -162,7 +165,7 @@ class BuildEnv:
         if cross_compilers:
             self.configInfo.set_compiler(env)
         if cross_path:
-            env["PATH"] = ":".join(self.configInfo.get_bin_dir() + [env["PATH"]])
+            env["PATH"][0:0] = self.configInfo.get_bin_dir()
         return env
 
     @property

kiwixbuild/configs/__init__.py

@@ -1,3 +1,3 @@
 from .base import *
 
-from . import android, armhf, musl, flatpak, i586, ios, native, neutral, win32, wasm
+from . import android, armhf, musl, flatpak, i586, ios, native, neutral, wasm

kiwixbuild/configs/android.py

@@ -45,7 +45,11 @@ class AndroidConfigInfo(ConfigInfo):
         return self.ndk_builder.install_path
 
     def get_cross_config(self):
-        extra_libs = ["-llog"]
+        extra_libs = [
+            "-llog",
+            "-Wl,--exclude-libs,libgcc.a",
+            "-Wl,--exclude-libs,libunwind.a",
+        ]
         extra_cflags = [
             "-I{}".format(include_dir) for include_dir in self.get_include_dirs()
         ]
@@ -147,6 +151,10 @@ class Android(MetaConfigInfo):
     name = "android"
     compatible_hosts = ["fedora", "debian"]
 
+    @property
+    def arch_name(self):
+        return "multi-linux-android"
+
     @property
     def subConfigNames(self):
         return ["android_{}".format(arch) for arch in option("android_arch")]

kiwixbuild/configs/armhf.py

@@ -6,7 +6,7 @@ from kiwixbuild._global import get_target_step
 
 # Base config for arm
 class ArmConfigInfo(ConfigInfo):
-    compatible_hosts = ["fedora", "debian"]
+    compatible_hosts = ["fedora", "debian", "almalinux"]
 
     def get_cross_config(self):
         return {
@@ -76,18 +76,15 @@ class ArmConfigInfo(ConfigInfo):
 
     def get_env(self):
         env = super().get_env()
-        env["LD_LIBRARY_PATH"] = ":".join(
+        env["LD_LIBRARY_PATH"][0:0] = [
-            [
             pj(self.root_path, self.arch_full, "lib64"),
             pj(self.root_path, "lib"),
-                env["LD_LIBRARY_PATH"],
         ]
-        )
         env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig")
         env["QEMU_LD_PREFIX"] = pj(self.root_path, self.arch_full, "libc")
         env["QEMU_SET_ENV"] = "LD_LIBRARY_PATH={}".format(
             ":".join(
-                [pj(self.root_path, self.arch_full, "lib"), env["LD_LIBRARY_PATH"]]
+                [pj(self.root_path, self.arch_full, "lib"), str(env["LD_LIBRARY_PATH"])]
             )
         )
         return env

kiwixbuild/configs/base.py

@@ -27,6 +27,10 @@ class ConfigInfo(metaclass=_MetaConfig):
     mixed = False
     libdir = None
 
+    @property
+    def arch_name(self):
+        return self.arch_full
+
     @classmethod
     def get_config(cls, name, targets=None):
         if name not in cls.all_running_configs:
@@ -131,7 +135,6 @@ def MixedMixin(static_name):
         static = False
 
         def add_targets(self, targetName, targets):
-            print(targetName)
             if option("target") == targetName:
                 return super().add_targets(targetName, targets)
             else:
@@ -158,15 +161,13 @@ def MixedMixin(static_name):
 
         def get_env(self):
             env = super().get_env()
-            env["PATH"] = ":".join(
+            env["PATH"].insert(0, pj(self.static_buildEnv.install_dir, "bin"))
-                [pj(self.static_buildEnv.install_dir, "bin")] + [env["PATH"]]
-            )
             pkgconfig_path = pj(
                 self.static_buildEnv.install_dir,
                 self.static_buildEnv.libprefix,
                 "pkgconfig",
             )
-            env["PKG_CONFIG_PATH"] = ":".join([env["PKG_CONFIG_PATH"], pkgconfig_path])
+            env["PKG_CONFIG_PATH"].append(pkgconfig_path)
             env["CPPFLAGS"] = " ".join(
                 [
                    "-I" + pj(self.static_buildEnv.install_dir, "include"),

kiwixbuild/configs/flatpak.py

@@ -4,6 +4,7 @@ from kiwixbuild._global import option, neutralEnv
 
 class FlatpakConfigInfo(ConfigInfo):
     name = "flatpak"
+    arch_name = "flatpak"
     build = "flatpak"
     static = ""
     toolchain_names = ["org.kde", "io.qt.qtwebengine"]

kiwixbuild/configs/ios.py

@@ -24,6 +24,10 @@ class AppleConfigInfo(ConfigInfo):
         super().__init__(*args, **kwargs)
         self._root_path = None
 
+    @property
+    def arch_name(self):
+        return self.target
+
     @property
     def root_path(self):
         if self._root_path is None:
@@ -209,6 +213,10 @@ class IOS(MetaConfigInfo):
     name = "iOS_multi"
     compatible_hosts = ["Darwin"]
 
+    @property
+    def arch_name(self):
+        return self.name
+
     @property
     def subConfigNames(self):
         return ["iOS_{}".format(arch) for arch in option("ios_arch")]
@@ -225,6 +233,10 @@ class AppleStaticAll(MetaConfigInfo):
     name = "apple_all_static"
     compatible_hosts = ["Darwin"]
 
+    @property
+    def arch_name(self):
+        return self.name
+
     @property
     def subConfigNames(self):
         return AppleXCFramework.subConfigNames

kiwixbuild/configs/musl.py

@@ -73,27 +73,22 @@ class MuslConfigInfo(ConfigInfo):
 
     def get_env(self):
         env = super().get_env()
-        env["LD_LIBRARY_PATH"] = ":".join(
+        env["LD_LIBRARY_PATH"][0:0] = [
-            [
             pj(self.root_path, self.arch_full, "lib64"),
             pj(self.root_path, "lib"),
-                env["LD_LIBRARY_PATH"],
         ]
-        )
         env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig")
         env["QEMU_LD_PREFIX"] = pj(self.root_path, self.arch_full, "libc")
         env["QEMU_SET_ENV"] = "LD_LIBRARY_PATH={}".format(
             ":".join(
-                [pj(self.root_path, self.arch_full, "lib"), env["LD_LIBRARY_PATH"]]
+                [pj(self.root_path, self.arch_full, "lib"), str(env["LD_LIBRARY_PATH"])]
             )
         )
         return env
 
     def set_comp_flags(self, env):
         super().set_comp_flags(env)
-        env["LD_LIBRARY_PATH"] = ":".join(
+        env["LD_LIBRARY_PATH"].insert(0, pj(self.root_path, self.arch_full, "lib"))
-            [pj(self.root_path, self.arch_full, "lib"), env["LD_LIBRARY_PATH"]]
-        )
         env["CFLAGS"] = (
             " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
             + env["CFLAGS"]

kiwixbuild/configs/native.py

@@ -3,6 +3,9 @@ from .base import ConfigInfo, MixedMixin
 from kiwixbuild.utils import pj
 from kiwixbuild._global import option, neutralEnv
 from kiwixbuild.configs.ios import MIN_MACOS_VERSION
+import sysconfig
+import platform
+import sys
 
 
 class NativeConfigInfo(ConfigInfo):
@@ -16,20 +19,26 @@ class NativeConfigInfo(ConfigInfo):
             env["CFLAGS"] += f"-mmacosx-version-min={MIN_MACOS_VERSION}"
         return env
 
+    @property
+    def arch_name(self):
+        if sys.platform == "darwin":
+            return f"{platform.machine()}-apple-darwin"
+        return sysconfig.get_platform()
+
 
 class NativeDyn(NativeConfigInfo):
     name = "native_dyn"
     static = False
-    compatible_hosts = ["fedora", "debian", "Darwin"]
+    compatible_hosts = ["fedora", "debian", "Darwin", "almalinux", "Windows"]
 
 
 class NativeStatic(NativeConfigInfo):
     name = "native_static"
     static = True
-    compatible_hosts = ["fedora", "debian"]
+    compatible_hosts = ["fedora", "debian", "Darwin", "almalinux", "Windows"]
 
 
 class NativeMixed(MixedMixin("native_static"), NativeConfigInfo):
     name = "native_mixed"
     static = False
-    compatible_hosts = ["fedora", "debian", "Darwin"]
+    compatible_hosts = ["fedora", "debian", "Darwin", "almalinux", "Windows"]

kiwixbuild/configs/neutral.py

@@ -3,6 +3,7 @@ from .base import ConfigInfo
 
 class NeutralConfigInfo(ConfigInfo):
     name = "neutral"
+    arch_name = "neutral"
     static = ""
     compatible_hosts = ["fedora", "debian", "Darwin"]
 

kiwixbuild/configs/wasm.py

@@ -79,9 +79,8 @@ class WasmConfigInfo(ConfigInfo):
 
     def get_env(self):
         env = super().get_env()
-        env["PATH"] = ":".join(
+        env["PATH"].extend(
             [
-                env["PATH"],
                 self.install_path,
                 pj(self.install_path, "upstream", "emscripten"),
                 pj(self.install_path, "node", "14.18.2_64bit", "bin"),

kiwixbuild/configs/win32.py

@@ -1,99 +0,0 @@
-import subprocess
-
-from .base import ConfigInfo
-from kiwixbuild.utils import which, pj
-from kiwixbuild._global import neutralEnv
-
-
-class Win32ConfigInfo(ConfigInfo):
-    build = "win32"
-    compatible_hosts = ["fedora", "debian"]
-    arch_full = "i686-w64-mingw32"
-    extra_libs = ["-lwinmm", "-lshlwapi", "-lws2_32", "-lssp"]
-
-    def get_cross_config(self):
-        return {
-            "exe_wrapper_def": self.exe_wrapper_def,
-            "binaries": self.binaries,
-            "root_path": self.root_path,
-            "extra_libs": self.extra_libs,
-            "extra_cflags": [
-                "-DWIN32",
-                *(
-                    "-I{}".format(include_dir)
-                    for include_dir in self.get_include_dirs()
-                ),
-            ],
-            "host_machine": {
-                "system": "Windows",
-                "lsystem": "windows",
-                "cpu_family": "x86",
-                "cpu": "i686",
-                "endian": "little",
-                "abi": "",
-            },
-        }
-
-    def finalize_setup(self):
-        super().finalize_setup()
-        self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt")
-        self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
-
-    @property
-    def root_path(self):
-        root_paths = {
-            "fedora": "/usr/i686-w64-mingw32/sys-root/mingw",
-            "debian": "/usr/i686-w64-mingw32",
-        }
-        return root_paths[neutralEnv("distname")]
-
-    @property
-    def binaries(self):
-        return {
-            k: which("{}-{}".format(self.arch_full, v))
-            for k, v in (
-                ("CC", "gcc"),
-                ("CXX", "g++"),
-                ("AR", "ar"),
-                ("STRIP", "strip"),
-                ("WINDRES", "windres"),
-                ("RANLIB", "ranlib"),
-                ("PKGCONFIG", "pkg-config"),
-            )
-        }
-
-    @property
-    def exe_wrapper_def(self):
-        try:
-            which("wine")
-        except subprocess.CalledProcessError:
-            return ""
-        else:
-            return "exe_wrapper = 'wine'"
-
-    @property
-    def configure_options(self):
-        yield f"--host={self.arch_full}"
-
-    def set_compiler(self, env):
-        for k, v in self.binaries.items():
-            env[k] = v
-
-    def get_bin_dir(self):
-        return [pj(self.root_path, "bin")]
-
-    def get_env(self):
-        env = super().get_env()
-        env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig")
-        env["LIBS"] = " ".join(self.extra_libs) + " " + env["LIBS"]
-        return env
-
-
-class Win32Dyn(Win32ConfigInfo):
-    name = "win32_dyn"
-    static = False
-
-
-class Win32Static(Win32ConfigInfo):
-    name = "win32_static"
-    static = True

kiwixbuild/dependencies/__init__.py

@@ -1,6 +1,7 @@
 from .base import *
 from . import (
     all_dependencies,
+    boostregex,
     tc_android_ndk,
     aria2,
     tc_armhf,

kiwixbuild/dependencies/all_dependencies.py

@@ -13,9 +13,28 @@ class AllBaseDependencies(Dependency):
     class Builder(NoopBuilder):
         @classmethod
         def get_dependencies(cls, configInfo, allDeps):
-            if configInfo.build == "wasm" or environ.get("OS_NAME") == "bionic":
+            if configInfo.build == "wasm" or environ.get("OS_NAME") == "manylinux":
                 return ["zlib", "lzma", "zstd", "icu4c", "xapian-core"]
 
+            if neutralEnv("distname") == "Windows":
+                base_deps = [
+                    "zlib",
+                    "zstd",
+                    "xapian-core",
+                    "zim-testing-suite",
+                    "icu4c",
+                    "boostregex",
+                    "docoptcpp",
+                ]
+
+                if not configInfo.name.endswith("_dyn"):
+                    base_deps += [
+                        "pugixml",
+                        "libcurl",
+                        "mustache",
+                        "libmicrohttpd",
+                    ]
+            else:
                 base_deps = [
                     "zlib",
                     "lzma",
@@ -31,11 +50,11 @@ class AllBaseDependencies(Dependency):
                 # Add specific dependencies depending of the config
                 if configInfo.build not in ("android", "iOS"):
                     # For zimtools
-                base_deps += ["docoptcpp"]
+                    base_deps += ["docoptcpp", "libmagic", "gumbo"]
-                if configInfo.build != "win32":
+                    if (
-                    # zimwriterfs
+                        configInfo.build == "native"
-                    base_deps += ["libmagic", "gumbo"]
+                        and neutralEnv("distname") != "Darwin"
-                if configInfo.build == "native" and neutralEnv("distname") != "Darwin":
+                    ):
                         # We compile kiwix-desktop only on native and not on `Darwin`
                         # So we need aria2 only there
                         base_deps += ["aria2"]

kiwixbuild/dependencies/aria2.py

@@ -1,16 +1,42 @@
-from .base import Dependency, ReleaseDownload, MakeBuilder
+from .base import Dependency, ReleaseDownload, MakeBuilder, NoopBuilder
 
-from kiwixbuild.utils import Remotefile, run_command
+from kiwixbuild.utils import Remotefile, run_command, pj
+import platform
+from shutil import copy2
 
+# Important: in case of aria2c update,
+# 'scripts/create_kiwix-desktop_appImage.sh' should not be forgotten!
 
 class Aria2(Dependency):
     name = "aria2"
 
+    if platform.system() == "Windows":
+
+        class Source(ReleaseDownload):
+            archive = Remotefile(
+                "aria2-1.37.0-win-64bit-build1.zip",
+                "67d015301eef0b612191212d564c5bb0a14b5b9c4796b76454276a4d28d9b288",
+                "https://dev.kiwix.org/kiwix-desktop/aria2-1.37.0-win-64bit-build1.zip",
+            )
+
+        class Builder(NoopBuilder):
+            def build(self):
+                self.command("copy_binary", self._copy_binary)
+
+            def _copy_binary(self, context):
+                context.try_skip(self.build_path)
+                copy2(
+                    pj(self.source_path, "aria2c.exe"),
+                    pj(self.buildEnv.install_dir, "bin"),
+                )
+
+    else:
+
         class Source(ReleaseDownload):
             archive = Remotefile(
                 "aria2-1.37.0.tar.xz",
                 "60a420ad7085eb616cb6e2bdf0a7206d68ff3d37fb5a956dc44242eb2f79b66b",
-            "https://github.com/aria2/aria2/releases/download/release-1.37.0/aria2-1.37.0.tar.xz",
+                "https://dev.kiwix.org/kiwix-desktop/aria2-1.37.0.tar.xz",
             )
 
             def _post_prepare_script(self, context):

kiwixbuild/dependencies/base.py

@@ -2,6 +2,7 @@ import subprocess
 import os
 import shutil
 import time
+import platform
 
 from kiwixbuild.utils import (
     pj,
@@ -131,8 +132,17 @@ class ReleaseDownload(Source):
 
     def _download(self, context):
         context.try_skip(neutralEnv("archive_dir"), self.full_name)
-        for archive in self.archives:
+        archive_iter = iter(self.archives)
+        archive = next(archive_iter, None)
+        while archive:
+            try:
                 neutralEnv("download")(archive)
+            except SkipCommand as e:
+                archive = next(archive_iter, None)
+                if not archive:
+                    raise e
+                continue
+            archive = next(archive_iter, None)
 
     def _extract(self, context):
         context.try_skip(self.extract_path)
@@ -314,8 +324,8 @@ class Builder:
             module["subdir"] = self.subsource_dir
         if getattr(self, "flatpack_build_options", None):
             module["build-options"] = self.flatpack_build_options
-        if getattr(self, "configure_option", ""):
+        if getattr(self, "configure_options", ""):
-            module["config-opts"] = self.configure_option.split(" ")
+            module["config-opts"] = list(self.configure_options)
 
     def get_env(self, *, cross_comp_flags, cross_compilers, cross_path):
         env = self.buildEnv.get_env(
@@ -370,7 +380,7 @@ class MakeBuilder(Builder):
     configure_options = []
     dynamic_configure_options = ["--enable-shared", "--disable-static"]
     static_configure_options = ["--enable-static", "--disable-shared"]
-    make_options = []
+    make_options = ["-j4"]
     install_options = []
     configure_script = "configure"
     configure_env = {
@@ -426,7 +436,6 @@ class MakeBuilder(Builder):
         command = [
             *self.buildEnv.make_wrapper,
             *neutralEnv("make_command"),
-            "-j4",
             *self.make_targets,
             *self.make_options,
         ]
@@ -483,6 +492,12 @@ class QMakeBuilder(MakeBuilder):
     qmake_targets = []
     flatpak_buildsystem = "qmake"
 
+    @property
+    def make_options(self):
+        if platform.system() == "Windows":
+            return []
+        return super().make_options
+
     @property
     def env_options(self):
         if "QMAKE_CC" in os.environ:
@@ -493,7 +508,7 @@ class QMakeBuilder(MakeBuilder):
     def _configure(self, context):
         context.try_skip(self.build_path)
         command = [
-            "qmake",
+            *neutralEnv("qmake_command"),
             *self.configure_options,
             *self.env_options,
             self.source_path,
@@ -509,8 +524,8 @@ class QMakeBuilder(MakeBuilder):
             *neutralEnv("git_command"),
             "archive",
             "-o",
-            f"{self.build_path}/{self.target_full_name()}.tar.gz",
+            f"{self.build_path}/{self.target.full_name()}.tar.gz",
-            f"--prefix={self.target_full_name()}/",
+            f"--prefix={self.target.full_name()}/",
             "HEAD",
         ]
         run_command(command, self.source_path, context)
@@ -523,6 +538,9 @@ class MesonBuilder(Builder):
 
     @property
     def build_type(self):
+        if platform.system() == "Windows":
+            return "release"
+
         return "release" if option("make_release") else "debug"
 
     @property
@@ -544,6 +562,7 @@ class MesonBuilder(Builder):
             cross_options += ["--cross-file", self.buildEnv.meson_crossfile]
         command = [
             *neutralEnv("meson_command"),
+            "setup",
             ".",
             self.build_path,
             f"--buildtype={self.build_type}",

kiwixbuild/dependencies/boostregex.py

@@ -0,0 +1,27 @@
+from .base import Dependency, ReleaseDownload, Builder as BaseBuilder
+
+from kiwixbuild.utils import Remotefile, pj
+from shutil import copytree
+
+
+class BoostRegex(Dependency):
+    name = "boostregex"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile(
+            "regex-boost-1.86.0.zip",
+            "",
+            "https://codeload.github.com/boostorg/regex/zip/refs/tags/boost-1.86.0",
+        )
+
+    class Builder(BaseBuilder):
+        def build(self):
+            self.command("copy_headers", self._copy_headers)
+
+        def _copy_headers(self, context):
+            context.try_skip(self.build_path)
+            copytree(
+                pj(self.source_path, "include", "boost"),
+                pj(self.buildEnv.install_dir, "include", "boost"),
+                dirs_exist_ok=True,
+            )

kiwixbuild/dependencies/docoptcpp.py

@@ -1,16 +1,34 @@
-from .base import Dependency, GitClone, CMakeBuilder
+from .base import Dependency, ReleaseDownload, MesonBuilder
 
 from kiwixbuild.utils import Remotefile
-
+from kiwixbuild._global import neutralEnv
 
 class docoptcpp(Dependency):
     name = "docoptcpp"
 
-    class Source(GitClone):
+    class Source(ReleaseDownload):
-        git_remote = "https://github.com/docopt/docopt.cpp.git"
+        name = "docoptcpp"
-        git_dir = "docopt.cpp"
+        src_archive = Remotefile(
-        force_full_clone = True
+            "v0.6.3.tar.gz",
-        git_ref = "3dd23e3280f213bacefdf5fcb04857bf52e90917"
+            "28af5a0c482c6d508d22b14d588a3b0bd9ff97135f99c2814a5aa3cbff1d6632",
+            "https://github.com/docopt/docopt.cpp/archive/v0.6.3.tar.gz",
+        )
 
-    class Builder(CMakeBuilder):
+        meson_archive = Remotefile(
-        make_install_targets = ["install"]
+            "docopt_0.6.3-3_patch.zip",
+            "1f641187f9d3f35b0a5ebd2011876ef8e9b04b69b7b163095dd7dfa16219ad01",
+            "https://wrapdb.mesonbuild.com/v2/docopt_0.6.3-3/get_patch",
+        )
+
+        archives = [src_archive, meson_archive]
+        patches = [
+            "docopt_meson_install_pkgconfig.patch",
+            "docopt_meson_use_boostregex.patch",
+        ]
+
+    class Builder(MesonBuilder):
+        @classmethod
+        def get_dependencies(cls, configInfo, allDeps):
+            if neutralEnv("distname") == "Windows":
+                return ["boostregex"]
+            return []

kiwixbuild/dependencies/icu4c.py

@@ -1,10 +1,105 @@
-from .base import Dependency, ReleaseDownload, MakeBuilder
+from .base import (
+    Dependency,
+    ReleaseDownload,
+    MakeBuilder,
+    Builder as BaseBuilder,
+)
 
 from kiwixbuild.utils import pj, SkipCommand, Remotefile, extract_archive
 from kiwixbuild._global import get_target_step, neutralEnv
 import os, shutil
 import fileinput
+import platform
 
+if platform.system() == "Windows":
+
+    class Icu(Dependency):
+        name = "icu4c"
+
+        class Source(ReleaseDownload):
+            archive = Remotefile(
+                "icu4c-74_1-Win64-MSVC2022.zip",
+                "",
+                "https://github.com/unicode-org/icu/releases/download/release-74-1/icu4c-74_1-Win64-MSVC2022.zip",
+            )
+
+        class Builder(BaseBuilder):
+            def build(self):
+                self.command("copy_headers", self._copy_headers)
+                self.command("copy_bins", self._copy_bin)
+                self.command("generate_pkg_config", self._generate_pkg_config)
+
+            def _copy_headers(self, context):
+                context.try_skip(self.build_path)
+                shutil.copytree(
+                    pj(self.source_path, "include", "unicode"),
+                    pj(self.buildEnv.install_dir, "include", "unicode"),
+                )
+
+            def _copy_bin(self, context):
+                context.try_skip(self.build_path)
+                shutil.copytree(
+                    pj(self.source_path, "lib64"),
+                    pj(self.buildEnv.install_dir, "lib"),
+                    dirs_exist_ok=True,
+                )
+
+                def ignore_non_dll(path, names):
+                    return [n for n in names if not n.endswith(".dll")]
+
+                shutil.copytree(
+                    pj(self.source_path, "bin64"),
+                    pj(self.buildEnv.install_dir, "bin"),
+                    ignore=ignore_non_dll,
+                    dirs_exist_ok=True,
+                )
+
+            def _generate_pkg_config(self, context):
+                context.try_skip(self.build_path)
+
+                pkg_config_template = """ Copyright (C) 2016 and later: Unicode, Inc. and others.
+# License & terms of use: http://www.unicode.org/copyright.html
+# Copyright (C) 2010-2013, International Business Machines Corporation. All Rights Reserved.
+
+# CFLAGS contains only anything end users should set
+CFLAGS = 
+# CXXFLAGS contains only anything end users should set
+CXXFLAGS =  -std=c++11
+# DEFS only contains those UCONFIG_CPPFLAGS which are not auto-set by platform.h
+DEFS = 
+prefix = {prefix}
+exec_prefix = ${{prefix}}
+libdir = ${{exec_prefix}}/lib
+includedir = ${{prefix}}/include
+baselibs = -lpthread -lm 
+UNICODE_VERSION=15.0
+ICUPREFIX=icu
+ICULIBSUFFIX=
+LIBICU=lib${{ICUPREFIX}}
+pkglibdir=${{libdir}}/icu${{ICULIBSUFFIX}}/73.1
+ICUDATA_NAME = icudt73l
+ICUDESC=International Components for Unicode
+
+Version: 73.1
+Cflags: -I${{includedir}}
+Description: International Components for Unicode: Internationalization library
+Name: icu-i18n
+Libs: -L${{libdir}} -licuin -licuuc -licudt"""
+
+                pkg_config_content = pkg_config_template.format(
+                    prefix=self.buildEnv.install_dir
+                )
+
+                os.makedirs(
+                    pj(self.buildEnv.install_dir, "lib", "pkgconfig"), exist_ok=True
+                )
+                with open(
+                    pj(self.buildEnv.install_dir, "lib", "pkgconfig", "icu-i18n.pc"),
+                    mode="w",
+                ) as f:
+                    f.write(pkg_config_content)
+
+else:
 
     class Icu(Dependency):
         name = "icu4c"
@@ -45,11 +140,7 @@ class Icu(Dependency):
 
             patches = [
                 "icu4c_fix_static_lib_name_mingw.patch",
-            #           "icu4c_android_elf64_st_info.patch",
-            #           "icu4c_custom_data.patch",
-            #           "icu4c_noxlocale.patch",
                 "icu4c_rpath.patch",
-            #           "icu4c_build_config.patch",
                 "icu4c_wasm.patch",
             ]
 
@@ -83,14 +174,17 @@ class Icu(Dependency):
 
             def set_env(self, env):
                 env["ICU_DATA_FILTER_FILE"] = pj(
-                os.path.dirname(os.path.realpath(__file__)), "icu4c_data_filter.json"
+                    os.path.dirname(os.path.realpath(__file__)),
+                    "icu4c_data_filter.json",
                 )
 
             def _post_configure_script(self, context):
                 if self.buildEnv.configInfo.build != "wasm":
                     context.skip()
                 context.try_skip(self.build_path)
-            for line in fileinput.input(pj(self.build_path, "Makefile"), inplace=True):
+                for line in fileinput.input(
+                    pj(self.build_path, "Makefile"), inplace=True
+                ):
                     if line == "#DATASUBDIR = data\n":
                         print("DATASUBDIR = data")
                     else:

kiwixbuild/dependencies/kiwix_desktop.py

@@ -1,4 +1,6 @@
+from kiwixbuild._global import option
 from .base import Dependency, GitClone, QMakeBuilder
+import platform
 
 
 class KiwixDesktop(Dependency):
@@ -11,18 +13,27 @@ class KiwixDesktop(Dependency):
 
     class Builder(QMakeBuilder):
         dependencies = ["qt", "qtwebengine", "libkiwix", "aria2"]
-        make_install_targets = ["install"]
         configure_env = None
 
-        flatpack_build_options = {"env": ["QMAKEPATH=/app/lib"]}
+        flatpack_build_options = {"env": {"QMAKEPATH": "/app"}}
+
+        @property
+        def make_targets(self):
+            if platform.system() == "Windows":
+                yield "release-all"
+            else:
+                yield from super().make_targets
+
+        @property
+        def make_install_targets(self):
+            if platform.system() == "Windows":
+                yield "release-install"
+            else:
+                yield "install"
 
         @property
         def configure_options(self):
-            if self.buildEnv.configInfo.name == "flatpak":
+            if self.buildEnv.configInfo.name != "flatpak":
-                yield "QMAKE_INCDIR+=/app/include/QtWebEngine"
-                yield "QMAKE_INCDIR+=/app/include/QtWebEngineCore"
-                yield "QMAKE_INCDIR+=/app/include/QtWebEngineWidgets"
-            else:
                 yield f"PREFIX={self.buildEnv.install_dir}"
             if self.buildEnv.configInfo.static:
                 yield "CONFIG+=static"

kiwixbuild/dependencies/kiwix_tools.py

@@ -10,7 +10,7 @@ class KiwixTools(Dependency):
         git_dir = "kiwix-tools"
 
     class Builder(MesonBuilder):
-        dependencies = ["libkiwix"]
+        dependencies = ["libkiwix", "docoptcpp"]
 
         @property
         def configure_options(self):

kiwixbuild/dependencies/libcurl.py

@@ -3,7 +3,7 @@ import os
 from .base import (
     Dependency,
     ReleaseDownload,
-    MakeBuilder,
+    MesonBuilder,
 )
 
 from kiwixbuild.utils import Remotefile, pj, Defaultdict, SkipCommand, run_command
@@ -15,30 +15,31 @@ class LibCurl(Dependency):
 
     class Source(ReleaseDownload):
         name = "libcurl"
-        archive = Remotefile(
+        src_archive = Remotefile(
-            "curl-7.67.0.tar.xz",
+            "curl-8.4.0.tar.xz",
-            "f5d2e7320379338c3952dcc7566a140abb49edb575f9f99272455785c40e536c",
+            "16c62a9c4af0f703d28bda6d7bbf37ba47055ad3414d70dec63e2e6336f2a82d",
-            "https://curl.haxx.se/download/curl-7.67.0.tar.xz",
+            "https://curl.se/download/curl-8.4.0.tar.xz",
         )
+        meson_archive = Remotefile(
+            "curl_8.4.0-2_patch.zip",
+            "bbb6ae75225c36ef9bb336cface729794c7c070c623a003fff40bd416042ff6e",
+            "https://dev.kiwix.org/libkiwix/curl_8.4.0-2_patch.zip",
+        )
+        archives = [src_archive, meson_archive]
 
-    class Builder(MakeBuilder):
+    class Builder(MesonBuilder):
         dependencies = ["zlib"]
         configure_options = [
-            *[
+            f"-D{p}=disabled"
-                f"--without-{p}"
             for p in (
-                    "libssh2",
+                "psl",
-                    "ssl",
+                "kerberos-auth",
-                    "libmetalink",
+                "gss-api",
-                    "librtmp",
+                "ssh",
-                    "nghttp2",
+                "rtmp",
-                    "libidn2",
+                "http2",
+                "idn",
                 "brotli",
-                )
-            ],
-            *[
-                f"--disable-{p}"
-                for p in (
                 "ftp",
                 "file",
                 "ldap",
@@ -52,7 +53,9 @@ class LibCurl(Dependency):
                 "smb",
                 "smtp",
                 "gopher",
-                    "manual",
+                "tool",
             )
-            ],
         ]
+
+        def _test(self, context):
+            context.skip("No Test")

kiwixbuild/dependencies/libkiwix.py

@@ -18,7 +18,6 @@ class Libkiwix(Dependency):
             "pugixml",
             "libzim",
             "zlib",
-            "lzma",
             "libcurl",
             "libmicrohttpd",
             "icu4c",

kiwixbuild/dependencies/libmagic.py

@@ -46,7 +46,5 @@ class LibMagic(Dependency):
                 cross_comp_flags=True, cross_compilers=True, cross_path=True
             )
             libmagic_native_builder = get_target_step("libmagic", "native_static")
-            env["PATH"] = ":".join(
+            env["PATH"].insert(0, pj(libmagic_native_builder.build_path, "src"))
-                [pj(libmagic_native_builder.build_path, "src"), env["PATH"]]
-            )
             run_command(command, self.build_path, context, env=env)

kiwixbuild/dependencies/libmicrohttpd.py

@@ -1,4 +1,4 @@
-from .base import Dependency, ReleaseDownload, MakeBuilder
+from .base import Dependency, ReleaseDownload, MesonBuilder
 
 from kiwixbuild.utils import Remotefile
 
@@ -7,17 +7,27 @@ class MicroHttpd(Dependency):
     name = "libmicrohttpd"
 
     class Source(ReleaseDownload):
-        archive = Remotefile(
+        src_archive = Remotefile(
             "libmicrohttpd-0.9.76.tar.gz",
             "f0b1547b5a42a6c0f724e8e1c1cb5ce9c4c35fb495e7d780b9930d35011ceb4c",
             "https://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.76.tar.gz",
         )
-
+        meson_archive = Remotefile(
-    class Builder(MakeBuilder):
+            "libmicrohttpd_0.9.16-3_patch.zip",
-        configure_options = [
+            "0954c094a0d4cfe0dd799d8df8a04face6669f7b4d51a7386a9c3e2d37b9c3b3",
-            "--disable-https",
+            "https://wrapdb.mesonbuild.com/v2/libmicrohttpd_0.9.76-3/get_patch",
-            "--without-libgcrypt",
+        )
-            "--without-libcurl",
+        archives = [src_archive, meson_archive]
-            "--disable-doc",
+        patches = [
-            "--disable-examples",
+            "libmicrohttpd_meson_pkgconfig.patch",
+            "libmicrohttpd_meson_timeval_tvsec_size.patch",
+            "libmicrohttpd_meson_winet6.patch",
+        ]
+
+    class Builder(MesonBuilder):
+        configure_options = [
+            "-Dgnutls=disabled",
+            "-Dgcrypt=disabled",
+            "-Dcurl=disabled",
+            "-Dexpat=disabled",
         ]

kiwixbuild/dependencies/libzim.py

@@ -1,5 +1,5 @@
 from .base import Dependency, GitClone, MesonBuilder
-from kiwixbuild._global import option, get_target_step
+from kiwixbuild._global import option, get_target_step, neutralEnv
 
 
 class Libzim(Dependency):
@@ -22,6 +22,8 @@ class Libzim(Dependency):
 
         @classmethod
         def get_dependencies(cls, configInfo, allDeps):
+            if neutralEnv("distname") == "Windows":
+                return ["zstd", "xapian-core", "icu4c", "zim-testing-suite"]
             deps = ["lzma", "zstd", "xapian-core", "icu4c"]
             if configInfo.name not in ("flatpak", "wasm"):
                 deps.append("zim-testing-suite")
@@ -30,6 +32,9 @@ class Libzim(Dependency):
         @property
         def configure_options(self):
             configInfo = self.buildEnv.configInfo
+            if neutralEnv("distname") == "Windows":
+                yield "-Dwith_xapian_fuller=false"
+                yield "-Dwerror=false"
             if configInfo.build == "android":
                 yield "-DUSE_BUFFER_HEADER=false"
                 yield "-Dstatic-linkage=true"

kiwixbuild/dependencies/lzma.py

@@ -1,28 +1,26 @@
-from .base import Dependency, ReleaseDownload, MakeBuilder
+from .base import (
+    Dependency,
+    ReleaseDownload,
+    MesonBuilder)
 
 from kiwixbuild.utils import Remotefile
 
 
 class lzma(Dependency):
-    name = "lzma"
+    name = 'lzma'
 
     class Source(ReleaseDownload):
-        archive = Remotefile(
+        src_archive = Remotefile(
             "xz-5.2.6.tar.gz",
             "a2105abee17bcd2ebd15ced31b4f5eda6e17efd6b10f921a01cda4a44c91b3a0",
             "https://altushost-swe.dl.sourceforge.net/project/lzmautils/xz-5.2.6.tar.gz",
         )
+        meson_patch = Remotefile(
+            "liblzma_5.2.6-3_patch.zip",
+            "1c71536d364e1a3ce6bea61266576f89cc5cce4d3b9e11f3494417dafa29780b",
+            "https://wrapdb.mesonbuild.com/v2/liblzma_5.2.6-3/get_patch",
+        )
+        archives = [src_archive, meson_patch]
+        patches = ['lzma_meson_install.patch']
 
-    class Builder(MakeBuilder):
+    Builder = MesonBuilder
-        @property
-        def configure_options(self):
-            return [
-                "--disable-xz",
-                "--disable-xzdec",
-                "--disable-lzmadec",
-                "--disable-lzmainfo",
-                "--disable-lzma-links",
-                "--disable-scripts",
-                "--disable-doc",
-                #                    "--disable-symbol-versions"
-            ]

kiwixbuild/dependencies/pugixml.py

@@ -15,5 +15,4 @@ class Pugixml(Dependency):
         flatpak_dest = "src"
 
     class Builder(MesonBuilder):
-        build_type = "release"
         strip_options = []

kiwixbuild/dependencies/qt.py

@@ -1,87 +1,25 @@
 import shutil
 
-from .base import Dependency, ReleaseDownload, MakeBuilder, QMakeBuilder
+from .base import Dependency, NoopBuilder, NoopSource
 
-from kiwixbuild.utils import Remotefile, pj, SkipCommand
+from kiwixbuild.utils import SkipCommand, colorize
 
 
 class Qt(Dependency):
     name = "qt"
 
-    class Source(ReleaseDownload):
+    Source = NoopSource
-        name = "qt"
-        source_dir = "qt-5.10.1"
-        archive = Remotefile(
-            "qt-everywhere-src-5.10.1.tar.xz",
-            "",
-            "http://ftp.oregonstate.edu/.1/blfs/conglomeration/qt5/qt-everywhere-src-5.10.1.tar.xz",
-        )
 
-    class Builder(MakeBuilder):
+    class Builder(NoopBuilder):
-        dependencies = ["icu4c", "zlib"]
+        def build(self):
-        dynamic_configure_options = ["-shared"]
+            error_msg = f"""WARNING: kiwix-build cannot build {self.name} for you.
-        static_configure_options = ["-static"]
+You must install it yourself using official Qt installer or your distribution system."""
-
+            print(colorize(error_msg, "WARNING"))
-        @property
-        def all_configure_options(self):
-            yield from self.configure_options
-            if self.buildEnv.configInfo.static:
-                yield from self.static_configure_options
-            else:
-                yield from self.dynamic_configure_options
-            if not self.target.force_native_build:
-                yield from self.buildEnv.configInfo.configure_options
-            yield from ("-prefix", self.buildEnv.install_dir)
-            yield from (
-                "-libdir",
-                pj(self.buildEnv.install_dir, self.buildEnv.libprefix),
-            )
-
-        @property
-        def configure_options(self):
-            skip_modules = [
-                "qt3d",
-                "qtcanvas3d",
-                "qtcharts",
-                "qtconnectivity",
-                "qtdatavis3d",
-                #    'qtdeclarative',
-                "qtdoc",
-                "qtgamepad",
-                "qtgraphicaleffects",
-                "qtlocation",
-                "qtmultimedia",
-                "qtnetworkauth",
-                "qtpurchasing",
-                #    'qtquickcontrols',
-                "qtquickcontrols2",
-                "qtremoteobjects",
-                "qtscript",
-                "qtscxml",
-                "qtsensors",
-                "qtserialbus",
-                "qtserialport",
-                "qtspeech",
-                "qtvirtualkeyboard",
-                "qtwayland",
-                "qtwebglplugin",
-                "qtwebsockets",
-                #                'qtwebview',
-            ]
-            yield "-recheck"
-            yield "-opensource"
-            yield "-confirm-license"
-            yield "-ccache"
-            yield from ("-make", "libs")
-            for module in skip_modules:
-                yield from ("-skip", module)
 
 
 class QtWebEngine(Dependency):
     name = "qtwebengine"
 
-    Source = Qt.Source
+    Source = NoopSource
 
-    class Builder(QMakeBuilder):
+    Builder = Qt.Builder
-        dependencies = ["qt"]
-        subsource_dir = "qtwebengine"

kiwixbuild/dependencies/tc_android_ndk.py

@@ -24,6 +24,10 @@ class android_ndk(Dependency):
         def source_dir(self):
             return self.target.full_name()
 
+        patches = [
+            "android-ndk-r21e-linux-x86_64-python3.12+.patch",
+        ]
+
     class Builder(Builder):
         @property
         def install_path(self):

kiwixbuild/dependencies/tc_armhf.py

@@ -9,9 +9,6 @@ base_url = (
 )
 
 
-aarch_base_url = "https://master.dl.sourceforge.net/project/raspberry-pi-cross-compilers/Bonus%20Raspberry%20Pi%20GCC%2064-Bit%20Toolchains/Raspberry%20Pi%20GCC%2064-Bit%20Cross-Compiler%20Toolchains/Stretch/GCC%206.3.0/"
-
-
 class armv6_toolchain(Dependency):
     dont_skip = True
     neutral = True
@@ -53,7 +50,6 @@ class aarch64_toolchain(Dependency):
         archive = Remotefile(
             "cross-gcc-6.3.0-pi_64.tar.gz",
             "1b048bb8886ad63d21797cd9129fc37b9ea0dfaac7e3c36f888aa16fbec1d320",
-            aarch_base_url + "cross-gcc-6.3.0-pi_64.tar.gz",
         )
 
     Builder = TcCopyBuilder

kiwixbuild/dependencies/xapian.py

@@ -1,12 +1,34 @@
-from .base import Dependency, ReleaseDownload, MakeBuilder
+from .base import Dependency, GitClone, ReleaseDownload, MakeBuilder, MesonBuilder
 
 from kiwixbuild.utils import Remotefile
 from kiwixbuild._global import neutralEnv
 
+import platform
+
 
 class Xapian(Dependency):
     name = "xapian-core"
 
+    if platform.system() == "Windows":
+
+        class Source(GitClone):
+            git_remote = "https://github.com/openzim/xapian-meson.git"
+            git_dir = "xapian-core"
+
+        class Builder(MesonBuilder):
+            configure_options = [
+                "-Denable-sse=false",
+                "-Denable-backend-chert=false",
+                "-Denable-backend-remote=false",
+            ]
+            subsource_dir = "xapian-core"
+
+            @classmethod
+            def get_dependencies(cls, configInfo, allDeps):
+                return ["zlib"]
+
+    else:
+
         class Source(ReleaseDownload):
             archive = Remotefile(
                 "xapian-core-1.4.23.tar.xz",
@@ -22,15 +44,12 @@ class Xapian(Dependency):
             ]
             configure_env = {
                 "_format_LDFLAGS": "{env.LDFLAGS} -L{buildEnv.install_dir}/{buildEnv.libprefix}",
-            "_format_CXXFLAGS": "{env.CXXFLAGS} -O3 -I{buildEnv.install_dir}/include",
+                "_format_CXXFLAGS": "{env.CXXFLAGS} -I{buildEnv.install_dir}/include",
             }
 
             @classmethod
             def get_dependencies(cls, configInfo, allDeps):
                 deps = ["zlib", "lzma"]
-            if (
+                if configInfo.build == "wasm" or neutralEnv("distname") == "Darwin":
-                configInfo.build in ("win32", "wasm")
-                or neutralEnv("distname") == "Darwin"
-            ):
                     return deps
                 return deps + ["uuid"]

kiwixbuild/dependencies/zim_testing_suite.py

@@ -9,9 +9,9 @@ class ZimTestingSuite(Dependency):
 
     class Source(ReleaseDownload):
         archive = Remotefile(
-            "zim-testing-suite-0.5.tar.gz",
+            "zim-testing-suite-0.8.0.tar.gz",
-            "3ffd7e0adf46e9a44cad463f4220d2406a700e95deeff936463be818acf47256",
+            "28f51449a3f9aea02652ca21f32c5598fd610d6cec3810fa552bd0c0f7a2d5fc",
-            "https://github.com/openzim/zim-testing-suite/releases/download/v0.5/zim-testing-suite-0.5.tar.gz",
+            "https://github.com/openzim/zim-testing-suite/releases/download/0.8.0/zim-testing-suite-0.8.0.tar.gz",
         )
 
     Builder = NoopBuilder

kiwixbuild/dependencies/zim_tools.py

@@ -1,4 +1,5 @@
 from .base import Dependency, GitClone, MesonBuilder
+from kiwixbuild._global import neutralEnv
 
 
 class ZimTools(Dependency):
@@ -13,14 +14,14 @@ class ZimTools(Dependency):
         @classmethod
         def get_dependencies(cls, configInfo, allDeps):
             base_deps = ["libzim", "docoptcpp", "mustache"]
-            if configInfo.build != "win32":
+            if neutralEnv("distname") != "Windows":
                 base_deps += ["libmagic", "gumbo"]
             return base_deps
 
         @property
         def configure_options(self):
-            # We don't build zimwriterfs on win32, and so we don't have magic
+            # We don't build zimwriterfs on Windows, and so we don't have magic
-            if self.buildEnv.configInfo.build != "win32":
+            if neutralEnv("distname") != "Windows":
                 yield f"-Dmagic-install-prefix={self.buildEnv.install_dir}"
             if self.buildEnv.configInfo.static:
                 yield "-Dstatic-linkage=true"

kiwixbuild/dependencies/zlib.py

@@ -1,63 +1,27 @@
 import shutil
 
-from .base import Dependency, ReleaseDownload, MakeBuilder
+from .base import (
+    Dependency,
+    ReleaseDownload,
+    MesonBuilder)
 
 from kiwixbuild.utils import Remotefile, pj, SkipCommand
+from kiwixbuild._global import neutralEnv
 
 
 class zlib(Dependency):
     name = "zlib"
 
     class Source(ReleaseDownload):
-        archive = Remotefile(
+        src_archive = Remotefile(
             "zlib-1.2.12.tar.gz",
-            "91844808532e5ce316b3c010929493c0244f3d37593afd6de04f71821d5136d9",
+            "91844808532e5ce316b3c010929493c0244f3d37593afd6de04f71821d5136d9"
         )
-        patches = ["zlib_std_libname.patch"]
+        meson_patch = Remotefile(
+            "zlib_1.2.12-1_patch.zip",
+            "8ec8344f3fe7b06ad4be768fd416694bc56cb4545ce78b0f1c18b3e72b3ec936",
+            "https://wrapdb.mesonbuild.com/v2/zlib_1.2.12-1/get_patch")
+        archives = [src_archive, meson_patch]
+        #patches = ['zlib_std_libname.patch']
 
-    class Builder(MakeBuilder):
+    Builder = MesonBuilder
-        dynamic_configure_options = ["--shared"]
-        static_configure_options = ["--static"]
-        make_install_targets = ["install"]
-
-        def _pre_build_script(self, context):
-            context.try_skip(self.build_path)
-            shutil.copytree(self.source_path, self.build_path)
-
-        def _configure(self, context):
-            if self.buildEnv.configInfo.build == "win32":
-                raise SkipCommand()
-            return super()._configure(context)
-
-        @property
-        def all_configure_options(self):
-            yield from self.configure_options
-            yield from self.static_configure_options if self.buildEnv.configInfo.static else self.dynamic_configure_options
-            yield from ("--prefix", self.buildEnv.install_dir)
-            yield from (
-                "--libdir",
-                pj(self.buildEnv.install_dir, self.buildEnv.libprefix),
-            )
-
-        @property
-        def make_options(self):
-            if self.buildEnv.configInfo.build != "win32":
-                return
-            yield "--makefile"
-            yield "win32/Makefile.gcc"
-            yield "PREFIX=i686-w64-mingw32-",
-            yield "SHARED_MODE={}".format(
-                "0" if self.buildEnv.configInfo.static else "1"
-            ),
-            yield "INCLUDE_PATH={}".format(pj(self.buildEnv.install_dir, "include")),
-            yield "LIBRARY_PATH={}".format(
-                pj(self.buildEnv.install_dir, self.buildEnv.libprefix)
-            ),
-            yield "BINARY_PATH={}".format(pj(self.buildEnv.install_dir, "bin"))
-
-        @property
-        def make_targets(self):
-            if self.buildEnv.configInfo.static:
-                return ["static"]
-            else:
-                return ["shared"]

kiwixbuild/dependencies/zstd.py

@@ -8,9 +8,9 @@ class zstd(Dependency):
 
     class Source(ReleaseDownload):
         archive = Remotefile(
-            "zstd-1.5.2.tar.gz",
+            "zstd-1.5.5.tar.gz",
-            "f7de13462f7a82c29ab865820149e778cbfe01087b3a55b5332707abf9db4a6e",
+            "98e9c3d949d1b924e28e01eccb7deed865eefebf25c2f21c702e5cd5b63b85e1",
-            "https://github.com/facebook/zstd/archive/refs/tags/v1.5.2.tar.gz",
+            "https://github.com/facebook/zstd/archive/refs/tags/v1.5.5.tar.gz",
         )
 
     class Builder(MesonBuilder):

kiwixbuild/flatpak_builder.py

@@ -37,19 +37,19 @@ MANIFEST = {
     "runtime": "org.kde.Platform",
     "runtime-version": base_deps_versions["org.kde"],
     "base": "io.qt.qtwebengine.BaseApp",
-    "base-version": base_deps_versions[
+    "base-version": base_deps_versions["io.qt.qtwebengine"],
-        "org.kde"
-    ],  # keep BaseApp (qwebengine) in sync with org.kde
     "sdk": "org.kde.Sdk",
     "command": "kiwix-desktop",
     "rename-icon": "kiwix-desktop",
     "finish-args": [
+        "--device=dri",
+        "--env=QTWEBENGINEPROCESS_PATH=/app/bin/QtWebEngineProcess",
         "--socket=wayland",
-        "--socket=x11",
+        "--socket=fallback-x11",
+        "--socket=pulseaudio",
         "--share=network",
         "--share=ipc",
-        "--device=dri",
+        "--filesystem=host:ro",
-        "--socket=pulseaudio",
     ],
     "cleanup": [
         "/include",
@@ -69,6 +69,7 @@ MANIFEST = {
         "/share/doc",
         "/share/man",
     ],
+    "cleanup-commands": ["/app/cleanup-BaseApp.sh"],
 }
 
 GET_REF_URL_API_TEMPLATE = "https://api.github.com/repos{repo}/git/refs/tags/{ref}"
@@ -158,10 +159,12 @@ class FlatpakBuilder:
                     module["no-autogen"] = True
                 module_sources = module.setdefault("sources", [])
                 if isinstance(source, ReleaseDownload):
+                    for archive in source.archives:
                         src = {
                             "type": "archive",
-                        "sha256": source.archive.sha256,
+                            "dest-filename": archive.name,
-                        "url": source.archive.url,
+                            "sha256": archive.sha256,
+                            "url": archive.url,
                         }
                         if hasattr(source, "flatpak_dest"):
                             src["dest"] = source.flatpak_dest
@@ -297,7 +300,7 @@ class FlatpakBuilder:
                     tlc = Dependency.all_deps[tlcName]
                     builderDef = (cfgName, tlcName)
                     builder = get_target_step(builderDef)
-                    print("build {} ({}):".format(builder.name, cfgName[0]))
+                    print("build {} ({}):".format(builder.name, cfgName))
                     add_target_step(builderDef, builder)
                     builder.build()
             print("[GENERATE FLATPAK MANIFEST]")

kiwixbuild/packages.py

@@ -50,7 +50,7 @@ PACKAGE_NAME_MAPPERS = {
     },
     "fedora_native_static": {
         "COMMON": _fedora_common + ["glibc-static", "libstdc++-static"],
-        "lzma": ["xz-devel", "xz-static"]
+        "lzma": ["xz-devel", "xz-static"],
         # Either there is no packages, or no static or too old
     },
     "fedora_i586_dyn": {
@@ -59,33 +59,6 @@ PACKAGE_NAME_MAPPERS = {
     "fedora_i586_static": {
         "COMMON": _fedora_common + ["glibc-devel.i686"],
     },
-    "fedora_win32_dyn": {
-        "COMMON": _fedora_common
-        + [
-            "mingw32-gcc-c++",
-            "mingw32-bzip2",
-            "mingw32-win-iconv",
-            "mingw32-winpthreads",
-            "wine",
-        ],
-        "zlib": ["mingw32-zlib"],
-        "lzma": ["mingw32-xz-libs"],
-        "libmicrohttpd": ["mingw32-libmicrohttpd"],
-    },
-    "fedora_win32_static": {
-        "COMMON": _fedora_common
-        + [
-            "mingw32-gcc-c++",
-            "mingw32-bzip2-static",
-            "mingw32-win-iconv-static",
-            "mingw32-winpthreads-static",
-            "wine",
-        ],
-        "zlib": ["mingw32-zlib-static"],
-        "lzma": ["mingw32-xz-libs-static"],
-        "libmicrohttpd": None,  # ['mingw32-libmicrohttpd-static'] packaging dependecy seems buggy, and some static lib are name libfoo.dll.a and
-        # gcc cannot found them.
-    },
     "fedora_armhf_static": {"COMMON": _fedora_common},
     "fedora_armhf_dyn": {"COMMON": _fedora_common},
     "fedora_android": {"COMMON": _fedora_common},
@@ -109,24 +82,6 @@ PACKAGE_NAME_MAPPERS = {
         "COMMON": _debian_common
         + ["libc6-dev-i386", "lib32stdc++6", "gcc-multilib", "g++-multilib"],
     },
-    "debian_win32_dyn": {
-        "COMMON": _debian_common
-        + [
-            "g++-mingw-w64-i686",
-            "gcc-mingw-w64-i686",
-            "gcc-mingw-w64-base",
-            "mingw-w64-tools",
-        ],
-    },
-    "debian_win32_static": {
-        "COMMON": _debian_common
-        + [
-            "g++-mingw-w64-i686",
-            "gcc-mingw-w64-i686",
-            "gcc-mingw-w64-base",
-            "mingw-w64-tools",
-        ],
-    },
     "debian_armhf_static": {
         "COMMON": _debian_common,
     },

kiwixbuild/patches/android-ndk-r21e-linux-x86_64-python3.12+.patch

@@ -0,0 +1,33 @@
+diff -ur android-ndk-r21e/build/tools/make_standalone_toolchain.py android-ndk-r21e.patched/build/tools/make_standalone_toolchain.py
+--- android-ndk-r21e/build/tools/make_standalone_toolchain.py	2025-03-04 20:48:14.681288830 +0400
++++ android-ndk-r21e.patched/build/tools/make_standalone_toolchain.py	2025-03-05 12:10:47.252578915 +0400
+@@ -1,4 +1,4 @@
+-#!/usr/bin/env python
++#!/usr/bin/env python3
+ #
+ # Copyright (C) 2016 The Android Open Source Project
+ #
+@@ -21,7 +21,6 @@
+ """
+ import argparse
+ import atexit
+-from distutils.dir_util import copy_tree
+ import inspect
+ import logging
+ import os
+@@ -221,13 +220,13 @@
+ 
+ def create_toolchain(install_path, arch, api, toolchain_path, host_tag):
+     """Create a standalone toolchain."""
+-    copy_tree(toolchain_path, install_path)
++    shutil.copytree(toolchain_path, install_path)
+     triple = get_triple(arch)
+     make_clang_scripts(install_path, arch, api, host_tag == 'windows-x86_64')
+     replace_gcc_wrappers(install_path, triple, host_tag == 'windows-x86_64')
+ 
+     prebuilt_path = os.path.join(NDK_DIR, 'prebuilt', host_tag)
+-    copy_tree(prebuilt_path, install_path)
++    shutil.copytree(prebuilt_path, install_path, dirs_exist_ok=True)
+ 
+     gdbserver_path = os.path.join(
+         NDK_DIR, 'prebuilt', 'android-' + arch, 'gdbserver')

kiwixbuild/patches/docopt_meson_install_pkgconfig.patch

@@ -0,0 +1,31 @@
+diff '--color=auto' -ur docoptcpp-0.6.2/meson.build docoptcpp-0.6.2_patched/meson.build
+--- docoptcpp-0.6.2/meson.build	2024-08-26 14:28:47.553448529 +0200
++++ docoptcpp-0.6.2_patched/meson.build	2024-08-26 14:10:47.232603427 +0200
+@@ -10,11 +10,25 @@
+ 
+ # bug with missing dllexport. fixed in next version.
+ if cpp.get_argument_syntax() == 'msvc'
+-  doclib = static_library('docopt', 'docopt.cpp')
++  doclib = static_library('docopt', 'docopt.cpp', install: true)
+ else
+-  doclib = library('docopt', 'docopt.cpp')
++  doclib = library('docopt', 'docopt.cpp', install: true)
+ endif
+ 
+ executable('docopt_example', 'examples/naval_fate.cpp', link_with: doclib)
+ docopt_dep = declare_dependency(include_directories: include_directories('.'),
+     link_with: doclib)
++
++install_headers(
++  'docopt.h',
++  'docopt_value.h',
++  subdir: 'docopt'
++)
++
++pkg_mod = import('pkgconfig')
++pkg_mod.generate(
++  doclib,
++  version: meson.project_version(),
++  name: 'docopt',
++  filebase: 'docopt'
++)

kiwixbuild/patches/docopt_meson_use_boostregex.patch

@@ -0,0 +1,14 @@
+diff -ur docoptcpp-0.6.2/meson.build docoptcpp-0.6.2_boostregex/meson.build
+--- docoptcpp-0.6.2/meson.build	2024-08-28 17:22:46.256716100 +0200
++++ docoptcpp-0.6.2_boostregex/meson.build	2024-08-28 17:02:47.932681000 +0200
+@@ -8,6 +8,10 @@
+   add_project_arguments('-DDOCOPT_DLL', '-DDOCOPT_EXPORTS', language: 'cpp')
+ endif
+ 
++if cpp.get_id() =='msvc'
++  add_project_arguments('-DDOCTOPT_USE_BOOST_REGEX', '-DBOOST_REGEX_STANDALONE', language: 'cpp')
++endif
++
+ # bug with missing dllexport. fixed in next version.
+ if cpp.get_argument_syntax() == 'msvc'
+   doclib = static_library('docopt', 'docopt.cpp', install: true)

kiwixbuild/patches/libmicrohttpd_meson_pkgconfig.patch

@@ -0,0 +1,29 @@
+diff '--color=auto' -ur libmicrohttpd-0.9.76_orig/meson.build libmicrohttpd-0.9.76/meson.build
+--- libmicrohttpd-0.9.76_orig/meson.build	2024-08-18 14:55:29.372805433 +0200
++++ libmicrohttpd-0.9.76/meson.build	2024-08-18 17:19:44.087444728 +0200
+@@ -6,6 +6,8 @@
+   default_options: ['warning_level=1'],
+ )
+ 
++pkg = import('pkgconfig')
++
+ add_project_arguments('-D_GNU_SOURCE', language: 'c')
+ 
+ incdirs = include_directories('src/include')
+@@ -267,6 +269,16 @@
+   install: true,
+ )
+ 
++install_headers(
++  'src/include/microhttpd.h',
++)
++
++pkg.generate(
++  libmicrohttpd,
++  description: 'Libmicrohttpd',
++  name: 'libmicrohttpd',
++)
++
+ depinc = include_directories('.', 'src/include')
+ libmicrohttpd_dep = declare_dependency(
+   include_directories: depinc,

kiwixbuild/patches/libmicrohttpd_meson_timeval_tvsec_size.patch

@@ -0,0 +1,31 @@
+diff '--color=auto' -ur libmicrohttpd-0.9.76/meson.build libmicrohttpd-0.9.76_patched/meson.build
+--- libmicrohttpd-0.9.76/meson.build	2024-08-22 15:17:59.217715872 +0200
++++ libmicrohttpd-0.9.76_patched/meson.build	2024-08-22 15:20:23.755358647 +0200
+@@ -126,7 +126,26 @@
+ foreach s : sizes
+   cdata.set('SIZEOF_@0@'.format(s.underscorify().to_upper()), cc.sizeof(s))
+ endforeach
+-cdata.set('SIZEOF_STRUCT_TIMEVAL_TV_SEC', cc.sizeof('time_t'))
++
++cdata.set(
++  'SIZEOF_STRUCT_TIMEVAL_TV_SEC',
++  cc.sizeof(
++    'test_var.tv_sec',
++    prefix: '''#ifdef HAVE_SYS_TIME_H
++#include <sys/time.h>
++#endif /* HAVE_SYS_TIME_H */
++#ifdef HAVE_TIME_H
++#include <time.h>
++#endif /* HAVE_TIME_H */
++#if HAVE_SYS_TYPES_H
++#include <sys/types.h>
++#endif /* HAVE_SYS_TYPES_H */
++#ifdef _WIN32
++#include <winsock.h>
++#endif /* _WIN32 */
++struct timeval test_var;'''
++  )
++)
+ cdata.set('SIZEOF_UINT64_T', 8)
+ 
+ cdata.set('HAVE_PIPE2_FUNC', cc.has_function('pipe2'))

kiwixbuild/patches/libmicrohttpd_meson_winet6.patch

@@ -0,0 +1,15 @@
+--- libmicrohttpd-0.9.76_orig/meson.build	2024-10-08 15:53:53.370828250 +0400
++++ libmicrohttpd-0.9.76/meson.build	2024-10-08 16:23:24.985668690 +0400
+@@ -77,7 +77,11 @@
+ endforeach
+
+ cdata.set('HAVE_ASSERT', cc.has_header_symbol('assert.h', 'assert'))
+-cdata.set10('HAVE_INET6', cc.has_header_symbol('netinet/in.h', 'struct in6_addr'))
++if host_machine.system() == 'windows'
++  cdata.set10('HAVE_INET6', 1)
++else
++  cdata.set10('HAVE_INET6', cc.has_header_symbol('netinet/in.h', 'struct in6_addr'))
++endif
+
+ functions = [
+   'accept4',

kiwixbuild/patches/lzma_meson_install.patch

@@ -0,0 +1,54 @@
+diff '--color=auto' -ur lzma-5.2.6_orig/src/liblzma/meson.build lzma-5.2.6/src/liblzma/meson.build
+--- lzma-5.2.6_orig/src/liblzma/meson.build	2023-11-23 14:31:26.110195070 +0100
++++ lzma-5.2.6/src/liblzma/meson.build	2023-12-06 17:04:49.325148650 +0100
+@@ -1,3 +1,5 @@
++pkg = import('pkgconfig')
++
+ lzma_sources = [
+   '../common/tuklib_physmem.c',
+   'common/common.c',
+@@ -121,12 +123,44 @@
+ lzmainc = include_directories('api', 'common',
+   'check', 'lz', 'rangecoder', 'lzma', 'delta', 'simple', '../common')
+ 
++
++install_headers(
++  'api/lzma.h',
++)
++
++install_headers(
++  'api/lzma/version.h',
++  'api/lzma/base.h',
++  'api/lzma/vli.h',
++  'api/lzma/check.h',
++  'api/lzma/filter.h',
++  'api/lzma/bcj.h',
++  'api/lzma/delta.h',
++  'api/lzma/lzma12.h',
++  'api/lzma/container.h',
++  'api/lzma/stream_flags.h',
++  'api/lzma/block.h',
++  'api/lzma/index.h',
++  'api/lzma/index_hash.h',
++  'api/lzma/hardware.h',
++  subdir: 'lzma'
++)
++
+ liblzma = library('lzma', lzma_sources,
+   main_dec_sources, main_enc_sources, check_sources,
+   simplefilter_sources, lzma1_sources,
+   lz_sources, delta_sources,
+   include_directories : [confinc, lzmainc],
+   c_args : ['-DHAVE_CONFIG_H', '-DTUKLIB_SYMBOL_PREFIX=lzma_'],
++  install: true
++)
++
++pkg.generate(liblzma,
++  name: 'liblzma',
++  filebase: 'liblzma',
++  description: 'The liblzma compression library',
++  version: meson.project_version(),
++  url: 'http://tukaani.org/xz/'
+ )
+ 
+ lzma_dep = declare_dependency(link_with : liblzma,

kiwixbuild/utils.py

@@ -13,7 +13,9 @@ from collections import namedtuple, defaultdict
 
 from kiwixbuild._global import neutralEnv, option
 
-pj = os.path.join
+
+def pj(*args):
+    return os.path.normpath(os.path.join(*args))
 
 
 COLORS = {
@@ -26,7 +28,7 @@ COLORS = {
 }
 
 
-REMOTE_PREFIX = "http://mirror.download.kiwix.org/dev/kiwix-build/"
+REMOTE_PREFIX = "https://dev.kiwix.org/kiwix-build/"
 
 
 def which(name):
@@ -61,9 +63,33 @@ class DefaultEnv(Defaultdict):
     def __getitem__(self, name):
         if name == b"PATH":
             raise KeyError
+        if name in ["PATH", "PKG_CONFIG_PATH", "LD_LIBRARY_PATH"]:
+            item = super().__getitem__(name)
+            if isinstance(item, PathArray):
+                return item
+            else:
+                item = PathArray(item)
+                self[name] = item
+                return item
         return super().__getitem__(name)
 
 
+def get_separator():
+    return ";" if neutralEnv("distname") == "Windows" else ":"
+
+
+class PathArray(list):
+    def __init__(self, value):
+        self.separator = get_separator()
+        if not value:
+            super().__init__([])
+        else:
+            super().__init__(value.split(self.separator))
+
+    def __str__(self):
+        return self.separator.join(self)
+
+
 def remove_duplicates(iterable, key_function=None):
     seen = set()
     if key_function is None:
@@ -77,7 +103,7 @@ def remove_duplicates(iterable, key_function=None):
 
 
 def get_sha256(path):
-    progress_chars = "/-\|"
+    progress_chars = "/-\\|"
     current = 0
     batch_size = 1024 * 8
     sha256 = hashlib.sha256()
@@ -137,7 +163,7 @@ def download_remote(what, where):
         context = None
     batch_size = 1024 * 8
     extra_args = {"context": context} if sys.version_info >= (3, 4, 3) else {}
-    progress_chars = "/-\|"
+    progress_chars = "/-\\|"
     try:
         with urllib.request.urlopen(what.url, **extra_args) as resource, open(
             file_path, "wb"
@@ -271,9 +297,22 @@ def extract_archive(archive_path, dest_dir, topdir=None, name=None):
                         if isdir(member):
                             continue
                         perm = (member.external_attr >> 16) & 0x1FF
+                        if perm:
                             os.chmod(pj(tmpdir, getname(member)), perm)
                 name = name or topdir
-                os.rename(pj(tmpdir, topdir), pj(dest_dir, name))
+                shutil.copytree(
+                    pj(tmpdir, topdir),
+                    pj(dest_dir, name),
+                    symlinks=True,
+                    dirs_exist_ok=True,
+                )
+                # Be sure that all directory in tmpdir are writable to allow correct suppersion of it
+                for root, dirs, _files in os.walk(tmpdir):
+                    for d in dirs:
+                        os.chmod(
+                            pj(root, d), stat.S_IWRITE | stat.S_IREAD | stat.S_IEXEC
+                        )
+
         else:
             if name:
                 dest_dir = pj(dest_dir, name)
@@ -295,6 +334,7 @@ def run_command(command, cwd, context, *, env=None, input=None):
         print("run command '{}'".format(command), file=log)
         print("current directory is '{}'".format(cwd), file=log)
         print("env is :", file=log)
+        env = {k: str(v) for k, v in env.items()}
         for k, v in env.items():
             print("  {} : {!r}".format(k, v), file=log)
 

kiwixbuild/versions.py

@@ -1,11 +1,11 @@
 # This file reference all the versions of the depedencies we use in kiwix-build.
 
 main_project_versions = {
-    "libzim": "9.1.0",
+    "libzim": "9.3.0",
-    "libkiwix": "13.1.0",
+    "libkiwix": "14.0.0",
     "kiwix-tools": "3.7.0",
-    "zim-tools": "3.4.0",
+    "zim-tools": "3.6.0",
-    "kiwix-desktop": "2.3.1",  # Also change KIWIX_DESKTOP_VERSION and KIWIX_DESKTOP_RELEASE in appveyor.yml
+    "kiwix-desktop": "2.4.1",
 }
 
 # This dictionnary specify what we need to build at each release process.
@@ -22,24 +22,18 @@ main_project_versions = {
 #     `(was ...)`.
 #    - Else, increment the value. If no value was present, see `(was ...)`.
 
-# For kiwix-desktop, if this is not None:
-# - set KIWIX_DESKTOP_RELEASE to 1
-# - set KIWIX_DESKTOP_VERSION to the version of the release (including release_versions)
-# If this is None:
-# - set KIWIX_DESKTOP_RELEASE to 0
-
 release_versions = {
-    "libzim": None,  # Depends of base deps (was 0)
+    "libzim": 1,  # Depends of base deps (was 0)
-    "libkiwix": None,  # Depends of libzim (was 0)
+    "libkiwix": None,  # Depends of libzim (was 1)
-    "kiwix-tools": 0,  # Depends of libkiwix and libzim (was None)
+    "kiwix-tools": None,  # Depends of libkiwix and libzim (was 2)
-    "zim-tools": None,  # Depends of libzim (was 0)
+    "zim-tools": 0,  # Depends of libzim (was None)
-    "kiwix-desktop": None,  # Depends of libkiwix and libzim (was 4)
+    "kiwix-desktop": None,  # Depends of libkiwix and libzim (was 0)
 }
 
 
 # This is the "version" of the whole base_deps_versions dict.
 # Change this when you change base_deps_versions.
-base_deps_meta_version = "98"
+base_deps_meta_version = "12"
 
 base_deps_versions = {
     "zlib": "1.2.12",
@@ -47,7 +41,7 @@ base_deps_versions = {
     "zstd": "1.5.2",
     "docoptcpp": "0.6.2",
     "uuid": "1.43.4",
-    "xapian-core": "1.4.23",
+    "xapian-core": "1.4.26",
     "mustache": "4.1",
     "pugixml": "1.2",
     "libmicrohttpd": "0.9.76",
@@ -56,10 +50,8 @@ base_deps_versions = {
     "libaria2": "1.37.0",
     "libmagic": "5.44",
     "android-ndk": "r21e",
-    "qt": "5.10.1",
+    "org.kde": "6.7",
-    "qtwebengine": "5.10.1",
+    "io.qt.qtwebengine": "6.7",
-    "org.kde": "5.15-21.08",
+    "zim-testing-suite": "0.8.0",
-    "io.qt.qtwebengine": "5.15-21.08",
-    "zim-testing-suite": "0.5",
     "emsdk": "3.1.41",
 }

scripts/create_kiwix-desktop_appImage.sh

@@ -14,32 +14,34 @@ fi
 ICONFILE=$SOURCEDIR/resources/icons/kiwix/scalable/kiwix-desktop.svg
 DESKTOPFILE=$SOURCEDIR/resources/org.kiwix.desktop.desktop
 
-# Create structure
+# Get linuxdeploy
-mkdir -p $APPDIR/usr/{bin,lib,share} $APPDIR/usr/share/applications $APPDIR/usr/share/icons/hicolor/48x48/apps
+wget --continue https://github.com/linuxdeploy/linuxdeploy/releases/download/1-alpha-20240109-1/linuxdeploy-x86_64.AppImage
-# Copy our files
+chmod u+x linuxdeploy-x86_64.AppImage
-cp $INSTALLDIR/bin/kiwix-desktop $APPDIR/usr/bin/
+wget --continue https://github.com/linuxdeploy/linuxdeploy-plugin-qt/releases/download/1-alpha-20240109-1/linuxdeploy-plugin-qt-x86_64.AppImage
-cp $INSTALLDIR/$SYSTEMLIBDIR/*.so* $APPDIR/usr/lib
+chmod u+x linuxdeploy-plugin-qt-x86_64.AppImage
-# Remove it as it break with linuxdeployqt (should we compile without it) ?
-rm -f $APPDIR/usr/lib/libmagic.so*
-# Copy nss lib (to not conflict with host's ones)
-cp -a /usr/$SYSTEMLIBDIR/nss $APPDIR/usr/lib
-cp $ICONFILE $APPDIR/usr/share/icons/hicolor/48x48/apps/kiwix-desktop.svg
-mkdir -p $APPDIR/usr/share/applications
-cp $DESKTOPFILE $APPDIR/usr/share/applications/kiwix-desktop.desktop
-
-# get the aria2
-wget --continue https://github.com/q3aql/aria2-static-builds/releases/download/v1.36.0/aria2-1.36.0-linux-gnu-64bit-build1.tar.bz2
-mkdir -p $APPDIR/usr/bin/ && tar -C $APPDIR/usr/bin/ -xf aria2-1.36.0-linux-gnu-64bit-build1.tar.bz2 aria2-1.36.0-linux-gnu-64bit-build1/aria2c --strip-components=1
-mkdir -p $APPDIR/etc/ssl/certs/ && tar -C $APPDIR/etc/ssl/certs/ -xf aria2-1.36.0-linux-gnu-64bit-build1.tar.bz2 aria2-1.36.0-linux-gnu-64bit-build1/ca-certificates.crt --strip-components=1
-
-# Get linuxdeployqt
-# Dispite the 'continuous' in the file name, it IS release 8
-wget --continue https://github.com/probonopd/linuxdeployqt/releases/download/continuous/linuxdeployqt-continuous-x86_64.AppImage -O linuxdeployqt
-chmod u+x linuxdeployqt
 
 # Fill with all deps libs and so
-./linuxdeployqt $APPDIR/usr/bin/kiwix-desktop -bundle-non-qt-libs -extra-plugins=imageformats,iconengines
+LD_LIBRARY_PATH=$INSTALLDIR/lib/x86_64-linux-gnu ./linuxdeploy-x86_64.AppImage \
+    --plugin=qt \
+    --appdir="$APPDIR" \
+    --executable=$INSTALLDIR/bin/kiwix-desktop \
+    --desktop-file=$DESKTOPFILE \
+    --icon-file=$ICONFILE \
+    --library=/usr/lib/x86_64-linux-gnu/libthai.so.0 \
+
+# Get the aria2c (https://github.com/abcfy2/aria2-static-build/releases/download/1.37.0/aria2-x86_64-linux-musl_libressl_static.zip)
+wget --continue https://dev.kiwix.org/kiwix-desktop/aria2-1.37.0-x86_64-linux-musl_libressl_static.zip
+mkdir -p $APPDIR/usr/bin/ && unzip aria2-1.37.0-x86_64-linux-musl_libressl_static.zip -d $APPDIR/usr/bin/
+
+# Copy the CA trustore from the hosting system
+mkdir -p $APPDIR/etc/ssl/certs/ && cp /etc/ssl/certs/ca-certificates.crt $APPDIR/etc/ssl/certs/
+
 # Fix the RPATH of QtWebEngineProcess [TODO] Fill a issue ?
 patchelf --set-rpath '$ORIGIN/../lib' $APPDIR/usr/libexec/QtWebEngineProcess
-# Build the image.
+
-./linuxdeployqt $APPDIR/usr/share/applications/kiwix-desktop.desktop -bundle-non-qt-libs -extra-plugins=imageformats,iconengines -appimage
+mv $APPDIR/{AppRun.wrapped,kiwix-desktop}
+sed -i 's/AppRun\.wrapped/kiwix-desktop/g' $APPDIR/AppRun
+wget --continue https://github.com/AppImage/AppImageKit/releases/download/13/appimagetool-x86_64.AppImage
+chmod u+x appimagetool-x86_64.AppImage
+
+./appimagetool-x86_64.AppImage AppDir Kiwix-"$VERSION"-x86_64.AppImage

scripts/package_kiwix-desktop_windows.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+
+import sys, subprocess, shutil, argparse, os
+from pathlib import Path
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument("install_dir")
+parser.add_argument("out_dir")
+parser.add_argument("--static_dir")
+parser.add_argument("archive_path", help="The full path of the archive to create")
+parser.add_argument("-s", "--sign", action="store_true")
+
+args = parser.parse_args()
+
+install_dir = Path(args.install_dir)
+if args.static_dir:
+    static_dir = Path(args.static_dir)
+else:
+    static_dir = install_dir
+
+out_dir = Path(args.out_dir)
+archive_path = Path(args.archive_path)
+
+out_dir.mkdir(parents=True, exist_ok=True)
+
+print(
+    f"""Packaging kiwix-desktop
+- From {install_dir}
+- Static dir is {static_dir}
+- Working dir {out_dir}
+- Archive path is {archive_path}
+- Python version {sys.version}"""
+)
+
+shutil.copy2(install_dir / "bin" / "kiwix-desktop.exe", out_dir)
+subprocess.run(["windeployqt", "--compiler-runtime", str(out_dir)], check=True)
+
+
+shutil.copy2(static_dir / "bin" / "aria2c.exe", out_dir)
+
+for dll in (static_dir / "bin").glob("*.dll"):
+    shutil.copy2(dll, out_dir)
+
+
+# Copy ssl stuff
+ssl_directory = Path("C:/") / "Program Files" / "OpenSSL"
+shutil.copy2(ssl_directory / "libcrypto-1_1-x64.dll", out_dir)
+shutil.copy2(ssl_directory / "libssl-1_1-x64.dll", out_dir)
+
+if args.sign:
+    # We assume here that signtool and certificate are properly configured.
+    # Env var `SIGNTOOL_THUMBPRINT` must contain thumbprint of the certificate to use.
+    command = [
+        os.getenv("SIGNTOOL_PATH", "signtool.exe"),
+        "sign",
+        "/fd",
+        "sha256",
+        "/tr",
+        "http://ts.ssl.com",
+        "/td",
+        "sha256",
+        "/sha1",
+        os.environ["SIGNTOOL_THUMBPRINT"],
+        str(out_dir / "kiwix-desktop.exe"),
+    ]
+    subprocess.run(command, check=True)
+
+print(
+    f"""Create archive
+- {archive_path.with_suffix('')}
+- From {out_dir.parent}
+- With {out_dir.name}"""
+)
+shutil.make_archive(
+    archive_path.with_suffix(""), "zip", root_dir=out_dir, base_dir=""
+)

scripts/upload_libkiwix_android_to_bintray.py

@@ -1,83 +0,0 @@
-#!/usr/bin/env python3
-
-import os, sys
-import json
-import requests
-
-bintray_auth = (os.environ['BINTRAY_USER'], os.environ['BINTRAY_PASS'])
-
-def create_version(version):
-    url = "https://api.bintray.com/packages/kiwix/kiwix/kiwixlib/versions"
-    payload = {
-        'name': version,
-        'desc': 'Release of libkiwix'
-    }
-    headers = {
-        'Content-Type': 'application/json'
-    }
-
-    r = requests.post(url, data=json.dumps(payload), headers=headers, auth=bintray_auth)
-    rcode = r.status_code
-
-    if rcode == 409:
-        print("Bintray version %s already exists, skipping." % version)
-        return True
-
-    rcode_family = rcode // 100
-    if rcode_family in (2, 3):
-        print("Bintray Version created!")
-        return True
-
-    print("ERROR : Bintray API response {}".format(rcode))
-    return False
-
-
-def upload(version, filepath, artefact):
-    url_template = "https://api.bintray.com/content/kiwix/kiwix/kiwixlib/{version}/org/kiwix/kiwixlib/kiwixlib/{version}/{artefact}"
-    parameters = {
-        'publish': 1,
-        'override': 1
-    }
-
-    # Upload the main artefact
-    url = url_template.format(version=version, artefact=artefact)
-    with open(filepath, 'rb') as f:
-        r = requests.put(url, data=f, auth=bintray_auth, params=parameters)
-
-    rcode = r.status_code
-    rcode_family = rcode // 100
-    if rcode_family not in (2, 3):
-        print("ERROR: Fail to upload artefact")
-        print(r.text)
-        return False
-
-    return True
-
-
-def upload_from_json(json_path):
-    basedir = os.path.dirname(json_path)
-    with open(str(json_path)) as f:
-        options = json.load(f)
-
-    if not create_version(options['version']):
-        raise RuntimeError("Cannot create version")
-
-    for file_ in options['files']:
-        path = os.path.join(basedir, file_)
-        if not upload(options['version'], path, file_):
-            raise RuntimeError("Cannot upload file {}".format(file_))
-
-
-if __name__ == "__main__":
-    try:
-        info_file = sys.argv[1]
-    except IndexError:
-        print("Usage {} infofile".format(sys.argv[0]))
-        sys.exit(-1)
-
-    print("Use info file {}".format(info_file))
-    try:
-        upload_from_json(info_file)
-    except RuntimeError as e:
-        sys.exit(str(e))
-

setup.py

@@ -36,7 +36,8 @@ setup(
     include_package_data=True,
     install_requires=[
         'meson',
-        'distro'
+        'distro',
+        'paramiko'
     ],
     entry_points={
         'console_scripts': [