Merge pull request #703 from kiwix/libzim_github_ci_windows

Libzim GitHub CI windows

.github/scripts/build_definition.py

@@ -21,6 +21,11 @@ BUILD_DEF = """
     | manylinux | native_mixed       | BP     |          |           |             |               | linux-x86_64-manylinux  |                        |
     | manylinux | aarch64_mixed      | BP     |          |           |             |               | linux-aarch64-manylinux |                        |
     ----------------------------------------------------------------------------------------------------------------------------------------------
+# On Windows, we build only libzim for now. And only native_mixed as xapian doesn't compile as dll
+    | windows   | native_static      | Bd     |          |           |             |               | win-x86_64              | win-x86_64-static      |
+    | windows   | native_dyn         | Bd     |          |           |             |               | win-x86_64              | win-x86_64-dyn         |
+    | windows   | native_mixed       | BPd    |          |           |             |               | win-x86_64              | win-x86_64-mixed       |
+    ----------------------------------------------------------------------------------------------------------------------------------------------
 # Osx builds, build binaries on native_dyn and native_static. On anyother things, build only the libraries
     | macos     | native_dyn         | d      | d        | dB        | B           |               |                         | macos-x86_64-dyn       |
     | macos     | native_static      |        |          | BP        | BP          |               | macos-x86_64            |                        |

.github/scripts/common.py

@@ -1,12 +1,13 @@
 import os
 from os import environ as _environ
-from pathlib import Path
+from pathlib import Path, PurePosixPath
 from datetime import date
 import tarfile
 import zipfile
 import subprocess
 import re
 import shutil
+import platform
 
 import requests
 
@@ -27,7 +28,7 @@ def get_build_dir(config) -> Path:
     command.append("--use-target-arch-name")
     return Path(
         subprocess.run(command, cwd=str(HOME), check=True, stdout=subprocess.PIPE)
-        .stdout[:-1]
+        .stdout.strip()
         .decode("utf8")
     )
 
@@ -41,7 +42,8 @@ SOURCE_DIR = HOME / "SOURCE"
 ARCHIVE_DIR = HOME / "ARCHIVE"
 TOOLCHAIN_DIR = BASE_DIR / "TOOLCHAINS"
 INSTALL_DIR = BASE_DIR / "INSTALL"
-TMP_DIR = Path(os.getenv("TMP_DIR", "/tmp"))
+default_tmp_dir = os.getenv("TEMP") if platform.system() == 'Windows' else "/tmp"
+TMP_DIR = Path(os.getenv("TMP_DIR", default_tmp_dir))
 KBUILD_SOURCE_DIR = HOME / "kiwix-build"
 
 _ref = _environ.get("GITHUB_REF", "").split("/")[-1]
@@ -195,8 +197,65 @@ def run_kiwix_build(
     subprocess.check_call(command, cwd=str(HOME), env=env)
     print_message("Build ended")
 
+try:
+    import paramiko
 
-def upload(file_to_upload, host, dest_path):
+    def upload(file_to_upload, host, dest_path):
+        if not file_to_upload.exists():
+            print_message("No {} to upload!", file_to_upload)
+            return
+
+        if ":" in host:
+            host, port = host.split(":", 1)
+        else:
+            port = "22"
+        if '@' in host:
+            user, host = host.split('@', 1)
+        else:
+            user = None
+
+        from contextlib import contextmanager
+
+        @contextmanager
+        def get_client():
+            client = paramiko.client.SSHClient()
+            client.set_missing_host_key_policy(paramiko.client.WarningPolicy)
+            print_message(f"Connect to {host}:{port}")
+            client.connect(host, port=port, username=user, key_filename=_environ.get("SSH_KEY"), look_for_keys=False, compress=True)
+            try:
+                yield client
+            finally:
+                client.close()
+
+        @contextmanager
+        def get_sftp():
+            with get_client() as client:
+                sftp = client.open_sftp()
+                try:
+                    yield sftp
+                finally:
+                    sftp.close()
+
+        dest_path = PurePosixPath(dest_path)
+        remote_file = dest_path.joinpath(file_to_upload.name)
+
+        with get_sftp() as sftp:
+            for part in list(reversed(dest_path.parents)) + [dest_path]:
+                part = str(part)
+                try:
+                    sftp.stat(part)
+                except FileNotFoundError:
+                    sftp.mkdir(part)
+
+            print_message(f"Sending archive {file_to_upload} to {remote_file}")
+            sftp.put(str(file_to_upload), str(remote_file), confirm=True)
+
+
+except ModuleNotFoundError:
+    # On old system (bionic) paramiko is really complex to install
+    # Keep the old implementaion on sush system.
+
+    def upload(file_to_upload, host, dest_path):
         if not file_to_upload.exists():
             print_message("No {} to upload!", file_to_upload)
             return

.github/scripts/upload_failure_logs.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+
+import tarfile
+from pathlib import Path
+from common import upload, OS_NAME, COMPILE_CONFIG, HOME
+
+ARCHIVE_NAME = Path(f"fail_log_{OS_NAME}_{COMPILE_CONFIG}.tar.gz")
+
+
+files_to_archive = []
+files_to_archive += HOME.glob("BUILD_*")
+files_to_archive += [HOME / "SOURCE", HOME / "LOGS", HOME / "TOOLCHAINS"]
+
+with tarfile.open(ARCHIVE_NAME, "w:xz") as tar:
+    for name in set(files_to_archive):
+        tar.add(str(name))
+
+upload(ARCHIVE_NAME, "ci@tmp.kiwix.org:30022", "/data/tmp/ci")

.github/scripts/upload_failure_logs.sh

@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-cd $HOME
-
-ARCHIVE_NAME=fail_log_${OS_NAME}_${COMPILE_CONFIG}.tar.gz
-tar -czf ${ARCHIVE_NAME} $HOME/BUILD_* $HOME/SOURCE $HOME/LOGS $HOME/TOOLCHAINS
-
-echo "Uploading archive $ARCHIVE_NAME"
-
-scp -c aes128-ctr -P 30022 -p -i ${SSH_KEY} \
-  -o PasswordAuthentication=no \
-  -o StrictHostKeyChecking=no \
-  $ARCHIVE_NAME \
-  ci@tmp.kiwix.org:/data/tmp/ci

.github/workflows/ci.yml

@@ -6,6 +6,61 @@ on:
    - cron: '0 1 * * *'
 
 jobs:
+  Windows:
+    strategy:
+      fail-fast: false
+    runs-on: windows-latest
+    env:
+      OS_NAME: windows
+      COMPILE_CONFIG: native_dyn
+      HOME: 'C:\\Users\\runneradmin'
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+    - name: Setup python 3.8
+      uses: actions/setup-python@v3
+      with:
+        python-version: '3.8'
+    - name: Install packages
+      run: |
+        choco.exe install pkgconfiglite ninja
+    - name: Install python modules
+      shell: bash
+      run: |
+        pip3 install meson pytest requests distro paramiko
+        pip3 install --no-deps $GITHUB_WORKSPACE
+    - name: Setup MSVC compiler
+      uses: bus1/cabuild/action/msdevshell@v1
+      with:
+        architecture: x64
+    - name: secret
+      shell: bash
+      run: |
+        echo "${{secrets.ssh_key}}" > $SSH_KEY
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+    - name: Ensure base deps
+      run: |
+        python .github\\scripts\\ensure_base_deps.py
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+    - name: Compile all deps
+      run: |
+        python .github\\scripts\\compile_all_deps.py
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+    - name: Build projects
+      run: |
+        python .github\\scripts\\build_projects.py
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+    - name: Upload failure logs
+      if: failure()
+      run: |
+        python .github\\scripts\\upload_failure_logs.py
+      env:
+        SSH_KEY: ${{ runner.temp }}/id_rsa
+
   Linux:
     strategy:
       fail-fast: false
@@ -63,6 +118,10 @@ jobs:
         pip3 install --user --no-deps .
       env:
         REP: ${{github.repository}}
+    - name: Install paramiko
+      if: ${{matrix.image_variant != 'bionic' }}
+      shell: bash
+      run: pip3 install --user paramiko
     - name: secret
       shell: bash
       run: |
@@ -91,7 +150,7 @@ jobs:
         COMPILE_CONFIG: ${{matrix.config}}
     - name: Upload failure logs
       if: failure()
-      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh
+      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py
       env:
         COMPILE_CONFIG: ${{matrix.config}}
 
@@ -112,6 +171,7 @@ jobs:
         git clone https://github.com/${REP}
         cd ./${REP##*/}
         git checkout --force ${GITHUB_SHA}
+        pip3 install --user paramiko
         pip3 install --user --no-deps .
       env:
         REP: ${{github.repository}}
@@ -136,7 +196,7 @@ jobs:
         kiwix-build/.github/scripts/build_projects.py
     - name: Upload failure logs
       if: failure()
-      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh
+      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py
 
   Macos:
     strategy:
@@ -172,7 +232,7 @@ jobs:
         brew install pkg-config ninja automake autoconf
     - name: Install python modules
       run: |
-        pip3 install meson pytest requests distro
+        pip3 install meson pytest requests distro paramiko
         pip3 install --no-deps $GITHUB_WORKSPACE
     - name: secret
       shell: bash
@@ -202,6 +262,6 @@ jobs:
         COMPILE_CONFIG: ${{matrix.config}}
     - name: Upload failure logs
       if: failure()
-      run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.sh
+      run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.py
       env:
         COMPILE_CONFIG: ${{matrix.config}}

kiwixbuild/buildenv.py

@@ -37,15 +37,6 @@ class NeutralEnv:
     def detect_platform(self):
         _platform = platform.system()
         self.distname = _platform
-        if _platform == "Windows":
-            print(
-                "ERROR: kiwix-build is not intented to run on Windows platform.\n"
-                "It should probably not work, but well, you still can have a try.",
-                file=sys.stderr,
-            )
-            cont = input("Do you want to continue ? [y/N]")
-            if cont.lower() != "y":
-                sys.exit(0)
         if _platform == "Linux":
             self.distname = distro.id()
             if self.distname == "ubuntu":
@@ -132,13 +123,12 @@ class BuildEnv:
     def get_env(self, *, cross_comp_flags, cross_compilers, cross_path):
         env = self.configInfo.get_env()
         pkgconfig_path = pj(self.install_dir, self.libprefix, "pkgconfig")
-        env["PKG_CONFIG_PATH"] = ":".join([env["PKG_CONFIG_PATH"], pkgconfig_path])
+        env["PKG_CONFIG_PATH"].append(pkgconfig_path)
 
-        env["PATH"] = ":".join([escape_path(pj(self.install_dir, "bin")), env["PATH"]])
+        env["PATH"].insert(0, pj(self.install_dir, "bin"))
 
-        env["LD_LIBRARY_PATH"] = ":".join(
+        env["LD_LIBRARY_PATH"].extend(
             [
-                env["LD_LIBRARY_PATH"],
                 pj(self.install_dir, "lib"),
                 pj(self.install_dir, self.libprefix),
             ]
@@ -170,7 +160,7 @@ class BuildEnv:
         if cross_compilers:
             self.configInfo.set_compiler(env)
         if cross_path:
-            env["PATH"] = ":".join(self.configInfo.get_bin_dir() + [env["PATH"]])
+            env["PATH"][0:0] = self.configInfo.get_bin_dir()
         return env
 
     @property

kiwixbuild/configs/armhf.py

@@ -76,18 +76,15 @@ class ArmConfigInfo(ConfigInfo):
 
     def get_env(self):
         env = super().get_env()
-        env["LD_LIBRARY_PATH"] = ":".join(
+        env["LD_LIBRARY_PATH"][0:0] = [
-            [
             pj(self.root_path, self.arch_full, "lib64"),
             pj(self.root_path, "lib"),
-                env["LD_LIBRARY_PATH"],
         ]
-        )
         env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig")
         env["QEMU_LD_PREFIX"] = pj(self.root_path, self.arch_full, "libc")
         env["QEMU_SET_ENV"] = "LD_LIBRARY_PATH={}".format(
             ":".join(
-                [pj(self.root_path, self.arch_full, "lib"), env["LD_LIBRARY_PATH"]]
+                [pj(self.root_path, self.arch_full, "lib"), str(env["LD_LIBRARY_PATH"])]
             )
         )
         return env

kiwixbuild/configs/base.py

@@ -161,15 +161,13 @@ def MixedMixin(static_name):
 
         def get_env(self):
             env = super().get_env()
-            env["PATH"] = ":".join(
+            env["PATH"].insert(0, pj(self.static_buildEnv.install_dir, "bin"))
-                [pj(self.static_buildEnv.install_dir, "bin")] + [env["PATH"]]
-            )
             pkgconfig_path = pj(
                 self.static_buildEnv.install_dir,
                 self.static_buildEnv.libprefix,
                 "pkgconfig",
             )
-            env["PKG_CONFIG_PATH"] = ":".join([env["PKG_CONFIG_PATH"], pkgconfig_path])
+            env["PKG_CONFIG_PATH"].append(pkgconfig_path)
             env["CPPFLAGS"] = " ".join(
                 [
                    "-I" + pj(self.static_buildEnv.install_dir, "include"),

kiwixbuild/configs/musl.py

@@ -73,27 +73,22 @@ class MuslConfigInfo(ConfigInfo):
 
     def get_env(self):
         env = super().get_env()
-        env["LD_LIBRARY_PATH"] = ":".join(
+        env["LD_LIBRARY_PATH"][0:0] = [
-            [
             pj(self.root_path, self.arch_full, "lib64"),
             pj(self.root_path, "lib"),
-                env["LD_LIBRARY_PATH"],
         ]
-        )
         env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig")
         env["QEMU_LD_PREFIX"] = pj(self.root_path, self.arch_full, "libc")
         env["QEMU_SET_ENV"] = "LD_LIBRARY_PATH={}".format(
             ":".join(
-                [pj(self.root_path, self.arch_full, "lib"), env["LD_LIBRARY_PATH"]]
+                [pj(self.root_path, self.arch_full, "lib"), str(env["LD_LIBRARY_PATH"])]
             )
         )
         return env
 
     def set_comp_flags(self, env):
         super().set_comp_flags(env)
-        env["LD_LIBRARY_PATH"] = ":".join(
+        env["LD_LIBRARY_PATH"].insert(0, pj(self.root_path, self.arch_full, "lib"))
-            [pj(self.root_path, self.arch_full, "lib"), env["LD_LIBRARY_PATH"]]
-        )
         env["CFLAGS"] = (
             " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
             + env["CFLAGS"]

kiwixbuild/configs/native.py

@@ -29,7 +29,7 @@ class NativeConfigInfo(ConfigInfo):
 class NativeDyn(NativeConfigInfo):
     name = "native_dyn"
     static = False
-    compatible_hosts = ["fedora", "debian", "Darwin", "almalinux"]
+    compatible_hosts = ["fedora", "debian", "Darwin", "almalinux", "Windows"]
 
 
 class NativeStatic(NativeConfigInfo):

kiwixbuild/configs/wasm.py

@@ -79,9 +79,8 @@ class WasmConfigInfo(ConfigInfo):
 
     def get_env(self):
         env = super().get_env()
-        env["PATH"] = ":".join(
+        env["PATH"].extend(
             [
-                env["PATH"],
                 self.install_path,
                 pj(self.install_path, "upstream", "emscripten"),
                 pj(self.install_path, "node", "14.18.2_64bit", "bin"),

kiwixbuild/dependencies/all_dependencies.py

@@ -13,6 +13,8 @@ class AllBaseDependencies(Dependency):
     class Builder(NoopBuilder):
         @classmethod
         def get_dependencies(cls, configInfo, allDeps):
+            if neutralEnv("distname") == "Windows":
+                return ["zlib", "zstd", "icu4c", "zim-testing-suite"]
             if configInfo.build == "wasm" or environ.get("OS_NAME") == "manylinux":
                 return ["zlib", "lzma", "zstd", "icu4c", "xapian-core"]
 

kiwixbuild/dependencies/base.py

@@ -131,8 +131,17 @@ class ReleaseDownload(Source):
 
     def _download(self, context):
         context.try_skip(neutralEnv("archive_dir"), self.full_name)
-        for archive in self.archives:
+        archive_iter = iter(self.archives)
+        archive = next(archive_iter, None)
+        while archive:
+            try:
                 neutralEnv("download")(archive)
+            except SkipCommand as e:
+                archive = next(archive_iter, None)
+                if not archive:
+                    raise e
+                continue
+            archive = next(archive_iter, None)
 
     def _extract(self, context):
         context.try_skip(self.extract_path)
@@ -544,6 +553,7 @@ class MesonBuilder(Builder):
             cross_options += ["--cross-file", self.buildEnv.meson_crossfile]
         command = [
             *neutralEnv("meson_command"),
+            "setup",
             ".",
             self.build_path,
             f"--buildtype={self.build_type}",

kiwixbuild/dependencies/icu4c.py

@@ -1,9 +1,10 @@
-from .base import Dependency, ReleaseDownload, MakeBuilder
+from .base import Dependency, ReleaseDownload, MakeBuilder, MesonBuilder
 
 from kiwixbuild.utils import pj, SkipCommand, Remotefile, extract_archive
 from kiwixbuild._global import get_target_step, neutralEnv
 import os, shutil
 import fileinput
+import platform
 
 
 class Icu(Dependency):
@@ -20,8 +21,13 @@ class Icu(Dependency):
             "ca1ee076163b438461e484421a7679fc33a64cd0a54f9d4b401893fa1eb42701",
             "https://github.com/unicode-org/icu/releases/download/release-73-2/icu4c-73_2-data.zip",
         )
+        meson_patch = Remotefile(
+            "icu_73.2-2_patch.zip",
+            "218a5f20b58b6b2372e636c2eb2d611a898fdc11be17d6c4f35a3cd54d472010",
+            "https://wrapdb.mesonbuild.com/v2/icu_73.2-2/get_patch",
+        )
 
-        archives = [archive_src, archive_data]
+        archives = [archive_src, archive_data, meson_patch]
 
         def _extract(self, context):
             context.try_skip(self.extract_path)
@@ -42,17 +48,30 @@ class Icu(Dependency):
                 topdir="data",
                 name="data",
             )
+            extract_archive(
+                pj(neutralEnv("archive_dir"), self.meson_patch.name),
+                neutralEnv("source_dir"),
+                topdir="icu",
+                name=self.source_dir,
+            )
 
         patches = [
             "icu4c_fix_static_lib_name_mingw.patch",
-            #           "icu4c_android_elf64_st_info.patch",
-            #           "icu4c_custom_data.patch",
-            #           "icu4c_noxlocale.patch",
             "icu4c_rpath.patch",
-            #           "icu4c_build_config.patch",
             "icu4c_wasm.patch",
         ]
 
+    if platform.system() == "Windows":
+
+        class Builder(MesonBuilder):
+            def set_env(self, env):
+                env["ICU_DATA_FILTER_FILE"] = pj(
+                    os.path.dirname(os.path.realpath(__file__)),
+                    "icu4c_data_filter.json",
+                )
+
+    else:
+
         class Builder(MakeBuilder):
             subsource_dir = "source"
             make_install_targets = ["install"]
@@ -83,14 +102,17 @@ class Icu(Dependency):
 
             def set_env(self, env):
                 env["ICU_DATA_FILTER_FILE"] = pj(
-                os.path.dirname(os.path.realpath(__file__)), "icu4c_data_filter.json"
+                    os.path.dirname(os.path.realpath(__file__)),
+                    "icu4c_data_filter.json",
                 )
 
             def _post_configure_script(self, context):
                 if self.buildEnv.configInfo.build != "wasm":
                     context.skip()
                 context.try_skip(self.build_path)
-            for line in fileinput.input(pj(self.build_path, "Makefile"), inplace=True):
+                for line in fileinput.input(
+                    pj(self.build_path, "Makefile"), inplace=True
+                ):
                     if line == "#DATASUBDIR = data\n":
                         print("DATASUBDIR = data")
                     else:

kiwixbuild/dependencies/libkiwix.py

@@ -18,7 +18,6 @@ class Libkiwix(Dependency):
             "pugixml",
             "libzim",
             "zlib",
-            "lzma",
             "libcurl",
             "libmicrohttpd",
             "icu4c",

kiwixbuild/dependencies/libmagic.py

@@ -46,7 +46,5 @@ class LibMagic(Dependency):
                 cross_comp_flags=True, cross_compilers=True, cross_path=True
             )
             libmagic_native_builder = get_target_step("libmagic", "native_static")
-            env["PATH"] = ":".join(
+            env["PATH"].insert(0, pj(libmagic_native_builder.build_path, "src"))
-                [pj(libmagic_native_builder.build_path, "src"), env["PATH"]]
-            )
             run_command(command, self.build_path, context, env=env)

kiwixbuild/dependencies/libzim.py

@@ -1,5 +1,5 @@
 from .base import Dependency, GitClone, MesonBuilder
-from kiwixbuild._global import option, get_target_step
+from kiwixbuild._global import option, get_target_step, neutralEnv
 
 
 class Libzim(Dependency):
@@ -10,6 +10,13 @@ class Libzim(Dependency):
         git_remote = "https://github.com/openzim/libzim.git"
         git_dir = "libzim"
 
+        @property
+        def git_ref(self):
+            if neutralEnv("distname") == "Windows":
+                return "libzim_github_ci_windows"
+            else:
+                return "main"
+
     class Builder(MesonBuilder):
         test_options = ["-t", "8"]
         strip_options = []
@@ -22,6 +29,8 @@ class Libzim(Dependency):
 
         @classmethod
         def get_dependencies(cls, configInfo, allDeps):
+            if neutralEnv("distname") == "Windows":
+                return ["zstd", "icu4c", "zim-testing-suite"]
             deps = ["lzma", "zstd", "xapian-core", "icu4c"]
             if configInfo.name not in ("flatpak", "wasm"):
                 deps.append("zim-testing-suite")
@@ -30,6 +39,9 @@ class Libzim(Dependency):
         @property
         def configure_options(self):
             configInfo = self.buildEnv.configInfo
+            if neutralEnv("distname") == "Windows":
+                yield "-Dwith_xapian=false"
+                yield "-Dwerror=false"
             if configInfo.build == "android":
                 yield "-DUSE_BUFFER_HEADER=false"
                 yield "-Dstatic-linkage=true"

kiwixbuild/dependencies/lzma.py

@@ -1,28 +1,26 @@
-from .base import Dependency, ReleaseDownload, MakeBuilder
+from .base import (
+    Dependency,
+    ReleaseDownload,
+    MesonBuilder)
 
 from kiwixbuild.utils import Remotefile
 
 
 class lzma(Dependency):
-    name = "lzma"
+    name = 'lzma'
 
     class Source(ReleaseDownload):
-        archive = Remotefile(
+        src_archive = Remotefile(
             "xz-5.2.6.tar.gz",
             "a2105abee17bcd2ebd15ced31b4f5eda6e17efd6b10f921a01cda4a44c91b3a0",
             "https://altushost-swe.dl.sourceforge.net/project/lzmautils/xz-5.2.6.tar.gz",
         )
+        meson_patch = Remotefile(
+            "liblzma_5.2.6-3_patch.zip",
+            "1c71536d364e1a3ce6bea61266576f89cc5cce4d3b9e11f3494417dafa29780b",
+            "https://wrapdb.mesonbuild.com/v2/liblzma_5.2.6-3/get_patch",
+        )
+        archives = [src_archive, meson_patch]
+        patches = ['lzma_meson_install.patch']
 
-    class Builder(MakeBuilder):
+    Builder = MesonBuilder
-        @property
-        def configure_options(self):
-            return [
-                "--disable-xz",
-                "--disable-xzdec",
-                "--disable-lzmadec",
-                "--disable-lzmainfo",
-                "--disable-lzma-links",
-                "--disable-scripts",
-                "--disable-doc",
-                #                    "--disable-symbol-versions"
-            ]

kiwixbuild/dependencies/xapian.py

@@ -27,6 +27,8 @@ class Xapian(Dependency):
 
         @classmethod
         def get_dependencies(cls, configInfo, allDeps):
+            if neutralEnv("distname") == "Windows":
+                return ["zlib"]
             deps = ["zlib", "lzma"]
             if (
                 configInfo.build in ("win32", "wasm")

kiwixbuild/dependencies/zlib.py

@@ -1,63 +1,27 @@
 import shutil
 
-from .base import Dependency, ReleaseDownload, MakeBuilder
+from .base import (
+    Dependency,
+    ReleaseDownload,
+    MesonBuilder)
 
 from kiwixbuild.utils import Remotefile, pj, SkipCommand
+from kiwixbuild._global import neutralEnv
 
 
 class zlib(Dependency):
     name = "zlib"
 
     class Source(ReleaseDownload):
-        archive = Remotefile(
+        src_archive = Remotefile(
             "zlib-1.2.12.tar.gz",
-            "91844808532e5ce316b3c010929493c0244f3d37593afd6de04f71821d5136d9",
+            "91844808532e5ce316b3c010929493c0244f3d37593afd6de04f71821d5136d9"
         )
-        patches = ["zlib_std_libname.patch"]
+        meson_patch = Remotefile(
+            "zlib_1.2.12-1_patch.zip",
+            "8ec8344f3fe7b06ad4be768fd416694bc56cb4545ce78b0f1c18b3e72b3ec936",
+            "https://wrapdb.mesonbuild.com/v2/zlib_1.2.12-1/get_patch")
+        archives = [src_archive, meson_patch]
+        #patches = ['zlib_std_libname.patch']
 
-    class Builder(MakeBuilder):
+    Builder = MesonBuilder
-        dynamic_configure_options = ["--shared"]
-        static_configure_options = ["--static"]
-        make_install_targets = ["install"]
-
-        def _pre_build_script(self, context):
-            context.try_skip(self.build_path)
-            shutil.copytree(self.source_path, self.build_path)
-
-        def _configure(self, context):
-            if self.buildEnv.configInfo.build == "win32":
-                raise SkipCommand()
-            return super()._configure(context)
-
-        @property
-        def all_configure_options(self):
-            yield from self.configure_options
-            yield from self.static_configure_options if self.buildEnv.configInfo.static else self.dynamic_configure_options
-            yield from ("--prefix", self.buildEnv.install_dir)
-            yield from (
-                "--libdir",
-                pj(self.buildEnv.install_dir, self.buildEnv.libprefix),
-            )
-
-        @property
-        def make_options(self):
-            if self.buildEnv.configInfo.build != "win32":
-                return
-            yield "--makefile"
-            yield "win32/Makefile.gcc"
-            yield "PREFIX=i686-w64-mingw32-",
-            yield "SHARED_MODE={}".format(
-                "0" if self.buildEnv.configInfo.static else "1"
-            ),
-            yield "INCLUDE_PATH={}".format(pj(self.buildEnv.install_dir, "include")),
-            yield "LIBRARY_PATH={}".format(
-                pj(self.buildEnv.install_dir, self.buildEnv.libprefix)
-            ),
-            yield "BINARY_PATH={}".format(pj(self.buildEnv.install_dir, "bin"))
-
-        @property
-        def make_targets(self):
-            if self.buildEnv.configInfo.static:
-                return ["static"]
-            else:
-                return ["shared"]

kiwixbuild/dependencies/zstd.py

@@ -8,9 +8,9 @@ class zstd(Dependency):
 
     class Source(ReleaseDownload):
         archive = Remotefile(
-            "zstd-1.5.2.tar.gz",
+            "zstd-1.5.5.tar.gz",
-            "f7de13462f7a82c29ab865820149e778cbfe01087b3a55b5332707abf9db4a6e",
+            "98e9c3d949d1b924e28e01eccb7deed865eefebf25c2f21c702e5cd5b63b85e1",
-            "https://github.com/facebook/zstd/archive/refs/tags/v1.5.2.tar.gz",
+            "https://github.com/facebook/zstd/archive/refs/tags/v1.5.5.tar.gz",
         )
 
     class Builder(MesonBuilder):

kiwixbuild/patches/lzma_meson_install.patch

@@ -0,0 +1,54 @@
+diff '--color=auto' -ur lzma-5.2.6_orig/src/liblzma/meson.build lzma-5.2.6/src/liblzma/meson.build
+--- lzma-5.2.6_orig/src/liblzma/meson.build	2023-11-23 14:31:26.110195070 +0100
++++ lzma-5.2.6/src/liblzma/meson.build	2023-12-06 17:04:49.325148650 +0100
+@@ -1,3 +1,5 @@
++pkg = import('pkgconfig')
++
+ lzma_sources = [
+   '../common/tuklib_physmem.c',
+   'common/common.c',
+@@ -121,12 +123,44 @@
+ lzmainc = include_directories('api', 'common',
+   'check', 'lz', 'rangecoder', 'lzma', 'delta', 'simple', '../common')
+ 
++
++install_headers(
++  'api/lzma.h',
++)
++
++install_headers(
++  'api/lzma/version.h',
++  'api/lzma/base.h',
++  'api/lzma/vli.h',
++  'api/lzma/check.h',
++  'api/lzma/filter.h',
++  'api/lzma/bcj.h',
++  'api/lzma/delta.h',
++  'api/lzma/lzma12.h',
++  'api/lzma/container.h',
++  'api/lzma/stream_flags.h',
++  'api/lzma/block.h',
++  'api/lzma/index.h',
++  'api/lzma/index_hash.h',
++  'api/lzma/hardware.h',
++  subdir: 'lzma'
++)
++
+ liblzma = library('lzma', lzma_sources,
+   main_dec_sources, main_enc_sources, check_sources,
+   simplefilter_sources, lzma1_sources,
+   lz_sources, delta_sources,
+   include_directories : [confinc, lzmainc],
+   c_args : ['-DHAVE_CONFIG_H', '-DTUKLIB_SYMBOL_PREFIX=lzma_'],
++  install: true
++)
++
++pkg.generate(liblzma,
++  name: 'liblzma',
++  filebase: 'liblzma',
++  description: 'The liblzma compression library',
++  version: meson.project_version(),
++  url: 'http://tukaani.org/xz/'
+ )
+ 
+ lzma_dep = declare_dependency(link_with : liblzma,

kiwixbuild/utils.py

@@ -63,9 +63,33 @@ class DefaultEnv(Defaultdict):
     def __getitem__(self, name):
         if name == b"PATH":
             raise KeyError
+        if name in ["PATH", "PKG_CONFIG_PATH", "LD_LIBRARY_PATH"]:
+            item = super().__getitem__(name)
+            if isinstance(item, PathArray):
+                return item
+            else:
+                item = PathArray(item)
+                self[name] = item
+                return item
         return super().__getitem__(name)
 
 
+def get_separator():
+    return ";" if neutralEnv("distname") == "Windows" else ":"
+
+
+class PathArray(list):
+    def __init__(self, value):
+        self.separator = get_separator()
+        if not value:
+            super().__init__([])
+        else:
+            super().__init__(value.split(self.separator))
+
+    def __str__(self):
+        return self.separator.join(self)
+
+
 def remove_duplicates(iterable, key_function=None):
     seen = set()
     if key_function is None:
@@ -273,9 +297,22 @@ def extract_archive(archive_path, dest_dir, topdir=None, name=None):
                         if isdir(member):
                             continue
                         perm = (member.external_attr >> 16) & 0x1FF
+                        if perm:
                             os.chmod(pj(tmpdir, getname(member)), perm)
                 name = name or topdir
-                os.rename(pj(tmpdir, topdir), pj(dest_dir, name))
+                shutil.copytree(
+                    pj(tmpdir, topdir),
+                    pj(dest_dir, name),
+                    symlinks=True,
+                    dirs_exist_ok=True,
+                )
+                # Be sure that all directory in tmpdir are writable to allow correct suppersion of it
+                for root, dirs, _files in os.walk(tmpdir):
+                    for d in dirs:
+                        os.chmod(
+                            pj(root, d), stat.S_IWRITE | stat.S_IREAD | stat.S_IEXEC
+                        )
+
         else:
             if name:
                 dest_dir = pj(dest_dir, name)
@@ -297,6 +334,7 @@ def run_command(command, cwd, context, *, env=None, input=None):
         print("run command '{}'".format(command), file=log)
         print("current directory is '{}'".format(cwd), file=log)
         print("env is :", file=log)
+        env = {k: str(v) for k, v in env.items()}
         for k, v in env.items():
             print("  {} : {!r}".format(k, v), file=log)
 

kiwixbuild/versions.py

@@ -39,7 +39,7 @@ release_versions = {
 
 # This is the "version" of the whole base_deps_versions dict.
 # Change this when you change base_deps_versions.
-base_deps_meta_version = "99"
+base_deps_meta_version = "00"
 
 base_deps_versions = {
     "zlib": "1.2.12",

setup.py

@@ -36,7 +36,8 @@ setup(
     include_package_data=True,
     install_requires=[
         'meson',
-        'distro'
+        'distro',
+        'paramiko'
     ],
     entry_points={
         'console_scripts': [