Merge pull request #819 from kiwix/release-zim-tools-3.6.0

Relase zim-tools 3.6.0

.github/configs/aqt.ini

@@ -0,0 +1,46 @@
+[DEFAULTS]
+
+[aqt]
+concurrency: 4
+baseurl: https://qt.mirror.constant.com/
+7zcmd: 7z
+print_stacktrace_on_error: False
+always_keep_archives: False
+archive_download_location: .
+min_module_size: 41
+
+[requests]
+connection_timeout: 3.5
+response_timeout: 30
+max_retries_on_connection_error: 5
+retry_backoff: 0.1
+max_retries_on_checksum_error: 5
+max_retries_to_retrieve_hash: 5
+hash_algorithm: sha256
+INSECURE_NOT_FOR_PRODUCTION_ignore_hash: False
+
+[mirrors]
+trusted_mirrors:
+    https://download.qt.io
+    https://qt.mirror.constant.com/
+    https://ftp.fau.de/qtproject/
+blacklist:
+    http://mirrors.ocf.berkeley.edu
+    http://mirrors.tuna.tsinghua.edu.cn
+    http://mirrors.geekpie.club
+fallbacks:
+    https://qtproject.mirror.liquidtelecom.com/
+    https://mirrors.aliyun.com/qt/
+    https://mirrors.ustc.edu.cn/qtproject/
+    https://ftp.jaist.ac.jp/pub/qtproject/
+    https://ftp.yz.yamagata-u.ac.jp/pub/qtproject/
+    https://qt-mirror.dannhauer.de/
+    https://ftp.fau.de/qtproject/
+    https://mirror.netcologne.de/qtproject/
+    https://mirrors.dotsrc.org/qtproject/
+    https://www.nic.funet.fi/pub/mirrors/download.qt-project.org/
+    https://master.qt.io/
+    https://mirrors.ukfast.co.uk/sites/qt.io/
+    https://ftp2.nluug.nl/languages/qt/
+    https://ftp1.nluug.nl/languages/qt/
+    https://qt.mirror.constant.com/

.github/scripts/build_definition.py

@@ -42,7 +42,7 @@ BUILD_DEF = """
     | focal     | native_static      | d      | d        | dBPSD     | dBPSD       |               | linux-x86_64            | linux-x86_64-static    |
     | focal     | native_mixed       | BPS    | BPS      |           |             |               | linux-x86_64            |                        |
     | focal     | native_dyn         | d      | d        | dB        | dB          |               |                         | linux-x86_64-dyn       |
-    | jammy     | native_dyn         | d      | d        |           |             | dBPS          |                         | linux-x86_64-dyn       |
+    | jammy     | native_dyn         |        |          |           |             | dBPS          |                         | linux-x86_64-dyn       |
     # libzim CI is building alpine_dyn but not us
     | focal     | android_arm        | dBP    | dBP      |           |             |               | android-arm             | android-arm            |
     | focal     | android_arm64      | dBP    | dBP      |           |             |               | android-arm64           | android-arm64          |

.github/scripts/common.py

@@ -544,7 +544,7 @@ def create_desktop_image(make_release):
     elif platform.system() == "Windows":
         archive_basename = "kiwix-desktop_windows_x64_{}".format(postfix)
         working_dir = INSTALL_DIR / archive_basename
-        build_path = working_dir.with_suffix(".zip")
+        build_path = Path(str(working_dir) + ".zip")
         app_name = build_path.name
         command = [
             "python",
@@ -595,7 +595,7 @@ def update_flathub_git():
     call(command)
     shutil.copy(str(BASE_DIR / "org.kiwix.desktop.json"), str(git_repo_dir))
     patch_dir = KBUILD_SOURCE_DIR / "kiwixbuild" / "patches"
-    for dep in ["pugixml"]:
+    for dep in ["pugixml", "libmicrohttpd"]:
         for f in patch_dir.glob("{}_*.patch".format(dep)):
             shutil.copy(str(f), str(git_repo_dir / "patches"))
     command = ["git", "add", "-A", "."]

.github/workflows/cd.yml

@@ -37,11 +37,14 @@ jobs:
         pip3 install meson pytest requests distro paramiko
         pip3 install --no-deps $GITHUB_WORKSPACE
     - name: Install QT
+      if: ${{ matrix.config == 'native_mixed' }}
       uses: jurplel/install-qt-action@v4
       with:
         version: 5.15.2
         modules: "qtwebengine"
         setup-python: false
+      env:
+        AQT_CONFIG: ${{ github.workspace }}/.github/configs/aqt.ini
     - name: Setup MSVC compiler
       uses: bus1/cabuild/action/msdevshell@v1
       with:
@@ -251,7 +254,7 @@ jobs:
         python-version: '3.10'
     - name: Install packages
       run:
-        brew install pkg-config ninja automake autoconf
+        brew install automake # ninja
     - name: Install python modules
       run: |
         pip3 install meson pytest requests distro
@@ -306,7 +309,7 @@ jobs:
     runs-on: ubuntu-22.04
     env:
       COMPILE_CONFIG: native_static
-      OS_NAME: linux
+      OS_NAME: focal
     steps:
     - name: Checkout code
       uses: actions/checkout@v4

.github/workflows/ci.yml

@@ -35,11 +35,14 @@ jobs:
         pip3 install meson pytest requests distro paramiko
         pip3 install --no-deps $GITHUB_WORKSPACE
     - name: Install QT
+      if: ${{ matrix.config == 'native_mixed' }}
       uses: jurplel/install-qt-action@v4
       with:
-        version: 5.15.2
-        modules: "qtwebengine"
+        version: 6.4.3
+        modules: "qtwebengine qtwebchannel qtpositioning"
         setup-python: false
+      env:
+        AQT_CONFIG: ${{ github.workspace }}/.github/configs/aqt.ini
     - name: Setup MSVC compiler
       uses: bus1/cabuild/action/msdevshell@v1
       with:
@@ -239,7 +242,7 @@ jobs:
         python-version: '3.10'
     - name: Install packages
       run:
-        brew install pkg-config ninja automake autoconf
+        brew install automake # ninja
     - name: Install python modules
       run: |
         pip3 install meson pytest requests distro paramiko

actions/dl_deps_archive/package-lock.json

@@ -114,9 +114,9 @@
       }
     },
     "node_modules/undici": {
-      "version": "5.28.2",
-      "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.2.tgz",
-      "integrity": "sha512-wh1pHJHnUeQV5Xa8/kyQhO7WFa8M34l026L5P/+2TYiakvGy5Rdc8jWZVyG7ieht/0WgJLEd3kcU5gKx+6GC8w==",
+      "version": "5.28.4",
+      "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.4.tgz",
+      "integrity": "sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g==",
       "dependencies": {
         "@fastify/busboy": "^2.0.0"
       },

kiwixbuild/dependencies/aria2.py

@@ -4,6 +4,8 @@ from kiwixbuild.utils import Remotefile, run_command, pj
 import platform
 from shutil import copy2
 
+# Important: in case of aria2c update,
+# 'scripts/create_kiwix-desktop_appImage.sh' should not be forgotten!
 
 class Aria2(Dependency):
     name = "aria2"
@@ -14,7 +16,7 @@ class Aria2(Dependency):
             archive = Remotefile(
                 "aria2-1.37.0-win-64bit-build1.zip",
                 "67d015301eef0b612191212d564c5bb0a14b5b9c4796b76454276a4d28d9b288",
-                "https://github.com/aria2/aria2/releases/download/release-1.37.0/aria2-1.37.0-win-64bit-build1.zip",
+                "https://dev.kiwix.org/kiwix-desktop/aria2-1.37.0-win-64bit-build1.zip",
             )
 
         class Builder(NoopBuilder):
@@ -34,7 +36,7 @@ class Aria2(Dependency):
             archive = Remotefile(
                 "aria2-1.37.0.tar.xz",
                 "60a420ad7085eb616cb6e2bdf0a7206d68ff3d37fb5a956dc44242eb2f79b66b",
-                "https://github.com/aria2/aria2/releases/download/release-1.37.0/aria2-1.37.0.tar.xz",
+                "https://dev.kiwix.org/kiwix-desktop/aria2-1.37.0.tar.xz",
             )
 
             def _post_prepare_script(self, context):

kiwixbuild/dependencies/base.py

@@ -524,8 +524,8 @@ class QMakeBuilder(MakeBuilder):
             *neutralEnv("git_command"),
             "archive",
             "-o",
-            f"{self.build_path}/{self.target_full_name()}.tar.gz",
-            f"--prefix={self.target_full_name()}/",
+            f"{self.build_path}/{self.target.full_name()}.tar.gz",
+            f"--prefix={self.target.full_name()}/",
             "HEAD",
         ]
         run_command(command, self.source_path, context)

kiwixbuild/dependencies/kiwix_desktop.py

@@ -15,7 +15,7 @@ class KiwixDesktop(Dependency):
         dependencies = ["qt", "qtwebengine", "libkiwix", "aria2"]
         configure_env = None
 
-        flatpack_build_options = {"env": {"QMAKEPATH": "/app/lib"}}
+        flatpack_build_options = {"env": {"QMAKEPATH": "/app"}}
 
         @property
         def make_targets(self):

kiwixbuild/dependencies/libcurl.py

@@ -36,7 +36,6 @@ class LibCurl(Dependency):
                 "kerberos-auth",
                 "gss-api",
                 "ssh",
-                "ssl",
                 "rtmp",
                 "http2",
                 "idn",

kiwixbuild/dependencies/tc_android_ndk.py

@@ -24,6 +24,10 @@ class android_ndk(Dependency):
         def source_dir(self):
             return self.target.full_name()
 
+        patches = [
+            "android-ndk-r21e-linux-x86_64-python3.12+.patch",
+        ]
+
     class Builder(Builder):
         @property
         def install_path(self):

kiwixbuild/dependencies/zim_testing_suite.py

@@ -9,9 +9,9 @@ class ZimTestingSuite(Dependency):
 
     class Source(ReleaseDownload):
         archive = Remotefile(
-            "zim-testing-suite-0.6.0.tar.gz",
-            "5aa91349a2791c862217b4d2ddd002f08589146ec047f10d5fa1f5fd85d0ea92",
-            "https://github.com/openzim/zim-testing-suite/releases/download/0.6.0/zim-testing-suite-0.6.0.tar.gz",
+            "zim-testing-suite-0.8.0.tar.gz",
+            "28f51449a3f9aea02652ca21f32c5598fd610d6cec3810fa552bd0c0f7a2d5fc",
+            "https://github.com/openzim/zim-testing-suite/releases/download/0.8.0/zim-testing-suite-0.8.0.tar.gz",
         )
 
     Builder = NoopBuilder

kiwixbuild/flatpak_builder.py

@@ -49,6 +49,7 @@ MANIFEST = {
         "--socket=pulseaudio",
         "--share=network",
         "--share=ipc",
+        "--filesystem=host:ro",
     ],
     "cleanup": [
         "/include",

kiwixbuild/patches/android-ndk-r21e-linux-x86_64-python3.12+.patch

@@ -0,0 +1,33 @@
+diff -ur android-ndk-r21e/build/tools/make_standalone_toolchain.py android-ndk-r21e.patched/build/tools/make_standalone_toolchain.py
+--- android-ndk-r21e/build/tools/make_standalone_toolchain.py	2025-03-04 20:48:14.681288830 +0400
++++ android-ndk-r21e.patched/build/tools/make_standalone_toolchain.py	2025-03-05 12:10:47.252578915 +0400
+@@ -1,4 +1,4 @@
+-#!/usr/bin/env python
++#!/usr/bin/env python3
+ #
+ # Copyright (C) 2016 The Android Open Source Project
+ #
+@@ -21,7 +21,6 @@
+ """
+ import argparse
+ import atexit
+-from distutils.dir_util import copy_tree
+ import inspect
+ import logging
+ import os
+@@ -221,13 +220,13 @@
+ 
+ def create_toolchain(install_path, arch, api, toolchain_path, host_tag):
+     """Create a standalone toolchain."""
+-    copy_tree(toolchain_path, install_path)
++    shutil.copytree(toolchain_path, install_path)
+     triple = get_triple(arch)
+     make_clang_scripts(install_path, arch, api, host_tag == 'windows-x86_64')
+     replace_gcc_wrappers(install_path, triple, host_tag == 'windows-x86_64')
+ 
+     prebuilt_path = os.path.join(NDK_DIR, 'prebuilt', host_tag)
+-    copy_tree(prebuilt_path, install_path)
++    shutil.copytree(prebuilt_path, install_path, dirs_exist_ok=True)
+ 
+     gdbserver_path = os.path.join(
+         NDK_DIR, 'prebuilt', 'android-' + arch, 'gdbserver')

kiwixbuild/versions.py

@@ -1,11 +1,11 @@
 # This file reference all the versions of the depedencies we use in kiwix-build.
 
 main_project_versions = {
-    "libzim": "9.2.3",
+    "libzim": "9.3.0",
     "libkiwix": "14.0.0",
     "kiwix-tools": "3.7.0",
-    "zim-tools": "3.5.0",
-    "kiwix-desktop": "2.4.0",
+    "zim-tools": "3.6.0",
+    "kiwix-desktop": "2.4.1",
 }
 
 # This dictionnary specify what we need to build at each release process.
@@ -23,17 +23,17 @@ main_project_versions = {
 #    - Else, increment the value. If no value was present, see `(was ...)`.
 
 release_versions = {
-    "libzim": None,  # Depends of base deps (was 2)
+    "libzim": 1,  # Depends of base deps (was 0)
     "libkiwix": None,  # Depends of libzim (was 1)
     "kiwix-tools": None,  # Depends of libkiwix and libzim (was 2)
-    "zim-tools": None,  # Depends of libzim (was 0)
-    "kiwix-desktop": 0,  # Depends of libkiwix and libzim (was None)
+    "zim-tools": 0,  # Depends of libzim (was None)
+    "kiwix-desktop": None,  # Depends of libkiwix and libzim (was 0)
 }
 
 
 # This is the "version" of the whole base_deps_versions dict.
 # Change this when you change base_deps_versions.
-base_deps_meta_version = "10"
+base_deps_meta_version = "12"
 
 base_deps_versions = {
     "zlib": "1.2.12",
@@ -50,8 +50,8 @@ base_deps_versions = {
     "libaria2": "1.37.0",
     "libmagic": "5.44",
     "android-ndk": "r21e",
-    "org.kde": "5.15-23.08",
-    "io.qt.qtwebengine": "5.15-23.08",
-    "zim-testing-suite": "0.6.0",
+    "org.kde": "6.7",
+    "io.qt.qtwebengine": "6.7",
+    "zim-testing-suite": "0.8.0",
     "emsdk": "3.1.41",
 }

scripts/create_kiwix-desktop_appImage.sh

@@ -29,10 +29,12 @@ LD_LIBRARY_PATH=$INSTALLDIR/lib/x86_64-linux-gnu ./linuxdeploy-x86_64.AppImage \
     --icon-file=$ICONFILE \
     --library=/usr/lib/x86_64-linux-gnu/libthai.so.0 \
 
-# get the aria2
-wget --continue https://github.com/q3aql/aria2-static-builds/releases/download/v1.36.0/aria2-1.36.0-linux-gnu-64bit-build1.tar.bz2
-mkdir -p $APPDIR/usr/bin/ && tar -C $APPDIR/usr/bin/ -xf aria2-1.36.0-linux-gnu-64bit-build1.tar.bz2 aria2-1.36.0-linux-gnu-64bit-build1/aria2c --strip-components=1
-mkdir -p $APPDIR/etc/ssl/certs/ && tar -C $APPDIR/etc/ssl/certs/ -xf aria2-1.36.0-linux-gnu-64bit-build1.tar.bz2 aria2-1.36.0-linux-gnu-64bit-build1/ca-certificates.crt --strip-components=1
+# Get the aria2c (https://github.com/abcfy2/aria2-static-build/releases/download/1.37.0/aria2-x86_64-linux-musl_libressl_static.zip)
+wget --continue https://dev.kiwix.org/kiwix-desktop/aria2-1.37.0-x86_64-linux-musl_libressl_static.zip
+mkdir -p $APPDIR/usr/bin/ && unzip aria2-1.37.0-x86_64-linux-musl_libressl_static.zip -d $APPDIR/usr/bin/
+
+# Copy the CA trustore from the hosting system
+mkdir -p $APPDIR/etc/ssl/certs/ && cp /etc/ssl/certs/ca-certificates.crt $APPDIR/etc/ssl/certs/
 
 # Fix the RPATH of QtWebEngineProcess [TODO] Fill a issue ?
 patchelf --set-rpath '$ORIGIN/../lib' $APPDIR/usr/libexec/QtWebEngineProcess