Refactor duplication, use qt6 by default on jammy

QT is installed only in a windows native_mixed job

.github/configs/aqt.ini

@@ -0,0 +1,46 @@
+[DEFAULTS]
+
+[aqt]
+concurrency: 4
+baseurl: https://qt.mirror.constant.com/
+7zcmd: 7z
+print_stacktrace_on_error: False
+always_keep_archives: False
+archive_download_location: .
+min_module_size: 41
+
+[requests]
+connection_timeout: 3.5
+response_timeout: 30
+max_retries_on_connection_error: 5
+retry_backoff: 0.1
+max_retries_on_checksum_error: 5
+max_retries_to_retrieve_hash: 5
+hash_algorithm: sha256
+INSECURE_NOT_FOR_PRODUCTION_ignore_hash: False
+
+[mirrors]
+trusted_mirrors:
+    https://download.qt.io
+    https://qt.mirror.constant.com/
+    https://ftp.fau.de/qtproject/
+blacklist:
+    http://mirrors.ocf.berkeley.edu
+    http://mirrors.tuna.tsinghua.edu.cn
+    http://mirrors.geekpie.club
+fallbacks:
+    https://qtproject.mirror.liquidtelecom.com/
+    https://mirrors.aliyun.com/qt/
+    https://mirrors.ustc.edu.cn/qtproject/
+    https://ftp.jaist.ac.jp/pub/qtproject/
+    https://ftp.yz.yamagata-u.ac.jp/pub/qtproject/
+    https://qt-mirror.dannhauer.de/
+    https://ftp.fau.de/qtproject/
+    https://mirror.netcologne.de/qtproject/
+    https://mirrors.dotsrc.org/qtproject/
+    https://www.nic.funet.fi/pub/mirrors/download.qt-project.org/
+    https://master.qt.io/
+    https://mirrors.ukfast.co.uk/sites/qt.io/
+    https://ftp2.nluug.nl/languages/qt/
+    https://ftp1.nluug.nl/languages/qt/
+    https://qt.mirror.constant.com/

.github/scripts/build_definition.py

@@ -42,7 +42,7 @@ BUILD_DEF = """
     | focal     | native_static      | d      | d        | dBPSD     | dBPSD       |               | linux-x86_64            | linux-x86_64-static    |
     | focal     | native_mixed       | BPS    | BPS      |           |             |               | linux-x86_64            |                        |
     | focal     | native_dyn         | d      | d        | dB        | dB          |               |                         | linux-x86_64-dyn       |
-    | jammy     | native_dyn         | d      | d        |           |             | dBPS          |                         | linux-x86_64-dyn       |
+    | jammy     | native_dyn         |        |          |           |             | dBPS          |                         | linux-x86_64-dyn       |
     # libzim CI is building alpine_dyn but not us
     | focal     | android_arm        | dBP    | dBP      |           |             |               | android-arm             | android-arm            |
     | focal     | android_arm64      | dBP    | dBP      |           |             |               | android-arm64           | android-arm64          |

.github/workflows/cd.yml

@@ -37,11 +37,14 @@ jobs:
         pip3 install meson pytest requests distro paramiko
         pip3 install --no-deps $GITHUB_WORKSPACE
     - name: Install QT
+      if: ${{ matrix.config == 'native_mixed' }}
       uses: jurplel/install-qt-action@v4
       with:
         version: 5.15.2
         modules: "qtwebengine"
         setup-python: false
+      env:
+        AQT_CONFIG: ${{ github.workspace }}/.github/configs/aqt.ini
     - name: Setup MSVC compiler
       uses: bus1/cabuild/action/msdevshell@v1
       with:
@@ -251,7 +254,7 @@ jobs:
         python-version: '3.10'
     - name: Install packages
       run:
-        brew install ninja automake
+        brew install automake # ninja
     - name: Install python modules
       run: |
         pip3 install meson pytest requests distro

.github/workflows/ci.yml

@@ -35,11 +35,14 @@ jobs:
         pip3 install meson pytest requests distro paramiko
         pip3 install --no-deps $GITHUB_WORKSPACE
     - name: Install QT
+      if: ${{ matrix.config == 'native_mixed' }}
       uses: jurplel/install-qt-action@v4
       with:
         version: 6.4.3
         modules: "qtwebengine qtwebchannel qtpositioning"
         setup-python: false
+      env:
+        AQT_CONFIG: ${{ github.workspace }}/.github/configs/aqt.ini
     - name: Setup MSVC compiler
       uses: bus1/cabuild/action/msdevshell@v1
       with:
@@ -239,7 +242,7 @@ jobs:
         python-version: '3.10'
     - name: Install packages
       run:
-        brew install ninja automake
+        brew install automake # ninja
     - name: Install python modules
       run: |
         pip3 install meson pytest requests distro paramiko

kiwixbuild/buildenv.py

@@ -42,10 +42,10 @@ class NeutralEnv:
     def detect_platform(self):
         _platform = platform.system()
         self.distname = _platform
+        self.codename = ""
         if _platform == "Linux":
             self.distname = distro.id()
-            if self.distname == "ubuntu":
+            self.codename = distro.codename()
-                self.distname = "debian"
 
     def download(self, what, where=None):
         where = where or self.archive_dir

kiwixbuild/builder.py

@@ -124,19 +124,16 @@ class Builder:
 
     def _get_packages(self):
         packages_list = []
-        for config in ConfigInfo.all_running_configs.values():
+        for runningConfig in ConfigInfo.all_running_configs.values():
-            mapper_name = "{host}_{config}".format(
+            for mapper_name in self._get_mapper_names_for_config(runningConfig):
-                host=neutralEnv("distname"), config=config
-            )
                 package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {})
                 packages_list += package_name_mapper.get("COMMON", [])
 
         to_drop = []
         for builderDef in self._targets:
-            configName, builderName = builderDef
+            builderConfig, builderName = builderDef
-            mapper_name = "{host}_{config}".format(
+
-                host=neutralEnv("distname"), config=configName
+            for mapper_name in self._get_mapper_names_for_config(builderConfig):
-            )
                 package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {})
                 packages = package_name_mapper.get(builderName)
                 if packages:
@@ -144,10 +141,16 @@ class Builder:
                     if packages is not True:
                         # True means "assume the dependency is install but do not try to install anything for it"
                         packages_list += packages
+
         for dep in to_drop:
             del self._targets[dep]
         return packages_list
 
+    def _get_mapper_names_for_config(self, config):
+        host = neutralEnv("distname")
+        codename = neutralEnv("codename")
+        return ( f"{host}_{config}", f"{host}_{codename}_{config}" )
+
     def install_packages(self):
         packages_to_have = self._get_packages()
         packages_to_have = remove_duplicates(packages_to_have)
@@ -160,7 +163,7 @@ class Builder:
         if distname in ("fedora", "redhat", "centos"):
             package_installer = "sudo dnf install {}"
             package_checker = "rpm -q --quiet {}"
-        elif distname in ("debian", "Ubuntu"):
+        elif distname in ("debian", "ubuntu"):
             package_installer = "sudo apt-get install {}"
             package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1'
         elif distname == "Darwin":

kiwixbuild/configs/android.py

@@ -7,7 +7,7 @@ class AndroidConfigInfo(ConfigInfo):
     build = "android"
     static = True
     toolchain_names = ["android-ndk"]
-    compatible_hosts = ["fedora", "debian"]
+    compatible_hosts = ["fedora", "debian", "ubuntu"]
 
     def __str__(self):
         return "android"

kiwixbuild/configs/armhf.py

@@ -6,7 +6,7 @@ from kiwixbuild._global import get_target_step
 
 # Base config for arm
 class ArmConfigInfo(ConfigInfo):
-    compatible_hosts = ["fedora", "debian", "almalinux"]
+    compatible_hosts = ["fedora", "debian", "ubuntu", "almalinux"]
 
     def get_cross_config(self):
         return {

kiwixbuild/configs/flatpak.py

@@ -8,7 +8,7 @@ class FlatpakConfigInfo(ConfigInfo):
     build = "flatpak"
     static = ""
     toolchain_names = ["org.kde", "io.qt.qtwebengine"]
-    compatible_hosts = ["debian", "fedora"]
+    compatible_hosts = ["debian", "ubuntu", "fedora"]
 
     def __str__(self):
         return "flatpak"

kiwixbuild/configs/i586.py

@@ -7,7 +7,7 @@ from kiwixbuild.utils import which, pj
 class I586ConfigInfo(ConfigInfo):
     build = "i586"
     arch_full = "i586-linux-gnu"
-    compatible_hosts = ["fedora", "debian"]
+    compatible_hosts = ["fedora", "debian", "ubuntu"]
 
     def get_cross_config(self):
         return {

kiwixbuild/configs/musl.py

@@ -5,7 +5,7 @@ from kiwixbuild._global import get_target_step
 
 
 class MuslConfigInfo(ConfigInfo):
-    compatible_hosts = ["fedora", "debian"]
+    compatible_hosts = ["fedora", "debian", "ubuntu"]
 
     def get_cross_config(self):
         return {

kiwixbuild/configs/native.py

@@ -29,16 +29,16 @@ class NativeConfigInfo(ConfigInfo):
 class NativeDyn(NativeConfigInfo):
     name = "native_dyn"
     static = False
-    compatible_hosts = ["fedora", "debian", "Darwin", "almalinux", "Windows"]
+    compatible_hosts = ["fedora", "debian", "ubuntu", "Darwin", "almalinux", "Windows"]
 
 
 class NativeStatic(NativeConfigInfo):
     name = "native_static"
     static = True
-    compatible_hosts = ["fedora", "debian", "Darwin", "almalinux", "Windows"]
+    compatible_hosts = ["fedora", "debian", "ubuntu", "Darwin", "almalinux", "Windows"]
 
 
 class NativeMixed(MixedMixin("native_static"), NativeConfigInfo):
     name = "native_mixed"
     static = False
-    compatible_hosts = ["fedora", "debian", "Darwin", "almalinux", "Windows"]
+    compatible_hosts = ["fedora", "debian", "ubuntu", "Darwin", "almalinux", "Windows"]

kiwixbuild/configs/neutral.py

@@ -5,7 +5,7 @@ class NeutralConfigInfo(ConfigInfo):
     name = "neutral"
     arch_name = "neutral"
     static = ""
-    compatible_hosts = ["fedora", "debian", "Darwin"]
+    compatible_hosts = ["fedora", "debian", "ubuntu", "Darwin"]
 
     def __str__(self):
         return "neutral"

kiwixbuild/configs/wasm.py

@@ -12,7 +12,7 @@ class WasmConfigInfo(ConfigInfo):
     libdir = "lib"
     # arch_full = 'wasm64-linux'
     toolchain_names = ["emsdk"]
-    compatible_hosts = ["fedora", "debian"]
+    compatible_hosts = ["fedora", "debian", "ubuntu"]
     exe_wrapper_def = ""
 
     def get_cross_config(self):

kiwixbuild/configs/win64.py

@@ -18,7 +18,7 @@ class Win64ConfigInfo(ConfigInfo):
         "-lkernel32",
     ]
     build = "win64"
-    compatible_hosts = ["fedora", "debian"]
+    compatible_hosts = ["fedora", "debian", "ubuntu"]
     arch_full = "x86_64-w64-mingw32"
 
     def get_cross_config(self):

kiwixbuild/dependencies/zim_testing_suite.py

@@ -9,9 +9,9 @@ class ZimTestingSuite(Dependency):
 
     class Source(ReleaseDownload):
         archive = Remotefile(
-            "zim-testing-suite-0.7.0.tar.gz",
+            "zim-testing-suite-0.8.0.tar.gz",
-            "b2da1d96973c9dfd8857f96850b40755a52813189ee961881bbddcd1e28aace4",
+            "28f51449a3f9aea02652ca21f32c5598fd610d6cec3810fa552bd0c0f7a2d5fc",
-            "https://github.com/openzim/zim-testing-suite/releases/download/0.7.0/zim-testing-suite-0.7.0.tar.gz",
+            "https://github.com/openzim/zim-testing-suite/releases/download/0.8.0/zim-testing-suite-0.8.0.tar.gz",
         )
 
     Builder = NoopBuilder

kiwixbuild/packages.py

@@ -91,6 +91,36 @@ PACKAGE_NAME_MAPPERS = {
     "debian_android": {
         "COMMON": _debian_common,
     },
+    "ubuntu_jammy_native_dyn": {
+        "COMMON": _debian_common,
+        "zlib": ["zlib1g-dev"],
+        "uuid": ["uuid-dev"],
+        "libmicrohttpd": ["libmicrohttpd-dev", "ccache"],
+        "qt": ["qt6-base-dev", "qt6-base-dev-tools", "libqt6webenginecore6-bin", "libqt6svg6", "qtchooser"],
+        "qtwebengine": ["qt6-webengine-dev"],
+        "aria2": ["aria2"],
+    },
+    "ubuntu_noble_native_dyn": {
+        "COMMON": _debian_common,
+        "zlib": ["zlib1g-dev"],
+        "uuid": ["uuid-dev"],
+        "libmicrohttpd": ["libmicrohttpd-dev", "ccache"],
+        "qt": ["qt6-base-dev", "qt6-base-dev-tools", "libqt6webenginecore6-bin", "libqt6svg6", "qtchooser"],
+        "qtwebengine": ["qt6-webengine-dev"],
+        "aria2": ["aria2"],
+    },
+    "ubuntu_oracular_native_dyn": {
+        "COMMON": _debian_common,
+        "zlib": ["zlib1g-dev"],
+        "uuid": ["uuid-dev"],
+        "libmicrohttpd": ["libmicrohttpd-dev", "ccache"],
+        "qt": ["qt6-base-dev", "qt6-base-dev-tools", "libqt6webenginecore6-bin", "libqt6svg6", "qtchooser"],
+        "qtwebengine": ["qt6-webengine-dev"],
+        "aria2": ["aria2"],
+    },
+    "ubuntu_native_static": {
+        "COMMON": _debian_common + ["libbz2-dev", "libmagic-dev"],
+    },
     "Darwin_native_dyn": {
         "COMMON": ["autoconf", "automake", "libtool", "cmake", "pkg-config"],
         "file": ["libmagic"],

kiwixbuild/versions.py

@@ -1,7 +1,7 @@
 # This file reference all the versions of the depedencies we use in kiwix-build.
 
 main_project_versions = {
-    "libzim": "9.2.3",
+    "libzim": "9.3.0",
     "libkiwix": "14.0.0",
     "kiwix-tools": "3.7.0",
     "zim-tools": "3.5.0",
@@ -23,11 +23,11 @@ main_project_versions = {
 #    - Else, increment the value. If no value was present, see `(was ...)`.
 
 release_versions = {
-    "libzim": None,  # Depends of base deps (was 2)
+    "libzim": 0,  # Depends of base deps (was 2)
     "libkiwix": None,  # Depends of libzim (was 1)
     "kiwix-tools": None,  # Depends of libkiwix and libzim (was 2)
     "zim-tools": None,  # Depends of libzim (was 0)
-    "kiwix-desktop": 0,  # Depends of libkiwix and libzim (was 0)
+    "kiwix-desktop": None,  # Depends of libkiwix and libzim (was 0)
 }
 
 
@@ -52,6 +52,6 @@ base_deps_versions = {
     "android-ndk": "r21e",
     "org.kde": "6.7",
     "io.qt.qtwebengine": "6.7",
-    "zim-testing-suite": "0.7.0",
+    "zim-testing-suite": "0.8.0",
     "emsdk": "3.1.41",
 }