Merge pull request #522 from kiwix/fix_ci

.github/ci_images/bionic_builder.dockerfile

@@ -30,7 +30,7 @@ RUN apt update -q \
 #    vim less grep \
   && apt-get clean -y \
   && rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/* \
-  && pip3 install meson==0.52.1 pytest gcovr requests distro
+  && pip3 install meson pytest gcovr requests distro
 
 # Create user
 RUN useradd --create-home runner

.github/ci_images/f35_builder.dockerfile

@@ -1,7 +1,7 @@
-FROM fedora:31
+FROM fedora:35
 
 ENV LANG C.UTF-8
-ENV OS_NAME f31
+ENV OS_NAME f35
 
 RUN dnf install -y --nodocs \
 # Base build tools
@@ -19,7 +19,7 @@ RUN dnf install -y --nodocs \
   && dnf remove -y "*-doc" \
   && dnf autoremove -y \
   && dnf clean all \
-  && pip3 install meson==0.52.1 pytest requests distro
+  && pip3 install meson pytest requests distro
 
 # Create user
 RUN useradd --create-home runner

.github/ci_images/focal_builder.dockerfile

@@ -25,7 +25,7 @@ RUN apt update -q \
 #    vim less grep \
   && apt-get clean -y \
   && rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/* \
-  && pip3 install meson==0.52.1 pytest gcovr requests distro
+  && pip3 install meson pytest gcovr requests distro
 
 # Create user
 RUN useradd --create-home runner

.github/ci_images/xenial_builder.dockerfile

@@ -1,30 +0,0 @@
-FROM ubuntu:xenial
-
-ENV LANG C.UTF-8
-ENV OS_NAME xenial
-
-RUN apt update -q \
-  && dpkg --add-architecture i386 \
-  && apt install -q -y --no-install-recommends \
-# Base build tools
-    build-essential automake libtool cmake ccache pkg-config autopoint patch \
-    python3-pip python3-setuptools python3-wheel git subversion wget unzip \
-    ninja-build openssh-client curl \
-# Python (2) is needed to install android-ndk
-    python \
-# Packaged dependencies
-    libbz2-dev libmagic-dev uuid-dev zlib1g-dev default-jdk \
-    libmicrohttpd-dev libgtest-dev \
-# Cross compile i586
-    libc6-dev-i386 lib32stdc++6 gcc-multilib g++-multilib \
-# Other tools (to remove)
-#    vim less grep \
-  && apt-get clean -y \
-  && rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/* \
-  && pip3 install meson==0.52.1 pytest 'markupsafe<2.0.0' 'jinja2<3.0.0' 'gcovr<5.0' requests distro
-
-# Create user
-RUN useradd --create-home runner
-USER runner
-WORKDIR /home/runner
-ENV PATH /home/runner/.local/bin:$PATH

.github/scripts/build_projects.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+
+
+from common import (
+    run_kiwix_build,
+    make_archive,
+    create_desktop_image,
+    fix_macos_rpath,
+    OS_NAME,
+    PLATFORM_TARGET,
+    DESKTOP,
+    notarize_macos_build,
+)
+
+if PLATFORM_TARGET.startswith("android_"):
+    TARGETS = ("libkiwix",)
+elif PLATFORM_TARGET.startswith("iOS") or PLATFORM_TARGET.startswith("macOS"):
+    TARGETS = ("libzim", "libkiwix")
+elif PLATFORM_TARGET.startswith("native_"):
+    if OS_NAME == "osx":
+        if PLATFORM_TARGET == "native_mixed":
+            TARGETS = ("libzim", )
+        else:
+            TARGETS = ("libzim", "zim-tools", "libkiwix")
+    else:
+        if DESKTOP:
+            TARGETS = ("kiwix-desktop",)
+        elif PLATFORM_TARGET == "native_mixed":
+            TARGETS = ("libzim",)
+        else:
+            TARGETS = ("zim-tools", "libkiwix", "kiwix-tools")
+elif PLATFORM_TARGET in ("win32_static", "armhf_static", "armhf_dyn", "i586_static"):
+    TARGETS = ("kiwix-tools",)
+elif PLATFORM_TARGET == "flatpak":
+    TARGETS = ("kiwix-desktop",)
+else:
+    TARGETS = ("libzim", "zim-tools", "libkiwix", "kiwix-tools")
+
+for target in TARGETS:
+    run_kiwix_build(target, platform=PLATFORM_TARGET)
+    if target == "kiwix-desktop":
+        create_desktop_image(make_release=False)
+    else:
+        if PLATFORM_TARGET == "native_mixed" and OS_NAME == "osx":
+            fix_macos_rpath(target)
+            notarize_macos_build(target)
+        make_archive(target, make_release=False)

.github/scripts/ensure_base_deps.py

@@ -33,7 +33,7 @@ def download_base_archive(base_name):
 ARCHIVE_NAME_TEMPLATE = "base_deps2_{os}_{platform}_{version}.tar.xz"
 
 if PLATFORM_TARGET == 'flatpak':
-    base_dep_archive_name = "base_deps2_{}_flatpak.tar.xz".format(OS_NAME)
+    base_dep_archive_name = "base_deps2_flatpak.tar.xz"
 else:
     base_dep_archive_name = ARCHIVE_NAME_TEMPLATE.format(
         os=OS_NAME,

.github/workflows/ci.yml

@@ -6,14 +6,14 @@ on:
    - cron: '0 1 * * *'
 
 env:
-  DOCKER_VERSION: 31
+  DOCKER_VERSION: 32
 
 jobs:
   Docker:
     strategy:
       fail-fast: false
       matrix:
-        variant: [xenial, bionic, f31, focal]
+        variant: [bionic, f35, focal]
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
@@ -46,27 +46,23 @@ jobs:
           - android_arm64
           - android_x86
           - android_x86_64
-        image_variant: ['xenial', 'bionic']
+        image_variant: ['bionic']
         lib_postfix: ['/x86_64-linux-gnu']
         include:
           - target: win32_static
-            image_variant: f31
+            image_variant: f35
             lib_postfix: '64'
           - target: win32_dyn
-            image_variant: f31
+            image_variant: f35
             lib_postfix: '64'
-        exclude:
-          - target: native_desktop
-            image_variant: xenial
-          - target: native_static
-            image_variant: bionic
     env:
       HOME: /home/runner
       SSH_KEY: /tmp/id_rsa
     runs-on: ubuntu-latest
     needs: Docker
     container:
-      image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-31"
+      image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-32"
+      options: "--device /dev/fuse --privileged"
     steps:
     - name: Checkout code
       shell: bash
@@ -97,12 +93,61 @@ jobs:
         kiwix-build/.github/scripts/compile_all_deps.py
       env:
         PLATFORM_TARGET: ${{matrix.target}}
+    - name: Build projects
+      shell: bash
+      run: |
+        cd $HOME
+        kiwix-build/.github/scripts/build_projects.py
+      env:
+        PLATFORM_TARGET: ${{matrix.target}}
     - name: Upload failure logs
       if: failure()
       run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh
       env:
         PLATFORM_TARGET: ${{matrix.target}}
 
+  Flatpak:
+    strategy:
+      fail-fast: false
+    env:
+      HOME: /home/runner
+      SSH_KEY: /tmp/id_rsa
+      PLATFORM_TARGET: flatpak
+      OS_NAME: focal
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      shell: bash
+      run: |
+        cd $HOME
+        git clone https://github.com/${REP}
+        cd ./${REP##*/}
+        git checkout --force ${GITHUB_SHA}
+        pip3 install --user --no-deps .
+      env:
+        REP: ${{github.repository}}
+    - name: Install flatpak tools
+      run: |
+        sudo apt-get update
+        sudo apt-get install flatpak-builder ninja-build meson
+    - name: secret
+      shell: bash
+      run: |
+        echo "${{secrets.ssh_key}}" > $SSH_KEY
+        chmod 600 $SSH_KEY
+    - name: Ensure base deps
+      shell: bash
+      run: |
+        cd $HOME
+        kiwix-build/.github/scripts/ensure_base_deps.py
+    - name: Build projects
+      shell: bash
+      run: |
+        cd $HOME
+        kiwix-build/.github/scripts/build_projects.py
+    - name: Upload failure logs
+      if: failure()
+      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh
 
   Macos:
     strategy:
@@ -132,7 +177,7 @@ jobs:
         brew install pkg-config ninja automake autoconf
     - name: Install python modules
       run: |
-        pip3 install meson==0.52.1 pytest requests distro
+        pip3 install meson pytest requests distro
         pip3 install --no-deps $GITHUB_WORKSPACE
     - name: secret
       shell: bash
@@ -153,6 +198,13 @@ jobs:
         $GITHUB_WORKSPACE/.github/scripts/compile_all_deps.py
       env:
         PLATFORM_TARGET: ${{matrix.target}}
+    - name: Build projects
+      shell: bash
+      run: |
+        cd $HOME
+        $GITHUB_WORKSPACE/.github/scripts/build_projects.py
+      env:
+        PLATFORM_TARGET: ${{matrix.target}}
     - name: Upload failure logs
       if: failure()
       run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.sh

.github/workflows/releaseNigthly.yml

@@ -8,14 +8,14 @@ on:
    - cron: '0 1 * * *'
 
 env:
-  DOCKER_VERSION: 31
+  DOCKER_VERSION: 32
 
 jobs:
   Docker:
     strategy:
       fail-fast: false
       matrix:
-        variant: [xenial, bionic, f31, focal]
+        variant: [bionic, f35, focal]
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
@@ -44,33 +44,29 @@ jobs:
           - win32_static
           - i586_static
           - android
-          - flatpak
         include:
           - target: native_static
-            image_variant: xenial
+            image_variant: bionic
             lib_postfix: '/x86_64-linux-gnu'
           - target: native_dyn
-            image_variant: xenial
+            image_variant: bionic
             lib_postfix: '/x86_64-linux-gnu'
           - target: native_mixed
-            image_variant: xenial
+            image_variant: bionic
             lib_postfix: '/x86_64-linux-gnu'
           - target: native_desktop
             image_variant: bionic
             lib_postfix: '/x86_64-linux-gnu'
           - target: armhf_static
-            image_variant: xenial
+            image_variant: bionic
             lib_postfix: '/x86_64-linux-gnu'
           - target: win32_static
-            image_variant: f31
+            image_variant: f35
             lib_postfix: '64'
           - target: i586_static
-            image_variant: xenial
+            image_variant: bionic
             lib_postfix: '/x86_64-linux-gnu'
           - target: android
-            image_variant: xenial
-            lib_postfix: '/x86_64-linux-gnu'
-          - target: flatpak
             image_variant: bionic
             lib_postfix: '/x86_64-linux-gnu'
     env:
@@ -79,7 +75,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: Docker
     container:
-      image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-31"
+      image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-32"
       options: "--device /dev/fuse --privileged"
     steps:
     - name: Checkout code
@@ -113,13 +109,56 @@ jobs:
         PLATFORM_TARGET: ${{matrix.target}}
         BINTRAY_USER: kiwix
         BINTRAY_PASS: ${{secrets.bintray_pass}}
-        GITHUB_PAT: ${{secrets.GHCR_TOKEN}}
     - name: Upload failure logs
       if: failure()
       run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh
       env:
         PLATFORM_TARGET: ${{matrix.target}}
 
+  Flatpak:
+    strategy:
+      fail-fast: false
+    env:
+      HOME: /home/runner
+      SSH_KEY: /tmp/id_rsa
+      PLATFORM_TARGET: flatpak
+      OS_NAME: focal
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      shell: bash
+      run: |
+        cd $HOME
+        git clone https://github.com/${REP}
+        cd ./${REP##*/}
+        git checkout --force ${GITHUB_SHA}
+        pip3 install --user --no-deps .
+      env:
+        REP: ${{github.repository}}
+    - name: Install flatpak tools
+      run: |
+        sudo apt-get update
+        sudo apt-get install flatpak-builder ninja-build meson
+    - name: secret
+      shell: bash
+      run: |
+        echo "${{secrets.ssh_key}}" > $SSH_KEY
+        chmod 600 $SSH_KEY
+    - name: Ensure base deps
+      shell: bash
+      run: |
+        cd $HOME
+        kiwix-build/.github/scripts/ensure_base_deps.py
+    - name: Build release
+      shell: bash
+      run: |
+        cd $HOME
+        kiwix-build/.github/scripts/build_release_nightly.py
+      env:
+        GITHUB_PAT: ${{secrets.GHCR_TOKEN}}
+    - name: Upload failure logs
+      if: failure()
+      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh
 
   Macos:
     strategy:
@@ -148,7 +187,7 @@ jobs:
         brew install pkg-config ninja automake autoconf
     - name: Install python modules
       run: |
-        pip3 install meson==0.52.1 pytest requests distro
+        pip3 install meson pytest requests distro
         pip3 install --no-deps $GITHUB_WORKSPACE
     - name: secret
       shell: bash

kiwixbuild/dependencies/flatpak.py

@@ -22,7 +22,7 @@ class org_kde(Dependency):
             run_command(command, self.buildEnv.build_dir, context, env=env)
 
         def _install_sdk(self, context):
-            command = "flatpak --user install -y {remote_name} {name}.Sdk//{version} {name}.Platform//{version}"
+            command = "flatpak --user install --noninteractive --verbose -y {remote_name} {name}.Sdk//{version} {name}.Platform//{version}"
             command = command.format(
                 remote_name = 'flathub',
                 name = self.target.name,

kiwixbuild/flatpak_builder.py

@@ -223,7 +223,7 @@ class FlatpakBuilder:
     def build(self):
         log = pj(self.platform.buildEnv.log_dir, 'cmd_build_flatpak.log')
         context = Context('build', log, False)
-        command = "flatpak-builder --user --ccache --force-clean --repo=repo builddir {id}.json"
+        command = "flatpak-builder --user --ccache --force-clean --keep-build-dirs --disable-rofiles-fuse --repo=repo builddir {id}.json"
         command = command.format(id = MANIFEST['app-id'])
         try:
             run_command(command, self.platform.buildEnv.build_dir, context, env=self.platform.get_env())

scripts/create_kiwix-desktop_appImage.sh

@@ -33,12 +33,13 @@ mkdir -p $APPDIR/usr/bin/ && tar -C $APPDIR/usr/bin/ -xf aria2-1.36.0-linux-gnu-
 mkdir -p $APPDIR/etc/ssl/certs/ && tar -C $APPDIR/etc/ssl/certs/ -xf aria2-1.36.0-linux-gnu-64bit-build1.tar.bz2 aria2-1.36.0-linux-gnu-64bit-build1/ca-certificates.crt --strip-components=1
 
 # Get linuxdeployqt
-wget --continue https://github.com/probonopd/linuxdeployqt/releases/download/6/linuxdeployqt-6-x86_64.AppImage
+# Dispite the 'continuous' in the file name, it IS release 8
-chmod a+x linuxdeployqt-6-x86_64.AppImage
+wget --continue https://github.com/probonopd/linuxdeployqt/releases/download/8/linuxdeployqt-continuous-x86_64.AppImage -O linuxdeployqt
+chmod u+x linuxdeployqt
 
 # Fill with all deps libs and so
-./linuxdeployqt-6-x86_64.AppImage $APPDIR/usr/bin/kiwix-desktop -unsupported-allow-new-glibc -bundle-non-qt-libs -extra-plugins=imageformats,iconengines
+./linuxdeployqt $APPDIR/usr/bin/kiwix-desktop -bundle-non-qt-libs -extra-plugins=imageformats,iconengines
 # Fix the RPATH of QtWebEngineProcess [TODO] Fill a issue ?
 patchelf --set-rpath '$ORIGIN/../lib' $APPDIR/usr/libexec/QtWebEngineProcess
 # Build the image.
-./linuxdeployqt-6-x86_64.AppImage $APPDIR/usr/share/applications/kiwix-desktop.desktop -unsupported-allow-new-glibc -bundle-non-qt-libs -extra-plugins=imageformats,iconengines -appimage
+./linuxdeployqt $APPDIR/usr/share/applications/kiwix-desktop.desktop -bundle-non-qt-libs -extra-plugins=imageformats,iconengines -appimage

setup.py

@@ -35,7 +35,7 @@ setup(
     packages=find_packages(),
     include_package_data=True,
     install_requires=[
-        'meson==0.52.1',
+        'meson',
         'distro'
     ],
     entry_points={