Merge pull request #522 from kiwix/fix_ci

2022-03-11 16:57:11 +01:00 · 2022-03-11 16:57:11 +01:00 · 077be3b035
parent 4f458e5954 d420b53f74
commit 077be3b035
12 changed files with 179 additions and 70 deletions
--- a/.github/ci_images/bionic_builder.dockerfile
+++ b/.github/ci_images/bionic_builder.dockerfile
@ -30,7 +30,7 @@ RUN apt update -q \
 #    vim less grep \
  && apt-get clean -y \
  && rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/* \
-  && pip3 install meson==0.52.1 pytest gcovr requests distro
+  && pip3 install meson pytest gcovr requests distro

 # Create user
 RUN useradd --create-home runner
--- a/.github/ci_images/f35_builder.dockerfile
+++ b/.github/ci_images/f35_builder.dockerfile
@ -1,7 +1,7 @@
-FROM fedora:31
+FROM fedora:35

 ENV LANG C.UTF-8
-ENV OS_NAME f31
+ENV OS_NAME f35

 RUN dnf install -y --nodocs \
 # Base build tools
@ -19,7 +19,7 @@ RUN dnf install -y --nodocs \
  && dnf remove -y "*-doc" \
  && dnf autoremove -y \
  && dnf clean all \
-  && pip3 install meson==0.52.1 pytest requests distro
+  && pip3 install meson pytest requests distro

 # Create user
 RUN useradd --create-home runner
--- a/.github/ci_images/focal_builder.dockerfile
+++ b/.github/ci_images/focal_builder.dockerfile
@ -25,7 +25,7 @@ RUN apt update -q \
 #    vim less grep \
  && apt-get clean -y \
  && rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/* \
-  && pip3 install meson==0.52.1 pytest gcovr requests distro
+  && pip3 install meson pytest gcovr requests distro

 # Create user
 RUN useradd --create-home runner
--- a/.github/ci_images/xenial_builder.dockerfile
+++ b/.github/ci_images/xenial_builder.dockerfile
@ -1,30 +0,0 @@
-FROM ubuntu:xenial
-
-ENV LANG C.UTF-8
-ENV OS_NAME xenial
-
-RUN apt update -q \
-  && dpkg --add-architecture i386 \
-  && apt install -q -y --no-install-recommends \
-# Base build tools
-    build-essential automake libtool cmake ccache pkg-config autopoint patch \
-    python3-pip python3-setuptools python3-wheel git subversion wget unzip \
-    ninja-build openssh-client curl \
-# Python (2) is needed to install android-ndk
-    python \
-# Packaged dependencies
-    libbz2-dev libmagic-dev uuid-dev zlib1g-dev default-jdk \
-    libmicrohttpd-dev libgtest-dev \
-# Cross compile i586
-    libc6-dev-i386 lib32stdc++6 gcc-multilib g++-multilib \
-# Other tools (to remove)
-#    vim less grep \
-  && apt-get clean -y \
-  && rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/* \
-  && pip3 install meson==0.52.1 pytest 'markupsafe<2.0.0' 'jinja2<3.0.0' 'gcovr<5.0' requests distro
-
-# Create user
-RUN useradd --create-home runner
-USER runner
-WORKDIR /home/runner
-ENV PATH /home/runner/.local/bin:$PATH
--- a/.github/scripts/build_projects.py
+++ b/.github/scripts/build_projects.py
@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+
+
+from common import (
+    run_kiwix_build,
+    make_archive,
+    create_desktop_image,
+    fix_macos_rpath,
+    OS_NAME,
+    PLATFORM_TARGET,
+    DESKTOP,
+    notarize_macos_build,
+)
+
+if PLATFORM_TARGET.startswith("android_"):
+    TARGETS = ("libkiwix",)
+elif PLATFORM_TARGET.startswith("iOS") or PLATFORM_TARGET.startswith("macOS"):
+    TARGETS = ("libzim", "libkiwix")
+elif PLATFORM_TARGET.startswith("native_"):
+    if OS_NAME == "osx":
+        if PLATFORM_TARGET == "native_mixed":
+            TARGETS = ("libzim", )
+        else:
+            TARGETS = ("libzim", "zim-tools", "libkiwix")
+    else:
+        if DESKTOP:
+            TARGETS = ("kiwix-desktop",)
+        elif PLATFORM_TARGET == "native_mixed":
+            TARGETS = ("libzim",)
+        else:
+            TARGETS = ("zim-tools", "libkiwix", "kiwix-tools")
+elif PLATFORM_TARGET in ("win32_static", "armhf_static", "armhf_dyn", "i586_static"):
+    TARGETS = ("kiwix-tools",)
+elif PLATFORM_TARGET == "flatpak":
+    TARGETS = ("kiwix-desktop",)
+else:
+    TARGETS = ("libzim", "zim-tools", "libkiwix", "kiwix-tools")
+
+for target in TARGETS:
+    run_kiwix_build(target, platform=PLATFORM_TARGET)
+    if target == "kiwix-desktop":
+        create_desktop_image(make_release=False)
+    else:
+        if PLATFORM_TARGET == "native_mixed" and OS_NAME == "osx":
+            fix_macos_rpath(target)
+            notarize_macos_build(target)
+        make_archive(target, make_release=False)
--- a/.github/scripts/ensure_base_deps.py
+++ b/.github/scripts/ensure_base_deps.py
@ -33,7 +33,7 @@ def download_base_archive(base_name):
 ARCHIVE_NAME_TEMPLATE = "base_deps2_{os}_{platform}_{version}.tar.xz"

 if PLATFORM_TARGET == 'flatpak':
-    base_dep_archive_name = "base_deps2_{}_flatpak.tar.xz".format(OS_NAME)
+    base_dep_archive_name = "base_deps2_flatpak.tar.xz"
 else:
    base_dep_archive_name = ARCHIVE_NAME_TEMPLATE.format(
        os=OS_NAME,
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@ -6,14 +6,14 @@ on:
   - cron: '0 1 * * *'

 env:
-  DOCKER_VERSION: 31
+  DOCKER_VERSION: 32

 jobs:
  Docker:
    strategy:
      fail-fast: false
      matrix:
-        variant: [xenial, bionic, f31, focal]
+        variant: [bionic, f35, focal]
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
@ -46,27 +46,23 @@ jobs:
          - android_arm64
          - android_x86
          - android_x86_64
-        image_variant: ['xenial', 'bionic']
+        image_variant: ['bionic']
        lib_postfix: ['/x86_64-linux-gnu']
        include:
          - target: win32_static
-            image_variant: f31
+            image_variant: f35
            lib_postfix: '64'
          - target: win32_dyn
-            image_variant: f31
+            image_variant: f35
            lib_postfix: '64'
-        exclude:
-          - target: native_desktop
-            image_variant: xenial
-          - target: native_static
-            image_variant: bionic
    env:
      HOME: /home/runner
      SSH_KEY: /tmp/id_rsa
    runs-on: ubuntu-latest
    needs: Docker
    container:
-      image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-31"
+      image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-32"
+      options: "--device /dev/fuse --privileged"
    steps:
    - name: Checkout code
      shell: bash
@ -97,12 +93,61 @@ jobs:
        kiwix-build/.github/scripts/compile_all_deps.py
      env:
        PLATFORM_TARGET: ${{matrix.target}}
+    - name: Build projects
+      shell: bash
+      run: |
+        cd $HOME
+        kiwix-build/.github/scripts/build_projects.py
+      env:
+        PLATFORM_TARGET: ${{matrix.target}}
    - name: Upload failure logs
      if: failure()
      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh
      env:
        PLATFORM_TARGET: ${{matrix.target}}

+  Flatpak:
+    strategy:
+      fail-fast: false
+    env:
+      HOME: /home/runner
+      SSH_KEY: /tmp/id_rsa
+      PLATFORM_TARGET: flatpak
+      OS_NAME: focal
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      shell: bash
+      run: |
+        cd $HOME
+        git clone https://github.com/${REP}
+        cd ./${REP##*/}
+        git checkout --force ${GITHUB_SHA}
+        pip3 install --user --no-deps .
+      env:
+        REP: ${{github.repository}}
+    - name: Install flatpak tools
+      run: |
+        sudo apt-get update
+        sudo apt-get install flatpak-builder ninja-build meson
+    - name: secret
+      shell: bash
+      run: |
+        echo "${{secrets.ssh_key}}" > $SSH_KEY
+        chmod 600 $SSH_KEY
+    - name: Ensure base deps
+      shell: bash
+      run: |
+        cd $HOME
+        kiwix-build/.github/scripts/ensure_base_deps.py
+    - name: Build projects
+      shell: bash
+      run: |
+        cd $HOME
+        kiwix-build/.github/scripts/build_projects.py
+    - name: Upload failure logs
+      if: failure()
+      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh

  Macos:
    strategy:
@ -132,7 +177,7 @@ jobs:
        brew install pkg-config ninja automake autoconf
    - name: Install python modules
      run: |
-        pip3 install meson==0.52.1 pytest requests distro
+        pip3 install meson pytest requests distro
        pip3 install --no-deps $GITHUB_WORKSPACE
    - name: secret
      shell: bash
@ -153,6 +198,13 @@ jobs:
        $GITHUB_WORKSPACE/.github/scripts/compile_all_deps.py
      env:
        PLATFORM_TARGET: ${{matrix.target}}
+    - name: Build projects
+      shell: bash
+      run: |
+        cd $HOME
+        $GITHUB_WORKSPACE/.github/scripts/build_projects.py
+      env:
+        PLATFORM_TARGET: ${{matrix.target}}
    - name: Upload failure logs
      if: failure()
      run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.sh
--- a/.github/workflows/releaseNigthly.yml
+++ b/.github/workflows/releaseNigthly.yml
@ -8,14 +8,14 @@ on:
   - cron: '0 1 * * *'

 env:
-  DOCKER_VERSION: 31
+  DOCKER_VERSION: 32

 jobs:
  Docker:
    strategy:
      fail-fast: false
      matrix:
-        variant: [xenial, bionic, f31, focal]
+        variant: [bionic, f35, focal]
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
@ -44,33 +44,29 @@ jobs:
          - win32_static
          - i586_static
          - android
-          - flatpak
        include:
          - target: native_static
-            image_variant: xenial
+            image_variant: bionic
            lib_postfix: '/x86_64-linux-gnu'
          - target: native_dyn
-            image_variant: xenial
+            image_variant: bionic
            lib_postfix: '/x86_64-linux-gnu'
          - target: native_mixed
-            image_variant: xenial
+            image_variant: bionic
            lib_postfix: '/x86_64-linux-gnu'
          - target: native_desktop
            image_variant: bionic
            lib_postfix: '/x86_64-linux-gnu'
          - target: armhf_static
-            image_variant: xenial
+            image_variant: bionic
            lib_postfix: '/x86_64-linux-gnu'
          - target: win32_static
-            image_variant: f31
+            image_variant: f35
            lib_postfix: '64'
          - target: i586_static
-            image_variant: xenial
+            image_variant: bionic
            lib_postfix: '/x86_64-linux-gnu'
          - target: android
-            image_variant: xenial
-            lib_postfix: '/x86_64-linux-gnu'
-          - target: flatpak
            image_variant: bionic
            lib_postfix: '/x86_64-linux-gnu'
    env:
@ -79,7 +75,7 @@ jobs:
    runs-on: ubuntu-latest
    needs: Docker
    container:
-      image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-31"
+      image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-32"
      options: "--device /dev/fuse --privileged"
    steps:
    - name: Checkout code
@ -113,13 +109,56 @@ jobs:
        PLATFORM_TARGET: ${{matrix.target}}
        BINTRAY_USER: kiwix
        BINTRAY_PASS: ${{secrets.bintray_pass}}
-        GITHUB_PAT: ${{secrets.GHCR_TOKEN}}
    - name: Upload failure logs
      if: failure()
      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh
      env:
        PLATFORM_TARGET: ${{matrix.target}}

+  Flatpak:
+    strategy:
+      fail-fast: false
+    env:
+      HOME: /home/runner
+      SSH_KEY: /tmp/id_rsa
+      PLATFORM_TARGET: flatpak
+      OS_NAME: focal
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      shell: bash
+      run: |
+        cd $HOME
+        git clone https://github.com/${REP}
+        cd ./${REP##*/}
+        git checkout --force ${GITHUB_SHA}
+        pip3 install --user --no-deps .
+      env:
+        REP: ${{github.repository}}
+    - name: Install flatpak tools
+      run: |
+        sudo apt-get update
+        sudo apt-get install flatpak-builder ninja-build meson
+    - name: secret
+      shell: bash
+      run: |
+        echo "${{secrets.ssh_key}}" > $SSH_KEY
+        chmod 600 $SSH_KEY
+    - name: Ensure base deps
+      shell: bash
+      run: |
+        cd $HOME
+        kiwix-build/.github/scripts/ensure_base_deps.py
+    - name: Build release
+      shell: bash
+      run: |
+        cd $HOME
+        kiwix-build/.github/scripts/build_release_nightly.py
+      env:
+        GITHUB_PAT: ${{secrets.GHCR_TOKEN}}
+    - name: Upload failure logs
+      if: failure()
+      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh

  Macos:
    strategy:
@ -148,7 +187,7 @@ jobs:
        brew install pkg-config ninja automake autoconf
    - name: Install python modules
      run: |
-        pip3 install meson==0.52.1 pytest requests distro
+        pip3 install meson pytest requests distro
        pip3 install --no-deps $GITHUB_WORKSPACE
    - name: secret
      shell: bash
--- a/kiwixbuild/dependencies/flatpak.py
+++ b/kiwixbuild/dependencies/flatpak.py
@ -22,7 +22,7 @@ class org_kde(Dependency):
            run_command(command, self.buildEnv.build_dir, context, env=env)

        def _install_sdk(self, context):
-            command = "flatpak --user install -y {remote_name} {name}.Sdk//{version} {name}.Platform//{version}"
+            command = "flatpak --user install --noninteractive --verbose -y {remote_name} {name}.Sdk//{version} {name}.Platform//{version}"
            command = command.format(
                remote_name = 'flathub',
                name = self.target.name,
--- a/kiwixbuild/flatpak_builder.py
+++ b/kiwixbuild/flatpak_builder.py
@ -223,7 +223,7 @@ class FlatpakBuilder:
    def build(self):
        log = pj(self.platform.buildEnv.log_dir, 'cmd_build_flatpak.log')
        context = Context('build', log, False)
-        command = "flatpak-builder --user --ccache --force-clean --repo=repo builddir {id}.json"
+        command = "flatpak-builder --user --ccache --force-clean --keep-build-dirs --disable-rofiles-fuse --repo=repo builddir {id}.json"
        command = command.format(id = MANIFEST['app-id'])
        try:
            run_command(command, self.platform.buildEnv.build_dir, context, env=self.platform.get_env())
--- a/scripts/create_kiwix-desktop_appImage.sh
+++ b/scripts/create_kiwix-desktop_appImage.sh
@ -33,12 +33,13 @@ mkdir -p $APPDIR/usr/bin/ && tar -C $APPDIR/usr/bin/ -xf aria2-1.36.0-linux-gnu-
 mkdir -p $APPDIR/etc/ssl/certs/ && tar -C $APPDIR/etc/ssl/certs/ -xf aria2-1.36.0-linux-gnu-64bit-build1.tar.bz2 aria2-1.36.0-linux-gnu-64bit-build1/ca-certificates.crt --strip-components=1

 # Get linuxdeployqt
-wget --continue https://github.com/probonopd/linuxdeployqt/releases/download/6/linuxdeployqt-6-x86_64.AppImage
-chmod a+x linuxdeployqt-6-x86_64.AppImage
+# Dispite the 'continuous' in the file name, it IS release 8
+wget --continue https://github.com/probonopd/linuxdeployqt/releases/download/8/linuxdeployqt-continuous-x86_64.AppImage -O linuxdeployqt
+chmod u+x linuxdeployqt

 # Fill with all deps libs and so
-./linuxdeployqt-6-x86_64.AppImage $APPDIR/usr/bin/kiwix-desktop -unsupported-allow-new-glibc -bundle-non-qt-libs -extra-plugins=imageformats,iconengines
+./linuxdeployqt $APPDIR/usr/bin/kiwix-desktop -bundle-non-qt-libs -extra-plugins=imageformats,iconengines
 # Fix the RPATH of QtWebEngineProcess [TODO] Fill a issue ?
 patchelf --set-rpath '$ORIGIN/../lib' $APPDIR/usr/libexec/QtWebEngineProcess
 # Build the image.
-./linuxdeployqt-6-x86_64.AppImage $APPDIR/usr/share/applications/kiwix-desktop.desktop -unsupported-allow-new-glibc -bundle-non-qt-libs -extra-plugins=imageformats,iconengines -appimage
+./linuxdeployqt $APPDIR/usr/share/applications/kiwix-desktop.desktop -bundle-non-qt-libs -extra-plugins=imageformats,iconengines -appimage
--- a/setup.py
+++ b/setup.py
@ -35,7 +35,7 @@ setup(
    packages=find_packages(),
    include_package_data=True,
    install_requires=[
-        'meson==0.52.1',
+        'meson',
        'distro'
    ],
    entry_points={