diff --git a/.github/ci_images/bionic_builder.dockerfile b/.github/ci_images/bionic_builder.dockerfile index c1e76f6..89291eb 100644 --- a/.github/ci_images/bionic_builder.dockerfile +++ b/.github/ci_images/bionic_builder.dockerfile @@ -30,7 +30,7 @@ RUN apt update -q \ # vim less grep \ && apt-get clean -y \ && rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/* \ - && pip3 install meson==0.52.1 pytest gcovr requests distro + && pip3 install meson pytest gcovr requests distro # Create user RUN useradd --create-home runner diff --git a/.github/ci_images/f31_builder.dockerfile b/.github/ci_images/f35_builder.dockerfile similarity index 89% rename from .github/ci_images/f31_builder.dockerfile rename to .github/ci_images/f35_builder.dockerfile index a96d7a5..bb3fc95 100644 --- a/.github/ci_images/f31_builder.dockerfile +++ b/.github/ci_images/f35_builder.dockerfile @@ -1,7 +1,7 @@ -FROM fedora:31 +FROM fedora:35 ENV LANG C.UTF-8 -ENV OS_NAME f31 +ENV OS_NAME f35 RUN dnf install -y --nodocs \ # Base build tools @@ -19,7 +19,7 @@ RUN dnf install -y --nodocs \ && dnf remove -y "*-doc" \ && dnf autoremove -y \ && dnf clean all \ - && pip3 install meson==0.52.1 pytest requests distro + && pip3 install meson pytest requests distro # Create user RUN useradd --create-home runner diff --git a/.github/ci_images/focal_builder.dockerfile b/.github/ci_images/focal_builder.dockerfile index 302ee13..3998210 100644 --- a/.github/ci_images/focal_builder.dockerfile +++ b/.github/ci_images/focal_builder.dockerfile @@ -25,7 +25,7 @@ RUN apt update -q \ # vim less grep \ && apt-get clean -y \ && rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/* \ - && pip3 install meson==0.52.1 pytest gcovr requests distro + && pip3 install meson pytest gcovr requests distro # Create user RUN useradd --create-home runner diff --git a/.github/ci_images/xenial_builder.dockerfile b/.github/ci_images/xenial_builder.dockerfile deleted file mode 100644 index 7b7ce9a..0000000 --- a/.github/ci_images/xenial_builder.dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -FROM ubuntu:xenial - -ENV LANG C.UTF-8 -ENV OS_NAME xenial - -RUN apt update -q \ - && dpkg --add-architecture i386 \ - && apt install -q -y --no-install-recommends \ -# Base build tools - build-essential automake libtool cmake ccache pkg-config autopoint patch \ - python3-pip python3-setuptools python3-wheel git subversion wget unzip \ - ninja-build openssh-client curl \ -# Python (2) is needed to install android-ndk - python \ -# Packaged dependencies - libbz2-dev libmagic-dev uuid-dev zlib1g-dev default-jdk \ - libmicrohttpd-dev libgtest-dev \ -# Cross compile i586 - libc6-dev-i386 lib32stdc++6 gcc-multilib g++-multilib \ -# Other tools (to remove) -# vim less grep \ - && apt-get clean -y \ - && rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/* \ - && pip3 install meson==0.52.1 pytest 'markupsafe<2.0.0' 'jinja2<3.0.0' 'gcovr<5.0' requests distro - -# Create user -RUN useradd --create-home runner -USER runner -WORKDIR /home/runner -ENV PATH /home/runner/.local/bin:$PATH diff --git a/.github/scripts/build_projects.py b/.github/scripts/build_projects.py new file mode 100755 index 0000000..6fd5851 --- /dev/null +++ b/.github/scripts/build_projects.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 + + +from common import ( + run_kiwix_build, + make_archive, + create_desktop_image, + fix_macos_rpath, + OS_NAME, + PLATFORM_TARGET, + DESKTOP, + notarize_macos_build, +) + +if PLATFORM_TARGET.startswith("android_"): + TARGETS = ("libkiwix",) +elif PLATFORM_TARGET.startswith("iOS") or PLATFORM_TARGET.startswith("macOS"): + TARGETS = ("libzim", "libkiwix") +elif PLATFORM_TARGET.startswith("native_"): + if OS_NAME == "osx": + if PLATFORM_TARGET == "native_mixed": + TARGETS = ("libzim", ) + else: + TARGETS = ("libzim", "zim-tools", "libkiwix") + else: + if DESKTOP: + TARGETS = ("kiwix-desktop",) + elif PLATFORM_TARGET == "native_mixed": + TARGETS = ("libzim",) + else: + TARGETS = ("zim-tools", "libkiwix", "kiwix-tools") +elif PLATFORM_TARGET in ("win32_static", "armhf_static", "armhf_dyn", "i586_static"): + TARGETS = ("kiwix-tools",) +elif PLATFORM_TARGET == "flatpak": + TARGETS = ("kiwix-desktop",) +else: + TARGETS = ("libzim", "zim-tools", "libkiwix", "kiwix-tools") + +for target in TARGETS: + run_kiwix_build(target, platform=PLATFORM_TARGET) + if target == "kiwix-desktop": + create_desktop_image(make_release=False) + else: + if PLATFORM_TARGET == "native_mixed" and OS_NAME == "osx": + fix_macos_rpath(target) + notarize_macos_build(target) + make_archive(target, make_release=False) diff --git a/.github/scripts/ensure_base_deps.py b/.github/scripts/ensure_base_deps.py index 4122d4b..f2bd484 100755 --- a/.github/scripts/ensure_base_deps.py +++ b/.github/scripts/ensure_base_deps.py @@ -33,7 +33,7 @@ def download_base_archive(base_name): ARCHIVE_NAME_TEMPLATE = "base_deps2_{os}_{platform}_{version}.tar.xz" if PLATFORM_TARGET == 'flatpak': - base_dep_archive_name = "base_deps2_{}_flatpak.tar.xz".format(OS_NAME) + base_dep_archive_name = "base_deps2_flatpak.tar.xz" else: base_dep_archive_name = ARCHIVE_NAME_TEMPLATE.format( os=OS_NAME, diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c69f45..1ae1d15 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,14 +6,14 @@ on: - cron: '0 1 * * *' env: - DOCKER_VERSION: 31 + DOCKER_VERSION: 32 jobs: Docker: strategy: fail-fast: false matrix: - variant: [xenial, bionic, f31, focal] + variant: [bionic, f35, focal] runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -46,27 +46,23 @@ jobs: - android_arm64 - android_x86 - android_x86_64 - image_variant: ['xenial', 'bionic'] + image_variant: ['bionic'] lib_postfix: ['/x86_64-linux-gnu'] include: - target: win32_static - image_variant: f31 + image_variant: f35 lib_postfix: '64' - target: win32_dyn - image_variant: f31 + image_variant: f35 lib_postfix: '64' - exclude: - - target: native_desktop - image_variant: xenial - - target: native_static - image_variant: bionic env: HOME: /home/runner SSH_KEY: /tmp/id_rsa runs-on: ubuntu-latest needs: Docker container: - image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-31" + image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-32" + options: "--device /dev/fuse --privileged" steps: - name: Checkout code shell: bash @@ -97,12 +93,61 @@ jobs: kiwix-build/.github/scripts/compile_all_deps.py env: PLATFORM_TARGET: ${{matrix.target}} + - name: Build projects + shell: bash + run: | + cd $HOME + kiwix-build/.github/scripts/build_projects.py + env: + PLATFORM_TARGET: ${{matrix.target}} - name: Upload failure logs if: failure() run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh env: PLATFORM_TARGET: ${{matrix.target}} + Flatpak: + strategy: + fail-fast: false + env: + HOME: /home/runner + SSH_KEY: /tmp/id_rsa + PLATFORM_TARGET: flatpak + OS_NAME: focal + runs-on: ubuntu-latest + steps: + - name: Checkout code + shell: bash + run: | + cd $HOME + git clone https://github.com/${REP} + cd ./${REP##*/} + git checkout --force ${GITHUB_SHA} + pip3 install --user --no-deps . + env: + REP: ${{github.repository}} + - name: Install flatpak tools + run: | + sudo apt-get update + sudo apt-get install flatpak-builder ninja-build meson + - name: secret + shell: bash + run: | + echo "${{secrets.ssh_key}}" > $SSH_KEY + chmod 600 $SSH_KEY + - name: Ensure base deps + shell: bash + run: | + cd $HOME + kiwix-build/.github/scripts/ensure_base_deps.py + - name: Build projects + shell: bash + run: | + cd $HOME + kiwix-build/.github/scripts/build_projects.py + - name: Upload failure logs + if: failure() + run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh Macos: strategy: @@ -132,7 +177,7 @@ jobs: brew install pkg-config ninja automake autoconf - name: Install python modules run: | - pip3 install meson==0.52.1 pytest requests distro + pip3 install meson pytest requests distro pip3 install --no-deps $GITHUB_WORKSPACE - name: secret shell: bash @@ -153,6 +198,13 @@ jobs: $GITHUB_WORKSPACE/.github/scripts/compile_all_deps.py env: PLATFORM_TARGET: ${{matrix.target}} + - name: Build projects + shell: bash + run: | + cd $HOME + $GITHUB_WORKSPACE/.github/scripts/build_projects.py + env: + PLATFORM_TARGET: ${{matrix.target}} - name: Upload failure logs if: failure() run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.sh diff --git a/.github/workflows/releaseNigthly.yml b/.github/workflows/releaseNigthly.yml index a4d0c10..047cbbf 100644 --- a/.github/workflows/releaseNigthly.yml +++ b/.github/workflows/releaseNigthly.yml @@ -8,14 +8,14 @@ on: - cron: '0 1 * * *' env: - DOCKER_VERSION: 31 + DOCKER_VERSION: 32 jobs: Docker: strategy: fail-fast: false matrix: - variant: [xenial, bionic, f31, focal] + variant: [bionic, f35, focal] runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -44,33 +44,29 @@ jobs: - win32_static - i586_static - android - - flatpak include: - target: native_static - image_variant: xenial + image_variant: bionic lib_postfix: '/x86_64-linux-gnu' - target: native_dyn - image_variant: xenial + image_variant: bionic lib_postfix: '/x86_64-linux-gnu' - target: native_mixed - image_variant: xenial + image_variant: bionic lib_postfix: '/x86_64-linux-gnu' - target: native_desktop image_variant: bionic lib_postfix: '/x86_64-linux-gnu' - target: armhf_static - image_variant: xenial + image_variant: bionic lib_postfix: '/x86_64-linux-gnu' - target: win32_static - image_variant: f31 + image_variant: f35 lib_postfix: '64' - target: i586_static - image_variant: xenial + image_variant: bionic lib_postfix: '/x86_64-linux-gnu' - target: android - image_variant: xenial - lib_postfix: '/x86_64-linux-gnu' - - target: flatpak image_variant: bionic lib_postfix: '/x86_64-linux-gnu' env: @@ -79,7 +75,7 @@ jobs: runs-on: ubuntu-latest needs: Docker container: - image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-31" + image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-32" options: "--device /dev/fuse --privileged" steps: - name: Checkout code @@ -113,13 +109,56 @@ jobs: PLATFORM_TARGET: ${{matrix.target}} BINTRAY_USER: kiwix BINTRAY_PASS: ${{secrets.bintray_pass}} - GITHUB_PAT: ${{secrets.GHCR_TOKEN}} - name: Upload failure logs if: failure() run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh env: PLATFORM_TARGET: ${{matrix.target}} + Flatpak: + strategy: + fail-fast: false + env: + HOME: /home/runner + SSH_KEY: /tmp/id_rsa + PLATFORM_TARGET: flatpak + OS_NAME: focal + runs-on: ubuntu-latest + steps: + - name: Checkout code + shell: bash + run: | + cd $HOME + git clone https://github.com/${REP} + cd ./${REP##*/} + git checkout --force ${GITHUB_SHA} + pip3 install --user --no-deps . + env: + REP: ${{github.repository}} + - name: Install flatpak tools + run: | + sudo apt-get update + sudo apt-get install flatpak-builder ninja-build meson + - name: secret + shell: bash + run: | + echo "${{secrets.ssh_key}}" > $SSH_KEY + chmod 600 $SSH_KEY + - name: Ensure base deps + shell: bash + run: | + cd $HOME + kiwix-build/.github/scripts/ensure_base_deps.py + - name: Build release + shell: bash + run: | + cd $HOME + kiwix-build/.github/scripts/build_release_nightly.py + env: + GITHUB_PAT: ${{secrets.GHCR_TOKEN}} + - name: Upload failure logs + if: failure() + run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh Macos: strategy: @@ -148,7 +187,7 @@ jobs: brew install pkg-config ninja automake autoconf - name: Install python modules run: | - pip3 install meson==0.52.1 pytest requests distro + pip3 install meson pytest requests distro pip3 install --no-deps $GITHUB_WORKSPACE - name: secret shell: bash diff --git a/kiwixbuild/dependencies/flatpak.py b/kiwixbuild/dependencies/flatpak.py index 943c777..4ff5265 100644 --- a/kiwixbuild/dependencies/flatpak.py +++ b/kiwixbuild/dependencies/flatpak.py @@ -22,7 +22,7 @@ class org_kde(Dependency): run_command(command, self.buildEnv.build_dir, context, env=env) def _install_sdk(self, context): - command = "flatpak --user install -y {remote_name} {name}.Sdk//{version} {name}.Platform//{version}" + command = "flatpak --user install --noninteractive --verbose -y {remote_name} {name}.Sdk//{version} {name}.Platform//{version}" command = command.format( remote_name = 'flathub', name = self.target.name, diff --git a/kiwixbuild/flatpak_builder.py b/kiwixbuild/flatpak_builder.py index 208ac77..797bbfc 100644 --- a/kiwixbuild/flatpak_builder.py +++ b/kiwixbuild/flatpak_builder.py @@ -223,7 +223,7 @@ class FlatpakBuilder: def build(self): log = pj(self.platform.buildEnv.log_dir, 'cmd_build_flatpak.log') context = Context('build', log, False) - command = "flatpak-builder --user --ccache --force-clean --repo=repo builddir {id}.json" + command = "flatpak-builder --user --ccache --force-clean --keep-build-dirs --disable-rofiles-fuse --repo=repo builddir {id}.json" command = command.format(id = MANIFEST['app-id']) try: run_command(command, self.platform.buildEnv.build_dir, context, env=self.platform.get_env()) diff --git a/scripts/create_kiwix-desktop_appImage.sh b/scripts/create_kiwix-desktop_appImage.sh index 39bd667..259ee49 100755 --- a/scripts/create_kiwix-desktop_appImage.sh +++ b/scripts/create_kiwix-desktop_appImage.sh @@ -33,12 +33,13 @@ mkdir -p $APPDIR/usr/bin/ && tar -C $APPDIR/usr/bin/ -xf aria2-1.36.0-linux-gnu- mkdir -p $APPDIR/etc/ssl/certs/ && tar -C $APPDIR/etc/ssl/certs/ -xf aria2-1.36.0-linux-gnu-64bit-build1.tar.bz2 aria2-1.36.0-linux-gnu-64bit-build1/ca-certificates.crt --strip-components=1 # Get linuxdeployqt -wget --continue https://github.com/probonopd/linuxdeployqt/releases/download/6/linuxdeployqt-6-x86_64.AppImage -chmod a+x linuxdeployqt-6-x86_64.AppImage +# Dispite the 'continuous' in the file name, it IS release 8 +wget --continue https://github.com/probonopd/linuxdeployqt/releases/download/8/linuxdeployqt-continuous-x86_64.AppImage -O linuxdeployqt +chmod u+x linuxdeployqt # Fill with all deps libs and so -./linuxdeployqt-6-x86_64.AppImage $APPDIR/usr/bin/kiwix-desktop -unsupported-allow-new-glibc -bundle-non-qt-libs -extra-plugins=imageformats,iconengines +./linuxdeployqt $APPDIR/usr/bin/kiwix-desktop -bundle-non-qt-libs -extra-plugins=imageformats,iconengines # Fix the RPATH of QtWebEngineProcess [TODO] Fill a issue ? patchelf --set-rpath '$ORIGIN/../lib' $APPDIR/usr/libexec/QtWebEngineProcess # Build the image. -./linuxdeployqt-6-x86_64.AppImage $APPDIR/usr/share/applications/kiwix-desktop.desktop -unsupported-allow-new-glibc -bundle-non-qt-libs -extra-plugins=imageformats,iconengines -appimage +./linuxdeployqt $APPDIR/usr/share/applications/kiwix-desktop.desktop -bundle-non-qt-libs -extra-plugins=imageformats,iconengines -appimage diff --git a/setup.py b/setup.py index c53a024..e3fad6c 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ setup( packages=find_packages(), include_package_data=True, install_requires=[ - 'meson==0.52.1', + 'meson', 'distro' ], entry_points={