From c55a96678e12514130c9fa496d34a890987f5927 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 19 Apr 2023 15:33:11 +0200 Subject: [PATCH 1/6] Build a minimalist bionic builder. Reuse the bionic builder (we should have remove in pr #588) and remove all deps not needed to build libzim. --- ci_images/bionic_builder.dockerfile | 27 +++------------------------ 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/ci_images/bionic_builder.dockerfile b/ci_images/bionic_builder.dockerfile index 4f09f2e..7aaf64e 100644 --- a/ci_images/bionic_builder.dockerfile +++ b/ci_images/bionic_builder.dockerfile @@ -4,42 +4,21 @@ ENV LANG C.UTF-8 ENV OS_NAME bionic RUN apt update -q \ - && dpkg --add-architecture i386 \ - && apt install -q -y --no-install-recommends software-properties-common \ - && add-apt-repository ppa:beineri/opt-qt-5.15.2-bionic \ && apt-get update \ && apt install -q -y --no-install-recommends \ # Base build tools build-essential automake libtool cmake ccache pkg-config autopoint patch \ python3-pip python3-setuptools python3-wheel git subversion wget unzip \ - ninja-build openssh-client curl libgl-dev \ -# Python (2) is needed to install android-ndk - python \ + ninja-build openssh-client curl \ # Packaged dependencies - libbz2-dev libmagic-dev uuid-dev zlib1g-dev \ - libmicrohttpd-dev aria2 libgtest-dev libgl-dev \ -# Devel package to compile python modules - libxml2-dev libxslt-dev python3-dev \ -# Qt packages - qt515base qt515webengine qt515svg qt515imageformats qt515wayland \ -# To create the appimage of kiwix-desktop - libfuse2 fuse patchelf \ -# Flatpak tools - elfutils flatpak flatpak-builder \ -# Cross compile i586 - libc6-dev-i386 lib32stdc++6 gcc-multilib g++-multilib \ + libbz2-dev uuid-dev zlib1g-dev \ + libgtest-dev \ # Other tools (to remove) # vim less grep \ && apt-get clean -y \ && rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/* \ && pip3 install meson pytest gcovr requests distro -# Set qt515 environment (the equivalent of "source /opt/qt515/bin/qt515-env.sh") -# RUN echo "source /opt/qt515/bin/qt515-env.sh" >> /home/runner/.bashrc -ENV PATH=/opt/qt515/bin:$PATH \ - LD_LIBRARY_PATH=/opt/qt515/lib/x86_64-linux-gnu:/opt/qt515/lib:$LD_LIBRARY_PATH \ - PKG_CONFIG_PATH=/opt/qt515/lib/pkgconfig:$PKG_CONFIG_PATH - # Create user RUN groupadd --gid 121 runner RUN useradd --uid 1001 --gid 121 --create-home runner From 7a0f490e007d4433df5e7e5d42b8ca201cd4530e Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 19 Apr 2023 15:33:54 +0200 Subject: [PATCH 2/6] Update CI workflows Update CI to build on native_mixed bionic and build the bionic docker image. --- .github/workflows/ci.yml | 3 +++ .github/workflows/container_images.yml | 2 +- .github/workflows/releaseNigthly.yml | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index db94518..69a9570 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,9 @@ jobs: image_variant: ['focal'] lib_postfix: ['/x86_64-linux-gnu'] include: + - target: native_mixed + image_variant: bionic + lib_postfix: '/x86_64-linux-gnu' - target: win32_static image_variant: f35 lib_postfix: '64' diff --git a/.github/workflows/container_images.yml b/.github/workflows/container_images.yml index c4a2fd0..e3f5e0b 100644 --- a/.github/workflows/container_images.yml +++ b/.github/workflows/container_images.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - variant: [f35, focal, alpine] + variant: [f35, focal, bionic, alpine] runs-on: ubuntu-22.04 steps: diff --git a/.github/workflows/releaseNigthly.yml b/.github/workflows/releaseNigthly.yml index 012d698..cee7918 100644 --- a/.github/workflows/releaseNigthly.yml +++ b/.github/workflows/releaseNigthly.yml @@ -38,6 +38,9 @@ jobs: - target: native_mixed image_variant: focal lib_postfix: '/x86_64-linux-gnu' + - target: native_mixed + image_variant: bionic + lib_postfix: '/x86_64-linux-gnu' - target: native_desktop image_variant: focal lib_postfix: '/x86_64-linux-gnu' From 4aa613c8e6184031013ec75eb73f932cc3a78a31 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 19 Apr 2023 15:34:39 +0200 Subject: [PATCH 3/6] Update CI build script Update CI scripts to build only libzim on native_mixed if os name is bionic. --- .github/scripts/build_projects.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/scripts/build_projects.py b/.github/scripts/build_projects.py index c84ed30..569327d 100755 --- a/.github/scripts/build_projects.py +++ b/.github/scripts/build_projects.py @@ -24,7 +24,10 @@ def select_build_target(): or PLATFORM_TARGET.startswith("macOS")): return ("libzim", "libkiwix") elif PLATFORM_TARGET.startswith("native_"): - if OS_NAME == "osx": + if OS_NAME == "bionic": + # PLATFORM_TARGET should always be "native_mixed" from CI jobs + return ("libzim", ) + elif OS_NAME == "osx": if PLATFORM_TARGET.endswith("_mixed"): return ("libzim", "libkiwix") else: From fffbd122c917cd1401d70c5a4bd56744aa266718 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 19 Apr 2023 15:35:13 +0200 Subject: [PATCH 4/6] =?UTF-8?q?Make=20bionic=20CI=C2=A0create=20archive=20?= =?UTF-8?q?containing=20bionic=20in=20the=20name.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/scripts/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/common.py b/.github/scripts/common.py index 87a2e59..647661f 100644 --- a/.github/scripts/common.py +++ b/.github/scripts/common.py @@ -47,7 +47,7 @@ else: RELEASE_OS_NAME = "macos" if OS_NAME == "osx" else "linux" PLATFORM_TO_RELEASE = { - "native_mixed": "{os}-x86_64".format(os=RELEASE_OS_NAME), + "native_mixed": "{os}-x86_64{extra}".format(os=RELEASE_OS_NAME, extra = "-bionic" if OS_NAME == "bionic" else ""), "native_static": "{os}-x86_64".format(os=RELEASE_OS_NAME), "win32_static": "win-i686", "armhf_static": "{os}-armhf".format(os=RELEASE_OS_NAME), From fc3b446c76105c8e8c6bfe372095177e887dbb90 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 25 Apr 2023 12:17:51 +0200 Subject: [PATCH 5/6] Also build libzim mixed for aarch64 on bionic. --- .github/scripts/build_projects.py | 9 ++++----- .github/scripts/common.py | 5 +++-- .github/workflows/ci.yml | 3 +++ .github/workflows/releaseNigthly.yml | 3 +++ 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/scripts/build_projects.py b/.github/scripts/build_projects.py index 569327d..4e7bb98 100755 --- a/.github/scripts/build_projects.py +++ b/.github/scripts/build_projects.py @@ -19,15 +19,14 @@ def select_build_target(): DESKTOP, OS_NAME ) - if (PLATFORM_TARGET.startswith("android_") + if OS_NAME == "bionic" and PLATFORM_TARGET.endswith("_mixed"): + return ("libzim", ) + elif (PLATFORM_TARGET.startswith("android_") or PLATFORM_TARGET.startswith("iOS") or PLATFORM_TARGET.startswith("macOS")): return ("libzim", "libkiwix") elif PLATFORM_TARGET.startswith("native_"): - if OS_NAME == "bionic": - # PLATFORM_TARGET should always be "native_mixed" from CI jobs - return ("libzim", ) - elif OS_NAME == "osx": + if OS_NAME == "osx": if PLATFORM_TARGET.endswith("_mixed"): return ("libzim", "libkiwix") else: diff --git a/.github/scripts/common.py b/.github/scripts/common.py index 647661f..a8abd93 100644 --- a/.github/scripts/common.py +++ b/.github/scripts/common.py @@ -45,15 +45,16 @@ else: DEV_BRANCH = None RELEASE_OS_NAME = "macos" if OS_NAME == "osx" else "linux" +EXTRA_NAME = "-bionic" if OS_NAME == "bionic" else "" PLATFORM_TO_RELEASE = { - "native_mixed": "{os}-x86_64{extra}".format(os=RELEASE_OS_NAME, extra = "-bionic" if OS_NAME == "bionic" else ""), + "native_mixed": "{os}-x86_64{extra}".format(os=RELEASE_OS_NAME, extra=EXTRA_NAME), "native_static": "{os}-x86_64".format(os=RELEASE_OS_NAME), "win32_static": "win-i686", "armhf_static": "{os}-armhf".format(os=RELEASE_OS_NAME), "armhf_mixed": "{os}-armhf".format(os=RELEASE_OS_NAME), "aarch64_static": "{os}-aarch64".format(os=RELEASE_OS_NAME), - "aarch64_mixed": "{os}-aarch64".format(os=RELEASE_OS_NAME), + "aarch64_mixed": "{os}-aarch64{extra}".format(os=RELEASE_OS_NAME, extra=EXTRA_NAME), "i586_static": "{os}-i586".format(os=RELEASE_OS_NAME), "macOS_arm64_static": "{os}-arm64".format(os=RELEASE_OS_NAME), "macOS_arm64_mixed": "{os}-arm64".format(os=RELEASE_OS_NAME), diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 69a9570..ac2aa69 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,9 @@ jobs: - target: native_mixed image_variant: bionic lib_postfix: '/x86_64-linux-gnu' + - target: aarch64_mixed + image_variant: bionic + lib_postfix: '/aarch64-linux-gnu' - target: win32_static image_variant: f35 lib_postfix: '64' diff --git a/.github/workflows/releaseNigthly.yml b/.github/workflows/releaseNigthly.yml index cee7918..ae974d6 100644 --- a/.github/workflows/releaseNigthly.yml +++ b/.github/workflows/releaseNigthly.yml @@ -59,6 +59,9 @@ jobs: - target: aarch64_mixed image_variant: focal lib_postfix: '/x86_64-linux-gnu' + - target: aarch64_mixed + image_variant: bionic + lib_postfix: '/aarch64-linux-gnu' - target: win32_static image_variant: f35 lib_postfix: '64' From fd85b80b2ca2741a71b14fdfc0c3554758fb34e1 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 26 Apr 2023 15:17:52 +0200 Subject: [PATCH 6/6] Move back to gcc 6.3.0 when compiling to aarch64. --- kiwixbuild/dependencies/armhf.py | 8 ++++---- kiwixbuild/versions.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kiwixbuild/dependencies/armhf.py b/kiwixbuild/dependencies/armhf.py index c0b350d..b1501d9 100644 --- a/kiwixbuild/dependencies/armhf.py +++ b/kiwixbuild/dependencies/armhf.py @@ -8,7 +8,7 @@ base_url = 'https://master.dl.sourceforge.net/project/raspberry-pi-cross-compile armhf_base_url = base_url + 'Raspberry%20Pi%20GCC%20Cross-Compiler%20Toolchains/Stretch/GCC%2010.3.0/Raspberry%20Pi%202%2C%203/' # This is Gcc 10.3.0 and ALL rapsberry Pi arch64 -aarch_base_url = base_url + 'Bonus%20Raspberry%20Pi%20GCC%2064-Bit%20Toolchains/Raspberry%20Pi%20GCC%2064-Bit%20Cross-Compiler%20Toolchains/Stretch/GCC%2010.3.0/' +aarch_base_url = base_url + 'Bonus%20Raspberry%20Pi%20GCC%2064-Bit%20Toolchains/Raspberry%20Pi%20GCC%2064-Bit%20Cross-Compiler%20Toolchains/Stretch/GCC%206.3.0/' class armhf_toolchain(Dependency): dont_skip = True @@ -29,8 +29,8 @@ class aarch64_toolchain(Dependency): name = "aarch64" class Source(ReleaseDownload): - archive = Remotefile('cross-gcc-10.3.0-pi_64.tar.gz', - '5b3fdb7ee8c496c377ab8b11d7ffd404b4d3041f4fdcfeebcbcb734d45a5f3e9', - aarch_base_url + 'cross-gcc-10.3.0-pi_64.tar.gz') + archive = Remotefile('cross-gcc-6.3.0-pi_64.tar.gz', + '1b048bb8886ad63d21797cd9129fc37b9ea0dfaac7e3c36f888aa16fbec1d320', + aarch_base_url + 'cross-gcc-6.3.0-pi_64.tar.gz') Builder = NoopBuilder diff --git a/kiwixbuild/versions.py b/kiwixbuild/versions.py index 08c994b..1337e79 100644 --- a/kiwixbuild/versions.py +++ b/kiwixbuild/versions.py @@ -39,7 +39,7 @@ release_versions = { # This is the "version" of the whole base_deps_versions dict. # Change this when you change base_deps_versions. -base_deps_meta_version = '85' +base_deps_meta_version = '86' base_deps_versions = { 'zlib' : '1.2.12',