From 82617f6b94a726b4f1f78e6c891f2c5e00bee01e Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 20 Jan 2020 22:55:21 +0100 Subject: [PATCH 01/10] Do not set `march` for android build. We already build to the right architecture by using the arch_full triplet gcc/g++/... We don't need to set march when building android. --- kiwixbuild/platforms/android.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/kiwixbuild/platforms/android.py b/kiwixbuild/platforms/android.py index c7e0b0e..1a8e3a4 100644 --- a/kiwixbuild/platforms/android.py +++ b/kiwixbuild/platforms/android.py @@ -40,9 +40,6 @@ class AndroidPlatformInfo(PlatformInfo): def get_cross_config(self): extra_libs = ['-llog'] extra_cflags = ['-I{}'.format(pj(self.buildEnv.install_dir, 'include'))] - if hasattr(self, 'march'): - extra_libs.append('-march={}'.format(self.march)) - extra_cflags.append('-march={}'.format(self.march)) return { 'exe_wrapper_def': '', 'install_path': self.install_path, @@ -65,11 +62,10 @@ class AndroidPlatformInfo(PlatformInfo): def set_env(self, env): root_path = pj(self.install_path, 'sysroot') - march = '-march={}'.format(self.march) if hasattr(self,'march') else '' env['PKG_CONFIG_LIBDIR'] = pj(root_path, 'lib', 'pkgconfig') - env['CFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} {} '.format(root_path, march) + env['CFLAGS'] - env['CXXFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} {} '.format(root_path, march) + env['CXXFLAGS'] - env['LDFLAGS'] = '--sysroot={} {} '.format(root_path, march) + env['LDFLAGS'] + env['CFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(root_path) + env['CFLAGS'] + env['CXXFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(root_path) + env['CXXFLAGS'] + env['LDFLAGS'] = '--sysroot={} '.format(root_path) + env['LDFLAGS'] #env['CFLAGS'] = ' -fPIC -D_FILE_OFFSET_BITS=64 -O3 '+env['CFLAGS'] #env['CXXFLAGS'] = (' -D__OPTIMIZE__ -fno-strict-aliasing ' # ' -DU_HAVE_NL_LANGINFO_CODESET=0 ' @@ -97,7 +93,6 @@ class AndroidArm(AndroidPlatformInfo): arch = cpu = 'arm' arch_full = 'arm-linux-androideabi' abi = 'armeabi-v7a' - march = 'armv7-a' class AndroidArm64(AndroidPlatformInfo): From 2774967e6d2eae0b1361e4d3f20112b5e55a0917 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 20 Jan 2020 21:28:39 +0100 Subject: [PATCH 02/10] Set the pkg-config for cross-compiling. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is needed for win32 on fedora. It is better to use a specific pkg-config than setting a PKG_CONFIG_LIBDIR. This avoid other CI (libzim/...) to set a env var specific to win32 build. --- kiwixbuild/platforms/android.py | 6 ++++-- kiwixbuild/platforms/armhf.py | 6 ++++-- kiwixbuild/platforms/i586.py | 3 ++- kiwixbuild/platforms/ios.py | 1 + kiwixbuild/platforms/win32.py | 3 ++- kiwixbuild/templates/meson_cross_file.txt | 2 +- 6 files changed, 14 insertions(+), 7 deletions(-) diff --git a/kiwixbuild/platforms/android.py b/kiwixbuild/platforms/android.py index 1a8e3a4..668a2ab 100644 --- a/kiwixbuild/platforms/android.py +++ b/kiwixbuild/platforms/android.py @@ -26,8 +26,10 @@ class AndroidPlatformInfo(PlatformInfo): def binaries(self): install_path = self.install_path - return {k:pj(install_path, 'bin', v) - for k,v in self.binaries_name.items()} + binaries = {k:pj(install_path, 'bin', v) + for k,v in self.binaries_name.items()} + binaries['PKGCONFIG'] = 'pkg-config' + return binaries @property def ndk_builder(self): diff --git a/kiwixbuild/platforms/armhf.py b/kiwixbuild/platforms/armhf.py index d07710d..6e0497d 100644 --- a/kiwixbuild/platforms/armhf.py +++ b/kiwixbuild/platforms/armhf.py @@ -49,8 +49,10 @@ class ArmhfPlatformInfo(PlatformInfo): ('RANLIB', 'ranlib'), ('LD', 'ld')) ) - return {k:pj(self.root_path, 'bin', v) - for k,v in binaries} + binaries = {k:pj(self.root_path, 'bin', v) + for k,v in binaries} + binaries['PKGCONFIG'] = 'pkg-config' + return binaries @property def exe_wrapper_def(self): diff --git a/kiwixbuild/platforms/i586.py b/kiwixbuild/platforms/i586.py index ec1160a..e090001 100644 --- a/kiwixbuild/platforms/i586.py +++ b/kiwixbuild/platforms/i586.py @@ -37,7 +37,8 @@ class I586PlatformInfo(PlatformInfo): ('AR', 'ar'), ('STRIP', 'strip'), ('RANLIB', 'ranlib'), - ('LD', 'ld')) + ('LD', 'ld'), + ('PKGCONFIG', 'pkg-config')) } def set_env(self, env): diff --git a/kiwixbuild/platforms/ios.py b/kiwixbuild/platforms/ios.py index 523d426..df08374 100644 --- a/kiwixbuild/platforms/ios.py +++ b/kiwixbuild/platforms/ios.py @@ -64,6 +64,7 @@ class iOSPlatformInfo(PlatformInfo): 'STRIP': '/usr/bin/strip', 'RANLIB': '/usr/bin/ranlib', 'LD': '/usr/bin/ld', + 'PKGCONFIG': 'pkg-config', } @property diff --git a/kiwixbuild/platforms/win32.py b/kiwixbuild/platforms/win32.py index b2f3fa6..17b37c4 100644 --- a/kiwixbuild/platforms/win32.py +++ b/kiwixbuild/platforms/win32.py @@ -49,7 +49,8 @@ class Win32PlatformInfo(PlatformInfo): ('AR', 'ar'), ('STRIP', 'strip'), ('WINDRES', 'windres'), - ('RANLIB', 'ranlib')) + ('RANLIB', 'ranlib'), + ('PKGCONFIG', 'pkg-config')) } @property diff --git a/kiwixbuild/templates/meson_cross_file.txt b/kiwixbuild/templates/meson_cross_file.txt index 6a664de..d5e6716 100644 --- a/kiwixbuild/templates/meson_cross_file.txt +++ b/kiwixbuild/templates/meson_cross_file.txt @@ -1,5 +1,5 @@ [binaries] -pkgconfig = 'pkg-config' +pkgconfig = '{binaries[PKGCONFIG]}' c = '{binaries[CC]}' ar = '{binaries[AR]}' cpp = '{binaries[CXX]}' From 2f151df63cea05464fb2288a38b465cf03b3a623 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 20 Jan 2020 21:30:39 +0100 Subject: [PATCH 03/10] Update version of meson. We should not force a specific version as it is a environment problem, but for now, set it. --- .travis.yml | 2 +- setup.py | 2 +- travis/xenial_builder.dockerfile | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6a9eb8c..6b78217 100644 --- a/.travis.yml +++ b/.travis.yml @@ -114,7 +114,7 @@ env: global: - NIGHTLY_DATE=$(date +%Y-%m-%d) - SSH_KEY=/tmp/private/travisci_builder_id_key - - DOCKER_VERSION=2 + - DOCKER_VERSION=4 matrix: - PLATFORM="flatpak" DESKTOP_ONLY=1 - PLATFORM="native_dyn" diff --git a/setup.py b/setup.py index eb1642d..0deb12f 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ setup( packages=find_packages(), include_package_data=True, install_requires=[ - 'meson==0.49.2' + 'meson==0.52.1' ], entry_points={ 'console_scripts': [ diff --git a/travis/xenial_builder.dockerfile b/travis/xenial_builder.dockerfile index 3508ace..e0b7bcb 100644 --- a/travis/xenial_builder.dockerfile +++ b/travis/xenial_builder.dockerfile @@ -20,7 +20,9 @@ RUN apt update -q && \ # vim less grep \ && \ apt-get clean -y && \ - rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/* + rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/* && \ + pip3 install --upgrade pip && \ + pip3 install meson==0.52.1 pytest gcovr # Create user RUN useradd --create-home ci_builder @@ -32,4 +34,4 @@ ENV TRAVIS_BUILD_DIR /home/ci_builder/kiwix-build ENV GRADLE_USER_HOME /home/ci_builder ENV TRAVIS_OS_NAME linux_xenial -CMD pip3 install --user ./kiwix-build && kiwix-build/travis/compile_all.py +CMD pip3 install --user --no-deps ./kiwix-build && kiwix-build/travis/compile_all.py From ce45966d40013c2fcead1f3d98a6a60dd4bec951 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 20 Jan 2020 22:58:10 +0100 Subject: [PATCH 04/10] Add a GithubAction CI to build the docker images. No really useful for now. But will be in next commit. --- .github/ci_images/bionic_builder.dockerfile | 33 +++++++++++++++++++++ .github/ci_images/f30_builder.dockerfile | 25 ++++++++++++++++ .github/ci_images/focal_builder.dockerfile | 33 +++++++++++++++++++++ .github/ci_images/xenial_builder.dockerfile | 30 +++++++++++++++++++ .github/workflows/base.yml | 28 +++++++++++++++++ 5 files changed, 149 insertions(+) create mode 100644 .github/ci_images/bionic_builder.dockerfile create mode 100644 .github/ci_images/f30_builder.dockerfile create mode 100644 .github/ci_images/focal_builder.dockerfile create mode 100644 .github/ci_images/xenial_builder.dockerfile create mode 100644 .github/workflows/base.yml diff --git a/.github/ci_images/bionic_builder.dockerfile b/.github/ci_images/bionic_builder.dockerfile new file mode 100644 index 0000000..c45cb78 --- /dev/null +++ b/.github/ci_images/bionic_builder.dockerfile @@ -0,0 +1,33 @@ +FROM ubuntu:bionic + +ENV LANG C.UTF-8 +ENV OS_NAME bionic + +RUN apt update -q \ + && apt install -q -y --no-install-recommends \ +# Base build tools + build-essential automake libtool cmake ccache pkg-config autopoint patch \ + python3-pip python3-setuptools python3-wheel git subversion wget unzip \ + ninja-build openssh-client \ +# Packaged dependencies + libbz2-dev libmagic-dev uuid-dev zlib1g-dev \ + libmicrohttpd-dev aria2 libgtest-dev \ +# Qt packages + libqt5gui5 qtbase5-dev qtwebengine5-dev libqt5svg5-dev qt5-image-formats-plugins qt5-default \ +# To create the appimage of kiwix-desktop + libfuse2 fuse patchelf \ +# Flatpak tools + elfutils flatpak flatpak-builder \ +# Cross win32 compiler + g++-mingw-w64-i686 gcc-mingw-w64-i686 gcc-mingw-w64-base mingw-w64-tools \ +# Other tools (to remove) +# vim less grep \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/* \ + && pip3 install meson==0.52.1 pytest gcovr + +# Create user +RUN useradd --create-home runner +USER runner +WORKDIR /home/runner +ENV PATH /home/runner/.local/bin:$PATH diff --git a/.github/ci_images/f30_builder.dockerfile b/.github/ci_images/f30_builder.dockerfile new file mode 100644 index 0000000..b21a9f3 --- /dev/null +++ b/.github/ci_images/f30_builder.dockerfile @@ -0,0 +1,25 @@ +FROM fedora:30 + +ENV LANG C.UTF-8 +ENV OS_NAME f30 + +RUN dnf install -y --nodocs \ +# Base build tools + make automake libtool cmake git-core subversion pkg-config gcc-c++ \ + wget unzip ninja-build ccache which patch gcovr xz openssh-clients \ +# Cross win32 compiler + mingw32-gcc-c++ mingw32-bzip2-static mingw32-win-iconv-static \ + mingw32-winpthreads-static mingw32-zlib-static mingw32-xz-libs-static \ + mingw32-libmicrohttpd \ +# Other tools (to remove) +# vim less grep + && dnf remove -y "*-doc" \ + && dnf autoremove -y \ + && dnf clean all \ + && pip3 install meson==0.52.1 pytest + +# Create user +RUN useradd --create-home runner +USER runner +WORKDIR /home/runner +ENV PATH /home/runner/.local/bin:$PATH diff --git a/.github/ci_images/focal_builder.dockerfile b/.github/ci_images/focal_builder.dockerfile new file mode 100644 index 0000000..b3b9a00 --- /dev/null +++ b/.github/ci_images/focal_builder.dockerfile @@ -0,0 +1,33 @@ +FROM ubuntu:focal + +ENV LANG C.UTF-8 +ENV OS_NAME focal + +RUN apt update -q \ + && apt install -q -y --no-install-recommends \ +# Base build tools + build-essential automake libtool cmake ccache pkg-config autopoint patch \ + python3-pip python3-setuptools python3-wheel git subversion wget unzip \ + ninja-build openssh-client \ +# Packaged dependencies + libbz2-dev libmagic-dev uuid-dev zlib1g-dev \ + libmicrohttpd-dev aria2 libgtest-dev \ +# Qt packages + libqt5gui5 qtbase5-dev qtwebengine5-dev libqt5svg5-dev qt5-image-formats-plugins qt5-default \ +# To create the appimage of kiwix-desktop + libfuse2 fuse patchelf \ +# Flatpak tools + elfutils flatpak flatpak-builder \ +# Cross win32 compiler + g++-mingw-w64-i686 gcc-mingw-w64-i686 gcc-mingw-w64-base mingw-w64-tools \ +# Other tools (to remove) +# vim less grep \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/* \ + && pip3 install meson==0.52.1 pytest gcovr + +# Create user +RUN useradd --create-home runner +USER runner +WORKDIR /home/runner +ENV PATH /home/runner/.local/bin:$PATH diff --git a/.github/ci_images/xenial_builder.dockerfile b/.github/ci_images/xenial_builder.dockerfile new file mode 100644 index 0000000..19a8653 --- /dev/null +++ b/.github/ci_images/xenial_builder.dockerfile @@ -0,0 +1,30 @@ +FROM ubuntu:xenial + +ENV LANG C.UTF-8 +ENV OS_NAME xenial + +RUN apt update -q \ + && dpkg --add-architecture i386 \ + && apt install -q -y --no-install-recommends \ +# Base build tools + build-essential automake libtool cmake ccache pkg-config autopoint patch \ + python3-pip python3-setuptools python3-wheel git subversion wget unzip \ + ninja-build openssh-client \ +# Python (2) is needed to install android-ndk + python \ +# Packaged dependencies + libbz2-dev libmagic-dev uuid-dev zlib1g-dev default-jdk \ + libmicrohttpd-dev libgtest-dev \ +# Cross compile i586 + libc6-dev-i386 lib32stdc++6 gcc-multilib g++-multilib \ +# Other tools (to remove) +# vim less grep \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/* \ + && pip3 install meson==0.52.1 pytest gcovr + +# Create user +RUN useradd --create-home runner +USER runner +WORKDIR /home/runner +ENV PATH /home/runner/.local/bin:$PATH diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml new file mode 100644 index 0000000..5ec2a8c --- /dev/null +++ b/.github/workflows/base.yml @@ -0,0 +1,28 @@ +name: Build Base + +on: [push] + +env: + DOCKER_VERSION: 25 + +jobs: + Docker: + strategy: + fail-fast: false + matrix: + variant: [xenial, bionic, f30, focal] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Build docker image + shell: bash + run: | + TAGNAME=${{matrix.variant}}-${DOCKER_VERSION} + if [ ! $(curl -sflL https://hub.docker.com/v2/repositories/kiwix/kiwix-build_ci/tags/${TAGNAME}) ] + then + echo "${{secrets.docker_password}}" | docker login -u "${{secrets.docker_username}}" --password-stdin + FULLTAGNAME=kiwix/kiwix-build_ci:${TAGNAME} + docker build -t ${FULLTAGNAME} - < ${GITHUB_WORKSPACE}/.github/ci_images/${{matrix.variant}}_builder.dockerfile + docker push ${FULLTAGNAME} + fi + From d37acc2cbe6a9b9d5db451f2c552d3ec906e3fc7 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 20 Jan 2020 23:01:00 +0100 Subject: [PATCH 05/10] Build basedeps archives on GithubAction. Use a specific script to download/build the base deps. Mainly based on script in travis/compile_all.py --- .github/scripts/common.py | 162 ++++++++++++++++++++++++++++ .github/scripts/ensure_base_deps.py | 72 +++++++++++++ .github/workflows/base.yml | 58 ++++++++++ 3 files changed, 292 insertions(+) create mode 100644 .github/scripts/common.py create mode 100755 .github/scripts/ensure_base_deps.py diff --git a/.github/scripts/common.py b/.github/scripts/common.py new file mode 100644 index 0000000..58ea5c6 --- /dev/null +++ b/.github/scripts/common.py @@ -0,0 +1,162 @@ +import os +from os import environ as _environ +from pathlib import Path +from datetime import date +import tarfile +import subprocess +import re + +from kiwixbuild.versions import base_deps_versions + + +PLATFORM_TARGET = _environ["PLATFORM_TARGET"] +OS_NAME = _environ["OS_NAME"] +HOME = Path(os.path.expanduser("~")) + +BASE_DIR = HOME / "BUILD_{}".format(PLATFORM_TARGET) +SOURCE_DIR = HOME / "SOURCE" +ARCHIVE_DIR = HOME / "ARCHIVE" +INSTALL_DIR = BASE_DIR / "INSTALL" +TMP_DIR = Path("/tmp") + +# [TODO] +KIWIX_DESKTOP_ONLY = False + +_ref = _environ.get("GITHUB_REF", "").split("/")[-1] +MAKE_RELEASE = re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", _ref) is not None + + +def print_message(message, *args, **kwargs): + message = message.format(*args, **kwargs) + message = "{0} {1} {0}".format("-" * 3, message) + print(message, flush=True) + + +MANIFEST_TEMPLATE = """{archive_name} +*************************** + +Dependencies archive for {target} on platform {platform} +Generated at {date} +""" + + +def write_manifest(manifest_file, archive_name, target, platform): + with manifest_file.open(mode="w") as f: + f.write( + MANIFEST_TEMPLATE.format( + archive_name=archive_name, + target=target, + platform=platform, + date=date.today().isoformat(), + ) + ) + + +def run_kiwix_build( + target, + platform, + build_deps_only=False, + target_only=False, + make_release=False, + make_dist=False, +): + command = ["kiwix-build"] + command.append(target) + command.append("--hide-progress") + if platform == "flatpak" or platform.startswith("win32_"): + command.append("--assume-packages-installed") + if target == "kiwix-lib-app" and platform.startswith("android_"): + command.extend(["--target-platform", "android", "--android-arch", platform[8:]]) + elif platform == "android": + command.extend(["--target-platform", "android"]) + for arch in ("arm", "arm64", "x86", "x86_64"): + command.extend(["--android-arch", arch]) + else: + command.extend(["--target-platform", platform]) + if build_deps_only: + command.append("--build-deps-only") + if target_only: + command.append("--build-nodeps") + if make_release: + command.append("--make-release") + if make_dist: + command.append("--make-dist") + print_message( + "Build {} (deps={}, release={}, dist={})", + target, + build_deps_only, + make_release, + make_dist, + ) + subprocess.check_call(command, cwd=str(HOME)) + print_message("Build ended") + + +def upload(file_to_upload, host, dest_path): + command = [ + "scp", + "-i", + _environ.get("SSH_KEY"), + "-o", + "StrictHostKeyChecking=no", + str(file_to_upload), + "{}:{}".format(host, dest_path), + ] + print_message("Sending archive with command {}", command) + subprocess.check_call(command) + + +def make_deps_archive(target=None, name=None, full=False): + archive_name = name or "deps2_{}_{}_{}.tar.xz".format( + OS_NAME, PLATFORM_TARGET, target + ) + print_message("Create archive {}.", archive_name) + files_to_archive = [INSTALL_DIR] + files_to_archive += HOME.glob("BUILD_*/LOGS") + if PLATFORM_TARGET == "native_mixed": + files_to_archive += [HOME / "BUILD_native_static" / "INSTALL"] + if PLATFORM_TARGET.startswith("android"): + files_to_archive.append(HOME / "BUILD_neutral" / "INSTALL") + if PLATFORM_TARGET == "android": + for arch in ("arm", "arm64", "x86", "x86_64"): + base_dir = HOME / "BUILD_android_{}".format(arch) + files_to_archive.append(base_dir / "INSTALL") + if (base_dir / "meson_cross_file.txt").exists(): + files_to_archive.append(base_dir / "meson_cross_file.txt") + files_to_archive += HOME.glob("BUILD_*/android-ndk*") + files_to_archive += HOME.glob("BUILD_*/android-sdk*") + if (BASE_DIR / "meson_cross_file.txt").exists(): + files_to_archive.append(BASE_DIR / "meson_cross_file.txt") + + manifest_file = BASE_DIR / "manifest.txt" + write_manifest(manifest_file, archive_name, target, PLATFORM_TARGET) + files_to_archive.append(manifest_file) + + relative_path = HOME + if full: + files_to_archive += ARCHIVE_DIR.glob(".*_ok") + files_to_archive += BASE_DIR.glob("*/.*_ok") + files_to_archive += (HOME / "BUILD_native_dyn").glob("*/.*_ok") + files_to_archive += (HOME / "BUILD_native_static").glob("*/.*_ok") + files_to_archive += HOME.glob("BUILD_android*/**/.*_ok") + files_to_archive += SOURCE_DIR.glob("*/.*_ok") + files_to_archive += [ + SOURCE_DIR / "pugixml-{}".format(base_deps_versions["pugixml"]) + ] + files_to_archive += HOME.glob( + "BUILD_*/pugixml-{}".format(base_deps_versions["pugixml"]) + ) + if PLATFORM_TARGET.startswith("armhf"): + files_to_archive += (SOURCE_DIR / "armhf").glob("*") + toolchains_subdirs = HOME.glob("BUILD_*/TOOLCHAINS/*/*") + for subdir in toolchains_subdirs: + if not subdir.match("tools"): + files_to_archive.append(subdir) + + archive_file = TMP_DIR / archive_name + with tarfile.open(str(archive_file), "w:xz") as tar: + for name in set(files_to_archive): + print(".{}".format(name), flush=True) + tar.add(str(name), arcname=str(name.relative_to(relative_path))) + + return archive_file diff --git a/.github/scripts/ensure_base_deps.py b/.github/scripts/ensure_base_deps.py new file mode 100755 index 0000000..6d0ccff --- /dev/null +++ b/.github/scripts/ensure_base_deps.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 + +import os +import tarfile +from urllib.request import urlopen +from urllib.error import URLError + +from kiwixbuild.versions import base_deps_meta_version + +from common import ( + print_message, + run_kiwix_build, + upload, + make_deps_archive, + HOME, + PLATFORM_TARGET, + OS_NAME, + MAKE_RELEASE, +) + + +def download_base_archive(base_name): + url = "http://tmp.kiwix.org/ci/{}".format(base_name) + file_path = str(HOME / base_name) + batch_size = 1024 * 1024 * 8 + with urlopen(url) as resource, open(file_path, "wb") as file: + while True: + batch = resource.read(batch_size) + if not batch: + break + print(".", end="", flush=True) + file.write(batch) + return file_path + + +ARCHIVE_NAME_TEMPLATE = "base_deps2_{os}_{platform}_{version}_{release}.tar.xz" +base_dep_archive_name = ARCHIVE_NAME_TEMPLATE.format( + os=OS_NAME, + platform=PLATFORM_TARGET, + version=base_deps_meta_version, + release="release" if MAKE_RELEASE else "debug", +) + +print_message("Getting archive {}", base_dep_archive_name) +try: + local_filename = download_base_archive(base_dep_archive_name) + with tarfile.open(local_filename) as f: + f.extractall(str(HOME)) + os.remove(str(local_filename)) +except URLError: + print_message("Cannot get archive. Build dependencies") + if PLATFORM_TARGET == "android": + for arch in ("arm", "arm64", "x86", "x86_64"): + archive_name = ARCHIVE_NAME_TEMPLATE.format( + os=OS_NAME, + platform="android_{}".format(arch), + version=base_deps_meta_version, + release="release" if MAKE_RELEASE else "debug", + ) + print_message("Getting archive {}", archive_name) + try: + local_filename = download_base_archive(archive_name) + with tarfile.open(local_filename) as f: + f.extractall(str(HOME)) + os.remove(str(local_filename)) + except URLError: + pass + else: + run_kiwix_build("alldependencies", platform=PLATFORM_TARGET) + archive_file = make_deps_archive(name=base_dep_archive_name, full=True) + upload(archive_file, "ci@tmp.kiwix.org", "/data/tmp/ci") + os.remove(str(archive_file)) diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml index 5ec2a8c..7a1b6b1 100644 --- a/.github/workflows/base.yml +++ b/.github/workflows/base.yml @@ -26,3 +26,61 @@ jobs: docker push ${FULLTAGNAME} fi + Linux: + strategy: + fail-fast: false + matrix: + target: + - native_static + - native_dyn + - android_arm + - android_arm64 + - win32_static + - win32_dyn + include: + - target: native_static + image_variant: xenial + lib_postfix: '/x86_64-linux-gnu' + - target: native_dyn + image_variant: xenial + lib_postfix: '/x86_64-linux-gnu' + - target: android_arm + image_variant: xenial + lib_postfix: '/x86_64-linux-gnu' + - target: android_arm64 + image_variant: xenial + lib_postfix: '/x86_64-linux-gnu' + - target: win32_static + image_variant: f30 + lib_postfix: '64' + - target: win32_dyn + image_variant: f30 + lib_postfix: '64' + env: + HOME: /home/runner + SSH_KEY: /tmp/id_rsa + runs-on: ubuntu-latest + needs: Docker + container: + image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-25" + steps: + - name: Checkout code + shell: bash + run: | + cd $HOME + git clone https://github.com/${REP} --depth=1 --branch ${GITHUB_REF##*/} + pip3 install --user --no-deps ./${REP##*/} + env: + REP: ${{github.repository}} + - name: secret + shell: bash + run: | + echo "${{secrets.ssh_key}}" > $SSH_KEY + chmod 600 $SSH_KEY + - name: Ensure base deps + shell: bash + run: | + cd $HOME + kiwix-build/.github/scripts/ensure_base_deps.py + env: + PLATFORM_TARGET: ${{matrix.target}} From c178c6ea35fb30abb795450663a79b763c8079dd Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 20 Jan 2020 23:02:12 +0100 Subject: [PATCH 06/10] Upload an archive of the build directory in case of failure. --- .github/scripts/upload_failure_logs.sh | 16 ++++++++++++++++ .github/workflows/base.yml | 5 +++++ 2 files changed, 21 insertions(+) create mode 100755 .github/scripts/upload_failure_logs.sh diff --git a/.github/scripts/upload_failure_logs.sh b/.github/scripts/upload_failure_logs.sh new file mode 100755 index 0000000..240a915 --- /dev/null +++ b/.github/scripts/upload_failure_logs.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -e + +cd $HOME + +ARCHIVE_NAME=fail_log_${OS_NAME}_${PLATFORM_TARGET}.tar.gz +tar -czf ${ARCHIVE_NAME} $HOME/BUILD_* $HOME/SOURCE $HOME/LOGS $HOME/TOOLCHAINS + +echo "Uploading archive $ARCHIVE_NAME" + +scp -p -i ${SSH_KEY} \ + -o PasswordAuthentication=no \ + -o StrictHostKeyChecking=no \ + $ARCHIVE_NAME \ + ci@tmp.kiwix.org:/data/tmp/ci diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml index 7a1b6b1..72971a8 100644 --- a/.github/workflows/base.yml +++ b/.github/workflows/base.yml @@ -84,3 +84,8 @@ jobs: kiwix-build/.github/scripts/ensure_base_deps.py env: PLATFORM_TARGET: ${{matrix.target}} + - name: Upload failure logs + if: failure() + run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh + env: + PLATFORM_TARGET: ${{matrix.target}} From 1d84266751edd4b5db772c07fdceb4faa0f86571 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 20 Jan 2020 23:03:43 +0100 Subject: [PATCH 07/10] Build all base dependencies. Use a custom script for this. Set `SKIP_BIG_MEMORY_TEST` to avoid launching big test on libzim. --- .github/scripts/common.py | 4 ++- .github/scripts/compile_all_deps.py | 40 +++++++++++++++++++++++++++++ .github/workflows/base.yml | 7 +++++ 3 files changed, 50 insertions(+), 1 deletion(-) create mode 100755 .github/scripts/compile_all_deps.py diff --git a/.github/scripts/common.py b/.github/scripts/common.py index 58ea5c6..c7fcaf9 100644 --- a/.github/scripts/common.py +++ b/.github/scripts/common.py @@ -88,7 +88,9 @@ def run_kiwix_build( make_release, make_dist, ) - subprocess.check_call(command, cwd=str(HOME)) + env = dict(_environ) + env['SKIP_BIG_MEMORY_TEST'] = '1' + subprocess.check_call(command, cwd=str(HOME), env=env) print_message("Build ended") diff --git a/.github/scripts/compile_all_deps.py b/.github/scripts/compile_all_deps.py new file mode 100755 index 0000000..2f91ae0 --- /dev/null +++ b/.github/scripts/compile_all_deps.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 + +import os + +from common import ( + run_kiwix_build, + make_deps_archive, + upload, + OS_NAME, + PLATFORM_TARGET, + KIWIX_DESKTOP_ONLY, +) + +if PLATFORM_TARGET == "android": + TARGETS = ("kiwix-lib-app",) +elif PLATFORM_TARGET.startswith("android_"): + TARGETS = ("libzim", "kiwix-lib") +elif PLATFORM_TARGET.startswith("iOS"): + TARGETS = ("libzim", "kiwix-lib") +elif PLATFORM_TARGET.startswith("native_"): + if OS_NAME == "osx": + TARGETS = ("libzim", "zimwriterfs", "zim-tools", "kiwix-lib") + else: + if PLATFORM_TARGET == "native_dyn" and KIWIX_DESKTOP_ONLY: + TARGETS = ("kiwix-desktop",) + elif PLATFORM_TARGET == "native_mixed": + TARGETS = ("libzim",) + else: + TARGETS = ("libzim", "zimwriterfs", "zim-tools", "kiwix-lib", "kiwix-tools") +elif PLATFORM_TARGET == "flatpak": + TARGETS = ("kiwix-desktop",) +else: + TARGETS = ("libzim", "zim-tools", "kiwix-lib", "kiwix-tools") + +for target in TARGETS: + if PLATFORM_TARGET not in ("android", "flatpak"): + run_kiwix_build(target, platform=PLATFORM_TARGET, build_deps_only=True) + archive_file = make_deps_archive(target=target) + upload(archive_file, "ci@tmp.kiwix.org", "/data/tmp/ci") + os.remove(str(archive_file)) diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml index 72971a8..02a996f 100644 --- a/.github/workflows/base.yml +++ b/.github/workflows/base.yml @@ -84,6 +84,13 @@ jobs: kiwix-build/.github/scripts/ensure_base_deps.py env: PLATFORM_TARGET: ${{matrix.target}} + - name: Compile all deps + shell: bash + run: | + cd $HOME + kiwix-build/.github/scripts/compile_all_deps.py + env: + PLATFORM_TARGET: ${{matrix.target}} - name: Upload failure logs if: failure() run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh From f73e0b2abf4eca8142238ee00a7dda7005c1be9a Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 20 Jan 2020 23:04:30 +0100 Subject: [PATCH 08/10] Build base deps on macos. --- .github/workflows/base.yml | 41 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml index 02a996f..2253f3e 100644 --- a/.github/workflows/base.yml +++ b/.github/workflows/base.yml @@ -96,3 +96,44 @@ jobs: run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh env: PLATFORM_TARGET: ${{matrix.target}} + + + Macos: + runs-on: macos-latest + env: + SSH_KEY: /tmp/id_rsa + OS_NAME: osx + PLATFORM_TARGET: native_dyn + steps: + - name: Checkout code + uses: actions/checkout@v1 + - name: Setup python 3.6 + uses: actions/setup-python@v1 + with: + python-version: '3.6' + - name: Install packages + uses: mstksg/get-package@v1 + with: + brew: pkg-config ninja + - name: Install python modules + run: | + pip3 install meson==0.52.1 pytest + pip3 install --no-deps $GITHUB_WORKSPACE + - name: secret + shell: bash + run: | + echo "${{secrets.ssh_key}}" > $SSH_KEY + chmod 600 $SSH_KEY + - name: Ensure base deps + shell: bash + run: | + cd $HOME + $GITHUB_WORKSPACE/.github/scripts/ensure_base_deps.py + - name: Compile all deps + shell: bash + run: | + cd $HOME + $GITHUB_WORKSPACE/.github/scripts/compile_all_deps.py + - name: Upload failure logs + if: failure() + run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.sh From 1b7a2c812ea1cb271fa6c0bc51705ba5d9977d4a Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 21 Jan 2020 10:10:45 +0100 Subject: [PATCH 09/10] Add missing platforms. No iOs for now. --- .github/scripts/common.py | 5 +++ .github/scripts/compile_all_deps.py | 18 ++++------ .github/scripts/ensure_base_deps.py | 1 - .github/workflows/base.yml | 52 ++++++++++++++++++++++++++--- 4 files changed, 59 insertions(+), 17 deletions(-) diff --git a/.github/scripts/common.py b/.github/scripts/common.py index c7fcaf9..de52aa9 100644 --- a/.github/scripts/common.py +++ b/.github/scripts/common.py @@ -10,6 +10,11 @@ from kiwixbuild.versions import base_deps_versions PLATFORM_TARGET = _environ["PLATFORM_TARGET"] +if PLATFORM_TARGET == "native_desktop": + PLATFORM_TARGET = "native_dyn" + DESKTOP = True +else: + DESKTOP = False OS_NAME = _environ["OS_NAME"] HOME = Path(os.path.expanduser("~")) diff --git a/.github/scripts/compile_all_deps.py b/.github/scripts/compile_all_deps.py index 2f91ae0..d6d5197 100755 --- a/.github/scripts/compile_all_deps.py +++ b/.github/scripts/compile_all_deps.py @@ -8,12 +8,11 @@ from common import ( upload, OS_NAME, PLATFORM_TARGET, + DESKTOP, KIWIX_DESKTOP_ONLY, ) -if PLATFORM_TARGET == "android": - TARGETS = ("kiwix-lib-app",) -elif PLATFORM_TARGET.startswith("android_"): +if PLATFORM_TARGET.startswith("android_"): TARGETS = ("libzim", "kiwix-lib") elif PLATFORM_TARGET.startswith("iOS"): TARGETS = ("libzim", "kiwix-lib") @@ -21,20 +20,17 @@ elif PLATFORM_TARGET.startswith("native_"): if OS_NAME == "osx": TARGETS = ("libzim", "zimwriterfs", "zim-tools", "kiwix-lib") else: - if PLATFORM_TARGET == "native_dyn" and KIWIX_DESKTOP_ONLY: + if DESKTOP: TARGETS = ("kiwix-desktop",) elif PLATFORM_TARGET == "native_mixed": TARGETS = ("libzim",) else: TARGETS = ("libzim", "zimwriterfs", "zim-tools", "kiwix-lib", "kiwix-tools") -elif PLATFORM_TARGET == "flatpak": - TARGETS = ("kiwix-desktop",) else: TARGETS = ("libzim", "zim-tools", "kiwix-lib", "kiwix-tools") for target in TARGETS: - if PLATFORM_TARGET not in ("android", "flatpak"): - run_kiwix_build(target, platform=PLATFORM_TARGET, build_deps_only=True) - archive_file = make_deps_archive(target=target) - upload(archive_file, "ci@tmp.kiwix.org", "/data/tmp/ci") - os.remove(str(archive_file)) + run_kiwix_build(target, platform=PLATFORM_TARGET, build_deps_only=True) + archive_file = make_deps_archive(target=target) + upload(archive_file, "ci@tmp.kiwix.org", "/data/tmp/ci") + os.remove(str(archive_file)) diff --git a/.github/scripts/ensure_base_deps.py b/.github/scripts/ensure_base_deps.py index 6d0ccff..004f837 100755 --- a/.github/scripts/ensure_base_deps.py +++ b/.github/scripts/ensure_base_deps.py @@ -18,7 +18,6 @@ from common import ( MAKE_RELEASE, ) - def download_base_archive(base_name): url = "http://tmp.kiwix.org/ci/{}".format(base_name) file_path = str(HOME / base_name) diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml index 2253f3e..c3073d4 100644 --- a/.github/workflows/base.yml +++ b/.github/workflows/base.yml @@ -33,10 +33,18 @@ jobs: target: - native_static - native_dyn - - android_arm - - android_arm64 + - native_mixed + - native_desktop + - armhf_static + - armhf_dyn - win32_static - win32_dyn + - i586_static + - i586_dyn + - android_arm + - android_arm64 + - android_x86 + - android_x86_64 include: - target: native_static image_variant: xenial @@ -44,10 +52,16 @@ jobs: - target: native_dyn image_variant: xenial lib_postfix: '/x86_64-linux-gnu' - - target: android_arm + - target: native_mixed image_variant: xenial lib_postfix: '/x86_64-linux-gnu' - - target: android_arm64 + - target: native_desktop + image_variant: bionic + lib_postfix: '/x86_64-linux-gnu' + - target: armhf_static + image_variant: xenial + lib_postfix: '/x86_64-linux-gnu' + - target: armhf_dyn image_variant: xenial lib_postfix: '/x86_64-linux-gnu' - target: win32_static @@ -56,6 +70,24 @@ jobs: - target: win32_dyn image_variant: f30 lib_postfix: '64' + - target: i586_static + image_variant: xenial + lib_postfix: '/x86_64-linux-gnu' + - target: i586_dyn + image_variant: xenial + lib_postfix: '/x86_64-linux-gnu' + - target: android_arm + image_variant: xenial + lib_postfix: '/x86_64-linux-gnu' + - target: android_arm64 + image_variant: xenial + lib_postfix: '/x86_64-linux-gnu' + - target: android_x86 + image_variant: xenial + lib_postfix: '/x86_64-linux-gnu' + - target: android_x86_64 + image_variant: xenial + lib_postfix: '/x86_64-linux-gnu' env: HOME: /home/runner SSH_KEY: /tmp/id_rsa @@ -99,11 +131,15 @@ jobs: Macos: + strategy: + fail-fast: false + matrix: + target: + - native_dyn runs-on: macos-latest env: SSH_KEY: /tmp/id_rsa OS_NAME: osx - PLATFORM_TARGET: native_dyn steps: - name: Checkout code uses: actions/checkout@v1 @@ -129,11 +165,17 @@ jobs: run: | cd $HOME $GITHUB_WORKSPACE/.github/scripts/ensure_base_deps.py + env: + PLATFORM_TARGET: ${{matrix.target}} - name: Compile all deps shell: bash run: | cd $HOME $GITHUB_WORKSPACE/.github/scripts/compile_all_deps.py + env: + PLATFORM_TARGET: ${{matrix.target}} - name: Upload failure logs if: failure() run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.sh + env: + PLATFORM_TARGET: ${{matrix.target}} From 70b3ed0c76bd48ecd79f80ad44c009635b7f12e5 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 21 Jan 2020 15:01:05 +0100 Subject: [PATCH 10/10] Update docker images --- .../{f30_builder.dockerfile => f31_builder.dockerfile} | 6 ++++-- .github/ci_images/xenial_builder.dockerfile | 2 +- .github/workflows/base.yml | 10 +++++----- 3 files changed, 10 insertions(+), 8 deletions(-) rename .github/ci_images/{f30_builder.dockerfile => f31_builder.dockerfile} (90%) diff --git a/.github/ci_images/f30_builder.dockerfile b/.github/ci_images/f31_builder.dockerfile similarity index 90% rename from .github/ci_images/f30_builder.dockerfile rename to .github/ci_images/f31_builder.dockerfile index b21a9f3..62ed448 100644 --- a/.github/ci_images/f30_builder.dockerfile +++ b/.github/ci_images/f31_builder.dockerfile @@ -1,7 +1,7 @@ -FROM fedora:30 +FROM fedora:31 ENV LANG C.UTF-8 -ENV OS_NAME f30 +ENV OS_NAME f31 RUN dnf install -y --nodocs \ # Base build tools @@ -11,6 +11,8 @@ RUN dnf install -y --nodocs \ mingw32-gcc-c++ mingw32-bzip2-static mingw32-win-iconv-static \ mingw32-winpthreads-static mingw32-zlib-static mingw32-xz-libs-static \ mingw32-libmicrohttpd \ +# python3 + python-unversioned-command \ # Other tools (to remove) # vim less grep && dnf remove -y "*-doc" \ diff --git a/.github/ci_images/xenial_builder.dockerfile b/.github/ci_images/xenial_builder.dockerfile index 19a8653..2b634a4 100644 --- a/.github/ci_images/xenial_builder.dockerfile +++ b/.github/ci_images/xenial_builder.dockerfile @@ -9,7 +9,7 @@ RUN apt update -q \ # Base build tools build-essential automake libtool cmake ccache pkg-config autopoint patch \ python3-pip python3-setuptools python3-wheel git subversion wget unzip \ - ninja-build openssh-client \ + ninja-build openssh-client curl \ # Python (2) is needed to install android-ndk python \ # Packaged dependencies diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml index c3073d4..0e2ad1b 100644 --- a/.github/workflows/base.yml +++ b/.github/workflows/base.yml @@ -3,14 +3,14 @@ name: Build Base on: [push] env: - DOCKER_VERSION: 25 + DOCKER_VERSION: 26 jobs: Docker: strategy: fail-fast: false matrix: - variant: [xenial, bionic, f30, focal] + variant: [xenial, bionic, f31, focal] runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -65,10 +65,10 @@ jobs: image_variant: xenial lib_postfix: '/x86_64-linux-gnu' - target: win32_static - image_variant: f30 + image_variant: f31 lib_postfix: '64' - target: win32_dyn - image_variant: f30 + image_variant: f31 lib_postfix: '64' - target: i586_static image_variant: xenial @@ -94,7 +94,7 @@ jobs: runs-on: ubuntu-latest needs: Docker container: - image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-25" + image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-26" steps: - name: Checkout code shell: bash