From d8288a94d5b5ccd0ceb94e7714aa1bb2ad2b2955 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 16 Apr 2019 17:00:55 +0200 Subject: [PATCH 1/9] Build on travis using xenial. --- .travis.yml | 2 +- kiwixbuild/versions.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c72a643..540c2b9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: cpp -dist: trusty +dist: xenial sudo: required services: - docker diff --git a/kiwixbuild/versions.py b/kiwixbuild/versions.py index a5b9f22..9ded59e 100644 --- a/kiwixbuild/versions.py +++ b/kiwixbuild/versions.py @@ -35,7 +35,7 @@ release_versions = { # This is the "version" of the whole base_deps_versions dict. # Change this when you change base_deps_versions. -base_deps_meta_version = '18' +base_deps_meta_version = '50' base_deps_versions = { From 318c524f4c71bf5cb950277bdb336635ea712550 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 30 Apr 2019 15:03:30 +0200 Subject: [PATCH 2/9] Use our own server to store libmagic. --- kiwixbuild/dependencies/libmagic.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kiwixbuild/dependencies/libmagic.py b/kiwixbuild/dependencies/libmagic.py index e99b41d..fb5d438 100644 --- a/kiwixbuild/dependencies/libmagic.py +++ b/kiwixbuild/dependencies/libmagic.py @@ -17,8 +17,7 @@ class LibMagic(Dependency): source_dir = "libmagic" archive_top_dir = 'file-5.35' archive = Remotefile('file-5.35.tar.gz', - '30c45e817440779be7aac523a905b123cba2a6ed0bf4f5439e1e99ba940b5546', - 'https://fossies.org/linux/misc/file-5.35.tar.gz') + '30c45e817440779be7aac523a905b123cba2a6ed0bf4f5439e1e99ba940b5546') class Builder(MakeBuilder): From f9d5e0aa212b42e56d9650022327cb7168a7b441 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 2 May 2019 14:17:08 +0200 Subject: [PATCH 3/9] Use an already made archive for the armhf toolchains instead of git clone. First, we don't need to clone all the git repository, we only need a sub-directory. Secondly, as we the base_deps archives only archive the sub-directory, subsequent run of kiwix-build in the ci will try to do a git update on a non git repository. By making ourselves a tar archive, we simplify the process. The archive on download.kiwix.org has been made with : `tar czf raspberrypy-tools.tar.gz raspberrypi-tools/arm-bcm2708/gcc-linaro-arm-linux-gnueabihf-raspbian-x64` `raspberrypi-tools` being a clone of `https://github.com/raspberrypi/tools` commit sha is 5caa7046982f0539cf5380f94da04b31129ed521 --- kiwixbuild/dependencies/armhf.py | 9 +++++---- kiwixbuild/platforms/armhf.py | 1 + travis/compile_all.py | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/kiwixbuild/dependencies/armhf.py b/kiwixbuild/dependencies/armhf.py index 4ba5200..b79735e 100644 --- a/kiwixbuild/dependencies/armhf.py +++ b/kiwixbuild/dependencies/armhf.py @@ -1,11 +1,12 @@ -from .base import Dependency, GitClone, NoopBuilder +from .base import Dependency, ReleaseDownload, NoopBuilder +from kiwixbuild.utils import Remotefile class armhf_toolchain(Dependency): neutral = True name = 'armhf' - class Source(GitClone): - git_remote = "https://github.com/raspberrypi/tools" - git_dir = "raspberrypi-tools" + class Source(ReleaseDownload): + archive = Remotefile('raspberrypi-tools.tar.gz', + 'e72b35436f2f23f2f7df322d6c318b9be57b21596b5ff0b8936af4ad94e04f2e') Builder = NoopBuilder diff --git a/kiwixbuild/platforms/armhf.py b/kiwixbuild/platforms/armhf.py index 98a1645..ec3b574 100644 --- a/kiwixbuild/platforms/armhf.py +++ b/kiwixbuild/platforms/armhf.py @@ -34,6 +34,7 @@ class ArmhfPlatformInfo(PlatformInfo): @property def root_path(self): return pj(self.tlc_source.source_path, + 'raspberrypi-tools', 'arm-bcm2708', 'gcc-linaro-{}-raspbian-x64'.format(self.arch_full)) diff --git a/travis/compile_all.py b/travis/compile_all.py index 137e0b9..f91440d 100755 --- a/travis/compile_all.py +++ b/travis/compile_all.py @@ -230,9 +230,9 @@ def make_deps_archive(target, full=False): base_deps_versions['pugixml'])] files_to_archive += HOME.glob('BUILD_*/pugixml-{}'.format( base_deps_versions['pugixml'])) - toolchains_subdirs = HOME.glob('**/TOOLCHAINS/*/*') if PLATFORM.startswith('armhf'): - files_to_archive += [SOURCE_DIR/'raspberrypi-tools'] + files_to_archive += (SOURCE_DIR/'armhf').glob('*') + toolchains_subdirs = HOME.glob('**/TOOLCHAINS/*/*') for subdir in toolchains_subdirs: if not subdir.match('tools'): files_to_archive.append(subdir) From 281adbd7689f1b3fef20ec10b97c300533be756b Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 18 Apr 2019 15:23:31 +0200 Subject: [PATCH 4/9] Build everything in a container. Docker containers don't have the SSH_KEY so we need to copy archives and git repo in a directory on the host and launch deploy.sh script from there. --- .travis.yml | 103 ++++++++++++------------------- kiwixbuild/packages.py | 4 +- kiwixbuild/versions.py | 2 +- travis/compile_all.py | 51 ++++++--------- travis/deploy.sh | 12 +++- travis/deploy_apk.sh | 1 - travis/make_release.sh | 1 - travis/upload_all_log.sh | 2 - travis/upload_base_deps.sh | 15 +++++ travis/xenial_builder.dockerfile | 41 ++++++++++++ 10 files changed, 131 insertions(+), 101 deletions(-) create mode 100755 travis/upload_base_deps.sh create mode 100644 travis/xenial_builder.dockerfile diff --git a/.travis.yml b/.travis.yml index 540c2b9..78fac47 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,11 +12,13 @@ if: type != push OR tag IS present before_install: - PATH=$PATH:${HOME}/bin:${HOME}/.local/bin +- mkdir -p /tmp/private $HOME/EXPORT/BASE +- chmod -R a+w $HOME/EXPORT - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then PATH=$PATH:$(brew --prefix)/opt/gettext/bin; fi - '[ -n "$encrypted_eba2f7543984_iv" ] && openssl aes-256-cbc -K $encrypted_eba2f7543984_key -iv $encrypted_eba2f7543984_iv - -in travis/travisci_builder_id_key.enc -out travis/travisci_builder_id_key -d' -- chmod 600 travis/travisci_builder_id_key + -in travis/travisci_builder_id_key.enc -out $SSH_KEY -d' +- chmod 600 $SSH_KEY before_cache: - rm -f $HOME/.gradle/caches/modules-2/modules-2.lock - rm -fr $HOME/.gradle/caches/*/plugin-resolution/ @@ -27,27 +29,37 @@ cache: - $HOME/.gradle/caches/ - $HOME/.gradle/wrapper/ - $HOME/.android/build-cache -install: if [[ $TRAVIS_OS_NAME != "linux" || $DESKTOP_ONLY != 1 ]]; then travis/install_extra_deps.sh; fi +install: if [[ $TRAVIS_OS_NAME != "linux" ]]; then travis/install_extra_deps.sh; fi script: - | - if [[ $TRAVIS_OS_NAME = "linux" && $DESKTOP_ONLY == 1 ]] + if [[ $TRAVIS_OS_NAME = "linux" ]] then - if [[ $PLATFORM = "flatpak" ]] - then - docker build -t kiwix/build -f travis/Dockerfile_flatpak . - else - docker build -t kiwix/build -f travis/Dockerfile . - fi - docker run -e PLATFORM -e NIGHTLY_DATE -e TRAVIS_EVENT_TYPE -e DESKTOP_ONLY -e TRAVIS_TAG --device /dev/fuse --cap-add ALL --privileged kiwix/build - else - if [[ $TRAVIS_OS_NAME = "linux" ]] - then - python3.5 travis/compile_all.py - else - python3 travis/compile_all.py - fi + PODMAN_FILE=xenial_builder.dockerfile + if [[ $PLATFORM = "flatpak" && $DESKTOP_ONLY = 1 ]]; then PODMAN_FILE=Dockerfile_flatpak; fi + if [[ $PLATFORM = "native_dyn" && $DESKTOP_ONLY = 1 ]]; then PODMAN_FILE=Dockerfile; fi fi -after_failure: travis/upload_all_log.sh +- | + if [[ $TRAVIS_OS_NAME = "linux" ]] + then + docker build -t kiwix/build -f travis/$PODMAN_FILE . + fi +- | + if [[ $TRAVIS_OS_NAME = "linux" ]] + then + docker run \ + -e PLATFORM -e NIGHTLY_DATE -e TRAVIS_EVENT_TYPE -e DESKTOP_ONLY -e TRAVIS_TAG \ + --device /dev/fuse --cap-add ALL --privileged \ + --mount=type=bind,src=$HOME/EXPORT,dst=/home/builder/EXPORT \ + --mount=type=bind,src=$HOME/.cache,dst=/home/builder/.cache \ + --mount=type=bind,src=$HOME/.gradle/caches,dst=/home/builder/.gradle/caches \ + kiwix/build + fi +- if [[ $TRAVIS_OS_NAME != "linux" ]]; then python3 travis/compile_all.py; fi +after_success: +- travis/upload_base_deps.sh +after_failure: +- travis/upload_base_deps.sh +- travis/upload_all_log.sh deploy: - provider: script skip_cleanup: true @@ -64,24 +76,25 @@ deploy: env: global: - NIGHTLY_DATE=$(date +%Y-%m-%d) + - SSH_KEY=/tmp/private/travisci_builder_id_key matrix: - PLATFORM="native_dyn" + - PLATFORM="native_static" + - PLATFORM="native_mixed" - PLATFORM="armhf_dyn" - PLATFORM="armhf_static" - - PLATFORM="native_mixed" + - PLATFORM="win32_dyn" + - PLATFORM="win32_static" + - PLATFORM="android" + - PLATFORM="android_arm" + - PLATFORM="android_arm64" + - PLATFORM="android_x86" + - PLATFORM="android_x86_64" addons: ssh_known_hosts: - download.kiwix.org - tmp.kiwix.org - download.openzim.org - apt: - packages: &common - - cmake - - autopoint - - python3.5 - - python3-pip - - ctpp2-utils - - libmagic-dev matrix: include: - env: PLATFORM="native_dyn" DESKTOP_ONLY=1 @@ -92,26 +105,6 @@ matrix: addons: apt: packages: [] - - env: PLATFORM="native_static" - addons: - apt: - packages: - - *common - - libbz2-dev - - zlib1g-dev - - uuid-dev - - libctpp2-dev - - env: PLATFORM="win32_dyn" - addons: &win32_addons - apt: - packages: - - *common - - g++-mingw-w64-i686 - - gcc-mingw-w64-i686 - - gcc-mingw-w64-base - - mingw-w64-tools - - env: PLATFORM="win32_static" - addons: *win32_addons - env: PLATFORM="i586_dyn" addons: &i586_addons apt: @@ -125,20 +118,6 @@ matrix: - libstdc++-6-dev:i386 - env: PLATFORM="i586_static" addons: *i586_addons - - env: PLATFORM="android" - addons: &android_addons - apt: - packages: - - *common - - default-jdk - - env: PLATFORM="android_arm" - addons: *android_addons - - env: PLATFORM="android_arm64" - addons: *android_addons - - env: PLATFORM="android_x86" - addons: *android_addons - - env: PLATFORM="android_x86_64" - addons: *android_addons - env: PLATFORM="native_dyn" os: osx - env: PLATFORM="iOS_arm64" diff --git a/kiwixbuild/packages.py b/kiwixbuild/packages.py index ad77415..af9ad7b 100644 --- a/kiwixbuild/packages.py +++ b/kiwixbuild/packages.py @@ -76,10 +76,10 @@ PACKAGE_NAME_MAPPERS = { 'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'], }, 'debian_i586_dyn': { - 'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib'], + 'COMMON': _debian_common + ['libc6-dev-i386', 'lib32stdc++6', 'gcc-multilib', 'g++-multilib'], }, 'debian_i586_static': { - 'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib'], + 'COMMON': _debian_common + ['libc6-dev-i386', 'lib32stdc++6', 'gcc-multilib', 'g++-multilib'], }, 'debian_win32_dyn': { 'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'], diff --git a/kiwixbuild/versions.py b/kiwixbuild/versions.py index 9ded59e..ee8edbd 100644 --- a/kiwixbuild/versions.py +++ b/kiwixbuild/versions.py @@ -35,7 +35,7 @@ release_versions = { # This is the "version" of the whole base_deps_versions dict. # Change this when you change base_deps_versions. -base_deps_meta_version = '50' +base_deps_meta_version = '51' base_deps_versions = { diff --git a/travis/compile_all.py b/travis/compile_all.py index f91440d..853638c 100755 --- a/travis/compile_all.py +++ b/travis/compile_all.py @@ -25,6 +25,8 @@ KBUILD_SOURCE_DIR = Path(environ['TRAVIS_BUILD_DIR']) KIWIX_DESKTOP_ONLY = environ.get('DESKTOP_ONLY') == '1' BASE_DIR = HOME/"BUILD_{}".format(PLATFORM) +BASE_EXPORT_DIR = HOME/"EXPORT"/"BASE" +GIT_EXPORT_DIR = HOME/"EXPORT"/"GIT" SOURCE_DIR = HOME/"SOURCE" ARCHIVE_DIR = HOME/"ARCHIVE" INSTALL_DIR = BASE_DIR/"INSTALL" @@ -34,7 +36,6 @@ NIGHTLY_ZIM_ARCHIVES_DIR = HOME/'NIGHTLY_ZIM_ARCHIVES'/NIGHTLY_DATE RELEASE_ZIM_ARCHIVES_DIR = HOME/'RELEASE_ZIM_ARCHIVES' DIST_KIWIX_ARCHIVES_DIR = HOME/'DIST_KIWIX_ARCHIVES' DIST_ZIM_ARCHIVES_DIR = HOME/'DIST_ZIM_ARCHIVES' -SSH_KEY = KBUILD_SOURCE_DIR/'travis'/'travisci_builder_id_key' BIN_EXT = '.exe' if PLATFORM.startswith('win32_') else '' @@ -193,8 +194,8 @@ def make_archive(project, platform): archive_add(arch, f) -def make_deps_archive(target, full=False): - archive_name = "deps_{}_{}_{}.tar.xz".format( +def make_deps_archive(target=None, name=None, full=False): + archive_name = name or "deps_{}_{}_{}.tar.xz".format( TRAVIS_OS_NAME, PLATFORM, target) print_message("Create archive {}.", archive_name) files_to_archive = [INSTALL_DIR] @@ -237,50 +238,43 @@ def make_deps_archive(target, full=False): if not subdir.match('tools'): files_to_archive.append(subdir) - with tarfile.open(str(relative_path/archive_name), 'w:xz') as tar: + with tarfile.open(str(BASE_EXPORT_DIR/archive_name), 'w:xz') as tar: for name in set(files_to_archive): - print('.', end='', flush=True) + print('.{}'.format(name), flush=True) tar.add(str(name), arcname=str(name.relative_to(relative_path))) - return relative_path/archive_name def update_flathub_git(): + GIT_REPO_DIR = GIT_EXPORT_DIR/"org.kiwix.desktop" env = dict(os.environ) - env['GIT_SSH_COMMAND'] = 'ssh -o StrictHostKeyChecking=no -i {}'.format(SSH_KEY) env['GIT_AUTHOR_NAME'] = env['GIT_COMMITTER_NAME'] = "KiwixBot" env['GIT_AUTHOR_EMAIL'] = env['GIT_COMMITTER_EMAIL'] = "kiwixbot@kymeria.fr" + def call(command, cwd=None): + cwd = cwd or GIT_REPO_DIR + subprocess.check_call(command, env=env, cwd=str(cwd)) command = ['git', 'clone', FLATPAK_GIT_REMOTE] - subprocess.check_call(command, env=env, cwd=str(HOME)) - shutil.copy(str(BASE_DIR/'org.kiwix.desktop.json'), - str(HOME/'org.kiwix.desktop')) + call(command, cwd=GIT_EXPORT_DIR) + shutil.copy(str(BASE_DIR/'org.kiwix.desktop.json'), str(GIT_REPO_DIR)) patch_dir = KBUILD_SOURCE_DIR/'kiwixbuild'/'patches' for dep in ('libaria2', 'mustache', 'pugixml', 'xapian'): for f in patch_dir.glob('{}_*.patch'.format(dep)): - shutil.copy(str(f), str(HOME/'org.kiwix.desktop'/'patches')) + shutil.copy(str(f), str(GIT_REPO_DIR/'patches')) command = ['git', 'add', '-A', '.'] - subprocess.check_call(command, env=env, cwd=str(HOME/'org.kiwix.desktop')) + call(command) command = ['git', 'commit', '-m', 'Update to version {}'.format(main_project_versions['kiwix-desktop'])] - subprocess.check_call(command, env=env, cwd=str(HOME/'org.kiwix.desktop')) - command = ['git', 'push'] - subprocess.check_call(command, env=env, cwd=str(HOME/'org.kiwix.desktop')) + call(command) -def scp(what, where): - print_message("Copy {} to {}", what, where) - command = ['scp', '-o', 'StrictHostKeyChecking=no', - '-i', str(SSH_KEY), - str(what), str(where)] - subprocess.check_call(command) - - for p in (NIGHTLY_KIWIX_ARCHIVES_DIR, NIGHTLY_ZIM_ARCHIVES_DIR, RELEASE_KIWIX_ARCHIVES_DIR, RELEASE_ZIM_ARCHIVES_DIR, DIST_KIWIX_ARCHIVES_DIR, - DIST_ZIM_ARCHIVES_DIR): + DIST_ZIM_ARCHIVES_DIR, + BASE_EXPORT_DIR, + GIT_EXPORT_DIR): try: p.mkdir(parents=True) except FileExistsError: @@ -331,11 +325,7 @@ if PLATFORM != 'flatpak': except URLError: pass run_kiwix_build('alldependencies', platform=PLATFORM) - if SSH_KEY.exists(): - archive = make_deps_archive('alldependencies', full=True) - destination = 'ci@tmp.kiwix.org:/data/tmp/ci/{}' - destination = destination.format(base_dep_archive_name) - scp(archive, destination) + make_deps_archive(name=base_dep_archive_name, full=True) # A basic compilation to be sure everything is working (for a PR) @@ -407,8 +397,7 @@ for target in TARGETS: run_kiwix_build(target, platform=PLATFORM, build_deps_only=True) - archive = make_deps_archive(target) - scp(archive, 'ci@tmp.kiwix.org:/data/tmp/ci/') + make_deps_archive(target=target) run_kiwix_build(target, platform=PLATFORM, diff --git a/travis/deploy.sh b/travis/deploy.sh index 543a13f..5809727 100755 --- a/travis/deploy.sh +++ b/travis/deploy.sh @@ -8,7 +8,6 @@ RELEASE_KIWIX_ARCHIVES_DIR=${HOME}/RELEASE_KIWIX_ARCHIVES RELEASE_ZIM_ARCHIVES_DIR=${HOME}/RELEASE_ZIM_ARCHIVES DIST_KIWIX_ARCHIVES_DIR=${HOME}/DIST_KIWIX_ARCHIVES DIST_ZIM_ARCHIVES_DIR=${HOME}/DIST_ZIM_ARCHIVES -SSH_KEY=${TRAVIS_BUILD_DIR}/travis/travisci_builder_id_key if [[ "$TRAVIS_EVENT_TYPE" = "cron" ]] then @@ -68,5 +67,16 @@ then ci@download.openzim.org:/data/openzim/release/${subdir} done fi + + GIT_REPOS=$(ls -d */) + if [[ "x$GIT_REPOS" != "x" ]] + then + for repo in $GIT_REPOS + do + (cd repo; + GIT_SSH_COMMAND"ssh -o StrictHostKeyChecking=no -i $SSH_KEY" git push + ) + done + fi fi diff --git a/travis/deploy_apk.sh b/travis/deploy_apk.sh index cabaa84..c17d748 100755 --- a/travis/deploy_apk.sh +++ b/travis/deploy_apk.sh @@ -4,7 +4,6 @@ set -e KEYSTORE_FILE=${TRAVIS_BUILD_DIR}/travis/kiwix-android.keystore GOOGLE_API_KEY=${TRAVIS_BUILD_DIR}/travis/googleplay_android_developer-5a411156212c.json -SSH_KEY=${TRAVIS_BUILD_DIR}/travis/travisci_builder_id_key cd ${HOME} diff --git a/travis/make_release.sh b/travis/make_release.sh index a875ab2..54be0b8 100755 --- a/travis/make_release.sh +++ b/travis/make_release.sh @@ -4,7 +4,6 @@ set -e KEYSTORE_FILE=${TRAVIS_BUILD_DIR}/travis/test_ks.ks GOOGLE_API_KEY=${TRAVIS_BUILD_DIR}/travis/googleplay_android_developer-5a411156212c.json -SSH_KEY=${TRAVIS_BUILD_DIR}/travis/travisci_builder_id_key cd ${HOME} diff --git a/travis/upload_all_log.sh b/travis/upload_all_log.sh index 1267b81..057f509 100755 --- a/travis/upload_all_log.sh +++ b/travis/upload_all_log.sh @@ -2,8 +2,6 @@ set -e -SSH_KEY=$(pwd)/travis/travisci_builder_id_key - cd $HOME tar -czf fail_log_${PLATFORM}.tar.gz BUILD_${PLATFORM} diff --git a/travis/upload_base_deps.sh b/travis/upload_base_deps.sh new file mode 100755 index 0000000..6c5f232 --- /dev/null +++ b/travis/upload_base_deps.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -e + +BASE_EXPORT_DIR=$HOME/EXPORT/BASE +ls $BASE_EXPORT_DIR + +BASE_ARCHIVES=$(find $BASE_EXPORT_DIR -type f) +echo $BASE_ARCHIVES +if [[ "x$BASE_ARCHIVES" != "x" ]] +then + scp -vrp -i ${SSH_KEY} -o StrictHostKeyChecking=no \ + $BASE_ARCHIVES \ + ci@tmp.kiwix.org:/data/tmp/ci +fi diff --git a/travis/xenial_builder.dockerfile b/travis/xenial_builder.dockerfile new file mode 100644 index 0000000..70c18bd --- /dev/null +++ b/travis/xenial_builder.dockerfile @@ -0,0 +1,41 @@ +FROM ubuntu:xenial + +ENV LANG C.UTF-8 + +RUN apt update -q +RUN \ + dpkg --add-architecture i386 && \ + apt install -q -y --no-install-recommends \ +# Base build tools + build-essential automake libtool cmake ccache pkg-config autopoint patch \ + python3-pip python3-setuptools python3-wheel git subversion wget unzip \ + ninja-build \ +# Python (2) is needed to install android-ndk + python \ +# Packaged dependencies + libbz2-dev libmagic-dev uuid-dev zlib1g-dev default-jdk \ + libmicrohttpd-dev \ +# Cross win32 compiler + g++-mingw-w64-i686 gcc-mingw-w64-i686 gcc-mingw-w64-base mingw-w64-tools \ +# Cross compile i586 + libc6-dev-i386 lib32stdc++6 gcc-multilib g++-multilib \ +# Other tools (to remove) + vim less grep && \ + apt-get clean -y && \ + rm -rf /usr/share/doc/* /var/cache/debconf/* + +# Create user +RUN useradd --create-home builder +USER builder +WORKDIR /home/builder +ENV PATH="/home/builder/.local/bin:${PATH}" + +# Install kiwix-build +COPY --chown=builder:builder . kiwix-build +RUN pip3 install --user -e ./kiwix-build + +ENV TRAVIS_BUILD_DIR /home/builder/kiwix-build +ENV GRADLE_USER_HOME /home/builder +ENV TRAVIS_OS_NAME linux_xenial + +CMD kiwix-build/travis/compile_all.py From 19ac17a637b34c1803332749615d2241849e3e93 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 2 May 2019 15:58:10 +0200 Subject: [PATCH 5/9] Build the docker images in a separated stage. Images are push to kiwix/kiwix-build_ci --- .travis.yml | 81 ++++++++++--------- ...Dockerfile => appimage_builder.dockerfile} | 0 ...ile_flatpak => flatpak_builder.dockerfile} | 0 3 files changed, 42 insertions(+), 39 deletions(-) rename travis/{Dockerfile => appimage_builder.dockerfile} (100%) rename travis/{Dockerfile_flatpak => flatpak_builder.dockerfile} (100%) diff --git a/.travis.yml b/.travis.yml index 78fac47..457a22a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,39 @@ branches: - /\d+\.\d+\.\d+$/ if: type != push OR tag IS present +stages: +- build_docker_images +- test + +jobs: + include: + - stage: build_docker_images + script: &build_images + - | + if [ ! $(curl -sflL https://hub.docker.com/v2/repositories/kiwix/kiwix-build_ci/tags/${VARIANT}-${TRAVIS_COMMIT}) ] + then + echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + docker build -t kiwix-build_ci -f travis/${VARIANT}_builder.dockerfile . + docker tag kiwix-build_ci kiwix/kiwix-build_ci:${VARIANT}-${TRAVIS_COMMIT} + docker push kiwix/kiwix-build_ci:${VARIANT}-${TRAVIS_COMMIT} + fi + env: VARIANT=xenial + - stage: build_docker_images + script: *build_images + env: VARIANT=flatpak + - stage: build_docker_images + script: *build_images + env: VARIANT=appimage + - stage: test + env: PLATFORM="native_dyn" + os: osx + - env: PLATFORM="iOS_arm64" + os: osx + - env: PLATFORM="iOS_i386" + os: osx + + + before_install: - PATH=$PATH:${HOME}/bin:${HOME}/.local/bin - mkdir -p /tmp/private $HOME/EXPORT/BASE @@ -34,25 +67,20 @@ script: - | if [[ $TRAVIS_OS_NAME = "linux" ]] then - PODMAN_FILE=xenial_builder.dockerfile - if [[ $PLATFORM = "flatpak" && $DESKTOP_ONLY = 1 ]]; then PODMAN_FILE=Dockerfile_flatpak; fi - if [[ $PLATFORM = "native_dyn" && $DESKTOP_ONLY = 1 ]]; then PODMAN_FILE=Dockerfile; fi + VARIANT=xenial + if [[ $PLATFORM = "flatpak" && $DESKTOP_ONLY = 1 ]]; then VARIANT=flatpak; fi + if [[ $PLATFORM = "native_dyn" && $DESKTOP_ONLY = 1 ]]; then VARIANT=appimage; fi fi - | if [[ $TRAVIS_OS_NAME = "linux" ]] then - docker build -t kiwix/build -f travis/$PODMAN_FILE . - fi -- | - if [[ $TRAVIS_OS_NAME = "linux" ]] - then - docker run \ + docker run --rm \ -e PLATFORM -e NIGHTLY_DATE -e TRAVIS_EVENT_TYPE -e DESKTOP_ONLY -e TRAVIS_TAG \ --device /dev/fuse --cap-add ALL --privileged \ --mount=type=bind,src=$HOME/EXPORT,dst=/home/builder/EXPORT \ --mount=type=bind,src=$HOME/.cache,dst=/home/builder/.cache \ --mount=type=bind,src=$HOME/.gradle/caches,dst=/home/builder/.gradle/caches \ - kiwix/build + kiwix/kiwix-build_ci:${VARIANT}-${TRAVIS_COMMIT} fi - if [[ $TRAVIS_OS_NAME != "linux" ]]; then python3 travis/compile_all.py; fi after_success: @@ -85,6 +113,10 @@ env: - PLATFORM="armhf_static" - PLATFORM="win32_dyn" - PLATFORM="win32_static" + - PLATFORM="i586_dyn" + - PLATFORM="i586_static" + - PLATFORM="native_dyn" DESKTOP_ONLY=1 + - PLATFORM="flatpak" DESKTOP_ONLY=1 - PLATFORM="android" - PLATFORM="android_arm" - PLATFORM="android_arm64" @@ -95,35 +127,6 @@ addons: - download.kiwix.org - tmp.kiwix.org - download.openzim.org -matrix: - include: - - env: PLATFORM="native_dyn" DESKTOP_ONLY=1 - addons: - apt: - packages: [] - - env: PLATFORM="flatpak" DESKTOP_ONLY=1 - addons: - apt: - packages: [] - - env: PLATFORM="i586_dyn" - addons: &i586_addons - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - *common - - gcc-multilib - - g++-multilib - - libc6-dev:i386 - - libstdc++-6-dev:i386 - - env: PLATFORM="i586_static" - addons: *i586_addons - - env: PLATFORM="native_dyn" - os: osx - - env: PLATFORM="iOS_arm64" - os: osx - - env: PLATFORM="iOS_i386" - os: osx notifications: irc: diff --git a/travis/Dockerfile b/travis/appimage_builder.dockerfile similarity index 100% rename from travis/Dockerfile rename to travis/appimage_builder.dockerfile diff --git a/travis/Dockerfile_flatpak b/travis/flatpak_builder.dockerfile similarity index 100% rename from travis/Dockerfile_flatpak rename to travis/flatpak_builder.dockerfile From 39d05d1b03fb85d382ea168bb512fa0690afc100 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 2 May 2019 18:17:02 +0200 Subject: [PATCH 6/9] Do not build the base archive for android platform. We already have a base archive per android_ platform. We don't need to create another archive that will mainly include the content of the other archives. --- travis/compile_all.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/travis/compile_all.py b/travis/compile_all.py index 853638c..1c52d75 100755 --- a/travis/compile_all.py +++ b/travis/compile_all.py @@ -324,8 +324,9 @@ if PLATFORM != 'flatpak': f.extractall(str(HOME)) except URLError: pass - run_kiwix_build('alldependencies', platform=PLATFORM) - make_deps_archive(name=base_dep_archive_name, full=True) + else: + run_kiwix_build('alldependencies', platform=PLATFORM) + make_deps_archive(name=base_dep_archive_name, full=True) # A basic compilation to be sure everything is working (for a PR) From 556095f8933765d4b1be89876f8b9b908c33f3ac Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 2 May 2019 18:22:13 +0200 Subject: [PATCH 7/9] Build android after all other build. Because we need the android_ build to be done before. --- .travis.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 457a22a..cba4ff0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,8 @@ if: type != push OR tag IS present stages: - build_docker_images -- test +- build +- android_build jobs: include: @@ -33,8 +34,9 @@ jobs: - stage: build_docker_images script: *build_images env: VARIANT=appimage - - stage: test - env: PLATFORM="native_dyn" + - stage: android_build + env: PLATFORM="android" + - env: PLATFORM="native_dyn" os: osx - env: PLATFORM="iOS_arm64" os: osx @@ -63,6 +65,7 @@ cache: - $HOME/.gradle/wrapper/ - $HOME/.android/build-cache install: if [[ $TRAVIS_OS_NAME != "linux" ]]; then travis/install_extra_deps.sh; fi +stage: build script: - | if [[ $TRAVIS_OS_NAME = "linux" ]] @@ -117,7 +120,6 @@ env: - PLATFORM="i586_static" - PLATFORM="native_dyn" DESKTOP_ONLY=1 - PLATFORM="flatpak" DESKTOP_ONLY=1 - - PLATFORM="android" - PLATFORM="android_arm" - PLATFORM="android_arm64" - PLATFORM="android_x86" From 2b2dd9f11834f6e282fdb707c5044910dfb7c02c Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 2 May 2019 20:14:12 +0200 Subject: [PATCH 8/9] Do not build the multi arch android app on pullrequest. We already build on all android arch on specific job. So no need to check recompile the same thing. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index cba4ff0..089d9c5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,6 +36,7 @@ jobs: env: VARIANT=appimage - stage: android_build env: PLATFORM="android" + if: type != pull_request - env: PLATFORM="native_dyn" os: osx - env: PLATFORM="iOS_arm64" From 0142dc94d4c557e7d9a4a06e1705b4c15ffeccfc Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 2 May 2019 20:33:22 +0200 Subject: [PATCH 9/9] [CI] Start to build flatpak first in the CI The flatpak build is the longer job. By starting it in the first jobs, we reduce a bit the total build time. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 089d9c5..ee4866e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -110,6 +110,7 @@ env: - NIGHTLY_DATE=$(date +%Y-%m-%d) - SSH_KEY=/tmp/private/travisci_builder_id_key matrix: + - PLATFORM="flatpak" DESKTOP_ONLY=1 - PLATFORM="native_dyn" - PLATFORM="native_static" - PLATFORM="native_mixed" @@ -120,7 +121,6 @@ env: - PLATFORM="i586_dyn" - PLATFORM="i586_static" - PLATFORM="native_dyn" DESKTOP_ONLY=1 - - PLATFORM="flatpak" DESKTOP_ONLY=1 - PLATFORM="android_arm" - PLATFORM="android_arm64" - PLATFORM="android_x86"