Build everything in a container.

Docker containers don't have the SSH_KEY so we need to copy archives and git repo in a directory on the host and launch deploy.sh script from there.
2019-04-18 15:23:31 +02:00 · 2019-04-18 15:23:31 +02:00 · 281adbd768
parent f9d5e0aa21
commit 281adbd768
10 changed files with 131 additions and 101 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -12,11 +12,13 @@ if: type != push OR tag IS present

 before_install:
 - PATH=$PATH:${HOME}/bin:${HOME}/.local/bin
+- mkdir -p /tmp/private $HOME/EXPORT/BASE
+- chmod -R a+w $HOME/EXPORT
 - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then PATH=$PATH:$(brew --prefix)/opt/gettext/bin; fi
 - '[ -n "$encrypted_eba2f7543984_iv" ] && openssl aes-256-cbc
  -K $encrypted_eba2f7543984_key -iv $encrypted_eba2f7543984_iv
-  -in travis/travisci_builder_id_key.enc -out travis/travisci_builder_id_key -d'
- chmod 600 travis/travisci_builder_id_key
+  -in travis/travisci_builder_id_key.enc -out $SSH_KEY -d'
+- chmod 600 $SSH_KEY
 before_cache:
  - rm -f $HOME/.gradle/caches/modules-2/modules-2.lock
  - rm -fr $HOME/.gradle/caches/*/plugin-resolution/
@ -27,27 +29,37 @@ cache:
    - $HOME/.gradle/caches/
    - $HOME/.gradle/wrapper/
    - $HOME/.android/build-cache
-install: if [[ $TRAVIS_OS_NAME != "linux" || $DESKTOP_ONLY != 1 ]]; then travis/install_extra_deps.sh; fi
+install: if [[ $TRAVIS_OS_NAME != "linux" ]]; then travis/install_extra_deps.sh; fi
 script:
 - |
-  if [[ $TRAVIS_OS_NAME = "linux" && $DESKTOP_ONLY == 1 ]]
+  if [[ $TRAVIS_OS_NAME = "linux" ]]
  then
-    if [[ $PLATFORM = "flatpak" ]]
-    then
-      docker build -t kiwix/build -f travis/Dockerfile_flatpak .
-    else
-      docker build -t kiwix/build -f travis/Dockerfile .
-    fi
-    docker run -e PLATFORM -e NIGHTLY_DATE -e TRAVIS_EVENT_TYPE -e DESKTOP_ONLY -e TRAVIS_TAG --device /dev/fuse --cap-add ALL --privileged kiwix/build
-  else
-    if [[ $TRAVIS_OS_NAME = "linux" ]]
-    then
-      python3.5 travis/compile_all.py
-    else
-      python3 travis/compile_all.py
-    fi
+    PODMAN_FILE=xenial_builder.dockerfile
+    if [[ $PLATFORM = "flatpak" && $DESKTOP_ONLY = 1 ]]; then PODMAN_FILE=Dockerfile_flatpak; fi
+    if [[ $PLATFORM = "native_dyn" && $DESKTOP_ONLY = 1 ]]; then PODMAN_FILE=Dockerfile; fi
  fi
-after_failure: travis/upload_all_log.sh
+- |
+  if [[ $TRAVIS_OS_NAME = "linux" ]]
+  then
+    docker build -t kiwix/build -f travis/$PODMAN_FILE .
+  fi
+- |
+  if [[ $TRAVIS_OS_NAME = "linux" ]]
+  then
+    docker run \
+      -e PLATFORM -e NIGHTLY_DATE -e TRAVIS_EVENT_TYPE -e DESKTOP_ONLY -e TRAVIS_TAG \
+      --device /dev/fuse --cap-add ALL --privileged \
+      --mount=type=bind,src=$HOME/EXPORT,dst=/home/builder/EXPORT \
+      --mount=type=bind,src=$HOME/.cache,dst=/home/builder/.cache \
+      --mount=type=bind,src=$HOME/.gradle/caches,dst=/home/builder/.gradle/caches \
+      kiwix/build
+  fi
+- if [[ $TRAVIS_OS_NAME != "linux" ]]; then python3 travis/compile_all.py; fi
+after_success:
+- travis/upload_base_deps.sh
+after_failure:
+- travis/upload_base_deps.sh
+- travis/upload_all_log.sh
 deploy:
  - provider: script
    skip_cleanup: true
@ -64,24 +76,25 @@ deploy:
 env:
  global:
    - NIGHTLY_DATE=$(date +%Y-%m-%d)
+    - SSH_KEY=/tmp/private/travisci_builder_id_key
  matrix:
    - PLATFORM="native_dyn"
+    - PLATFORM="native_static"
+    - PLATFORM="native_mixed"
    - PLATFORM="armhf_dyn"
    - PLATFORM="armhf_static"
-    - PLATFORM="native_mixed"
+    - PLATFORM="win32_dyn"
+    - PLATFORM="win32_static"
+    - PLATFORM="android"
+    - PLATFORM="android_arm"
+    - PLATFORM="android_arm64"
+    - PLATFORM="android_x86"
+    - PLATFORM="android_x86_64"
 addons:
  ssh_known_hosts:
  - download.kiwix.org
  - tmp.kiwix.org
  - download.openzim.org
-  apt:
-    packages: &common
-      - cmake
-      - autopoint
-      - python3.5
-      - python3-pip
-      - ctpp2-utils
-      - libmagic-dev
 matrix:
  include:
    - env: PLATFORM="native_dyn" DESKTOP_ONLY=1
@ -92,26 +105,6 @@ matrix:
      addons:
        apt:
          packages: []
-    - env: PLATFORM="native_static"
-      addons:
-        apt:
-          packages:
-            - *common
-            - libbz2-dev
-            - zlib1g-dev
-            - uuid-dev
-            - libctpp2-dev
-    - env: PLATFORM="win32_dyn"
-      addons: &win32_addons
-        apt:
-          packages:
-            - *common
-            - g++-mingw-w64-i686
-            - gcc-mingw-w64-i686
-            - gcc-mingw-w64-base
-            - mingw-w64-tools
-    - env: PLATFORM="win32_static"
-      addons: *win32_addons
    - env: PLATFORM="i586_dyn"
      addons: &i586_addons
        apt:
@ -125,20 +118,6 @@ matrix:
            - libstdc++-6-dev:i386
    - env: PLATFORM="i586_static"
      addons: *i586_addons
-    - env: PLATFORM="android"
-      addons: &android_addons
-        apt:
-          packages:
-            - *common
-            - default-jdk
-    - env: PLATFORM="android_arm"
-      addons: *android_addons
-    - env: PLATFORM="android_arm64"
-      addons: *android_addons
-    - env: PLATFORM="android_x86"
-      addons: *android_addons
-    - env: PLATFORM="android_x86_64"
-      addons: *android_addons
    - env: PLATFORM="native_dyn"
      os: osx
    - env: PLATFORM="iOS_arm64"
--- a/kiwixbuild/packages.py
+++ b/kiwixbuild/packages.py
@ -76,10 +76,10 @@ PACKAGE_NAME_MAPPERS = {
        'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'],
    },
    'debian_i586_dyn': {
-        'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib'],
+        'COMMON': _debian_common + ['libc6-dev-i386', 'lib32stdc++6', 'gcc-multilib', 'g++-multilib'],
    },
    'debian_i586_static': {
-        'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib'],
+        'COMMON': _debian_common + ['libc6-dev-i386', 'lib32stdc++6', 'gcc-multilib', 'g++-multilib'],
    },
    'debian_win32_dyn': {
        'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'],
--- a/kiwixbuild/versions.py
+++ b/kiwixbuild/versions.py
@ -35,7 +35,7 @@ release_versions = {

 # This is the "version" of the whole base_deps_versions dict.
 # Change this when you change base_deps_versions.
-base_deps_meta_version = '50'
+base_deps_meta_version = '51'


 base_deps_versions = {
--- a/travis/compile_all.py
+++ b/travis/compile_all.py
@ -25,6 +25,8 @@ KBUILD_SOURCE_DIR = Path(environ['TRAVIS_BUILD_DIR'])
 KIWIX_DESKTOP_ONLY = environ.get('DESKTOP_ONLY') == '1'

 BASE_DIR = HOME/"BUILD_{}".format(PLATFORM)
+BASE_EXPORT_DIR = HOME/"EXPORT"/"BASE"
+GIT_EXPORT_DIR = HOME/"EXPORT"/"GIT"
 SOURCE_DIR = HOME/"SOURCE"
 ARCHIVE_DIR = HOME/"ARCHIVE"
 INSTALL_DIR = BASE_DIR/"INSTALL"
@ -34,7 +36,6 @@ NIGHTLY_ZIM_ARCHIVES_DIR = HOME/'NIGHTLY_ZIM_ARCHIVES'/NIGHTLY_DATE
 RELEASE_ZIM_ARCHIVES_DIR = HOME/'RELEASE_ZIM_ARCHIVES'
 DIST_KIWIX_ARCHIVES_DIR = HOME/'DIST_KIWIX_ARCHIVES'
 DIST_ZIM_ARCHIVES_DIR = HOME/'DIST_ZIM_ARCHIVES'
-SSH_KEY = KBUILD_SOURCE_DIR/'travis'/'travisci_builder_id_key'

 BIN_EXT = '.exe' if PLATFORM.startswith('win32_') else ''

@ -193,8 +194,8 @@ def make_archive(project, platform):
            archive_add(arch, f)


-def make_deps_archive(target, full=False):
-    archive_name = "deps_{}_{}_{}.tar.xz".format(
+def make_deps_archive(target=None, name=None, full=False):
+    archive_name = name or "deps_{}_{}_{}.tar.xz".format(
        TRAVIS_OS_NAME, PLATFORM, target)
    print_message("Create archive {}.", archive_name)
    files_to_archive = [INSTALL_DIR]
@ -237,50 +238,43 @@ def make_deps_archive(target, full=False):
            if not subdir.match('tools'):
                files_to_archive.append(subdir)

-    with tarfile.open(str(relative_path/archive_name), 'w:xz') as tar:
+    with tarfile.open(str(BASE_EXPORT_DIR/archive_name), 'w:xz') as tar:
        for name in set(files_to_archive):
-            print('.', end='', flush=True)
+            print('.{}'.format(name), flush=True)
            tar.add(str(name), arcname=str(name.relative_to(relative_path)))
-    return relative_path/archive_name


 def update_flathub_git():
+    GIT_REPO_DIR = GIT_EXPORT_DIR/"org.kiwix.desktop"
    env = dict(os.environ)
-    env['GIT_SSH_COMMAND'] = 'ssh -o StrictHostKeyChecking=no -i {}'.format(SSH_KEY)
    env['GIT_AUTHOR_NAME'] = env['GIT_COMMITTER_NAME'] = "KiwixBot"
    env['GIT_AUTHOR_EMAIL'] = env['GIT_COMMITTER_EMAIL'] = "kiwixbot@kymeria.fr"
+    def call(command, cwd=None):
+        cwd = cwd or GIT_REPO_DIR
+        subprocess.check_call(command, env=env, cwd=str(cwd))
    command = ['git', 'clone', FLATPAK_GIT_REMOTE]
-    subprocess.check_call(command, env=env, cwd=str(HOME))
-    shutil.copy(str(BASE_DIR/'org.kiwix.desktop.json'),
-                str(HOME/'org.kiwix.desktop'))
+    call(command, cwd=GIT_EXPORT_DIR)
+    shutil.copy(str(BASE_DIR/'org.kiwix.desktop.json'), str(GIT_REPO_DIR))
    patch_dir = KBUILD_SOURCE_DIR/'kiwixbuild'/'patches'
    for dep in ('libaria2', 'mustache', 'pugixml', 'xapian'):
        for f in patch_dir.glob('{}_*.patch'.format(dep)):
-            shutil.copy(str(f), str(HOME/'org.kiwix.desktop'/'patches'))
+            shutil.copy(str(f), str(GIT_REPO_DIR/'patches'))
    command = ['git', 'add', '-A', '.']
-    subprocess.check_call(command, env=env, cwd=str(HOME/'org.kiwix.desktop'))
+    call(command)
    command = ['git', 'commit', '-m',
               'Update to version {}'.format(main_project_versions['kiwix-desktop'])]
-    subprocess.check_call(command, env=env, cwd=str(HOME/'org.kiwix.desktop'))
-    command = ['git', 'push']
-    subprocess.check_call(command, env=env, cwd=str(HOME/'org.kiwix.desktop'))
+    call(command)



-def scp(what, where):
-    print_message("Copy {} to {}", what, where)
-    command = ['scp', '-o', 'StrictHostKeyChecking=no',
-                      '-i', str(SSH_KEY),
-                      str(what), str(where)]
-    subprocess.check_call(command)
-
-
 for p in (NIGHTLY_KIWIX_ARCHIVES_DIR,
          NIGHTLY_ZIM_ARCHIVES_DIR,
          RELEASE_KIWIX_ARCHIVES_DIR,
          RELEASE_ZIM_ARCHIVES_DIR,
          DIST_KIWIX_ARCHIVES_DIR,
-          DIST_ZIM_ARCHIVES_DIR):
+          DIST_ZIM_ARCHIVES_DIR,
+          BASE_EXPORT_DIR,
+          GIT_EXPORT_DIR):
    try:
        p.mkdir(parents=True)
    except FileExistsError:
@ -331,11 +325,7 @@ if PLATFORM != 'flatpak':
                except URLError:
                    pass
        run_kiwix_build('alldependencies', platform=PLATFORM)
-        if SSH_KEY.exists():
-            archive = make_deps_archive('alldependencies', full=True)
-            destination = 'ci@tmp.kiwix.org:/data/tmp/ci/{}'
-            destination = destination.format(base_dep_archive_name)
-            scp(archive, destination)
+        make_deps_archive(name=base_dep_archive_name, full=True)


 # A basic compilation to be sure everything is working (for a PR)
@ -407,8 +397,7 @@ for target in TARGETS:
        run_kiwix_build(target,
                        platform=PLATFORM,
                        build_deps_only=True)
-        archive = make_deps_archive(target)
-        scp(archive, 'ci@tmp.kiwix.org:/data/tmp/ci/')
+        make_deps_archive(target=target)

    run_kiwix_build(target,
                    platform=PLATFORM,
--- a/travis/deploy.sh
+++ b/travis/deploy.sh
@ -8,7 +8,6 @@ RELEASE_KIWIX_ARCHIVES_DIR=${HOME}/RELEASE_KIWIX_ARCHIVES
 RELEASE_ZIM_ARCHIVES_DIR=${HOME}/RELEASE_ZIM_ARCHIVES
 DIST_KIWIX_ARCHIVES_DIR=${HOME}/DIST_KIWIX_ARCHIVES
 DIST_ZIM_ARCHIVES_DIR=${HOME}/DIST_ZIM_ARCHIVES
-SSH_KEY=${TRAVIS_BUILD_DIR}/travis/travisci_builder_id_key

 if [[ "$TRAVIS_EVENT_TYPE" = "cron" ]]
 then
@ -68,5 +67,16 @@ then
        ci@download.openzim.org:/data/openzim/release/${subdir}
    done
  fi
+
+  GIT_REPOS=$(ls -d */)
+  if [[ "x$GIT_REPOS" != "x" ]]
+  then
+    for repo in $GIT_REPOS
+    do
+      (cd repo;
+      GIT_SSH_COMMAND"ssh -o StrictHostKeyChecking=no -i $SSH_KEY" git push
+      )
+    done
+  fi
 fi

--- a/travis/deploy_apk.sh
+++ b/travis/deploy_apk.sh
@ -4,7 +4,6 @@ set -e

 KEYSTORE_FILE=${TRAVIS_BUILD_DIR}/travis/kiwix-android.keystore
 GOOGLE_API_KEY=${TRAVIS_BUILD_DIR}/travis/googleplay_android_developer-5a411156212c.json
-SSH_KEY=${TRAVIS_BUILD_DIR}/travis/travisci_builder_id_key

 cd ${HOME}

--- a/travis/make_release.sh
+++ b/travis/make_release.sh
@ -4,7 +4,6 @@ set -e

 KEYSTORE_FILE=${TRAVIS_BUILD_DIR}/travis/test_ks.ks
 GOOGLE_API_KEY=${TRAVIS_BUILD_DIR}/travis/googleplay_android_developer-5a411156212c.json
-SSH_KEY=${TRAVIS_BUILD_DIR}/travis/travisci_builder_id_key

 cd ${HOME}

--- a/travis/upload_all_log.sh
+++ b/travis/upload_all_log.sh
@ -2,8 +2,6 @@

 set -e

-SSH_KEY=$(pwd)/travis/travisci_builder_id_key
-
 cd $HOME

 tar -czf fail_log_${PLATFORM}.tar.gz BUILD_${PLATFORM}
--- a/travis/upload_base_deps.sh
+++ b/travis/upload_base_deps.sh
@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -e
+
+BASE_EXPORT_DIR=$HOME/EXPORT/BASE
+ls $BASE_EXPORT_DIR
+
+BASE_ARCHIVES=$(find $BASE_EXPORT_DIR -type f)
+echo $BASE_ARCHIVES
+if [[ "x$BASE_ARCHIVES" != "x" ]]
+then
+  scp -vrp -i ${SSH_KEY} -o StrictHostKeyChecking=no \
+    $BASE_ARCHIVES \
+    ci@tmp.kiwix.org:/data/tmp/ci
+fi
--- a/travis/xenial_builder.dockerfile
+++ b/travis/xenial_builder.dockerfile
@ -0,0 +1,41 @@
+FROM ubuntu:xenial
+
+ENV LANG C.UTF-8
+
+RUN apt update -q
+RUN \
+  dpkg --add-architecture i386 && \
+  apt install -q -y --no-install-recommends \
+# Base build tools
+    build-essential automake libtool cmake ccache pkg-config autopoint patch \
+    python3-pip python3-setuptools python3-wheel git subversion wget unzip \
+    ninja-build \
+# Python (2) is needed to install android-ndk
+    python \
+# Packaged dependencies
+    libbz2-dev libmagic-dev uuid-dev zlib1g-dev default-jdk \
+    libmicrohttpd-dev \
+# Cross win32 compiler
+    g++-mingw-w64-i686 gcc-mingw-w64-i686 gcc-mingw-w64-base mingw-w64-tools \
+# Cross compile i586
+    libc6-dev-i386 lib32stdc++6 gcc-multilib g++-multilib \
+# Other tools (to remove)
+    vim less grep && \
+  apt-get clean -y && \
+  rm -rf /usr/share/doc/* /var/cache/debconf/*
+
+# Create user
+RUN useradd --create-home builder
+USER builder
+WORKDIR /home/builder
+ENV PATH="/home/builder/.local/bin:${PATH}"
+
+# Install kiwix-build
+COPY --chown=builder:builder . kiwix-build
+RUN pip3 install --user -e ./kiwix-build
+
+ENV TRAVIS_BUILD_DIR /home/builder/kiwix-build
+ENV GRADLE_USER_HOME /home/builder
+ENV TRAVIS_OS_NAME   linux_xenial
+
+CMD kiwix-build/travis/compile_all.py