Merge pull request #318 from kiwix/podman

Docker

.travis.yml

@@ -1,6 +1,6 @@
 language: cpp
 
-dist: trusty
+dist: xenial
 sudo: required
 services:
 - docker
@@ -10,13 +10,51 @@ branches:
   - /\d+\.\d+\.\d+$/
 if: type != push OR tag IS present
 
+stages:
+- build_docker_images
+- build
+- android_build
+
+jobs:
+  include:
+    - stage: build_docker_images
+      script: &build_images
+        - |
+          if [ ! $(curl -sflL https://hub.docker.com/v2/repositories/kiwix/kiwix-build_ci/tags/${VARIANT}-${TRAVIS_COMMIT}) ]
+          then
+            echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+            docker build -t kiwix-build_ci -f travis/${VARIANT}_builder.dockerfile .
+            docker tag kiwix-build_ci kiwix/kiwix-build_ci:${VARIANT}-${TRAVIS_COMMIT}
+            docker push kiwix/kiwix-build_ci:${VARIANT}-${TRAVIS_COMMIT}
+          fi
+      env: VARIANT=xenial
+    - stage: build_docker_images
+      script: *build_images
+      env: VARIANT=flatpak
+    - stage: build_docker_images
+      script: *build_images
+      env: VARIANT=appimage
+    - stage: android_build
+      env: PLATFORM="android"
+      if: type != pull_request
+    - env: PLATFORM="native_dyn"
+      os: osx
+    - env: PLATFORM="iOS_arm64"
+      os: osx
+    - env: PLATFORM="iOS_i386"
+      os: osx
+
+
+
 before_install:
 - PATH=$PATH:${HOME}/bin:${HOME}/.local/bin
+- mkdir -p /tmp/private $HOME/EXPORT/BASE
+- chmod -R a+w $HOME/EXPORT
 - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then PATH=$PATH:$(brew --prefix)/opt/gettext/bin; fi
 - '[ -n "$encrypted_eba2f7543984_iv" ] && openssl aes-256-cbc
   -K $encrypted_eba2f7543984_key -iv $encrypted_eba2f7543984_iv
-  -in travis/travisci_builder_id_key.enc -out travis/travisci_builder_id_key -d'
+  -in travis/travisci_builder_id_key.enc -out $SSH_KEY -d'
-- chmod 600 travis/travisci_builder_id_key
+- chmod 600 $SSH_KEY
 before_cache:
   - rm -f $HOME/.gradle/caches/modules-2/modules-2.lock
   - rm -fr $HOME/.gradle/caches/*/plugin-resolution/
@@ -27,27 +65,33 @@ cache:
     - $HOME/.gradle/caches/
     - $HOME/.gradle/wrapper/
     - $HOME/.android/build-cache
-install: if [[ $TRAVIS_OS_NAME != "linux" || $DESKTOP_ONLY != 1 ]]; then travis/install_extra_deps.sh; fi
+install: if [[ $TRAVIS_OS_NAME != "linux" ]]; then travis/install_extra_deps.sh; fi
+stage: build
 script:
 - |
-  if [[ $TRAVIS_OS_NAME = "linux" && $DESKTOP_ONLY == 1 ]]
-  then
-    if [[ $PLATFORM = "flatpak" ]]
-    then
-      docker build -t kiwix/build -f travis/Dockerfile_flatpak .
-    else
-      docker build -t kiwix/build -f travis/Dockerfile .
-    fi
-    docker run -e PLATFORM -e NIGHTLY_DATE -e TRAVIS_EVENT_TYPE -e DESKTOP_ONLY -e TRAVIS_TAG --device /dev/fuse --cap-add ALL --privileged kiwix/build
-  else
   if [[ $TRAVIS_OS_NAME = "linux" ]]
   then
-      python3.5 travis/compile_all.py
+    VARIANT=xenial
-    else
+    if [[ $PLATFORM = "flatpak" && $DESKTOP_ONLY = 1 ]]; then VARIANT=flatpak; fi
-      python3 travis/compile_all.py
+    if [[ $PLATFORM = "native_dyn" && $DESKTOP_ONLY = 1 ]]; then VARIANT=appimage; fi
   fi
+- |
+  if [[ $TRAVIS_OS_NAME = "linux" ]]
+  then
+    docker run --rm \
+      -e PLATFORM -e NIGHTLY_DATE -e TRAVIS_EVENT_TYPE -e DESKTOP_ONLY -e TRAVIS_TAG \
+      --device /dev/fuse --cap-add ALL --privileged \
+      --mount=type=bind,src=$HOME/EXPORT,dst=/home/builder/EXPORT \
+      --mount=type=bind,src=$HOME/.cache,dst=/home/builder/.cache \
+      --mount=type=bind,src=$HOME/.gradle/caches,dst=/home/builder/.gradle/caches \
+      kiwix/kiwix-build_ci:${VARIANT}-${TRAVIS_COMMIT}
   fi
-after_failure: travis/upload_all_log.sh
+- if [[ $TRAVIS_OS_NAME != "linux" ]]; then python3 travis/compile_all.py; fi
+after_success:
+- travis/upload_base_deps.sh
+after_failure:
+- travis/upload_base_deps.sh
+- travis/upload_all_log.sh
 deploy:
   - provider: script
     skip_cleanup: true
@@ -64,87 +108,28 @@ deploy:
 env:
   global:
     - NIGHTLY_DATE=$(date +%Y-%m-%d)
+    - SSH_KEY=/tmp/private/travisci_builder_id_key
   matrix:
+    - PLATFORM="flatpak" DESKTOP_ONLY=1
     - PLATFORM="native_dyn"
+    - PLATFORM="native_static"
+    - PLATFORM="native_mixed"
     - PLATFORM="armhf_dyn"
     - PLATFORM="armhf_static"
-    - PLATFORM="native_mixed"
+    - PLATFORM="win32_dyn"
+    - PLATFORM="win32_static"
+    - PLATFORM="i586_dyn"
+    - PLATFORM="i586_static"
+    - PLATFORM="native_dyn" DESKTOP_ONLY=1
+    - PLATFORM="android_arm"
+    - PLATFORM="android_arm64"
+    - PLATFORM="android_x86"
+    - PLATFORM="android_x86_64"
 addons:
   ssh_known_hosts:
   - download.kiwix.org
   - tmp.kiwix.org
   - download.openzim.org
-  apt:
-    packages: &common
-      - cmake
-      - autopoint
-      - python3.5
-      - python3-pip
-      - ctpp2-utils
-      - libmagic-dev
-matrix:
-  include:
-    - env: PLATFORM="native_dyn" DESKTOP_ONLY=1
-      addons:
-        apt:
-          packages: []
-    - env: PLATFORM="flatpak" DESKTOP_ONLY=1
-      addons:
-        apt:
-          packages: []
-    - env: PLATFORM="native_static"
-      addons:
-        apt:
-          packages:
-            - *common
-            - libbz2-dev
-            - zlib1g-dev
-            - uuid-dev
-            - libctpp2-dev
-    - env: PLATFORM="win32_dyn"
-      addons: &win32_addons
-        apt:
-          packages:
-            - *common
-            - g++-mingw-w64-i686
-            - gcc-mingw-w64-i686
-            - gcc-mingw-w64-base
-            - mingw-w64-tools
-    - env: PLATFORM="win32_static"
-      addons: *win32_addons
-    - env: PLATFORM="i586_dyn"
-      addons: &i586_addons
-        apt:
-          sources:
-            - ubuntu-toolchain-r-test
-          packages:
-            - *common
-            - gcc-multilib
-            - g++-multilib
-            - libc6-dev:i386
-            - libstdc++-6-dev:i386
-    - env: PLATFORM="i586_static"
-      addons: *i586_addons
-    - env: PLATFORM="android"
-      addons: &android_addons
-        apt:
-          packages:
-            - *common
-            - default-jdk
-    - env: PLATFORM="android_arm"
-      addons: *android_addons
-    - env: PLATFORM="android_arm64"
-      addons: *android_addons
-    - env: PLATFORM="android_x86"
-      addons: *android_addons
-    - env: PLATFORM="android_x86_64"
-      addons: *android_addons
-    - env: PLATFORM="native_dyn"
-      os: osx
-    - env: PLATFORM="iOS_arm64"
-      os: osx
-    - env: PLATFORM="iOS_i386"
-      os: osx
 
 notifications:
   irc:

kiwixbuild/dependencies/armhf.py

@@ -1,11 +1,12 @@
-from .base import Dependency, GitClone, NoopBuilder
+from .base import Dependency, ReleaseDownload, NoopBuilder
+from kiwixbuild.utils import Remotefile
 
 class armhf_toolchain(Dependency):
     neutral = True
     name = 'armhf'
 
-    class Source(GitClone):
+    class Source(ReleaseDownload):
-        git_remote = "https://github.com/raspberrypi/tools"
+        archive = Remotefile('raspberrypi-tools.tar.gz',
-        git_dir = "raspberrypi-tools"
+                             'e72b35436f2f23f2f7df322d6c318b9be57b21596b5ff0b8936af4ad94e04f2e')
 
     Builder = NoopBuilder

kiwixbuild/dependencies/libmagic.py

@@ -17,8 +17,7 @@ class LibMagic(Dependency):
         source_dir = "libmagic"
         archive_top_dir = 'file-5.35'
         archive = Remotefile('file-5.35.tar.gz',
-                             '30c45e817440779be7aac523a905b123cba2a6ed0bf4f5439e1e99ba940b5546',
+                             '30c45e817440779be7aac523a905b123cba2a6ed0bf4f5439e1e99ba940b5546')
-                             'https://fossies.org/linux/misc/file-5.35.tar.gz')
 
     class Builder(MakeBuilder):
 

kiwixbuild/packages.py

@@ -76,10 +76,10 @@ PACKAGE_NAME_MAPPERS = {
         'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'],
     },
     'debian_i586_dyn': {
-        'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib'],
+        'COMMON': _debian_common + ['libc6-dev-i386', 'lib32stdc++6', 'gcc-multilib', 'g++-multilib'],
     },
     'debian_i586_static': {
-        'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib'],
+        'COMMON': _debian_common + ['libc6-dev-i386', 'lib32stdc++6', 'gcc-multilib', 'g++-multilib'],
     },
     'debian_win32_dyn': {
         'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'],

kiwixbuild/platforms/armhf.py

@@ -34,6 +34,7 @@ class ArmhfPlatformInfo(PlatformInfo):
     @property
     def root_path(self):
         return pj(self.tlc_source.source_path,
+                  'raspberrypi-tools',
                   'arm-bcm2708',
                   'gcc-linaro-{}-raspbian-x64'.format(self.arch_full))
 

kiwixbuild/versions.py

@@ -35,7 +35,7 @@ release_versions = {
 
 # This is the "version" of the whole base_deps_versions dict.
 # Change this when you change base_deps_versions.
-base_deps_meta_version = '18'
+base_deps_meta_version = '51'
 
 
 base_deps_versions = {

travis/compile_all.py

@@ -25,6 +25,8 @@ KBUILD_SOURCE_DIR = Path(environ['TRAVIS_BUILD_DIR'])
 KIWIX_DESKTOP_ONLY = environ.get('DESKTOP_ONLY') == '1'
 
 BASE_DIR = HOME/"BUILD_{}".format(PLATFORM)
+BASE_EXPORT_DIR = HOME/"EXPORT"/"BASE"
+GIT_EXPORT_DIR = HOME/"EXPORT"/"GIT"
 SOURCE_DIR = HOME/"SOURCE"
 ARCHIVE_DIR = HOME/"ARCHIVE"
 INSTALL_DIR = BASE_DIR/"INSTALL"
@@ -34,7 +36,6 @@ NIGHTLY_ZIM_ARCHIVES_DIR = HOME/'NIGHTLY_ZIM_ARCHIVES'/NIGHTLY_DATE
 RELEASE_ZIM_ARCHIVES_DIR = HOME/'RELEASE_ZIM_ARCHIVES'
 DIST_KIWIX_ARCHIVES_DIR = HOME/'DIST_KIWIX_ARCHIVES'
 DIST_ZIM_ARCHIVES_DIR = HOME/'DIST_ZIM_ARCHIVES'
-SSH_KEY = KBUILD_SOURCE_DIR/'travis'/'travisci_builder_id_key'
 
 BIN_EXT = '.exe' if PLATFORM.startswith('win32_') else ''
 
@@ -193,8 +194,8 @@ def make_archive(project, platform):
             archive_add(arch, f)
 
 
-def make_deps_archive(target, full=False):
+def make_deps_archive(target=None, name=None, full=False):
-    archive_name = "deps_{}_{}_{}.tar.xz".format(
+    archive_name = name or "deps_{}_{}_{}.tar.xz".format(
         TRAVIS_OS_NAME, PLATFORM, target)
     print_message("Create archive {}.", archive_name)
     files_to_archive = [INSTALL_DIR]
@@ -230,57 +231,50 @@ def make_deps_archive(target, full=False):
             base_deps_versions['pugixml'])]
         files_to_archive += HOME.glob('BUILD_*/pugixml-{}'.format(
             base_deps_versions['pugixml']))
-        toolchains_subdirs = HOME.glob('**/TOOLCHAINS/*/*')
         if PLATFORM.startswith('armhf'):
-            files_to_archive += [SOURCE_DIR/'raspberrypi-tools']
+            files_to_archive += (SOURCE_DIR/'armhf').glob('*')
+        toolchains_subdirs = HOME.glob('**/TOOLCHAINS/*/*')
         for subdir in toolchains_subdirs:
             if not subdir.match('tools'):
                 files_to_archive.append(subdir)
 
-    with tarfile.open(str(relative_path/archive_name), 'w:xz') as tar:
+    with tarfile.open(str(BASE_EXPORT_DIR/archive_name), 'w:xz') as tar:
         for name in set(files_to_archive):
-            print('.', end='', flush=True)
+            print('.{}'.format(name), flush=True)
             tar.add(str(name), arcname=str(name.relative_to(relative_path)))
-    return relative_path/archive_name
 
 
 def update_flathub_git():
+    GIT_REPO_DIR = GIT_EXPORT_DIR/"org.kiwix.desktop"
     env = dict(os.environ)
-    env['GIT_SSH_COMMAND'] = 'ssh -o StrictHostKeyChecking=no -i {}'.format(SSH_KEY)
     env['GIT_AUTHOR_NAME'] = env['GIT_COMMITTER_NAME'] = "KiwixBot"
     env['GIT_AUTHOR_EMAIL'] = env['GIT_COMMITTER_EMAIL'] = "kiwixbot@kymeria.fr"
+    def call(command, cwd=None):
+        cwd = cwd or GIT_REPO_DIR
+        subprocess.check_call(command, env=env, cwd=str(cwd))
     command = ['git', 'clone', FLATPAK_GIT_REMOTE]
-    subprocess.check_call(command, env=env, cwd=str(HOME))
+    call(command, cwd=GIT_EXPORT_DIR)
-    shutil.copy(str(BASE_DIR/'org.kiwix.desktop.json'),
+    shutil.copy(str(BASE_DIR/'org.kiwix.desktop.json'), str(GIT_REPO_DIR))
-                str(HOME/'org.kiwix.desktop'))
     patch_dir = KBUILD_SOURCE_DIR/'kiwixbuild'/'patches'
     for dep in ('libaria2', 'mustache', 'pugixml', 'xapian'):
         for f in patch_dir.glob('{}_*.patch'.format(dep)):
-            shutil.copy(str(f), str(HOME/'org.kiwix.desktop'/'patches'))
+            shutil.copy(str(f), str(GIT_REPO_DIR/'patches'))
     command = ['git', 'add', '-A', '.']
-    subprocess.check_call(command, env=env, cwd=str(HOME/'org.kiwix.desktop'))
+    call(command)
     command = ['git', 'commit', '-m',
                'Update to version {}'.format(main_project_versions['kiwix-desktop'])]
-    subprocess.check_call(command, env=env, cwd=str(HOME/'org.kiwix.desktop'))
+    call(command)
-    command = ['git', 'push']
-    subprocess.check_call(command, env=env, cwd=str(HOME/'org.kiwix.desktop'))
 
 
 
-def scp(what, where):
-    print_message("Copy {} to {}", what, where)
-    command = ['scp', '-o', 'StrictHostKeyChecking=no',
-                      '-i', str(SSH_KEY),
-                      str(what), str(where)]
-    subprocess.check_call(command)
-
-
 for p in (NIGHTLY_KIWIX_ARCHIVES_DIR,
           NIGHTLY_ZIM_ARCHIVES_DIR,
           RELEASE_KIWIX_ARCHIVES_DIR,
           RELEASE_ZIM_ARCHIVES_DIR,
           DIST_KIWIX_ARCHIVES_DIR,
-          DIST_ZIM_ARCHIVES_DIR):
+          DIST_ZIM_ARCHIVES_DIR,
+          BASE_EXPORT_DIR,
+          GIT_EXPORT_DIR):
     try:
         p.mkdir(parents=True)
     except FileExistsError:
@@ -330,12 +324,9 @@ if PLATFORM != 'flatpak':
                         f.extractall(str(HOME))
                 except URLError:
                     pass
+        else:
             run_kiwix_build('alldependencies', platform=PLATFORM)
-        if SSH_KEY.exists():
+            make_deps_archive(name=base_dep_archive_name, full=True)
-            archive = make_deps_archive('alldependencies', full=True)
-            destination = 'ci@tmp.kiwix.org:/data/tmp/ci/{}'
-            destination = destination.format(base_dep_archive_name)
-            scp(archive, destination)
 
 
 # A basic compilation to be sure everything is working (for a PR)
@@ -407,8 +398,7 @@ for target in TARGETS:
         run_kiwix_build(target,
                         platform=PLATFORM,
                         build_deps_only=True)
-        archive = make_deps_archive(target)
+        make_deps_archive(target=target)
-        scp(archive, 'ci@tmp.kiwix.org:/data/tmp/ci/')
 
     run_kiwix_build(target,
                     platform=PLATFORM,

travis/deploy.sh

@@ -8,7 +8,6 @@ RELEASE_KIWIX_ARCHIVES_DIR=${HOME}/RELEASE_KIWIX_ARCHIVES
 RELEASE_ZIM_ARCHIVES_DIR=${HOME}/RELEASE_ZIM_ARCHIVES
 DIST_KIWIX_ARCHIVES_DIR=${HOME}/DIST_KIWIX_ARCHIVES
 DIST_ZIM_ARCHIVES_DIR=${HOME}/DIST_ZIM_ARCHIVES
-SSH_KEY=${TRAVIS_BUILD_DIR}/travis/travisci_builder_id_key
 
 if [[ "$TRAVIS_EVENT_TYPE" = "cron" ]]
 then
@@ -68,5 +67,16 @@ then
         ci@download.openzim.org:/data/openzim/release/${subdir}
     done
   fi
+
+  GIT_REPOS=$(ls -d */)
+  if [[ "x$GIT_REPOS" != "x" ]]
+  then
+    for repo in $GIT_REPOS
+    do
+      (cd repo;
+      GIT_SSH_COMMAND"ssh -o StrictHostKeyChecking=no -i $SSH_KEY" git push
+      )
+    done
+  fi
 fi
 

travis/deploy_apk.sh

@@ -4,7 +4,6 @@ set -e
 
 KEYSTORE_FILE=${TRAVIS_BUILD_DIR}/travis/kiwix-android.keystore
 GOOGLE_API_KEY=${TRAVIS_BUILD_DIR}/travis/googleplay_android_developer-5a411156212c.json
-SSH_KEY=${TRAVIS_BUILD_DIR}/travis/travisci_builder_id_key
 
 cd ${HOME}
 

travis/make_release.sh

@@ -4,7 +4,6 @@ set -e
 
 KEYSTORE_FILE=${TRAVIS_BUILD_DIR}/travis/test_ks.ks
 GOOGLE_API_KEY=${TRAVIS_BUILD_DIR}/travis/googleplay_android_developer-5a411156212c.json
-SSH_KEY=${TRAVIS_BUILD_DIR}/travis/travisci_builder_id_key
 
 cd ${HOME}
 

travis/upload_all_log.sh

@@ -2,8 +2,6 @@
 
 set -e
 
-SSH_KEY=$(pwd)/travis/travisci_builder_id_key
-
 cd $HOME
 
 tar -czf fail_log_${PLATFORM}.tar.gz BUILD_${PLATFORM}

travis/upload_base_deps.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -e
+
+BASE_EXPORT_DIR=$HOME/EXPORT/BASE
+ls $BASE_EXPORT_DIR
+
+BASE_ARCHIVES=$(find $BASE_EXPORT_DIR -type f)
+echo $BASE_ARCHIVES
+if [[ "x$BASE_ARCHIVES" != "x" ]]
+then
+  scp -vrp -i ${SSH_KEY} -o StrictHostKeyChecking=no \
+    $BASE_ARCHIVES \
+    ci@tmp.kiwix.org:/data/tmp/ci
+fi

travis/xenial_builder.dockerfile

@@ -0,0 +1,41 @@
+FROM ubuntu:xenial
+
+ENV LANG C.UTF-8
+
+RUN apt update -q
+RUN \
+  dpkg --add-architecture i386 && \
+  apt install -q -y --no-install-recommends \
+# Base build tools
+    build-essential automake libtool cmake ccache pkg-config autopoint patch \
+    python3-pip python3-setuptools python3-wheel git subversion wget unzip \
+    ninja-build \
+# Python (2) is needed to install android-ndk
+    python \
+# Packaged dependencies
+    libbz2-dev libmagic-dev uuid-dev zlib1g-dev default-jdk \
+    libmicrohttpd-dev \
+# Cross win32 compiler
+    g++-mingw-w64-i686 gcc-mingw-w64-i686 gcc-mingw-w64-base mingw-w64-tools \
+# Cross compile i586
+    libc6-dev-i386 lib32stdc++6 gcc-multilib g++-multilib \
+# Other tools (to remove)
+    vim less grep && \
+  apt-get clean -y && \
+  rm -rf /usr/share/doc/* /var/cache/debconf/*
+
+# Create user
+RUN useradd --create-home builder
+USER builder
+WORKDIR /home/builder
+ENV PATH="/home/builder/.local/bin:${PATH}"
+
+# Install kiwix-build
+COPY --chown=builder:builder . kiwix-build
+RUN pip3 install --user -e ./kiwix-build
+
+ENV TRAVIS_BUILD_DIR /home/builder/kiwix-build
+ENV GRADLE_USER_HOME /home/builder
+ENV TRAVIS_OS_NAME   linux_xenial
+
+CMD kiwix-build/travis/compile_all.py