diff --git a/.travis.yml b/.travis.yml index c72a643..ee4866e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: cpp -dist: trusty +dist: xenial sudo: required services: - docker @@ -10,13 +10,51 @@ branches: - /\d+\.\d+\.\d+$/ if: type != push OR tag IS present +stages: +- build_docker_images +- build +- android_build + +jobs: + include: + - stage: build_docker_images + script: &build_images + - | + if [ ! $(curl -sflL https://hub.docker.com/v2/repositories/kiwix/kiwix-build_ci/tags/${VARIANT}-${TRAVIS_COMMIT}) ] + then + echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + docker build -t kiwix-build_ci -f travis/${VARIANT}_builder.dockerfile . + docker tag kiwix-build_ci kiwix/kiwix-build_ci:${VARIANT}-${TRAVIS_COMMIT} + docker push kiwix/kiwix-build_ci:${VARIANT}-${TRAVIS_COMMIT} + fi + env: VARIANT=xenial + - stage: build_docker_images + script: *build_images + env: VARIANT=flatpak + - stage: build_docker_images + script: *build_images + env: VARIANT=appimage + - stage: android_build + env: PLATFORM="android" + if: type != pull_request + - env: PLATFORM="native_dyn" + os: osx + - env: PLATFORM="iOS_arm64" + os: osx + - env: PLATFORM="iOS_i386" + os: osx + + + before_install: - PATH=$PATH:${HOME}/bin:${HOME}/.local/bin +- mkdir -p /tmp/private $HOME/EXPORT/BASE +- chmod -R a+w $HOME/EXPORT - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then PATH=$PATH:$(brew --prefix)/opt/gettext/bin; fi - '[ -n "$encrypted_eba2f7543984_iv" ] && openssl aes-256-cbc -K $encrypted_eba2f7543984_key -iv $encrypted_eba2f7543984_iv - -in travis/travisci_builder_id_key.enc -out travis/travisci_builder_id_key -d' -- chmod 600 travis/travisci_builder_id_key + -in travis/travisci_builder_id_key.enc -out $SSH_KEY -d' +- chmod 600 $SSH_KEY before_cache: - rm -f $HOME/.gradle/caches/modules-2/modules-2.lock - rm -fr $HOME/.gradle/caches/*/plugin-resolution/ @@ -27,27 +65,33 @@ cache: - $HOME/.gradle/caches/ - $HOME/.gradle/wrapper/ - $HOME/.android/build-cache -install: if [[ $TRAVIS_OS_NAME != "linux" || $DESKTOP_ONLY != 1 ]]; then travis/install_extra_deps.sh; fi +install: if [[ $TRAVIS_OS_NAME != "linux" ]]; then travis/install_extra_deps.sh; fi +stage: build script: - | - if [[ $TRAVIS_OS_NAME = "linux" && $DESKTOP_ONLY == 1 ]] + if [[ $TRAVIS_OS_NAME = "linux" ]] then - if [[ $PLATFORM = "flatpak" ]] - then - docker build -t kiwix/build -f travis/Dockerfile_flatpak . - else - docker build -t kiwix/build -f travis/Dockerfile . - fi - docker run -e PLATFORM -e NIGHTLY_DATE -e TRAVIS_EVENT_TYPE -e DESKTOP_ONLY -e TRAVIS_TAG --device /dev/fuse --cap-add ALL --privileged kiwix/build - else - if [[ $TRAVIS_OS_NAME = "linux" ]] - then - python3.5 travis/compile_all.py - else - python3 travis/compile_all.py - fi + VARIANT=xenial + if [[ $PLATFORM = "flatpak" && $DESKTOP_ONLY = 1 ]]; then VARIANT=flatpak; fi + if [[ $PLATFORM = "native_dyn" && $DESKTOP_ONLY = 1 ]]; then VARIANT=appimage; fi fi -after_failure: travis/upload_all_log.sh +- | + if [[ $TRAVIS_OS_NAME = "linux" ]] + then + docker run --rm \ + -e PLATFORM -e NIGHTLY_DATE -e TRAVIS_EVENT_TYPE -e DESKTOP_ONLY -e TRAVIS_TAG \ + --device /dev/fuse --cap-add ALL --privileged \ + --mount=type=bind,src=$HOME/EXPORT,dst=/home/builder/EXPORT \ + --mount=type=bind,src=$HOME/.cache,dst=/home/builder/.cache \ + --mount=type=bind,src=$HOME/.gradle/caches,dst=/home/builder/.gradle/caches \ + kiwix/kiwix-build_ci:${VARIANT}-${TRAVIS_COMMIT} + fi +- if [[ $TRAVIS_OS_NAME != "linux" ]]; then python3 travis/compile_all.py; fi +after_success: +- travis/upload_base_deps.sh +after_failure: +- travis/upload_base_deps.sh +- travis/upload_all_log.sh deploy: - provider: script skip_cleanup: true @@ -64,87 +108,28 @@ deploy: env: global: - NIGHTLY_DATE=$(date +%Y-%m-%d) + - SSH_KEY=/tmp/private/travisci_builder_id_key matrix: + - PLATFORM="flatpak" DESKTOP_ONLY=1 - PLATFORM="native_dyn" + - PLATFORM="native_static" + - PLATFORM="native_mixed" - PLATFORM="armhf_dyn" - PLATFORM="armhf_static" - - PLATFORM="native_mixed" + - PLATFORM="win32_dyn" + - PLATFORM="win32_static" + - PLATFORM="i586_dyn" + - PLATFORM="i586_static" + - PLATFORM="native_dyn" DESKTOP_ONLY=1 + - PLATFORM="android_arm" + - PLATFORM="android_arm64" + - PLATFORM="android_x86" + - PLATFORM="android_x86_64" addons: ssh_known_hosts: - download.kiwix.org - tmp.kiwix.org - download.openzim.org - apt: - packages: &common - - cmake - - autopoint - - python3.5 - - python3-pip - - ctpp2-utils - - libmagic-dev -matrix: - include: - - env: PLATFORM="native_dyn" DESKTOP_ONLY=1 - addons: - apt: - packages: [] - - env: PLATFORM="flatpak" DESKTOP_ONLY=1 - addons: - apt: - packages: [] - - env: PLATFORM="native_static" - addons: - apt: - packages: - - *common - - libbz2-dev - - zlib1g-dev - - uuid-dev - - libctpp2-dev - - env: PLATFORM="win32_dyn" - addons: &win32_addons - apt: - packages: - - *common - - g++-mingw-w64-i686 - - gcc-mingw-w64-i686 - - gcc-mingw-w64-base - - mingw-w64-tools - - env: PLATFORM="win32_static" - addons: *win32_addons - - env: PLATFORM="i586_dyn" - addons: &i586_addons - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - *common - - gcc-multilib - - g++-multilib - - libc6-dev:i386 - - libstdc++-6-dev:i386 - - env: PLATFORM="i586_static" - addons: *i586_addons - - env: PLATFORM="android" - addons: &android_addons - apt: - packages: - - *common - - default-jdk - - env: PLATFORM="android_arm" - addons: *android_addons - - env: PLATFORM="android_arm64" - addons: *android_addons - - env: PLATFORM="android_x86" - addons: *android_addons - - env: PLATFORM="android_x86_64" - addons: *android_addons - - env: PLATFORM="native_dyn" - os: osx - - env: PLATFORM="iOS_arm64" - os: osx - - env: PLATFORM="iOS_i386" - os: osx notifications: irc: diff --git a/kiwixbuild/dependencies/armhf.py b/kiwixbuild/dependencies/armhf.py index 4ba5200..b79735e 100644 --- a/kiwixbuild/dependencies/armhf.py +++ b/kiwixbuild/dependencies/armhf.py @@ -1,11 +1,12 @@ -from .base import Dependency, GitClone, NoopBuilder +from .base import Dependency, ReleaseDownload, NoopBuilder +from kiwixbuild.utils import Remotefile class armhf_toolchain(Dependency): neutral = True name = 'armhf' - class Source(GitClone): - git_remote = "https://github.com/raspberrypi/tools" - git_dir = "raspberrypi-tools" + class Source(ReleaseDownload): + archive = Remotefile('raspberrypi-tools.tar.gz', + 'e72b35436f2f23f2f7df322d6c318b9be57b21596b5ff0b8936af4ad94e04f2e') Builder = NoopBuilder diff --git a/kiwixbuild/dependencies/libmagic.py b/kiwixbuild/dependencies/libmagic.py index e99b41d..fb5d438 100644 --- a/kiwixbuild/dependencies/libmagic.py +++ b/kiwixbuild/dependencies/libmagic.py @@ -17,8 +17,7 @@ class LibMagic(Dependency): source_dir = "libmagic" archive_top_dir = 'file-5.35' archive = Remotefile('file-5.35.tar.gz', - '30c45e817440779be7aac523a905b123cba2a6ed0bf4f5439e1e99ba940b5546', - 'https://fossies.org/linux/misc/file-5.35.tar.gz') + '30c45e817440779be7aac523a905b123cba2a6ed0bf4f5439e1e99ba940b5546') class Builder(MakeBuilder): diff --git a/kiwixbuild/packages.py b/kiwixbuild/packages.py index ad77415..af9ad7b 100644 --- a/kiwixbuild/packages.py +++ b/kiwixbuild/packages.py @@ -76,10 +76,10 @@ PACKAGE_NAME_MAPPERS = { 'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'], }, 'debian_i586_dyn': { - 'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib'], + 'COMMON': _debian_common + ['libc6-dev-i386', 'lib32stdc++6', 'gcc-multilib', 'g++-multilib'], }, 'debian_i586_static': { - 'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib'], + 'COMMON': _debian_common + ['libc6-dev-i386', 'lib32stdc++6', 'gcc-multilib', 'g++-multilib'], }, 'debian_win32_dyn': { 'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'], diff --git a/kiwixbuild/platforms/armhf.py b/kiwixbuild/platforms/armhf.py index 98a1645..ec3b574 100644 --- a/kiwixbuild/platforms/armhf.py +++ b/kiwixbuild/platforms/armhf.py @@ -34,6 +34,7 @@ class ArmhfPlatformInfo(PlatformInfo): @property def root_path(self): return pj(self.tlc_source.source_path, + 'raspberrypi-tools', 'arm-bcm2708', 'gcc-linaro-{}-raspbian-x64'.format(self.arch_full)) diff --git a/kiwixbuild/versions.py b/kiwixbuild/versions.py index a5b9f22..ee8edbd 100644 --- a/kiwixbuild/versions.py +++ b/kiwixbuild/versions.py @@ -35,7 +35,7 @@ release_versions = { # This is the "version" of the whole base_deps_versions dict. # Change this when you change base_deps_versions. -base_deps_meta_version = '18' +base_deps_meta_version = '51' base_deps_versions = { diff --git a/travis/Dockerfile b/travis/appimage_builder.dockerfile similarity index 100% rename from travis/Dockerfile rename to travis/appimage_builder.dockerfile diff --git a/travis/compile_all.py b/travis/compile_all.py index 137e0b9..1c52d75 100755 --- a/travis/compile_all.py +++ b/travis/compile_all.py @@ -25,6 +25,8 @@ KBUILD_SOURCE_DIR = Path(environ['TRAVIS_BUILD_DIR']) KIWIX_DESKTOP_ONLY = environ.get('DESKTOP_ONLY') == '1' BASE_DIR = HOME/"BUILD_{}".format(PLATFORM) +BASE_EXPORT_DIR = HOME/"EXPORT"/"BASE" +GIT_EXPORT_DIR = HOME/"EXPORT"/"GIT" SOURCE_DIR = HOME/"SOURCE" ARCHIVE_DIR = HOME/"ARCHIVE" INSTALL_DIR = BASE_DIR/"INSTALL" @@ -34,7 +36,6 @@ NIGHTLY_ZIM_ARCHIVES_DIR = HOME/'NIGHTLY_ZIM_ARCHIVES'/NIGHTLY_DATE RELEASE_ZIM_ARCHIVES_DIR = HOME/'RELEASE_ZIM_ARCHIVES' DIST_KIWIX_ARCHIVES_DIR = HOME/'DIST_KIWIX_ARCHIVES' DIST_ZIM_ARCHIVES_DIR = HOME/'DIST_ZIM_ARCHIVES' -SSH_KEY = KBUILD_SOURCE_DIR/'travis'/'travisci_builder_id_key' BIN_EXT = '.exe' if PLATFORM.startswith('win32_') else '' @@ -193,8 +194,8 @@ def make_archive(project, platform): archive_add(arch, f) -def make_deps_archive(target, full=False): - archive_name = "deps_{}_{}_{}.tar.xz".format( +def make_deps_archive(target=None, name=None, full=False): + archive_name = name or "deps_{}_{}_{}.tar.xz".format( TRAVIS_OS_NAME, PLATFORM, target) print_message("Create archive {}.", archive_name) files_to_archive = [INSTALL_DIR] @@ -230,57 +231,50 @@ def make_deps_archive(target, full=False): base_deps_versions['pugixml'])] files_to_archive += HOME.glob('BUILD_*/pugixml-{}'.format( base_deps_versions['pugixml'])) - toolchains_subdirs = HOME.glob('**/TOOLCHAINS/*/*') if PLATFORM.startswith('armhf'): - files_to_archive += [SOURCE_DIR/'raspberrypi-tools'] + files_to_archive += (SOURCE_DIR/'armhf').glob('*') + toolchains_subdirs = HOME.glob('**/TOOLCHAINS/*/*') for subdir in toolchains_subdirs: if not subdir.match('tools'): files_to_archive.append(subdir) - with tarfile.open(str(relative_path/archive_name), 'w:xz') as tar: + with tarfile.open(str(BASE_EXPORT_DIR/archive_name), 'w:xz') as tar: for name in set(files_to_archive): - print('.', end='', flush=True) + print('.{}'.format(name), flush=True) tar.add(str(name), arcname=str(name.relative_to(relative_path))) - return relative_path/archive_name def update_flathub_git(): + GIT_REPO_DIR = GIT_EXPORT_DIR/"org.kiwix.desktop" env = dict(os.environ) - env['GIT_SSH_COMMAND'] = 'ssh -o StrictHostKeyChecking=no -i {}'.format(SSH_KEY) env['GIT_AUTHOR_NAME'] = env['GIT_COMMITTER_NAME'] = "KiwixBot" env['GIT_AUTHOR_EMAIL'] = env['GIT_COMMITTER_EMAIL'] = "kiwixbot@kymeria.fr" + def call(command, cwd=None): + cwd = cwd or GIT_REPO_DIR + subprocess.check_call(command, env=env, cwd=str(cwd)) command = ['git', 'clone', FLATPAK_GIT_REMOTE] - subprocess.check_call(command, env=env, cwd=str(HOME)) - shutil.copy(str(BASE_DIR/'org.kiwix.desktop.json'), - str(HOME/'org.kiwix.desktop')) + call(command, cwd=GIT_EXPORT_DIR) + shutil.copy(str(BASE_DIR/'org.kiwix.desktop.json'), str(GIT_REPO_DIR)) patch_dir = KBUILD_SOURCE_DIR/'kiwixbuild'/'patches' for dep in ('libaria2', 'mustache', 'pugixml', 'xapian'): for f in patch_dir.glob('{}_*.patch'.format(dep)): - shutil.copy(str(f), str(HOME/'org.kiwix.desktop'/'patches')) + shutil.copy(str(f), str(GIT_REPO_DIR/'patches')) command = ['git', 'add', '-A', '.'] - subprocess.check_call(command, env=env, cwd=str(HOME/'org.kiwix.desktop')) + call(command) command = ['git', 'commit', '-m', 'Update to version {}'.format(main_project_versions['kiwix-desktop'])] - subprocess.check_call(command, env=env, cwd=str(HOME/'org.kiwix.desktop')) - command = ['git', 'push'] - subprocess.check_call(command, env=env, cwd=str(HOME/'org.kiwix.desktop')) + call(command) -def scp(what, where): - print_message("Copy {} to {}", what, where) - command = ['scp', '-o', 'StrictHostKeyChecking=no', - '-i', str(SSH_KEY), - str(what), str(where)] - subprocess.check_call(command) - - for p in (NIGHTLY_KIWIX_ARCHIVES_DIR, NIGHTLY_ZIM_ARCHIVES_DIR, RELEASE_KIWIX_ARCHIVES_DIR, RELEASE_ZIM_ARCHIVES_DIR, DIST_KIWIX_ARCHIVES_DIR, - DIST_ZIM_ARCHIVES_DIR): + DIST_ZIM_ARCHIVES_DIR, + BASE_EXPORT_DIR, + GIT_EXPORT_DIR): try: p.mkdir(parents=True) except FileExistsError: @@ -330,12 +324,9 @@ if PLATFORM != 'flatpak': f.extractall(str(HOME)) except URLError: pass - run_kiwix_build('alldependencies', platform=PLATFORM) - if SSH_KEY.exists(): - archive = make_deps_archive('alldependencies', full=True) - destination = 'ci@tmp.kiwix.org:/data/tmp/ci/{}' - destination = destination.format(base_dep_archive_name) - scp(archive, destination) + else: + run_kiwix_build('alldependencies', platform=PLATFORM) + make_deps_archive(name=base_dep_archive_name, full=True) # A basic compilation to be sure everything is working (for a PR) @@ -407,8 +398,7 @@ for target in TARGETS: run_kiwix_build(target, platform=PLATFORM, build_deps_only=True) - archive = make_deps_archive(target) - scp(archive, 'ci@tmp.kiwix.org:/data/tmp/ci/') + make_deps_archive(target=target) run_kiwix_build(target, platform=PLATFORM, diff --git a/travis/deploy.sh b/travis/deploy.sh index 543a13f..5809727 100755 --- a/travis/deploy.sh +++ b/travis/deploy.sh @@ -8,7 +8,6 @@ RELEASE_KIWIX_ARCHIVES_DIR=${HOME}/RELEASE_KIWIX_ARCHIVES RELEASE_ZIM_ARCHIVES_DIR=${HOME}/RELEASE_ZIM_ARCHIVES DIST_KIWIX_ARCHIVES_DIR=${HOME}/DIST_KIWIX_ARCHIVES DIST_ZIM_ARCHIVES_DIR=${HOME}/DIST_ZIM_ARCHIVES -SSH_KEY=${TRAVIS_BUILD_DIR}/travis/travisci_builder_id_key if [[ "$TRAVIS_EVENT_TYPE" = "cron" ]] then @@ -68,5 +67,16 @@ then ci@download.openzim.org:/data/openzim/release/${subdir} done fi + + GIT_REPOS=$(ls -d */) + if [[ "x$GIT_REPOS" != "x" ]] + then + for repo in $GIT_REPOS + do + (cd repo; + GIT_SSH_COMMAND"ssh -o StrictHostKeyChecking=no -i $SSH_KEY" git push + ) + done + fi fi diff --git a/travis/deploy_apk.sh b/travis/deploy_apk.sh index cabaa84..c17d748 100755 --- a/travis/deploy_apk.sh +++ b/travis/deploy_apk.sh @@ -4,7 +4,6 @@ set -e KEYSTORE_FILE=${TRAVIS_BUILD_DIR}/travis/kiwix-android.keystore GOOGLE_API_KEY=${TRAVIS_BUILD_DIR}/travis/googleplay_android_developer-5a411156212c.json -SSH_KEY=${TRAVIS_BUILD_DIR}/travis/travisci_builder_id_key cd ${HOME} diff --git a/travis/Dockerfile_flatpak b/travis/flatpak_builder.dockerfile similarity index 100% rename from travis/Dockerfile_flatpak rename to travis/flatpak_builder.dockerfile diff --git a/travis/make_release.sh b/travis/make_release.sh index a875ab2..54be0b8 100755 --- a/travis/make_release.sh +++ b/travis/make_release.sh @@ -4,7 +4,6 @@ set -e KEYSTORE_FILE=${TRAVIS_BUILD_DIR}/travis/test_ks.ks GOOGLE_API_KEY=${TRAVIS_BUILD_DIR}/travis/googleplay_android_developer-5a411156212c.json -SSH_KEY=${TRAVIS_BUILD_DIR}/travis/travisci_builder_id_key cd ${HOME} diff --git a/travis/upload_all_log.sh b/travis/upload_all_log.sh index 1267b81..057f509 100755 --- a/travis/upload_all_log.sh +++ b/travis/upload_all_log.sh @@ -2,8 +2,6 @@ set -e -SSH_KEY=$(pwd)/travis/travisci_builder_id_key - cd $HOME tar -czf fail_log_${PLATFORM}.tar.gz BUILD_${PLATFORM} diff --git a/travis/upload_base_deps.sh b/travis/upload_base_deps.sh new file mode 100755 index 0000000..6c5f232 --- /dev/null +++ b/travis/upload_base_deps.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -e + +BASE_EXPORT_DIR=$HOME/EXPORT/BASE +ls $BASE_EXPORT_DIR + +BASE_ARCHIVES=$(find $BASE_EXPORT_DIR -type f) +echo $BASE_ARCHIVES +if [[ "x$BASE_ARCHIVES" != "x" ]] +then + scp -vrp -i ${SSH_KEY} -o StrictHostKeyChecking=no \ + $BASE_ARCHIVES \ + ci@tmp.kiwix.org:/data/tmp/ci +fi diff --git a/travis/xenial_builder.dockerfile b/travis/xenial_builder.dockerfile new file mode 100644 index 0000000..70c18bd --- /dev/null +++ b/travis/xenial_builder.dockerfile @@ -0,0 +1,41 @@ +FROM ubuntu:xenial + +ENV LANG C.UTF-8 + +RUN apt update -q +RUN \ + dpkg --add-architecture i386 && \ + apt install -q -y --no-install-recommends \ +# Base build tools + build-essential automake libtool cmake ccache pkg-config autopoint patch \ + python3-pip python3-setuptools python3-wheel git subversion wget unzip \ + ninja-build \ +# Python (2) is needed to install android-ndk + python \ +# Packaged dependencies + libbz2-dev libmagic-dev uuid-dev zlib1g-dev default-jdk \ + libmicrohttpd-dev \ +# Cross win32 compiler + g++-mingw-w64-i686 gcc-mingw-w64-i686 gcc-mingw-w64-base mingw-w64-tools \ +# Cross compile i586 + libc6-dev-i386 lib32stdc++6 gcc-multilib g++-multilib \ +# Other tools (to remove) + vim less grep && \ + apt-get clean -y && \ + rm -rf /usr/share/doc/* /var/cache/debconf/* + +# Create user +RUN useradd --create-home builder +USER builder +WORKDIR /home/builder +ENV PATH="/home/builder/.local/bin:${PATH}" + +# Install kiwix-build +COPY --chown=builder:builder . kiwix-build +RUN pip3 install --user -e ./kiwix-build + +ENV TRAVIS_BUILD_DIR /home/builder/kiwix-build +ENV GRADLE_USER_HOME /home/builder +ENV TRAVIS_OS_NAME linux_xenial + +CMD kiwix-build/travis/compile_all.py