diff --git a/.travis.yml b/.travis.yml index 540c2b9..78fac47 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,11 +12,13 @@ if: type != push OR tag IS present before_install: - PATH=$PATH:${HOME}/bin:${HOME}/.local/bin +- mkdir -p /tmp/private $HOME/EXPORT/BASE +- chmod -R a+w $HOME/EXPORT - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then PATH=$PATH:$(brew --prefix)/opt/gettext/bin; fi - '[ -n "$encrypted_eba2f7543984_iv" ] && openssl aes-256-cbc -K $encrypted_eba2f7543984_key -iv $encrypted_eba2f7543984_iv - -in travis/travisci_builder_id_key.enc -out travis/travisci_builder_id_key -d' -- chmod 600 travis/travisci_builder_id_key + -in travis/travisci_builder_id_key.enc -out $SSH_KEY -d' +- chmod 600 $SSH_KEY before_cache: - rm -f $HOME/.gradle/caches/modules-2/modules-2.lock - rm -fr $HOME/.gradle/caches/*/plugin-resolution/ @@ -27,27 +29,37 @@ cache: - $HOME/.gradle/caches/ - $HOME/.gradle/wrapper/ - $HOME/.android/build-cache -install: if [[ $TRAVIS_OS_NAME != "linux" || $DESKTOP_ONLY != 1 ]]; then travis/install_extra_deps.sh; fi +install: if [[ $TRAVIS_OS_NAME != "linux" ]]; then travis/install_extra_deps.sh; fi script: - | - if [[ $TRAVIS_OS_NAME = "linux" && $DESKTOP_ONLY == 1 ]] + if [[ $TRAVIS_OS_NAME = "linux" ]] then - if [[ $PLATFORM = "flatpak" ]] - then - docker build -t kiwix/build -f travis/Dockerfile_flatpak . - else - docker build -t kiwix/build -f travis/Dockerfile . - fi - docker run -e PLATFORM -e NIGHTLY_DATE -e TRAVIS_EVENT_TYPE -e DESKTOP_ONLY -e TRAVIS_TAG --device /dev/fuse --cap-add ALL --privileged kiwix/build - else - if [[ $TRAVIS_OS_NAME = "linux" ]] - then - python3.5 travis/compile_all.py - else - python3 travis/compile_all.py - fi + PODMAN_FILE=xenial_builder.dockerfile + if [[ $PLATFORM = "flatpak" && $DESKTOP_ONLY = 1 ]]; then PODMAN_FILE=Dockerfile_flatpak; fi + if [[ $PLATFORM = "native_dyn" && $DESKTOP_ONLY = 1 ]]; then PODMAN_FILE=Dockerfile; fi fi -after_failure: travis/upload_all_log.sh +- | + if [[ $TRAVIS_OS_NAME = "linux" ]] + then + docker build -t kiwix/build -f travis/$PODMAN_FILE . + fi +- | + if [[ $TRAVIS_OS_NAME = "linux" ]] + then + docker run \ + -e PLATFORM -e NIGHTLY_DATE -e TRAVIS_EVENT_TYPE -e DESKTOP_ONLY -e TRAVIS_TAG \ + --device /dev/fuse --cap-add ALL --privileged \ + --mount=type=bind,src=$HOME/EXPORT,dst=/home/builder/EXPORT \ + --mount=type=bind,src=$HOME/.cache,dst=/home/builder/.cache \ + --mount=type=bind,src=$HOME/.gradle/caches,dst=/home/builder/.gradle/caches \ + kiwix/build + fi +- if [[ $TRAVIS_OS_NAME != "linux" ]]; then python3 travis/compile_all.py; fi +after_success: +- travis/upload_base_deps.sh +after_failure: +- travis/upload_base_deps.sh +- travis/upload_all_log.sh deploy: - provider: script skip_cleanup: true @@ -64,24 +76,25 @@ deploy: env: global: - NIGHTLY_DATE=$(date +%Y-%m-%d) + - SSH_KEY=/tmp/private/travisci_builder_id_key matrix: - PLATFORM="native_dyn" + - PLATFORM="native_static" + - PLATFORM="native_mixed" - PLATFORM="armhf_dyn" - PLATFORM="armhf_static" - - PLATFORM="native_mixed" + - PLATFORM="win32_dyn" + - PLATFORM="win32_static" + - PLATFORM="android" + - PLATFORM="android_arm" + - PLATFORM="android_arm64" + - PLATFORM="android_x86" + - PLATFORM="android_x86_64" addons: ssh_known_hosts: - download.kiwix.org - tmp.kiwix.org - download.openzim.org - apt: - packages: &common - - cmake - - autopoint - - python3.5 - - python3-pip - - ctpp2-utils - - libmagic-dev matrix: include: - env: PLATFORM="native_dyn" DESKTOP_ONLY=1 @@ -92,26 +105,6 @@ matrix: addons: apt: packages: [] - - env: PLATFORM="native_static" - addons: - apt: - packages: - - *common - - libbz2-dev - - zlib1g-dev - - uuid-dev - - libctpp2-dev - - env: PLATFORM="win32_dyn" - addons: &win32_addons - apt: - packages: - - *common - - g++-mingw-w64-i686 - - gcc-mingw-w64-i686 - - gcc-mingw-w64-base - - mingw-w64-tools - - env: PLATFORM="win32_static" - addons: *win32_addons - env: PLATFORM="i586_dyn" addons: &i586_addons apt: @@ -125,20 +118,6 @@ matrix: - libstdc++-6-dev:i386 - env: PLATFORM="i586_static" addons: *i586_addons - - env: PLATFORM="android" - addons: &android_addons - apt: - packages: - - *common - - default-jdk - - env: PLATFORM="android_arm" - addons: *android_addons - - env: PLATFORM="android_arm64" - addons: *android_addons - - env: PLATFORM="android_x86" - addons: *android_addons - - env: PLATFORM="android_x86_64" - addons: *android_addons - env: PLATFORM="native_dyn" os: osx - env: PLATFORM="iOS_arm64" diff --git a/kiwixbuild/packages.py b/kiwixbuild/packages.py index ad77415..af9ad7b 100644 --- a/kiwixbuild/packages.py +++ b/kiwixbuild/packages.py @@ -76,10 +76,10 @@ PACKAGE_NAME_MAPPERS = { 'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'], }, 'debian_i586_dyn': { - 'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib'], + 'COMMON': _debian_common + ['libc6-dev-i386', 'lib32stdc++6', 'gcc-multilib', 'g++-multilib'], }, 'debian_i586_static': { - 'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib'], + 'COMMON': _debian_common + ['libc6-dev-i386', 'lib32stdc++6', 'gcc-multilib', 'g++-multilib'], }, 'debian_win32_dyn': { 'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'], diff --git a/kiwixbuild/versions.py b/kiwixbuild/versions.py index 9ded59e..ee8edbd 100644 --- a/kiwixbuild/versions.py +++ b/kiwixbuild/versions.py @@ -35,7 +35,7 @@ release_versions = { # This is the "version" of the whole base_deps_versions dict. # Change this when you change base_deps_versions. -base_deps_meta_version = '50' +base_deps_meta_version = '51' base_deps_versions = { diff --git a/travis/compile_all.py b/travis/compile_all.py index f91440d..853638c 100755 --- a/travis/compile_all.py +++ b/travis/compile_all.py @@ -25,6 +25,8 @@ KBUILD_SOURCE_DIR = Path(environ['TRAVIS_BUILD_DIR']) KIWIX_DESKTOP_ONLY = environ.get('DESKTOP_ONLY') == '1' BASE_DIR = HOME/"BUILD_{}".format(PLATFORM) +BASE_EXPORT_DIR = HOME/"EXPORT"/"BASE" +GIT_EXPORT_DIR = HOME/"EXPORT"/"GIT" SOURCE_DIR = HOME/"SOURCE" ARCHIVE_DIR = HOME/"ARCHIVE" INSTALL_DIR = BASE_DIR/"INSTALL" @@ -34,7 +36,6 @@ NIGHTLY_ZIM_ARCHIVES_DIR = HOME/'NIGHTLY_ZIM_ARCHIVES'/NIGHTLY_DATE RELEASE_ZIM_ARCHIVES_DIR = HOME/'RELEASE_ZIM_ARCHIVES' DIST_KIWIX_ARCHIVES_DIR = HOME/'DIST_KIWIX_ARCHIVES' DIST_ZIM_ARCHIVES_DIR = HOME/'DIST_ZIM_ARCHIVES' -SSH_KEY = KBUILD_SOURCE_DIR/'travis'/'travisci_builder_id_key' BIN_EXT = '.exe' if PLATFORM.startswith('win32_') else '' @@ -193,8 +194,8 @@ def make_archive(project, platform): archive_add(arch, f) -def make_deps_archive(target, full=False): - archive_name = "deps_{}_{}_{}.tar.xz".format( +def make_deps_archive(target=None, name=None, full=False): + archive_name = name or "deps_{}_{}_{}.tar.xz".format( TRAVIS_OS_NAME, PLATFORM, target) print_message("Create archive {}.", archive_name) files_to_archive = [INSTALL_DIR] @@ -237,50 +238,43 @@ def make_deps_archive(target, full=False): if not subdir.match('tools'): files_to_archive.append(subdir) - with tarfile.open(str(relative_path/archive_name), 'w:xz') as tar: + with tarfile.open(str(BASE_EXPORT_DIR/archive_name), 'w:xz') as tar: for name in set(files_to_archive): - print('.', end='', flush=True) + print('.{}'.format(name), flush=True) tar.add(str(name), arcname=str(name.relative_to(relative_path))) - return relative_path/archive_name def update_flathub_git(): + GIT_REPO_DIR = GIT_EXPORT_DIR/"org.kiwix.desktop" env = dict(os.environ) - env['GIT_SSH_COMMAND'] = 'ssh -o StrictHostKeyChecking=no -i {}'.format(SSH_KEY) env['GIT_AUTHOR_NAME'] = env['GIT_COMMITTER_NAME'] = "KiwixBot" env['GIT_AUTHOR_EMAIL'] = env['GIT_COMMITTER_EMAIL'] = "kiwixbot@kymeria.fr" + def call(command, cwd=None): + cwd = cwd or GIT_REPO_DIR + subprocess.check_call(command, env=env, cwd=str(cwd)) command = ['git', 'clone', FLATPAK_GIT_REMOTE] - subprocess.check_call(command, env=env, cwd=str(HOME)) - shutil.copy(str(BASE_DIR/'org.kiwix.desktop.json'), - str(HOME/'org.kiwix.desktop')) + call(command, cwd=GIT_EXPORT_DIR) + shutil.copy(str(BASE_DIR/'org.kiwix.desktop.json'), str(GIT_REPO_DIR)) patch_dir = KBUILD_SOURCE_DIR/'kiwixbuild'/'patches' for dep in ('libaria2', 'mustache', 'pugixml', 'xapian'): for f in patch_dir.glob('{}_*.patch'.format(dep)): - shutil.copy(str(f), str(HOME/'org.kiwix.desktop'/'patches')) + shutil.copy(str(f), str(GIT_REPO_DIR/'patches')) command = ['git', 'add', '-A', '.'] - subprocess.check_call(command, env=env, cwd=str(HOME/'org.kiwix.desktop')) + call(command) command = ['git', 'commit', '-m', 'Update to version {}'.format(main_project_versions['kiwix-desktop'])] - subprocess.check_call(command, env=env, cwd=str(HOME/'org.kiwix.desktop')) - command = ['git', 'push'] - subprocess.check_call(command, env=env, cwd=str(HOME/'org.kiwix.desktop')) + call(command) -def scp(what, where): - print_message("Copy {} to {}", what, where) - command = ['scp', '-o', 'StrictHostKeyChecking=no', - '-i', str(SSH_KEY), - str(what), str(where)] - subprocess.check_call(command) - - for p in (NIGHTLY_KIWIX_ARCHIVES_DIR, NIGHTLY_ZIM_ARCHIVES_DIR, RELEASE_KIWIX_ARCHIVES_DIR, RELEASE_ZIM_ARCHIVES_DIR, DIST_KIWIX_ARCHIVES_DIR, - DIST_ZIM_ARCHIVES_DIR): + DIST_ZIM_ARCHIVES_DIR, + BASE_EXPORT_DIR, + GIT_EXPORT_DIR): try: p.mkdir(parents=True) except FileExistsError: @@ -331,11 +325,7 @@ if PLATFORM != 'flatpak': except URLError: pass run_kiwix_build('alldependencies', platform=PLATFORM) - if SSH_KEY.exists(): - archive = make_deps_archive('alldependencies', full=True) - destination = 'ci@tmp.kiwix.org:/data/tmp/ci/{}' - destination = destination.format(base_dep_archive_name) - scp(archive, destination) + make_deps_archive(name=base_dep_archive_name, full=True) # A basic compilation to be sure everything is working (for a PR) @@ -407,8 +397,7 @@ for target in TARGETS: run_kiwix_build(target, platform=PLATFORM, build_deps_only=True) - archive = make_deps_archive(target) - scp(archive, 'ci@tmp.kiwix.org:/data/tmp/ci/') + make_deps_archive(target=target) run_kiwix_build(target, platform=PLATFORM, diff --git a/travis/deploy.sh b/travis/deploy.sh index 543a13f..5809727 100755 --- a/travis/deploy.sh +++ b/travis/deploy.sh @@ -8,7 +8,6 @@ RELEASE_KIWIX_ARCHIVES_DIR=${HOME}/RELEASE_KIWIX_ARCHIVES RELEASE_ZIM_ARCHIVES_DIR=${HOME}/RELEASE_ZIM_ARCHIVES DIST_KIWIX_ARCHIVES_DIR=${HOME}/DIST_KIWIX_ARCHIVES DIST_ZIM_ARCHIVES_DIR=${HOME}/DIST_ZIM_ARCHIVES -SSH_KEY=${TRAVIS_BUILD_DIR}/travis/travisci_builder_id_key if [[ "$TRAVIS_EVENT_TYPE" = "cron" ]] then @@ -68,5 +67,16 @@ then ci@download.openzim.org:/data/openzim/release/${subdir} done fi + + GIT_REPOS=$(ls -d */) + if [[ "x$GIT_REPOS" != "x" ]] + then + for repo in $GIT_REPOS + do + (cd repo; + GIT_SSH_COMMAND"ssh -o StrictHostKeyChecking=no -i $SSH_KEY" git push + ) + done + fi fi diff --git a/travis/deploy_apk.sh b/travis/deploy_apk.sh index cabaa84..c17d748 100755 --- a/travis/deploy_apk.sh +++ b/travis/deploy_apk.sh @@ -4,7 +4,6 @@ set -e KEYSTORE_FILE=${TRAVIS_BUILD_DIR}/travis/kiwix-android.keystore GOOGLE_API_KEY=${TRAVIS_BUILD_DIR}/travis/googleplay_android_developer-5a411156212c.json -SSH_KEY=${TRAVIS_BUILD_DIR}/travis/travisci_builder_id_key cd ${HOME} diff --git a/travis/make_release.sh b/travis/make_release.sh index a875ab2..54be0b8 100755 --- a/travis/make_release.sh +++ b/travis/make_release.sh @@ -4,7 +4,6 @@ set -e KEYSTORE_FILE=${TRAVIS_BUILD_DIR}/travis/test_ks.ks GOOGLE_API_KEY=${TRAVIS_BUILD_DIR}/travis/googleplay_android_developer-5a411156212c.json -SSH_KEY=${TRAVIS_BUILD_DIR}/travis/travisci_builder_id_key cd ${HOME} diff --git a/travis/upload_all_log.sh b/travis/upload_all_log.sh index 1267b81..057f509 100755 --- a/travis/upload_all_log.sh +++ b/travis/upload_all_log.sh @@ -2,8 +2,6 @@ set -e -SSH_KEY=$(pwd)/travis/travisci_builder_id_key - cd $HOME tar -czf fail_log_${PLATFORM}.tar.gz BUILD_${PLATFORM} diff --git a/travis/upload_base_deps.sh b/travis/upload_base_deps.sh new file mode 100755 index 0000000..6c5f232 --- /dev/null +++ b/travis/upload_base_deps.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -e + +BASE_EXPORT_DIR=$HOME/EXPORT/BASE +ls $BASE_EXPORT_DIR + +BASE_ARCHIVES=$(find $BASE_EXPORT_DIR -type f) +echo $BASE_ARCHIVES +if [[ "x$BASE_ARCHIVES" != "x" ]] +then + scp -vrp -i ${SSH_KEY} -o StrictHostKeyChecking=no \ + $BASE_ARCHIVES \ + ci@tmp.kiwix.org:/data/tmp/ci +fi diff --git a/travis/xenial_builder.dockerfile b/travis/xenial_builder.dockerfile new file mode 100644 index 0000000..70c18bd --- /dev/null +++ b/travis/xenial_builder.dockerfile @@ -0,0 +1,41 @@ +FROM ubuntu:xenial + +ENV LANG C.UTF-8 + +RUN apt update -q +RUN \ + dpkg --add-architecture i386 && \ + apt install -q -y --no-install-recommends \ +# Base build tools + build-essential automake libtool cmake ccache pkg-config autopoint patch \ + python3-pip python3-setuptools python3-wheel git subversion wget unzip \ + ninja-build \ +# Python (2) is needed to install android-ndk + python \ +# Packaged dependencies + libbz2-dev libmagic-dev uuid-dev zlib1g-dev default-jdk \ + libmicrohttpd-dev \ +# Cross win32 compiler + g++-mingw-w64-i686 gcc-mingw-w64-i686 gcc-mingw-w64-base mingw-w64-tools \ +# Cross compile i586 + libc6-dev-i386 lib32stdc++6 gcc-multilib g++-multilib \ +# Other tools (to remove) + vim less grep && \ + apt-get clean -y && \ + rm -rf /usr/share/doc/* /var/cache/debconf/* + +# Create user +RUN useradd --create-home builder +USER builder +WORKDIR /home/builder +ENV PATH="/home/builder/.local/bin:${PATH}" + +# Install kiwix-build +COPY --chown=builder:builder . kiwix-build +RUN pip3 install --user -e ./kiwix-build + +ENV TRAVIS_BUILD_DIR /home/builder/kiwix-build +ENV GRADLE_USER_HOME /home/builder +ENV TRAVIS_OS_NAME linux_xenial + +CMD kiwix-build/travis/compile_all.py