diff --git a/.travis.yml b/.travis.yml index fe092af..5298758 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,7 @@ cache: - $HOME/.gradle/wrapper/ - $HOME/.android/build-cache install: travis/install_extra_deps.sh -script: travis_wait 30 travis/compile_all.sh +script: travis_wait 30 travis/compile_all.py deploy: provider: script skip_cleanup: true diff --git a/travis/compile_all.py b/travis/compile_all.py new file mode 100755 index 0000000..e9bb166 --- /dev/null +++ b/travis/compile_all.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 + +import sys, os +import shutil +from os import environ +from pathlib import Path +from datetime import date +import tarfile +import subprocess +import re + +PLATFORM = environ['PLATFORM'] + +def home(): + return Path(os.path.expanduser('~')) + +BASE_DIR = home()/"BUILD_{}".format(PLATFORM) +NIGHTLY_ARCHIVES_DIR = home()/'NIGTHLY_ARCHIVES' +RELEASE_ARCHIVES_DIR = home()/'RELEASE_ARCHIVES' +DIST_KIWIX_ARCHIVES_DIR = home()/'DIST_KIWIX_ARCHIVES' +DIST_ZIM_ARCHIVES_DIR = home()/'DIST_ZIM_ARCHIVES' +SSH_KEY = Path(environ['TRAVIS_BUILD_DIR'])/'travis'/'travis_ci_builder_id_key' + +VERSIONS = { + 'kiwix-lib': '1.0.2', + 'kiwix-tools': '0.3.0', + 'libzim': '3.0.0', + 'zim-tools': '0.0.1', + 'zimwriterfs': '1.0' +} + + +def write_manifest(manifest_file, archive_name, target, platform): + manifest_file.write_text('''{archive_name} +*************************** + +Dependencies archive for {target} on platform {platform} +Generated at {date} +'''.format( + archive_name=archive_name, + target=target, + platform=platform, + date=date.today().isoformat())) + + +def run_kiwix_build(target, platform, build_deps_only=False, make_release=False, make_dist=False): + command = [str(Path(environ['TRAVIS_BUILD_DIR'])/'kiwix-build.py')] + command.append(target) + command.append('--hide-progress') + command.extend(['--target-platform', platform]) + if build_deps_only: + command.append('--build-deps-only') + if make_release: + command.append('--make-release') + if make_dist: + command.append('--make-dist') + subprocess.check_call(command, cwd=str(home())) + + +def make_archive(archive_name, file_to_archives): + archive_dir = RELEASE_ARCHIVES_DIR if make_release else NIGHTLY_ARCHIVES_DIR + archive = archive_dir/'{}.tar.gz'.format(archive_name) + base_bin_dir = BASE_DIR/'INSTALL'/'bin' + with tarfile.open(str(archive), 'w:gz') as arch: + for f in file_to_archives: + arch.add(str(base_bin_dir/f), arcname=str(f)) + + +def scp(what, where): + command = ['scp', '-i', SSH_KEY, what, where] + subprocess.check_call(command) + + +for p in (NIGHTLY_ARCHIVES_DIR, RELEASE_ARCHIVES_DIR, DIST_KIWIX_ARCHIVES_DIR, DIST_ZIM_ARCHIVES_DIR): + try: + p.mkdir(parents=True) + except FileExistsError: + pass + +make_release = re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", environ.get('TRAVIS_TAG', '')) is not None + +# A basic compilation to be sure everything is working (for a PR) +if environ['TRAVIS_EVENT_TYPE'] != 'cron' and not make_release: + if PLATFORM.startswith('android'): + TARGETS = ('kiwix-android',) + elif PLATFORM.startswith('native_'): + TARGETS = ('kiwix-tools', 'zim-tools', 'zimwriterfs') + else: + TARGETS = ('kiwix-tools', ) + + for target in TARGETS: + run_kiwix_build(target, + platform=PLATFORM) + + sys.exit(0) + +if PLATFORM.startswith('android'): + TARGETS = ('libzim', 'kiwix-lib', 'kiwix-android') +elif PLATFORM.startswith('native_'): + TARGETS = ('libzim', 'zimwriterfs', 'zim-tools', 'kiwix-lib', 'kiwix-tools') +else: + TARGETS = ('libzim', 'kiwix-lib', 'kiwix-tools') + +for target in TARGETS: + if environ['TRAVIS_EVENT_TYPE'] == 'cron': + run_kiwix_build(target, + platform=PLATFORM, + build_deps_only=True) + (BASE_DIR/'.install_package_ok').unlink() + + archive_name = "deps_{}_{}.tar.gz".format(PLATFORM, target) + files_to_archive = [BASE_DIR/'INSTALL'] + files_to_archive += BASE_DIR.glob('**/android-ndk*') + if (BASE_DIR/'meson_cross_file.txt').exists(): + files_to_archive.append(BASE_DIR/'meson_cross_file.txt') + + manifest_file = BASE_DIR/'manifest.txt' + write_manifest(manifest_file, archive_name, target, PLATFORM) + files_to_archive.append(manifest_file) + with tarfile.open(str(BASE_DIR/archive_name), 'w:gz') as tar: + for name in files_to_archive: + tar.add(str(name)) + scp(str(archive_name), 'nightlybot@download.kiwix.org:/var/www/tmp.kiwix.org/ci/') + + run_kiwix_build(target, + platform=PLATFORM, + make_release=make_release) + if make_release and PLATFORM == 'native_dyn': + run_kiwix_build(target, + platform=PLATFORM, + make_release=True, + make_dist=True) + (BASE_DIR/'.install_packages_ok').unlink() + + +# We have build everything. Now create archives for public deployement. +kiwix_tools_bins = ('kiwix-install', 'kiwix-manage', 'kiwix-read', 'kiwix-search', 'kiwix-serve') +zim_tools_bins = ('zimbench', 'zimdump', 'zimsearch', 'zimdiff', 'zimpatch', 'zimsplit') +zimwriterfs_bins = ('zimwriterfs',) + +_date = date.today().isoformat() +kiwix_tools_postfix = VERSIONS['kiwix-tools'] if make_release else _date +zim_tools_postfix = VERSIONS['zim-tools'] if make_release else _date +zimwriterfs_postfix = VERSIONS['zimwriterfs'] if make_release else _date + +if make_release and PLATFORM == 'native_dyn': + for target in TARGETS: + if target in ('kiwix-lib', 'kiwix-tools'): + out_dir = DIST_KIWIX_ARCHIVES_DIR + else: + out_dir = DIST_ZIM_ARCHIVES_DIR + + if target in ('kiwix-lib', 'kiwix-tools', 'libzim'): + shutil.copy(str(BASE_DIR/target/'meson-dist'/'{}-{}.tar.xz'.format(target, VERSIONS[target])), + str(out_dir)) +elif PLATFORM == 'native_static': + make_archive('kiwix-tools_linux64-{}'.format(kiwix_tools_postfix), kiwix_tools_bins) + make_archive('zim-tools_linux64-{}'.format(zim_tools_postfix), zim_tools_bins) + make_archive('zimwriterfs_linux64-{}'.format(zimwriterfs_postfix), zimwriterfs_bins) +elif PLATFORM == 'win32_static': + make_archive('kiwix-tools_win32-{}'.format(kiwix_tools_postfix), + ('{}.exe'.format(b) for b in kiwix_tools_bins)) +elif PLATFORM == 'armhf_static': + make_archive('kiwix-tools_armhf-{}.'.format(kiwix_tools_postfix), kiwix_tools_bins) +elif PLATFORM.startswith('android_'): + APK_NAME = "kiwix-{}".format(PLATFORM) + source_debug_dir = BASE_DIR/'kiwix-android'/'app'/'build'/'outputs'/'apk'/'kiwix'/'debug' + source_release_dir = BASE_DIR/'kiwix-android'/'app'/'build'/'outputs'/'apk'/'kiwix'/'release' + shutil.copy(str(source_debug_dir/'app-kiwix-debug.apk'), + str(NIGHTLY_ARCHIVES_DIR/"{}-debug.apk".format(APK_NAME))) + shutil.copy(str(source_release_dir/'app-kiwix-release-unsigned.apk'), + str(NIGHTLY_ARCHIVES_DIR/"{}-release_signed".format(APK_NAME))) + diff --git a/travis/compile_all.sh b/travis/compile_all.sh deleted file mode 100755 index fe8fba8..0000000 --- a/travis/compile_all.sh +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env bash - -set -e - -BASE_DIR="BUILD_${PLATFORM}" -NIGHTLY_ARCHIVES_DIR=${HOME}/NIGHTLY_ARCHIVES -RELEASE_ARCHIVES_DIR=${HOME}/RELEASE_ARCHIVES -SSH_KEY=${TRAVIS_BUILD_DIR}/travis/travisci_builder_id_key - -mkdir -p ${NIGHTLY_ARCHIVES_DIR} -mkdir -p ${RELEASE_ARCHIVES_DIR} - -function make_archive { - if [[ "$MAKE_RELEASE" == "0" ]] - then - ARCHIVE_PATH="${NIGHTLY_ARCHIVES_DIR}/${1}_$(date +%Y-%m-%d).tar.gz" - else - ARCHIVE_PATH="${RELEASE_ARCHIVES_DIR}/${1}-${TRAVIS_TAG}.tar.gz" - fi - ( - cd ${BASE_DIR}/INSTALL/bin - tar -czf "${ARCHIVE_PATH}" $2 - ) -} - -cd ${HOME} - -if [[ $TRAVIS_TAG =~ ^[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+ ]] -then - MAKE_RELEASE=1 -else - MAKE_RELEASE=0 -fi - -if [[ "$TRAVIS_EVENT_TYPE" == "cron" || "$MAKE_RELEASE" == "1" ]] -then - if [[ ${PLATFORM} = android* ]] - then - TARGETS="libzim kiwix-lib kiwix-android" - elif [[ ${PLATFORM} =~ native_* ]] - then - TARGETS="libzim zimwriterfs zim-tools kiwix-lib kiwix-tools" - else - TARGETS="libzim kiwix-lib kiwix-tools" - fi - - for TARGET in ${TARGETS} - do - echo $TARGET - if [[ "$TRAVIS_EVENT_TYPE" == "cron" ]] - then - ${TRAVIS_BUILD_DIR}/kiwix-build.py \ - --target-platform $PLATFORM \ - --build-deps-only \ - --hide-progress \ - ${TARGET} - rm ${BASE_DIR}/.install_packages_ok - - ( - cd ${BASE_DIR} - if [ -f meson_cross_file.txt ] - then - MESON_FILE=meson_cross_file.txt - fi - ANDROID_NDK_DIR=$(find . -name "android-ndk*") - ARCHIVE_NAME="deps_${PLATFORM}_${TARGET}.tar.gz" - - cat < manifest.txt -${ARCHIVE_NAME} -********************************* - -Dependencies archive for ${TARGET} on platform ${PLATFORM} -Generated at $(date) -EOF - - tar -czf ${ARCHIVE_NAME} INSTALL manifest.txt ${MESON_FILE} ${ANDROID_NDK_DIR} - scp -i ${SSH_KEY} ${ARCHIVE_NAME} nightlybot@download.kiwix.org:/var/www/tmp.kiwix.org/ci/ - ) - fi - - if [[ "$MAKE_RELEASE" == "1" ]] - then - ${TRAVIS_BUILD_DIR}/kiwix-build.py \ - --hide-progress \ - --make-release \ - --target-platform $PLATFORM ${TARGET} - if [[ "$PLATFORM" == "native_dyn" ]] - then - ${TRAVIS_BUILD_DIR}/kiwix-build.py \ - --hide-progress \ - --make-release \ - --make-dist \ - --target-platform $PLATFORM ${TARGET} - fi - else - ${TRAVIS_BUILD_DIR}/kiwix-build.py \ - --hide-progress \ - --target-platform $PLATFORM ${TARGET} - fi - rm ${BASE_DIR}/.install_packages_ok - done - - # We have build every thing. Now create archives for public deployement. - case ${PLATFORM} in - native_dyn) - #[TODO] Copy archive somewhere - #scp ${BASE_DIR}/${TARGET}/${TARGET}-${TARGET-version}.tar.gz ${SOMEWHERE} - ;; - native_static) - make_archive kiwix-tools_linux64 "kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve" - make_archive zim-tools_linux64 "zimbench zimdump zimsearch zimdiff zimpatch zimsplit" - make_archive zimwriterfs_linux64 "zimwriterfs" - ;; - win32_static) - make_archive kiwix-tools_win32 "kiwix-install.exe kiwix-manage.exe kiwix-read.exe kiwix-search.exe kiwix-serve.exe" - ;; - armhf_static) - make_archive kiwix-tools_armhf "kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve" - ;; - android_*) - APK_NAME="kiwix-${PLATFORM}" - cp ${BASE_DIR}/kiwix-android/app/build/outputs/apk/kiwix/debug/app-kiwix-debug.apk ${NIGHTLY_ARCHIVES_DIR}/${APK_NAME}-debug.apk - cp ${BASE_DIR}/kiwix-android/app/build/outputs/apk/kiwix/release/app-kiwix-release-unsigned.apk ${NIGHTLY_ARCHIVES_DIR}/${APK_NAME}-release-unsigned.apk - ;; - esac - -else - # No a cron job, we just have to build to be sure nothing is broken. - if [[ ${PLATFORM} = android* ]] - then - TARGETS="kiwix-android" - elif [[ ${PLATFORM} =~ native_* ]] - then - TARGETS="kiwix-tools zim-tools zimwriterfs" - else - TARGETS="kiwix-tools" - fi - for TARGET in ${TARGETS} - do - ${TRAVIS_BUILD_DIR}/kiwix-build.py \ - --target-platform $PLATFORM \ - --hide-progress \ - ${TARGET} - done -fi diff --git a/travis/deploy.sh b/travis/deploy.sh index 5659fc7..9b06fab 100755 --- a/travis/deploy.sh +++ b/travis/deploy.sh @@ -4,6 +4,8 @@ set -e NIGHTLY_ARCHIVES_DIR=${HOME}/NIGHTLY_ARCHIVES RELEASE_ARCHIVES_DIR=${HOME}/RELEASE_ARCHIVES +DIST_KIWIX_ARCHIVES_DIR=${HOME}/DIST_KIWIX_ARCHIVES +DIST_ZIM_ARCHIVES_DIR=${HOME}/DIST_ZIM_ARCHIVES SSH_KEY=travis/travisci_builder_id_key NIGHTLY_ARCHIVES=$(find $NIGHTLY_ARCHIVES_DIR -type f) @@ -21,3 +23,20 @@ then ${RELEASE_ARCHIVES} \ nightlybot@downoald.kiwix.org:/var/www/download.kiwix.org/releases fi + +DIST_KIWIX_ARCHIVES=$(find $DIST_KIWIX_ARCHIVES_DIR -type f) +if [[ "x$DIST_KIWIX_ARCHIVES" != "x" ]] +then + scp -vrp -i ${SSH_KEY} \ + ${DIST_KIWIX_ARCHIVES} \ + nightlybot@download.kiwix.org:/var/www/download.kiwix.org/release +fi + +DIST_ZIM_ARCHIVES=$(find $DIST_ZIM_ARCHIVES_DIR -type f) +if [[ "x$DIST_ZIM_ARCHIVES" != "x" ]] +then + scp -vrp -i ${SSH_KEY} \ + ${DIST_ZIM_ARCHIVES} \ + nightlybot@download.openzim.org:/var/www/download.openzim.org/release +fi +