New travis script written in python and handle release deployement.
`compile_all.sh` script starts to be very complex. Let's rewrite it in python. In the same time, update the travis scripts to handle deployement of release versions.
This commit is contained in:
Matthieu Gautier
parent
47b00fe5dc
commit
06cf4d44d0
|
|
@ -17,7 +17,7 @@ cache:
|
||||||
- $HOME/.gradle/wrapper/
|
- $HOME/.gradle/wrapper/
|
||||||
- $HOME/.android/build-cache
|
- $HOME/.android/build-cache
|
||||||
install: travis/install_extra_deps.sh
|
install: travis/install_extra_deps.sh
|
||||||
script: travis_wait 30 travis/compile_all.sh
|
script: travis_wait 30 travis/compile_all.py
|
||||||
deploy:
|
deploy:
|
||||||
provider: script
|
provider: script
|
||||||
skip_cleanup: true
|
skip_cleanup: true
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,173 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import sys, os
|
||||||
|
import shutil
|
||||||
|
from os import environ
|
||||||
|
from pathlib import Path
|
||||||
|
from datetime import date
|
||||||
|
import tarfile
|
||||||
|
import subprocess
|
||||||
|
import re
|
||||||
|
|
||||||
|
PLATFORM = environ['PLATFORM']
|
||||||
|
|
||||||
|
def home():
|
||||||
|
return Path(os.path.expanduser('~'))
|
||||||
|
|
||||||
|
BASE_DIR = home()/"BUILD_{}".format(PLATFORM)
|
||||||
|
NIGHTLY_ARCHIVES_DIR = home()/'NIGTHLY_ARCHIVES'
|
||||||
|
RELEASE_ARCHIVES_DIR = home()/'RELEASE_ARCHIVES'
|
||||||
|
DIST_KIWIX_ARCHIVES_DIR = home()/'DIST_KIWIX_ARCHIVES'
|
||||||
|
DIST_ZIM_ARCHIVES_DIR = home()/'DIST_ZIM_ARCHIVES'
|
||||||
|
SSH_KEY = Path(environ['TRAVIS_BUILD_DIR'])/'travis'/'travis_ci_builder_id_key'
|
||||||
|
|
||||||
|
VERSIONS = {
|
||||||
|
'kiwix-lib': '1.0.2',
|
||||||
|
'kiwix-tools': '0.3.0',
|
||||||
|
'libzim': '3.0.0',
|
||||||
|
'zim-tools': '0.0.1',
|
||||||
|
'zimwriterfs': '1.0'
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def write_manifest(manifest_file, archive_name, target, platform):
|
||||||
|
manifest_file.write_text('''{archive_name}
|
||||||
|
***************************
|
||||||
|
|
||||||
|
Dependencies archive for {target} on platform {platform}
|
||||||
|
Generated at {date}
|
||||||
|
'''.format(
|
||||||
|
archive_name=archive_name,
|
||||||
|
target=target,
|
||||||
|
platform=platform,
|
||||||
|
date=date.today().isoformat()))
|
||||||
|
|
||||||
|
|
||||||
|
def run_kiwix_build(target, platform, build_deps_only=False, make_release=False, make_dist=False):
|
||||||
|
command = [str(Path(environ['TRAVIS_BUILD_DIR'])/'kiwix-build.py')]
|
||||||
|
command.append(target)
|
||||||
|
command.append('--hide-progress')
|
||||||
|
command.extend(['--target-platform', platform])
|
||||||
|
if build_deps_only:
|
||||||
|
command.append('--build-deps-only')
|
||||||
|
if make_release:
|
||||||
|
command.append('--make-release')
|
||||||
|
if make_dist:
|
||||||
|
command.append('--make-dist')
|
||||||
|
subprocess.check_call(command, cwd=str(home()))
|
||||||
|
|
||||||
|
|
||||||
|
def make_archive(archive_name, file_to_archives):
|
||||||
|
archive_dir = RELEASE_ARCHIVES_DIR if make_release else NIGHTLY_ARCHIVES_DIR
|
||||||
|
archive = archive_dir/'{}.tar.gz'.format(archive_name)
|
||||||
|
base_bin_dir = BASE_DIR/'INSTALL'/'bin'
|
||||||
|
with tarfile.open(str(archive), 'w:gz') as arch:
|
||||||
|
for f in file_to_archives:
|
||||||
|
arch.add(str(base_bin_dir/f), arcname=str(f))
|
||||||
|
|
||||||
|
|
||||||
|
def scp(what, where):
|
||||||
|
command = ['scp', '-i', SSH_KEY, what, where]
|
||||||
|
subprocess.check_call(command)
|
||||||
|
|
||||||
|
|
||||||
|
for p in (NIGHTLY_ARCHIVES_DIR, RELEASE_ARCHIVES_DIR, DIST_KIWIX_ARCHIVES_DIR, DIST_ZIM_ARCHIVES_DIR):
|
||||||
|
try:
|
||||||
|
p.mkdir(parents=True)
|
||||||
|
except FileExistsError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
make_release = re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", environ.get('TRAVIS_TAG', '')) is not None
|
||||||
|
|
||||||
|
# A basic compilation to be sure everything is working (for a PR)
|
||||||
|
if environ['TRAVIS_EVENT_TYPE'] != 'cron' and not make_release:
|
||||||
|
if PLATFORM.startswith('android'):
|
||||||
|
TARGETS = ('kiwix-android',)
|
||||||
|
elif PLATFORM.startswith('native_'):
|
||||||
|
TARGETS = ('kiwix-tools', 'zim-tools', 'zimwriterfs')
|
||||||
|
else:
|
||||||
|
TARGETS = ('kiwix-tools', )
|
||||||
|
|
||||||
|
for target in TARGETS:
|
||||||
|
run_kiwix_build(target,
|
||||||
|
platform=PLATFORM)
|
||||||
|
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
if PLATFORM.startswith('android'):
|
||||||
|
TARGETS = ('libzim', 'kiwix-lib', 'kiwix-android')
|
||||||
|
elif PLATFORM.startswith('native_'):
|
||||||
|
TARGETS = ('libzim', 'zimwriterfs', 'zim-tools', 'kiwix-lib', 'kiwix-tools')
|
||||||
|
else:
|
||||||
|
TARGETS = ('libzim', 'kiwix-lib', 'kiwix-tools')
|
||||||
|
|
||||||
|
for target in TARGETS:
|
||||||
|
if environ['TRAVIS_EVENT_TYPE'] == 'cron':
|
||||||
|
run_kiwix_build(target,
|
||||||
|
platform=PLATFORM,
|
||||||
|
build_deps_only=True)
|
||||||
|
(BASE_DIR/'.install_package_ok').unlink()
|
||||||
|
|
||||||
|
archive_name = "deps_{}_{}.tar.gz".format(PLATFORM, target)
|
||||||
|
files_to_archive = [BASE_DIR/'INSTALL']
|
||||||
|
files_to_archive += BASE_DIR.glob('**/android-ndk*')
|
||||||
|
if (BASE_DIR/'meson_cross_file.txt').exists():
|
||||||
|
files_to_archive.append(BASE_DIR/'meson_cross_file.txt')
|
||||||
|
|
||||||
|
manifest_file = BASE_DIR/'manifest.txt'
|
||||||
|
write_manifest(manifest_file, archive_name, target, PLATFORM)
|
||||||
|
files_to_archive.append(manifest_file)
|
||||||
|
with tarfile.open(str(BASE_DIR/archive_name), 'w:gz') as tar:
|
||||||
|
for name in files_to_archive:
|
||||||
|
tar.add(str(name))
|
||||||
|
scp(str(archive_name), 'nightlybot@download.kiwix.org:/var/www/tmp.kiwix.org/ci/')
|
||||||
|
|
||||||
|
run_kiwix_build(target,
|
||||||
|
platform=PLATFORM,
|
||||||
|
make_release=make_release)
|
||||||
|
if make_release and PLATFORM == 'native_dyn':
|
||||||
|
run_kiwix_build(target,
|
||||||
|
platform=PLATFORM,
|
||||||
|
make_release=True,
|
||||||
|
make_dist=True)
|
||||||
|
(BASE_DIR/'.install_packages_ok').unlink()
|
||||||
|
|
||||||
|
|
||||||
|
# We have build everything. Now create archives for public deployement.
|
||||||
|
kiwix_tools_bins = ('kiwix-install', 'kiwix-manage', 'kiwix-read', 'kiwix-search', 'kiwix-serve')
|
||||||
|
zim_tools_bins = ('zimbench', 'zimdump', 'zimsearch', 'zimdiff', 'zimpatch', 'zimsplit')
|
||||||
|
zimwriterfs_bins = ('zimwriterfs',)
|
||||||
|
|
||||||
|
_date = date.today().isoformat()
|
||||||
|
kiwix_tools_postfix = VERSIONS['kiwix-tools'] if make_release else _date
|
||||||
|
zim_tools_postfix = VERSIONS['zim-tools'] if make_release else _date
|
||||||
|
zimwriterfs_postfix = VERSIONS['zimwriterfs'] if make_release else _date
|
||||||
|
|
||||||
|
if make_release and PLATFORM == 'native_dyn':
|
||||||
|
for target in TARGETS:
|
||||||
|
if target in ('kiwix-lib', 'kiwix-tools'):
|
||||||
|
out_dir = DIST_KIWIX_ARCHIVES_DIR
|
||||||
|
else:
|
||||||
|
out_dir = DIST_ZIM_ARCHIVES_DIR
|
||||||
|
|
||||||
|
if target in ('kiwix-lib', 'kiwix-tools', 'libzim'):
|
||||||
|
shutil.copy(str(BASE_DIR/target/'meson-dist'/'{}-{}.tar.xz'.format(target, VERSIONS[target])),
|
||||||
|
str(out_dir))
|
||||||
|
elif PLATFORM == 'native_static':
|
||||||
|
make_archive('kiwix-tools_linux64-{}'.format(kiwix_tools_postfix), kiwix_tools_bins)
|
||||||
|
make_archive('zim-tools_linux64-{}'.format(zim_tools_postfix), zim_tools_bins)
|
||||||
|
make_archive('zimwriterfs_linux64-{}'.format(zimwriterfs_postfix), zimwriterfs_bins)
|
||||||
|
elif PLATFORM == 'win32_static':
|
||||||
|
make_archive('kiwix-tools_win32-{}'.format(kiwix_tools_postfix),
|
||||||
|
('{}.exe'.format(b) for b in kiwix_tools_bins))
|
||||||
|
elif PLATFORM == 'armhf_static':
|
||||||
|
make_archive('kiwix-tools_armhf-{}.'.format(kiwix_tools_postfix), kiwix_tools_bins)
|
||||||
|
elif PLATFORM.startswith('android_'):
|
||||||
|
APK_NAME = "kiwix-{}".format(PLATFORM)
|
||||||
|
source_debug_dir = BASE_DIR/'kiwix-android'/'app'/'build'/'outputs'/'apk'/'kiwix'/'debug'
|
||||||
|
source_release_dir = BASE_DIR/'kiwix-android'/'app'/'build'/'outputs'/'apk'/'kiwix'/'release'
|
||||||
|
shutil.copy(str(source_debug_dir/'app-kiwix-debug.apk'),
|
||||||
|
str(NIGHTLY_ARCHIVES_DIR/"{}-debug.apk".format(APK_NAME)))
|
||||||
|
shutil.copy(str(source_release_dir/'app-kiwix-release-unsigned.apk'),
|
||||||
|
str(NIGHTLY_ARCHIVES_DIR/"{}-release_signed".format(APK_NAME)))
|
||||||
|
|
||||||
|
|
@ -1,145 +0,0 @@
|
||||||
#!/usr/bin/env bash
|
|
||||||
|
|
||||||
set -e
|
|
||||||
|
|
||||||
BASE_DIR="BUILD_${PLATFORM}"
|
|
||||||
NIGHTLY_ARCHIVES_DIR=${HOME}/NIGHTLY_ARCHIVES
|
|
||||||
RELEASE_ARCHIVES_DIR=${HOME}/RELEASE_ARCHIVES
|
|
||||||
SSH_KEY=${TRAVIS_BUILD_DIR}/travis/travisci_builder_id_key
|
|
||||||
|
|
||||||
mkdir -p ${NIGHTLY_ARCHIVES_DIR}
|
|
||||||
mkdir -p ${RELEASE_ARCHIVES_DIR}
|
|
||||||
|
|
||||||
function make_archive {
|
|
||||||
if [[ "$MAKE_RELEASE" == "0" ]]
|
|
||||||
then
|
|
||||||
ARCHIVE_PATH="${NIGHTLY_ARCHIVES_DIR}/${1}_$(date +%Y-%m-%d).tar.gz"
|
|
||||||
else
|
|
||||||
ARCHIVE_PATH="${RELEASE_ARCHIVES_DIR}/${1}-${TRAVIS_TAG}.tar.gz"
|
|
||||||
fi
|
|
||||||
(
|
|
||||||
cd ${BASE_DIR}/INSTALL/bin
|
|
||||||
tar -czf "${ARCHIVE_PATH}" $2
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
cd ${HOME}
|
|
||||||
|
|
||||||
if [[ $TRAVIS_TAG =~ ^[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+ ]]
|
|
||||||
then
|
|
||||||
MAKE_RELEASE=1
|
|
||||||
else
|
|
||||||
MAKE_RELEASE=0
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ "$TRAVIS_EVENT_TYPE" == "cron" || "$MAKE_RELEASE" == "1" ]]
|
|
||||||
then
|
|
||||||
if [[ ${PLATFORM} = android* ]]
|
|
||||||
then
|
|
||||||
TARGETS="libzim kiwix-lib kiwix-android"
|
|
||||||
elif [[ ${PLATFORM} =~ native_* ]]
|
|
||||||
then
|
|
||||||
TARGETS="libzim zimwriterfs zim-tools kiwix-lib kiwix-tools"
|
|
||||||
else
|
|
||||||
TARGETS="libzim kiwix-lib kiwix-tools"
|
|
||||||
fi
|
|
||||||
|
|
||||||
for TARGET in ${TARGETS}
|
|
||||||
do
|
|
||||||
echo $TARGET
|
|
||||||
if [[ "$TRAVIS_EVENT_TYPE" == "cron" ]]
|
|
||||||
then
|
|
||||||
${TRAVIS_BUILD_DIR}/kiwix-build.py \
|
|
||||||
--target-platform $PLATFORM \
|
|
||||||
--build-deps-only \
|
|
||||||
--hide-progress \
|
|
||||||
${TARGET}
|
|
||||||
rm ${BASE_DIR}/.install_packages_ok
|
|
||||||
|
|
||||||
(
|
|
||||||
cd ${BASE_DIR}
|
|
||||||
if [ -f meson_cross_file.txt ]
|
|
||||||
then
|
|
||||||
MESON_FILE=meson_cross_file.txt
|
|
||||||
fi
|
|
||||||
ANDROID_NDK_DIR=$(find . -name "android-ndk*")
|
|
||||||
ARCHIVE_NAME="deps_${PLATFORM}_${TARGET}.tar.gz"
|
|
||||||
|
|
||||||
cat <<EOF > manifest.txt
|
|
||||||
${ARCHIVE_NAME}
|
|
||||||
*********************************
|
|
||||||
|
|
||||||
Dependencies archive for ${TARGET} on platform ${PLATFORM}
|
|
||||||
Generated at $(date)
|
|
||||||
EOF
|
|
||||||
|
|
||||||
tar -czf ${ARCHIVE_NAME} INSTALL manifest.txt ${MESON_FILE} ${ANDROID_NDK_DIR}
|
|
||||||
scp -i ${SSH_KEY} ${ARCHIVE_NAME} nightlybot@download.kiwix.org:/var/www/tmp.kiwix.org/ci/
|
|
||||||
)
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ "$MAKE_RELEASE" == "1" ]]
|
|
||||||
then
|
|
||||||
${TRAVIS_BUILD_DIR}/kiwix-build.py \
|
|
||||||
--hide-progress \
|
|
||||||
--make-release \
|
|
||||||
--target-platform $PLATFORM ${TARGET}
|
|
||||||
if [[ "$PLATFORM" == "native_dyn" ]]
|
|
||||||
then
|
|
||||||
${TRAVIS_BUILD_DIR}/kiwix-build.py \
|
|
||||||
--hide-progress \
|
|
||||||
--make-release \
|
|
||||||
--make-dist \
|
|
||||||
--target-platform $PLATFORM ${TARGET}
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
${TRAVIS_BUILD_DIR}/kiwix-build.py \
|
|
||||||
--hide-progress \
|
|
||||||
--target-platform $PLATFORM ${TARGET}
|
|
||||||
fi
|
|
||||||
rm ${BASE_DIR}/.install_packages_ok
|
|
||||||
done
|
|
||||||
|
|
||||||
# We have build every thing. Now create archives for public deployement.
|
|
||||||
case ${PLATFORM} in
|
|
||||||
native_dyn)
|
|
||||||
#[TODO] Copy archive somewhere
|
|
||||||
#scp ${BASE_DIR}/${TARGET}/${TARGET}-${TARGET-version}.tar.gz ${SOMEWHERE}
|
|
||||||
;;
|
|
||||||
native_static)
|
|
||||||
make_archive kiwix-tools_linux64 "kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve"
|
|
||||||
make_archive zim-tools_linux64 "zimbench zimdump zimsearch zimdiff zimpatch zimsplit"
|
|
||||||
make_archive zimwriterfs_linux64 "zimwriterfs"
|
|
||||||
;;
|
|
||||||
win32_static)
|
|
||||||
make_archive kiwix-tools_win32 "kiwix-install.exe kiwix-manage.exe kiwix-read.exe kiwix-search.exe kiwix-serve.exe"
|
|
||||||
;;
|
|
||||||
armhf_static)
|
|
||||||
make_archive kiwix-tools_armhf "kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve"
|
|
||||||
;;
|
|
||||||
android_*)
|
|
||||||
APK_NAME="kiwix-${PLATFORM}"
|
|
||||||
cp ${BASE_DIR}/kiwix-android/app/build/outputs/apk/kiwix/debug/app-kiwix-debug.apk ${NIGHTLY_ARCHIVES_DIR}/${APK_NAME}-debug.apk
|
|
||||||
cp ${BASE_DIR}/kiwix-android/app/build/outputs/apk/kiwix/release/app-kiwix-release-unsigned.apk ${NIGHTLY_ARCHIVES_DIR}/${APK_NAME}-release-unsigned.apk
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
else
|
|
||||||
# No a cron job, we just have to build to be sure nothing is broken.
|
|
||||||
if [[ ${PLATFORM} = android* ]]
|
|
||||||
then
|
|
||||||
TARGETS="kiwix-android"
|
|
||||||
elif [[ ${PLATFORM} =~ native_* ]]
|
|
||||||
then
|
|
||||||
TARGETS="kiwix-tools zim-tools zimwriterfs"
|
|
||||||
else
|
|
||||||
TARGETS="kiwix-tools"
|
|
||||||
fi
|
|
||||||
for TARGET in ${TARGETS}
|
|
||||||
do
|
|
||||||
${TRAVIS_BUILD_DIR}/kiwix-build.py \
|
|
||||||
--target-platform $PLATFORM \
|
|
||||||
--hide-progress \
|
|
||||||
${TARGET}
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
|
|
@ -4,6 +4,8 @@ set -e
|
||||||
|
|
||||||
NIGHTLY_ARCHIVES_DIR=${HOME}/NIGHTLY_ARCHIVES
|
NIGHTLY_ARCHIVES_DIR=${HOME}/NIGHTLY_ARCHIVES
|
||||||
RELEASE_ARCHIVES_DIR=${HOME}/RELEASE_ARCHIVES
|
RELEASE_ARCHIVES_DIR=${HOME}/RELEASE_ARCHIVES
|
||||||
|
DIST_KIWIX_ARCHIVES_DIR=${HOME}/DIST_KIWIX_ARCHIVES
|
||||||
|
DIST_ZIM_ARCHIVES_DIR=${HOME}/DIST_ZIM_ARCHIVES
|
||||||
SSH_KEY=travis/travisci_builder_id_key
|
SSH_KEY=travis/travisci_builder_id_key
|
||||||
|
|
||||||
NIGHTLY_ARCHIVES=$(find $NIGHTLY_ARCHIVES_DIR -type f)
|
NIGHTLY_ARCHIVES=$(find $NIGHTLY_ARCHIVES_DIR -type f)
|
||||||
|
|
@ -21,3 +23,20 @@ then
|
||||||
${RELEASE_ARCHIVES} \
|
${RELEASE_ARCHIVES} \
|
||||||
nightlybot@downoald.kiwix.org:/var/www/download.kiwix.org/releases
|
nightlybot@downoald.kiwix.org:/var/www/download.kiwix.org/releases
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
DIST_KIWIX_ARCHIVES=$(find $DIST_KIWIX_ARCHIVES_DIR -type f)
|
||||||
|
if [[ "x$DIST_KIWIX_ARCHIVES" != "x" ]]
|
||||||
|
then
|
||||||
|
scp -vrp -i ${SSH_KEY} \
|
||||||
|
${DIST_KIWIX_ARCHIVES} \
|
||||||
|
nightlybot@download.kiwix.org:/var/www/download.kiwix.org/release
|
||||||
|
fi
|
||||||
|
|
||||||
|
DIST_ZIM_ARCHIVES=$(find $DIST_ZIM_ARCHIVES_DIR -type f)
|
||||||
|
if [[ "x$DIST_ZIM_ARCHIVES" != "x" ]]
|
||||||
|
then
|
||||||
|
scp -vrp -i ${SSH_KEY} \
|
||||||
|
${DIST_ZIM_ARCHIVES} \
|
||||||
|
nightlybot@download.openzim.org:/var/www/download.openzim.org/release
|
||||||
|
fi
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue