smallsolar
/
kiwix-build
mirror of https://github.com/kiwix/kiwix-build.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #103 from kiwix/travis_update 

New travis script written in python.
This commit is contained in:
Matthieu Gautier 2018-02-02 11:41:50 +01:00 committed by GitHub
parent 170ba5e3b1 06cf4d44d0
commit 524da9c4fe
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 195 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.travis.yml
View File

@ -17,7 +17,7 @@ cache:
- $HOME/.gradle/wrapper/
- $HOME/.android/build-cache
install: travis/install_extra_deps.sh
script: travis_wait 30 travis/compile_all.sh
script: travis_wait 30 travis/compile_all.py
deploy:
provider: script
skip_cleanup: true

3
dependencies.py
View File

@ -291,6 +291,7 @@ class ZimTools(Dependency):
class Source(GitClone):
git_remote = "https://github.com/openzim/zim-tools.git"
git_dir = "zim-tools"
release_git_ref = "0.0.1"
class Builder(MesonBuilder):
@property
@ -341,7 +342,7 @@ class Kiwixlib(Dependency):
class Source(GitClone):
git_remote = "https://github.com/kiwix/kiwix-lib.git"
git_dir = "kiwix-lib"
release_git_ref = "1.0.1"
release_git_ref = "1.0.2"
class Builder(MesonBuilder):
@property

173
travis/compile_all.py Executable file
View File

@ -0,0 +1,173 @@
#!/usr/bin/env python3
import sys, os
import shutil
from os import environ
from pathlib import Path
from datetime import date
import tarfile
import subprocess
import re
PLATFORM = environ['PLATFORM']
def home():
return Path(os.path.expanduser('~'))
BASE_DIR = home()/"BUILD_{}".format(PLATFORM)
NIGHTLY_ARCHIVES_DIR = home()/'NIGTHLY_ARCHIVES'
RELEASE_ARCHIVES_DIR = home()/'RELEASE_ARCHIVES'
DIST_KIWIX_ARCHIVES_DIR = home()/'DIST_KIWIX_ARCHIVES'
DIST_ZIM_ARCHIVES_DIR = home()/'DIST_ZIM_ARCHIVES'
SSH_KEY = Path(environ['TRAVIS_BUILD_DIR'])/'travis'/'travis_ci_builder_id_key'
VERSIONS = {
'kiwix-lib': '1.0.2',
'kiwix-tools': '0.3.0',
'libzim': '3.0.0',
'zim-tools': '0.0.1',
'zimwriterfs': '1.0'
}
def write_manifest(manifest_file, archive_name, target, platform):
manifest_file.write_text('''{archive_name}
***************************
Dependencies archive for {target} on platform {platform}
Generated at {date}
'''.format(
archive_name=archive_name,
target=target,
platform=platform,
date=date.today().isoformat()))
def run_kiwix_build(target, platform, build_deps_only=False, make_release=False, make_dist=False):
command = [str(Path(environ['TRAVIS_BUILD_DIR'])/'kiwix-build.py')]
command.append(target)
command.append('--hide-progress')
command.extend(['--target-platform', platform])
if build_deps_only:
command.append('--build-deps-only')
if make_release:
command.append('--make-release')
if make_dist:
command.append('--make-dist')
subprocess.check_call(command, cwd=str(home()))
def make_archive(archive_name, file_to_archives):
archive_dir = RELEASE_ARCHIVES_DIR if make_release else NIGHTLY_ARCHIVES_DIR
archive = archive_dir/'{}.tar.gz'.format(archive_name)
base_bin_dir = BASE_DIR/'INSTALL'/'bin'
with tarfile.open(str(archive), 'w:gz') as arch:
for f in file_to_archives:
arch.add(str(base_bin_dir/f), arcname=str(f))
def scp(what, where):
command = ['scp', '-i', SSH_KEY, what, where]
subprocess.check_call(command)
for p in (NIGHTLY_ARCHIVES_DIR, RELEASE_ARCHIVES_DIR, DIST_KIWIX_ARCHIVES_DIR, DIST_ZIM_ARCHIVES_DIR):
try:
p.mkdir(parents=True)
except FileExistsError:
pass
make_release = re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", environ.get('TRAVIS_TAG', '')) is not None
# A basic compilation to be sure everything is working (for a PR)
if environ['TRAVIS_EVENT_TYPE'] != 'cron' and not make_release:
if PLATFORM.startswith('android'):
TARGETS = ('kiwix-android',)
elif PLATFORM.startswith('native_'):
TARGETS = ('kiwix-tools', 'zim-tools', 'zimwriterfs')
else:
TARGETS = ('kiwix-tools', )
for target in TARGETS:
run_kiwix_build(target,
platform=PLATFORM)
sys.exit(0)
if PLATFORM.startswith('android'):
TARGETS = ('libzim', 'kiwix-lib', 'kiwix-android')
elif PLATFORM.startswith('native_'):
TARGETS = ('libzim', 'zimwriterfs', 'zim-tools', 'kiwix-lib', 'kiwix-tools')
else:
TARGETS = ('libzim', 'kiwix-lib', 'kiwix-tools')
for target in TARGETS:
if environ['TRAVIS_EVENT_TYPE'] == 'cron':
run_kiwix_build(target,
platform=PLATFORM,
build_deps_only=True)
(BASE_DIR/'.install_package_ok').unlink()
archive_name = "deps_{}_{}.tar.gz".format(PLATFORM, target)
files_to_archive = [BASE_DIR/'INSTALL']
files_to_archive += BASE_DIR.glob('**/android-ndk*')
if (BASE_DIR/'meson_cross_file.txt').exists():
files_to_archive.append(BASE_DIR/'meson_cross_file.txt')
manifest_file = BASE_DIR/'manifest.txt'
write_manifest(manifest_file, archive_name, target, PLATFORM)
files_to_archive.append(manifest_file)
with tarfile.open(str(BASE_DIR/archive_name), 'w:gz') as tar:
for name in files_to_archive:
tar.add(str(name))
scp(str(archive_name), 'nightlybot@download.kiwix.org:/var/www/tmp.kiwix.org/ci/')
run_kiwix_build(target,
platform=PLATFORM,
make_release=make_release)
if make_release and PLATFORM == 'native_dyn':
run_kiwix_build(target,
platform=PLATFORM,
make_release=True,
make_dist=True)
(BASE_DIR/'.install_packages_ok').unlink()
# We have build everything. Now create archives for public deployement.
kiwix_tools_bins = ('kiwix-install', 'kiwix-manage', 'kiwix-read', 'kiwix-search', 'kiwix-serve')
zim_tools_bins = ('zimbench', 'zimdump', 'zimsearch', 'zimdiff', 'zimpatch', 'zimsplit')
zimwriterfs_bins = ('zimwriterfs',)
_date = date.today().isoformat()
kiwix_tools_postfix = VERSIONS['kiwix-tools'] if make_release else _date
zim_tools_postfix = VERSIONS['zim-tools'] if make_release else _date
zimwriterfs_postfix = VERSIONS['zimwriterfs'] if make_release else _date
if make_release and PLATFORM == 'native_dyn':
for target in TARGETS:
if target in ('kiwix-lib', 'kiwix-tools'):
out_dir = DIST_KIWIX_ARCHIVES_DIR
else:
out_dir = DIST_ZIM_ARCHIVES_DIR
if target in ('kiwix-lib', 'kiwix-tools', 'libzim'):
shutil.copy(str(BASE_DIR/target/'meson-dist'/'{}-{}.tar.xz'.format(target, VERSIONS[target])),
str(out_dir))
elif PLATFORM == 'native_static':
make_archive('kiwix-tools_linux64-{}'.format(kiwix_tools_postfix), kiwix_tools_bins)
make_archive('zim-tools_linux64-{}'.format(zim_tools_postfix), zim_tools_bins)
make_archive('zimwriterfs_linux64-{}'.format(zimwriterfs_postfix), zimwriterfs_bins)
elif PLATFORM == 'win32_static':
make_archive('kiwix-tools_win32-{}'.format(kiwix_tools_postfix),
('{}.exe'.format(b) for b in kiwix_tools_bins))
elif PLATFORM == 'armhf_static':
make_archive('kiwix-tools_armhf-{}.'.format(kiwix_tools_postfix), kiwix_tools_bins)
elif PLATFORM.startswith('android_'):
APK_NAME = "kiwix-{}".format(PLATFORM)
source_debug_dir = BASE_DIR/'kiwix-android'/'app'/'build'/'outputs'/'apk'/'kiwix'/'debug'
source_release_dir = BASE_DIR/'kiwix-android'/'app'/'build'/'outputs'/'apk'/'kiwix'/'release'
shutil.copy(str(source_debug_dir/'app-kiwix-debug.apk'),
str(NIGHTLY_ARCHIVES_DIR/"{}-debug.apk".format(APK_NAME)))
shutil.copy(str(source_release_dir/'app-kiwix-release-unsigned.apk'),
str(NIGHTLY_ARCHIVES_DIR/"{}-release_signed".format(APK_NAME)))

145
travis/compile_all.sh
View File

@ -1,145 +0,0 @@
#!/usr/bin/env bash
set -e
BASE_DIR="BUILD_${PLATFORM}"
NIGHTLY_ARCHIVES_DIR=${HOME}/NIGHTLY_ARCHIVES
RELEASE_ARCHIVES_DIR=${HOME}/RELEASE_ARCHIVES
SSH_KEY=${TRAVIS_BUILD_DIR}/travis/travisci_builder_id_key
mkdir -p ${NIGHTLY_ARCHIVES_DIR}
mkdir -p ${RELEASE_ARCHIVES_DIR}
function make_archive {
if [[ "$MAKE_RELEASE" == "0" ]]
then
ARCHIVE_PATH="${NIGHTLY_ARCHIVES_DIR}/${1}_$(date +%Y-%m-%d).tar.gz"
else
ARCHIVE_PATH="${RELEASE_ARCHIVES_DIR}/${1}-${TRAVIS_TAG}.tar.gz"
fi
(
cd ${BASE_DIR}/INSTALL/bin
tar -czf "${ARCHIVE_PATH}" $2
)
}
cd ${HOME}
if [[ $TRAVIS_TAG =~ ^[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+ ]]
then
MAKE_RELEASE=1
else
MAKE_RELEASE=0
fi
if [[ "$TRAVIS_EVENT_TYPE" == "cron" || "$MAKE_RELEASE" == "1" ]]
then
if [[ ${PLATFORM} = android* ]]
then
TARGETS="libzim kiwix-lib kiwix-android"
elif [[ ${PLATFORM} =~ native_* ]]
then
TARGETS="libzim zimwriterfs zim-tools kiwix-lib kiwix-tools"
else
TARGETS="libzim kiwix-lib kiwix-tools"
fi
for TARGET in ${TARGETS}
do
echo $TARGET
if [[ "$TRAVIS_EVENT_TYPE" == "cron" ]]
then
${TRAVIS_BUILD_DIR}/kiwix-build.py \
--target-platform $PLATFORM \
--build-deps-only \
--hide-progress \
${TARGET}
rm ${BASE_DIR}/.install_packages_ok
(
cd ${BASE_DIR}
if [ -f meson_cross_file.txt ]
then
MESON_FILE=meson_cross_file.txt
fi
ANDROID_NDK_DIR=$(find . -name "android-ndk*")
ARCHIVE_NAME="deps_${PLATFORM}_${TARGET}.tar.gz"
cat <<EOF > manifest.txt
${ARCHIVE_NAME}
*********************************
Dependencies archive for ${TARGET} on platform ${PLATFORM}
Generated at $(date)
EOF
tar -czf ${ARCHIVE_NAME} INSTALL manifest.txt ${MESON_FILE} ${ANDROID_NDK_DIR}
scp -i ${SSH_KEY} ${ARCHIVE_NAME} nightlybot@download.kiwix.org:/var/www/tmp.kiwix.org/ci/
)
fi
if [[ "$MAKE_RELEASE" == "1" ]]
then
${TRAVIS_BUILD_DIR}/kiwix-build.py \
--hide-progress \
--make-release \
--target-platform $PLATFORM ${TARGET}
if [[ "$PLATFORM" == "native_dyn" ]]
then
${TRAVIS_BUILD_DIR}/kiwix-build.py \
--hide-progress \
--make-release \
--make-dist \
--target-platform $PLATFORM ${TARGET}
fi
else
${TRAVIS_BUILD_DIR}/kiwix-build.py \
--hide-progress \
--target-platform $PLATFORM ${TARGET}
fi
rm ${BASE_DIR}/.install_packages_ok
done
# We have build every thing. Now create archives for public deployement.
case ${PLATFORM} in
native_dyn)
#[TODO] Copy archive somewhere
#scp ${BASE_DIR}/${TARGET}/${TARGET}-${TARGET-version}.tar.gz ${SOMEWHERE}
;;
native_static)
make_archive kiwix-tools_linux64 "kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve"
make_archive zim-tools_linux64 "zimbench zimdump zimsearch zimdiff zimpatch zimsplit"
make_archive zimwriterfs_linux64 "zimwriterfs"
;;
win32_static)
make_archive kiwix-tools_win32 "kiwix-install.exe kiwix-manage.exe kiwix-read.exe kiwix-search.exe kiwix-serve.exe"
;;
armhf_static)
make_archive kiwix-tools_armhf "kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve"
;;
android_*)
APK_NAME="kiwix-${PLATFORM}"
cp ${BASE_DIR}/kiwix-android/app/build/outputs/apk/kiwix/debug/app-kiwix-debug.apk ${NIGHTLY_ARCHIVES_DIR}/${APK_NAME}-debug.apk
cp ${BASE_DIR}/kiwix-android/app/build/outputs/apk/kiwix/release/app-kiwix-release-unsigned.apk ${NIGHTLY_ARCHIVES_DIR}/${APK_NAME}-release-unsigned.apk
;;
esac
else
# No a cron job, we just have to build to be sure nothing is broken.
if [[ ${PLATFORM} = android* ]]
then
TARGETS="kiwix-android"
elif [[ ${PLATFORM} =~ native_* ]]
then
TARGETS="kiwix-tools zim-tools zimwriterfs"
else
TARGETS="kiwix-tools"
fi
for TARGET in ${TARGETS}
do
${TRAVIS_BUILD_DIR}/kiwix-build.py \
--target-platform $PLATFORM \
--hide-progress \
${TARGET}
done
fi

19
travis/deploy.sh
View File

@ -4,6 +4,8 @@ set -e
NIGHTLY_ARCHIVES_DIR=${HOME}/NIGHTLY_ARCHIVES
RELEASE_ARCHIVES_DIR=${HOME}/RELEASE_ARCHIVES
DIST_KIWIX_ARCHIVES_DIR=${HOME}/DIST_KIWIX_ARCHIVES
DIST_ZIM_ARCHIVES_DIR=${HOME}/DIST_ZIM_ARCHIVES
SSH_KEY=travis/travisci_builder_id_key
NIGHTLY_ARCHIVES=$(find $NIGHTLY_ARCHIVES_DIR -type f)
@ -21,3 +23,20 @@ then
${RELEASE_ARCHIVES} \
nightlybot@downoald.kiwix.org:/var/www/download.kiwix.org/releases
fi
DIST_KIWIX_ARCHIVES=$(find $DIST_KIWIX_ARCHIVES_DIR -type f)
if [[ "x$DIST_KIWIX_ARCHIVES" != "x" ]]
then
scp -vrp -i ${SSH_KEY} \
${DIST_KIWIX_ARCHIVES} \
nightlybot@download.kiwix.org:/var/www/download.kiwix.org/release
fi
DIST_ZIM_ARCHIVES=$(find $DIST_ZIM_ARCHIVES_DIR -type f)
if [[ "x$DIST_ZIM_ARCHIVES" != "x" ]]
then
scp -vrp -i ${SSH_KEY} \
${DIST_ZIM_ARCHIVES} \
nightlybot@download.openzim.org:/var/www/download.openzim.org/release
fi