kiwix-build/travis/compile_all.py

522 lines
20 KiB
Python
Executable File

#!/usr/bin/env python3
import sys, os
import shutil
from os import environ
from pathlib import Path
from datetime import date
import tarfile, zipfile
import subprocess
import re
import json
from urllib.request import urlopen
from urllib.error import URLError
from kiwixbuild.versions import (
main_project_versions,
release_versions,
base_deps_versions,
base_deps_meta_version)
PLATFORM = environ['PLATFORM']
TRAVIS_OS_NAME = environ['TRAVIS_OS_NAME']
HOME = Path(os.path.expanduser('~'))
NIGHTLY_DATE = environ['NIGHTLY_DATE']
KBUILD_SOURCE_DIR = Path(environ['TRAVIS_BUILD_DIR'])
KIWIX_DESKTOP_ONLY = environ.get('DESKTOP_ONLY') == '1'
BASE_DIR = HOME/"BUILD_{}".format(PLATFORM)
EXPORT_DIR = HOME/'EXPORT'
BASE_EXPORT_DIR = EXPORT_DIR/"BASE"
GIT_EXPORT_DIR = EXPORT_DIR/"GIT"
SOURCE_DIR = HOME/"SOURCE"
ARCHIVE_DIR = HOME/"ARCHIVE"
INSTALL_DIR = BASE_DIR/"INSTALL"
NIGHTLY_KIWIX_ARCHIVES_DIR = EXPORT_DIR/'NIGHTLY_KIWIX_ARCHIVES'/NIGHTLY_DATE
RELEASE_KIWIX_ARCHIVES_DIR = EXPORT_DIR/'RELEASE_KIWIX_ARCHIVES'
NIGHTLY_ZIM_ARCHIVES_DIR = EXPORT_DIR/'NIGHTLY_ZIM_ARCHIVES'/NIGHTLY_DATE
RELEASE_ZIM_ARCHIVES_DIR = EXPORT_DIR/'RELEASE_ZIM_ARCHIVES'
DIST_KIWIX_ARCHIVES_DIR = EXPORT_DIR/'DIST_KIWIX_ARCHIVES'
DIST_ZIM_ARCHIVES_DIR = EXPORT_DIR/'DIST_ZIM_ARCHIVES'
BINTRAY_ARCHIVES_DIR = EXPORT_DIR/'BINTRAY_ARCHIVES'
BIN_EXT = '.exe' if PLATFORM.startswith('win32_') else ''
# We have build everything. Now create archives for public deployement.
EXPORT_FILES = {
'kiwix-tools':
(INSTALL_DIR/'bin', [f+BIN_EXT for f in ('kiwix-manage', 'kiwix-read', 'kiwix-search', 'kiwix-serve')]),
'zim-tools':
(INSTALL_DIR/'bin', [f+BIN_EXT for f in ('zimbench', 'zimcheck', 'zimdump', 'zimsearch', 'zimdiff', 'zimpatch', 'zimsplit')]),
'zimwriterfs':
(INSTALL_DIR/'bin', ['zimwriterfs'+BIN_EXT]),
'libzim':
(INSTALL_DIR, ('lib/x86_64-linux-gnu/libzim.so.{}'.format(main_project_versions['libzim']),
'lib/x86_64-linux-gnu/libzim.so.{}'.format(main_project_versions['libzim'][0]),
'include/zim/**/*.h'))
}
FLATPAK_HTTP_GIT_REMOTE = 'https://github.com/flathub/org.kiwix.desktop.git'
FLATPAK_GIT_REMOTE = 'git@github.com:flathub/org.kiwix.desktop.git'
_date = date.today().isoformat()
def print_message(message, *args, **kwargs):
message = message.format(*args, **kwargs)
message = "{0} {1} {0}".format('-'*3, message)
print(message, flush=True)
def write_manifest(manifest_file, archive_name, target, platform):
with manifest_file.open(mode='w') as f:
f.write('''{archive_name}
***************************
Dependencies archive for {target} on platform {platform}
Generated at {date}
'''.format(
archive_name=archive_name,
target=target,
platform=platform,
date=date.today().isoformat()))
def run_kiwix_build(target, platform,
build_deps_only=False,
target_only=False,
make_release=False,
make_dist=False):
command = ['kiwix-build']
command.append(target)
command.append('--hide-progress')
if platform == 'flatpak' or platform.startswith('win32_'):
command.append('--assume-packages-installed')
if target == 'kiwix-android' and platform.startswith('android_'):
command.extend(['--target-platform', 'android', '--android-arch', platform[8:]])
elif platform == 'android':
command.extend(['--target-platform', 'android'])
for arch in ('arm', 'arm64', 'x86', 'x86_64'):
command.extend(['--android-arch', arch])
else:
command.extend(['--target-platform', platform])
if build_deps_only:
command.append('--build-deps-only')
if target_only:
command.append('--build-nodeps')
if make_release:
command.append('--make-release')
if make_dist:
command.append('--make-dist')
print_message("Build {} (deps={}, release={}, dist={})",
target, build_deps_only, make_release, make_dist)
subprocess.check_call(command, cwd=str(HOME))
print_message("Build ended")
def create_desktop_image():
print_message("creating desktop image")
if make_release:
postfix = main_project_versions['kiwix-desktop']
extra_postfix = release_versions.get('kiwix-desktop')
if extra_postfix is None:
# We should not make archives for release not wanted
return
if extra_postfix:
postfix = "{}-{}".format(postfix, extra_postfix)
archive_dir = RELEASE_KIWIX_ARCHIVES_DIR/'kiwix-desktop'
src_dir = SOURCE_DIR/'kiwix-desktop_release'
else:
postfix = _date
archive_dir = NIGHTLY_KIWIX_ARCHIVES_DIR
src_dir = SOURCE_DIR/'kiwix-desktop'
if PLATFORM == 'flatpak':
build_path = BASE_DIR/'org.kiwix.desktop.flatpak'
app_name = 'org.kiwix.desktop.{}.flatpak'.format(postfix)
print_message("archive is ", build_path)
else:
build_path = HOME/'Kiwix-{}-x86_64.AppImage'.format(postfix)
app_name = "kiwix-desktop_x86_64_{}.appimage".format(postfix)
command = ['kiwix-build/scripts/create_kiwix-desktop_appImage.sh',
str(INSTALL_DIR), str(src_dir), str(HOME/'AppDir')]
env = dict(os.environ)
env['VERSION'] = postfix
print_message("Build AppImage of kiwix-desktop")
subprocess.check_call(command, cwd=str(HOME), env=env)
try:
archive_dir.mkdir(parents=True)
except FileExistsError:
pass
print_message("Copy Build to {}".format(archive_dir/app_name))
shutil.copy(str(build_path), str(archive_dir/app_name))
def make_archive(project, platform):
base_dir, export_files = EXPORT_FILES[project]
if make_release:
postfix = main_project_versions[project]
extra_postfix = release_versions.get(project)
if extra_postfix is None:
# We should not make archives for release not wanted
return
if extra_postfix:
postfix = "{}-{}".format(postfix, extra_postfix)
if project in ('kiwix-lib', 'kiwix-tools'):
archive_dir = RELEASE_KIWIX_ARCHIVES_DIR/project
else:
archive_dir = RELEASE_ZIM_ARCHIVES_DIR/project
else:
postfix = _date
if project in ('kiwix-lib', 'kiwix-tools'):
archive_dir = NIGHTLY_KIWIX_ARCHIVES_DIR
else:
archive_dir = NIGHTLY_ZIM_ARCHIVES_DIR
try:
archive_dir.mkdir(parents=True)
except FileExistsError:
pass
archive_name = "{}_{}-{}".format(project, platform, postfix)
files_to_archive = []
for export_file in export_files:
files_to_archive.extend(base_dir.glob(export_file))
if platform == "win-i686":
open_archive = lambda a : zipfile.ZipFile(str(a), 'w', compression=zipfile.ZIP_DEFLATED)
archive_add = lambda a, f : a.write(str(f), arcname=str(f.relative_to(base_dir)))
archive_ext = ".zip"
else:
open_archive = lambda a : tarfile.open(str(a), 'w:gz')
archive_add = lambda a, f : a.add(str(f), arcname="{}/{}".format(archive_name, str(f.relative_to(base_dir))))
archive_ext = ".tar.gz"
archive = archive_dir/'{}{}'.format(archive_name, archive_ext)
print_message("create archive {} with {}", archive, files_to_archive)
with open_archive(archive) as arch:
for f in files_to_archive:
archive_add(arch, f)
def make_deps_archive(target=None, name=None, full=False):
archive_name = name or "deps_{}_{}_{}.tar.xz".format(
TRAVIS_OS_NAME, PLATFORM, target)
print_message("Create archive {}.", archive_name)
files_to_archive = [INSTALL_DIR]
files_to_archive += HOME.glob('BUILD_*/LOGS')
if PLATFORM == 'native_mixed':
files_to_archive += [HOME/'BUILD_native_static'/'INSTALL']
if PLATFORM.startswith('android'):
files_to_archive.append(HOME/'BUILD_neutral'/'INSTALL')
if PLATFORM == 'android':
for arch in ('arm', 'arm64', 'x86', 'x86_64'):
base_dir = HOME/"BUILD_android_{}".format(arch)
files_to_archive.append(base_dir/'INSTALL')
if (base_dir/'meson_cross_file.txt').exists():
files_to_archive.append(base_dir/'meson_cross_file.txt')
files_to_archive += HOME.glob('BUILD_*/android-ndk*')
files_to_archive += HOME.glob('BUILD_*/android-sdk*')
if (BASE_DIR/'meson_cross_file.txt').exists():
files_to_archive.append(BASE_DIR/'meson_cross_file.txt')
manifest_file = BASE_DIR/'manifest.txt'
write_manifest(manifest_file, archive_name, target, PLATFORM)
files_to_archive.append(manifest_file)
relative_path = HOME
if full:
files_to_archive += ARCHIVE_DIR.glob(".*_ok")
files_to_archive += BASE_DIR.glob('*/.*_ok')
files_to_archive += (HOME/"BUILD_native_dyn").glob('*/.*_ok')
files_to_archive += (HOME/"BUILD_native_static").glob('*/.*_ok')
files_to_archive += HOME.glob('BUILD_android*/**/.*_ok')
files_to_archive += SOURCE_DIR.glob('*/.*_ok')
files_to_archive += [SOURCE_DIR/'pugixml-{}'.format(
base_deps_versions['pugixml'])]
files_to_archive += HOME.glob('BUILD_*/pugixml-{}'.format(
base_deps_versions['pugixml']))
if PLATFORM.startswith('armhf'):
files_to_archive += (SOURCE_DIR/'armhf').glob('*')
toolchains_subdirs = HOME.glob('**/TOOLCHAINS/*/*')
for subdir in toolchains_subdirs:
if not subdir.match('tools'):
files_to_archive.append(subdir)
with tarfile.open(str(BASE_EXPORT_DIR/archive_name), 'w:xz') as tar:
for name in set(files_to_archive):
print('.{}'.format(name), flush=True)
tar.add(str(name), arcname=str(name.relative_to(relative_path)))
def make_flatpak_cache_archive():
archive_name = "base_deps_{os}_{platform}.tar.xz".format(
os=TRAVIS_OS_NAME,
platform=PLATFORM)
cache = BASE_DIR/'.flatpak-builder'
print_message("make flatpak cache archive")
nb_files = 0
def pseudo_filter(tarinfo):
nonlocal nb_files
nb_files += 1
if (nb_files % 1000) == 0:
print('.', flush=True)
return tarinfo
with tarfile.open(str(BASE_EXPORT_DIR/archive_name), 'w:xz') as tar:
tar.add(str(cache), arcname=str(cache.relative_to(BASE_DIR)), filter=pseudo_filter)
def update_flathub_git():
GIT_REPO_DIR = GIT_EXPORT_DIR/"org.kiwix.desktop"
env = dict(os.environ)
env['GIT_AUTHOR_NAME'] = env['GIT_COMMITTER_NAME'] = "KiwixBot"
env['GIT_AUTHOR_EMAIL'] = env['GIT_COMMITTER_EMAIL'] = "kiwixbot@kymeria.fr"
def call(command, cwd=None):
cwd = cwd or GIT_REPO_DIR
print_message("call ", command)
subprocess.check_call(command, env=env, cwd=str(cwd))
command = ['git', 'clone', FLATPAK_HTTP_GIT_REMOTE]
call(command, cwd=GIT_EXPORT_DIR)
shutil.copy(str(BASE_DIR/'org.kiwix.desktop.json'), str(GIT_REPO_DIR))
patch_dir = KBUILD_SOURCE_DIR/'kiwixbuild'/'patches'
for dep in ('libaria2', 'mustache', 'pugixml', 'xapian'):
for f in patch_dir.glob('{}_*.patch'.format(dep)):
shutil.copy(str(f), str(GIT_REPO_DIR/'patches'))
command = ['git', 'add', '-A', '.']
call(command)
command = ['git', 'commit', '-m',
'Update to version {}'.format(main_project_versions['kiwix-desktop'])]
call(command)
command = ['git', 'config', 'remote.origin.pushurl', FLATPAK_GIT_REMOTE]
call(command)
for p in (NIGHTLY_KIWIX_ARCHIVES_DIR,
NIGHTLY_ZIM_ARCHIVES_DIR,
RELEASE_KIWIX_ARCHIVES_DIR,
RELEASE_ZIM_ARCHIVES_DIR,
DIST_KIWIX_ARCHIVES_DIR,
DIST_ZIM_ARCHIVES_DIR,
BINTRAY_ARCHIVES_DIR,
BASE_EXPORT_DIR,
GIT_EXPORT_DIR):
try:
p.mkdir(parents=True)
except FileExistsError:
pass
make_release = re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", environ.get('TRAVIS_TAG', '')) is not None
def download_base_archive(base_name):
url = 'http://tmp.kiwix.org/ci/{}'.format(base_name)
file_path = str(HOME/base_name)
batch_size = 1024*1024*8
with urlopen(url) as resource, open(file_path, 'wb') as file:
while True:
batch = resource.read(batch_size)
if not batch:
break
print(".", end="", flush=True)
file.write(batch)
return file_path
if PLATFORM != 'flatpak':
# The first thing we need to do is to (potentially) download already compiled base dependencies.
base_dep_archive_name = "base_deps_{os}_{platform}_{version}_{release}.tar.xz".format(
os=TRAVIS_OS_NAME,
platform=PLATFORM,
version=base_deps_meta_version,
release='release' if make_release else 'debug')
print_message("Getting archive {}", base_dep_archive_name)
try:
local_filename = download_base_archive(base_dep_archive_name)
with tarfile.open(local_filename) as f:
f.extractall(str(HOME))
except URLError:
print_message("Cannot get archive. Build dependencies")
if PLATFORM == 'android':
for arch in ('arm', 'arm64', 'x86', 'x86_64'):
archive_name = "base_deps_{os}_android_{arch}_{version}_{release}.tar.xz".format(
os=TRAVIS_OS_NAME,
arch=arch,
version=base_deps_meta_version,
release='release' if make_release else 'debug')
print_message("Getting archive {}", archive_name)
try:
local_filename = download_base_archive(archive_name)
with tarfile.open(local_filename) as f:
f.extractall(str(HOME))
except URLError:
pass
else:
run_kiwix_build('alldependencies', platform=PLATFORM)
make_deps_archive(name=base_dep_archive_name, full=True)
else:
base_dep_archive_name = "base_deps_{os}_{platform}.tar.xz".format(
os=TRAVIS_OS_NAME,
platform=PLATFORM)
print_message("Getting archive {}", base_dep_archive_name)
try:
local_filename = download_base_archive(base_dep_archive_name)
BASE_DIR.mkdir(parents=True)
with tarfile.open(local_filename) as f:
f.extractall(str(BASE_DIR))
except URLError:
print_message("Cannot get archive. Move on")
# A basic compilation to be sure everything is working (for a PR)
if environ['TRAVIS_EVENT_TYPE'] != 'cron' and not make_release:
if PLATFORM.startswith('android'):
TARGETS = ('kiwix-lib',)
elif PLATFORM.startswith('iOS'):
TARGETS = ('kiwix-lib',)
elif PLATFORM.startswith('native_'):
if TRAVIS_OS_NAME == "osx":
TARGETS = ('kiwix-lib', 'zim-tools', 'zimwriterfs')
elif PLATFORM == 'native_dyn' and KIWIX_DESKTOP_ONLY:
TARGETS = ('kiwix-desktop', )
elif PLATFORM == 'native_mixed':
TARGETS = ('libzim', )
else:
TARGETS = ('kiwix-tools', 'zim-tools', 'zimwriterfs')
elif PLATFORM == 'flatpak':
TARGETS = ('kiwix-desktop', )
else:
TARGETS = ('kiwix-tools', )
for target in TARGETS:
run_kiwix_build(target,
platform=PLATFORM)
if PLATFORM == 'native_mixed':
make_archive('libzim', 'linux-x86_64')
elif PLATFORM == 'native_static':
for target in ('kiwix-tools', 'zim-tools', 'zimwriterfs'):
make_archive(target, 'linux-x86_64')
elif PLATFORM == 'win32_static':
make_archive('kiwix-tools', 'win-i686')
elif PLATFORM == 'armhf_static':
make_archive('kiwix-tools', 'linux-armhf')
elif PLATFORM == 'i586_static':
make_archive('kiwix-tools', 'linux-i586')
elif PLATFORM == 'flatpak':
make_flatpak_cache_archive()
sys.exit(0)
TARGETS = tuple()
if PLATFORM == 'android':
TARGETS = ('kiwix-lib-app',)
elif PLATFORM.startswith('android_'):
TARGETS = ('libzim', 'kiwix-lib')
elif PLATFORM.startswith('iOS'):
TARGETS = ('libzim', 'kiwix-lib')
elif PLATFORM.startswith('native_'):
if TRAVIS_OS_NAME == "osx":
TARGETS = ('libzim', 'zimwriterfs', 'zim-tools', 'kiwix-lib')
else:
if PLATFORM == 'native_dyn' and KIWIX_DESKTOP_ONLY:
TARGETS = ('kiwix-desktop', )
elif PLATFORM == 'native_mixed':
TARGETS = ('libzim', )
else:
TARGETS = ('libzim', 'zimwriterfs', 'zim-tools', 'kiwix-lib', 'kiwix-tools')
elif PLATFORM == 'flatpak':
TARGETS = ('kiwix-desktop', )
else:
TARGETS = ('libzim', 'zim-tools', 'kiwix-lib', 'kiwix-tools')
for target in TARGETS:
if environ['TRAVIS_EVENT_TYPE'] == 'cron' and PLATFORM not in ('android', 'flatpak'):
run_kiwix_build(target,
platform=PLATFORM,
build_deps_only=True)
make_deps_archive(target=target)
if target == 'kiwix-lib-app':
extra_postfix = release_versions.get('kiwix-lib')
if extra_postfix:
env['KIWIXLIB_BUILDVERSION'] = extra_postfix
run_kiwix_build(target,
platform=PLATFORM,
make_release=make_release)
print_message("target is ", target)
if target == 'kiwix-desktop':
create_desktop_image()
if make_release and PLATFORM == 'native_dyn' and release_versions.get(target) == 0:
run_kiwix_build(target,
platform=PLATFORM,
make_release=True,
make_dist=True)
# We have build everything. Now create archives for public deployement.
if make_release and PLATFORM == 'native_dyn':
for target in TARGETS:
if release_versions.get(target) != 0:
# Do not release project not in release_versions
continue
if target in ('kiwix-lib', 'kiwix-tools', 'kiwix-desktop'):
out_dir = DIST_KIWIX_ARCHIVES_DIR
else:
out_dir = DIST_ZIM_ARCHIVES_DIR
if target in ('kiwix-lib', 'kiwix-tools', 'libzim', 'zim-tools', 'zimwriterfs', 'kiwix-desktop'):
try:
(out_dir/target).mkdir(parents=True)
except FileExistsError:
pass
full_target_name = "{}-{}".format(target, main_project_versions[target])
if target != 'kiwix-desktop':
in_file = BASE_DIR/full_target_name/'meson-dist'/'{}.tar.xz'.format(
full_target_name)
else:
in_file = BASE_DIR/full_target_name/'{}.tar.gz'.format(full_target_name)
if in_file.exists():
shutil.copy(str(in_file), str(out_dir/target))
elif PLATFORM == 'native_mixed':
make_archive('libzim', 'linux-x86_64')
elif PLATFORM == 'native_static':
for target in ('kiwix-tools', 'zim-tools', 'zimwriterfs'):
make_archive(target, 'linux-x86_64')
elif PLATFORM == 'win32_static':
make_archive('kiwix-tools', 'win-i686')
elif PLATFORM == 'armhf_static':
make_archive('kiwix-tools', 'linux-armhf')
elif PLATFORM == 'i586_static':
make_archive('kiwix-tools', 'linux-i586')
elif make_release and PLATFORM == 'flatpak':
update_flathub_git()
elif PLATFORM == 'android' and 'kiwix-lib-app' in TARGETS:
if make_release and release_versions.get('kiwix-lib') is not None:
postfix = main_project_versions['kiwix-lib']
extra_postfix = release_versions.get('kiwix-lib')
if extra_postfix:
postfix = "{}-{}".format(postfix, extra_postfix)
basename = "kiwixlib-{}".format(postfix)
output_release_dir = HOME/'BUILD_android'/'kiwix-lib-app'/'kiwixLibAndroid'/'build'
shutil.copy(str(output_release_dir/'outputs'/'aar'/'kiwixLibAndroid-release.aar'),
str(BINTRAY_ARCHIVES_DIR/(basename+'.aar')))
shutil.copy(str(output_release_dir/'pom.xml'),
str(BINTRAY_ARCHIVES_DIR/(basename+'.pom')))
json_filename = '{}_bintray_info.json'.format(basename)
data = {
'version': postfix,
'files': [basename+ext for ext in ('.aar', '.pom')]
}
with open(str(BINTRAY_ARCHIVES_DIR/json_filename), 'w') as f:
json.dump(data, f)