diff --git a/.travis.yml b/.travis.yml index ebd8fc0..b0328b1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,11 +17,13 @@ before_cache: cache: ccache: true directories: + - $HOME/.cache/pip - $HOME/.gradle/caches/ - $HOME/.gradle/wrapper/ - $HOME/.android/build-cache install: travis/install_extra_deps.sh -script: travis_wait 30 travis/compile_all.py +script: travis/compile_all.py +after_failure: travis/upload_all_log.sh deploy: - provider: script skip_cleanup: true @@ -74,3 +76,16 @@ addons: - ubuntu-toolchain-r-test packages: - g++-5 + - cmake + - python3-pip + - zlib1g-dev + - libjpeg-dev + - libmagic-dev + - ctpp2-utils + - libctpp2-dev + - libmicrohttpd-dev + - g++-mingw-w64-i686 + - gcc-mingw-w64-i686 + - gcc-mingw-w64-base + - mingw-w64-tools + - default-jdk diff --git a/dependencies.py b/dependencies.py index b29a698..7eb0e57 100644 --- a/dependencies.py +++ b/dependencies.py @@ -6,10 +6,12 @@ from dependency_utils import ( ReleaseDownload, GitClone, SvnClone, + NoopSource, MakeBuilder, CMakeBuilder, MesonBuilder, GradleBuilder, + NoopBuilder, Builder as BaseBuilder) from utils import Remotefile, pj, SkipCommand, copy_tree, add_execution_right @@ -31,7 +33,6 @@ from utils import Remotefile, pj, SkipCommand, copy_tree, add_execution_right class zlib(Dependency): name = 'zlib' - version = '1.2.8' class Source(ReleaseDownload): archive = Remotefile('zlib-1.2.8.tar.gz', @@ -78,7 +79,6 @@ class zlib(Dependency): class lzma(Dependency): name = 'lzma' - version = '5.0.4' class Source(ReleaseDownload): archive = Remotefile('xz-5.0.4.tar.bz2', @@ -91,7 +91,6 @@ class lzma(Dependency): class UUID(Dependency): name = 'uuid' - version = "1.43.4" class Source(ReleaseDownload): archive = Remotefile('e2fsprogs-libs-1.43.4.tar.gz', @@ -110,7 +109,6 @@ class UUID(Dependency): class Xapian(Dependency): name = "xapian-core" - version = "1.4.5" class Source(ReleaseDownload): archive = Remotefile('xapian-core-1.4.5.tar.xz', @@ -132,7 +130,6 @@ class Xapian(Dependency): class CTPP2(Dependency): name = "ctpp2" - version = "2.8.3" class Source(ReleaseDownload): name = "ctpp2" @@ -175,7 +172,6 @@ class CTPP2C(CTPP2): class Pugixml(Dependency): name = "pugixml" - version = "1.2" class Source(ReleaseDownload): archive = Remotefile('pugixml-1.2.tar.gz', @@ -187,7 +183,6 @@ class Pugixml(Dependency): class MicroHttpd(Dependency): name = "libmicrohttpd" - version = "0.9.46" class Source(ReleaseDownload): archive = Remotefile('libmicrohttpd-0.9.46.tar.gz', @@ -200,7 +195,6 @@ class MicroHttpd(Dependency): class Gumbo(Dependency): name = "gumbo" - version = "0.10.1" class Source(ReleaseDownload): archive = Remotefile('gumbo-0.10.1.tar.gz', @@ -217,7 +211,6 @@ class Gumbo(Dependency): class Icu(Dependency): name = "icu4c" - version = "58.2" class Source(SvnClone): name = "icu4c" @@ -279,9 +272,9 @@ class Libzim(Dependency): class Source(GitClone): git_remote = "https://github.com/openzim/libzim.git" git_dir = "libzim" - release_git_ref = "3.0.0" - Builder = MesonBuilder + class Builder(MesonBuilder): + test_option = "-t 8" class ZimTools(Dependency): @@ -291,7 +284,6 @@ class ZimTools(Dependency): class Source(GitClone): git_remote = "https://github.com/openzim/zim-tools.git" git_dir = "zim-tools" - release_git_ref = "0.0.1" class Builder(MesonBuilder): @property @@ -343,7 +335,6 @@ class Kiwixlib(Dependency): class Source(GitClone): git_remote = "https://github.com/kiwix/kiwix-lib.git" git_dir = "kiwix-lib" - release_git_ref = "1.0.2" class Builder(MesonBuilder): @property @@ -367,7 +358,6 @@ class KiwixTools(Dependency): class Source(GitClone): git_remote = "https://github.com/kiwix/kiwix-tools.git" git_dir = "kiwix-tools" - release_git_ref = "0.3.0" class Builder(MesonBuilder): @property @@ -379,7 +369,6 @@ class KiwixTools(Dependency): class Gradle(Dependency): name = "Gradle" - version = "3.4" class Source(ReleaseDownload): archive = Remotefile('gradle-4.1-bin.zip', @@ -400,6 +389,29 @@ class Gradle(Dependency): pj(self.buildEnv.install_dir, "lib")) +class AllBaseDependencies(Dependency): + name = "alldependencies" + + @property + def dependencies(self): + base_deps = ['zlib', 'lzma', 'xapian-core', 'gumbo', 'pugixml', 'libmicrohttpd'] + if self.buildEnv.platform_info.build != 'native': + base_deps += ["icu4c_cross-compile"] + else: + base_deps += ["icu4c"] + if ( self.buildEnv.platform_info.build != 'android' + and self.buildEnv.distname != 'Darwin'): + base_deps += ['ctpp2c', 'ctpp2'] + if self.buildEnv.platform_info.build == 'android': + base_deps += ['Gradle'] + + return base_deps + + + Source = NoopSource + Builder = NoopBuilder + + class KiwixAndroid(Dependency): name = "kiwix-android" dependencies = ["Gradle", "kiwix-lib"] diff --git a/dependency_utils.py b/dependency_utils.py index 5326bd4..d218974 100644 --- a/dependency_utils.py +++ b/dependency_utils.py @@ -3,6 +3,7 @@ import os import shutil from utils import pj, Context, SkipCommand, extract_archive, Defaultdict, StopBuild +from dependency_versions import main_project_versions, base_deps_versions SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -20,7 +21,6 @@ class Dependency(metaclass=_MetaDependency): all_deps = {} dependencies = [] force_native_build = False - version = None def __init__(self, buildEnv): self.buildEnv = buildEnv @@ -28,6 +28,10 @@ class Dependency(metaclass=_MetaDependency): self.builder = self.Builder(self) self.skip = False + @property + def version(self): + return base_deps_versions.get(self.name, None) + @property def full_name(self): if self.version: @@ -94,6 +98,11 @@ class Source: return self.target.command(*args, **kwargs) +class NoopSource(Source): + def prepare(self): + pass + + class ReleaseDownload(Source): archive_top_dir = None @@ -124,7 +133,10 @@ class ReleaseDownload(Source): class GitClone(Source): base_git_ref = "master" - release_git_ref = "master" + + @property + def release_git_ref(self): + return main_project_versions.get(self.name, "master") @property def source_dir(self): @@ -236,6 +248,14 @@ class Builder: self.command('make_dist', self._make_dist) +class NoopBuilder(Builder): + def build(self): + pass + + def make_dist(self): + pass + + class MakeBuilder(Builder): configure_option = "" dynamic_configure_option = "--enable-shared --disable-static" @@ -331,6 +351,7 @@ class CMakeBuilder(MakeBuilder): class MesonBuilder(Builder): configure_option = "" + test_option = "" @property def library_type(self): @@ -372,7 +393,7 @@ class MesonBuilder(Builder): and not self.buildEnv.platform_info.static) ): raise SkipCommand() - command = "{} --verbose".format(self.buildEnv.mesontest_command) + command = "{} --verbose {}".format(self.buildEnv.mesontest_command, self.test_option) self.buildEnv.run_command(command, self.build_path, context) def _install(self, context): diff --git a/dependency_versions.py b/dependency_versions.py new file mode 100644 index 0000000..b0cb2e3 --- /dev/null +++ b/dependency_versions.py @@ -0,0 +1,27 @@ + +main_project_versions = { + 'kiwix-lib': '1.0.2', + 'kiwix-tools': '0.3.0', + 'libzim': '3.0.0', + 'zim-tools': '0.0.1', + 'zimwriterfs': '1.1', +} + + +# This is the "version" of the whole base_deps_versions dict. +# Change this when you change base_deps_versions. +base_deps_meta_version = '0' + + +base_deps_versions = { + 'zlib' : '1.2.8', + 'lzma' : '5.0.4', + 'uuid' : '1.43.4', + 'xapian-core' : '1.4.5', + 'ctpp2' : '2.8.3', + 'pugixml' : '1.2', + 'libmicrohttpd' : '0.9.46', + 'gumbo' : '0.10.1', + 'icu4c' : '58.2', + 'Gradle' : '3.4', +} diff --git a/travis/compile_all.py b/travis/compile_all.py index 0e4893f..1f068e5 100755 --- a/travis/compile_all.py +++ b/travis/compile_all.py @@ -8,6 +8,11 @@ from datetime import date import tarfile import subprocess import re +from urllib.request import urlretrieve +from urllib.error import URLError + +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import dependency_versions PLATFORM = environ['PLATFORM'] @@ -15,6 +20,9 @@ def home(): return Path(os.path.expanduser('~')) BASE_DIR = home()/"BUILD_{}".format(PLATFORM) +SOURCE_DIR = home()/"SOURCE" +ARCHIVE_DIR = home()/"ARCHIVE" +TOOLCHAINS_DIR = home()/"TOOLCHAINS" NIGHTLY_ARCHIVES_DIR = home()/'NIGHTLY_ARCHIVES' RELEASE_KIWIX_ARCHIVES_DIR = home()/'RELEASE_KIWIX_ARCHIVES' RELEASE_ZIM_ARCHIVES_DIR = home()/'RELEASE_ZIM_ARCHIVES' @@ -22,14 +30,6 @@ DIST_KIWIX_ARCHIVES_DIR = home()/'DIST_KIWIX_ARCHIVES' DIST_ZIM_ARCHIVES_DIR = home()/'DIST_ZIM_ARCHIVES' SSH_KEY = Path(environ['TRAVIS_BUILD_DIR'])/'travis'/'travisci_builder_id_key' -VERSIONS = { - 'kiwix-lib': '1.0.2', - 'kiwix-tools': '0.3.0', - 'libzim': '3.0.0', - 'zim-tools': '0.0.1', - 'zimwriterfs': '1.1' -} - # We have build everything. Now create archives for public deployement. BINARIES = { 'kiwix-tools': ('kiwix-install', 'kiwix-manage', 'kiwix-read', 'kiwix-search', 'kiwix-serve'), @@ -66,6 +66,8 @@ def run_kiwix_build(target, platform, build_deps_only=False, make_release=False, command.append('--make-release') if make_dist: command.append('--make-dist') + print("--- Build {} (deps={}, release={}, dist={}) ---".format( + target, build_deps_only, make_release, make_dist), flush=True) subprocess.check_call(command, cwd=str(home())) @@ -74,7 +76,7 @@ def make_archive(project, platform): if platform == "win32": file_to_archives = ['{}.exe'.format(f) for f in file_to_archives] if make_release: - postfix = VERSIONS[project] + postfix = dependency_versions.main_project_versions[project] if project in ('kiwix-lib', 'kiwix-tools'): archive_dir = RELEASE_KIWIX_ARCHIVES_DIR/project else: @@ -94,8 +96,40 @@ def make_archive(project, platform): arch.add(str(base_bin_dir/f), arcname=str(f)) +def make_deps_archive(target, full=False): + (BASE_DIR/'.install_packages_ok').unlink() + + archive_name = "deps_{}_{}.tar.gz".format(PLATFORM, target) + files_to_archive = [BASE_DIR/'INSTALL'] + files_to_archive += BASE_DIR.glob('**/android-ndk*') + if (BASE_DIR/'meson_cross_file.txt').exists(): + files_to_archive.append(BASE_DIR/'meson_cross_file.txt') + + manifest_file = BASE_DIR/'manifest.txt' + write_manifest(manifest_file, archive_name, target, PLATFORM) + files_to_archive.append(manifest_file) + + relative_path = BASE_DIR + if full: + files_to_archive += [ARCHIVE_DIR] + files_to_archive += BASE_DIR.glob('*/.*_ok') + files_to_archive += SOURCE_DIR.glob('*/.*_ok') + files_to_archive += [SOURCE_DIR/'pugixml-{}'.format( + dependency_versions.base_deps_versions['pugixml'])] + files_to_archive += [BASE_DIR/'pugixml-{}'.format( + dependency_versions.base_deps_versions['pugixml'])] + if (TOOLCHAINS_DIR).exists(): + files_to_archive.append(TOOLCHAINS_DIR) + relative_path = home() + + with tarfile.open(str(relative_path/archive_name), 'w:gz') as tar: + for name in files_to_archive: + tar.add(str(name), arcname=str(name.relative_to(relative_path))) + return relative_path/archive_name + + def scp(what, where): - command = ['scp', '-i', str(SSH_KEY), what, where] + command = ['scp', '-i', str(SSH_KEY), str(what), str(where)] subprocess.check_call(command) @@ -111,6 +145,27 @@ for p in (NIGHTLY_ARCHIVES_DIR, make_release = re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", environ.get('TRAVIS_TAG', '')) is not None +# The first thing we need to do is to (potentially) download already compiled base dependencies. +BASE_DEP_VERSION = dependency_versions.base_deps_meta_version +base_dep_archive_name = "base_deps_{}_{}.tar.gz".format(PLATFORM, BASE_DEP_VERSION) + +print("--- Getting archive {} ---".format(base_dep_archive_name), flush=True) +try: + local_filename, headers = urlretrieve( + 'http://tmp.kiwix.org/ci/{}'.format(base_dep_archive_name)) + with tarfile.open(local_filename) as f: + f.extractall(str(home())) +except URLError: + print("--- Cannot get archive. Build dependencies ---", flush=True) + run_kiwix_build('alldependencies', platform=PLATFORM) + archive = make_deps_archive('alldependencies', full=True) + destination = 'nightlybot@download.kiwix.org:/var/www/tmp.kiwix.org/ci/{}' + destination = destination.format(base_dep_archive_name) + scp(archive, destination) + + + + # A basic compilation to be sure everything is working (for a PR) if environ['TRAVIS_EVENT_TYPE'] != 'cron' and not make_release: if PLATFORM.startswith('android'): @@ -143,21 +198,8 @@ for target in TARGETS: run_kiwix_build(target, platform=PLATFORM, build_deps_only=True) - (BASE_DIR/'.install_packages_ok').unlink() - - archive_name = "deps_{}_{}.tar.gz".format(PLATFORM, target) - files_to_archive = [BASE_DIR/'INSTALL'] - files_to_archive += BASE_DIR.glob('**/android-ndk*') - if (BASE_DIR/'meson_cross_file.txt').exists(): - files_to_archive.append(BASE_DIR/'meson_cross_file.txt') - - manifest_file = BASE_DIR/'manifest.txt' - write_manifest(manifest_file, archive_name, target, PLATFORM) - files_to_archive.append(manifest_file) - with tarfile.open(str(BASE_DIR/archive_name), 'w:gz') as tar: - for name in files_to_archive: - tar.add(str(name), arcname=str(name.relative_to(BASE_DIR))) - scp(str(BASE_DIR/archive_name), 'nightlybot@download.kiwix.org:/var/www/tmp.kiwix.org/ci/') + archive = make_deps_archive(target) + scp(archive, 'nightlybot@download.kiwix.org:/var/www/tmp.kiwix.org/ci/') run_kiwix_build(target, platform=PLATFORM, @@ -171,10 +213,6 @@ for target in TARGETS: # We have build everything. Now create archives for public deployement. -kiwix_tools_postfix = VERSIONS['kiwix-tools'] if make_release else _date -zim_tools_postfix = VERSIONS['zim-tools'] if make_release else _date -zimwriterfs_postfix = VERSIONS['zimwriterfs'] if make_release else _date - if make_release and PLATFORM == 'native_dyn': for target in TARGETS: if target in ('kiwix-lib', 'kiwix-tools'): @@ -183,10 +221,14 @@ if make_release and PLATFORM == 'native_dyn': out_dir = DIST_ZIM_ARCHIVES_DIR if target in ('kiwix-lib', 'kiwix-tools', 'libzim', 'zim-tools'): - shutil.copy(str(BASE_DIR/target/'meson-dist'/'{}-{}.tar.xz'.format(target, VERSIONS[target])), + shutil.copy(str(BASE_DIR/target/'meson-dist'/'{}-{}.tar.xz'.format( + target, + dependency_versions.main_project_versions[target])), str(out_dir)) if target in ('zimwriterfs',): - shutil.copy(str(BASE_DIR/target/'{}-{}.tar.gz'.format(target, VERSIONS[target])), + shutil.copy(str(BASE_DIR/target/'{}-{}.tar.gz'.format( + target, + dependency_versions.main_project_versions[target])), str(out_dir)) elif PLATFORM == 'native_static': for target in ('kiwix-tools', 'zim-tools', 'zimwriterfs'): diff --git a/travis/install_extra_deps.sh b/travis/install_extra_deps.sh index 987c2d0..107021a 100755 --- a/travis/install_extra_deps.sh +++ b/travis/install_extra_deps.sh @@ -4,17 +4,13 @@ set -e orig_dir=$(pwd) -sudo apt-get update -qq -sudo apt-get install -qq python3-pip zlib1g-dev libjpeg-dev -pip3 install --user meson==0.43.0 -pip3 install --user pillow +pip3 install --user --upgrade pip wheel +pip3 install --user pillow meson==0.43.0 # ninja -git clone git://github.com/ninja-build/ninja.git -cd ninja -git checkout release -./configure.py --bootstrap -sudo cp ninja /bin +wget https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip +unzip ninja-linux.zip ninja +cp ninja $HOME/bin cd $orig_dir diff --git a/travis/upload_all_log.sh b/travis/upload_all_log.sh new file mode 100755 index 0000000..87c4838 --- /dev/null +++ b/travis/upload_all_log.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -e + +SSH_KEY=$(pwd)/travis/travisci_builder_id_key + +cd $HOME + +tar -czf fail_log_${PLATFORM}.tar.gz BUILD_${PLATFORM} + +scp -vrp -i ${SSH_KEY} \ + fail_log_${PLATFORM}.tar.gz \ + nightlybot@download.kiwix.org:/var/www/tmp.kiwix.org/ci