From f0391a638bff558c95e528f5c9831df604219251 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Fri, 23 Feb 2018 15:24:47 +0100 Subject: [PATCH 01/12] Move projects' versions in a separated file. This way, versions can be imported by external scripts (ie travis) --- dependencies.py | 14 -------------- dependency_utils.py | 11 +++++++++-- dependency_versions.py | 21 +++++++++++++++++++++ 3 files changed, 30 insertions(+), 16 deletions(-) create mode 100644 dependency_versions.py diff --git a/dependencies.py b/dependencies.py index b29a698..66d9f05 100644 --- a/dependencies.py +++ b/dependencies.py @@ -31,7 +31,6 @@ from utils import Remotefile, pj, SkipCommand, copy_tree, add_execution_right class zlib(Dependency): name = 'zlib' - version = '1.2.8' class Source(ReleaseDownload): archive = Remotefile('zlib-1.2.8.tar.gz', @@ -78,7 +77,6 @@ class zlib(Dependency): class lzma(Dependency): name = 'lzma' - version = '5.0.4' class Source(ReleaseDownload): archive = Remotefile('xz-5.0.4.tar.bz2', @@ -91,7 +89,6 @@ class lzma(Dependency): class UUID(Dependency): name = 'uuid' - version = "1.43.4" class Source(ReleaseDownload): archive = Remotefile('e2fsprogs-libs-1.43.4.tar.gz', @@ -110,7 +107,6 @@ class UUID(Dependency): class Xapian(Dependency): name = "xapian-core" - version = "1.4.5" class Source(ReleaseDownload): archive = Remotefile('xapian-core-1.4.5.tar.xz', @@ -132,7 +128,6 @@ class Xapian(Dependency): class CTPP2(Dependency): name = "ctpp2" - version = "2.8.3" class Source(ReleaseDownload): name = "ctpp2" @@ -175,7 +170,6 @@ class CTPP2C(CTPP2): class Pugixml(Dependency): name = "pugixml" - version = "1.2" class Source(ReleaseDownload): archive = Remotefile('pugixml-1.2.tar.gz', @@ -187,7 +181,6 @@ class Pugixml(Dependency): class MicroHttpd(Dependency): name = "libmicrohttpd" - version = "0.9.46" class Source(ReleaseDownload): archive = Remotefile('libmicrohttpd-0.9.46.tar.gz', @@ -200,7 +193,6 @@ class MicroHttpd(Dependency): class Gumbo(Dependency): name = "gumbo" - version = "0.10.1" class Source(ReleaseDownload): archive = Remotefile('gumbo-0.10.1.tar.gz', @@ -217,7 +209,6 @@ class Gumbo(Dependency): class Icu(Dependency): name = "icu4c" - version = "58.2" class Source(SvnClone): name = "icu4c" @@ -279,7 +270,6 @@ class Libzim(Dependency): class Source(GitClone): git_remote = "https://github.com/openzim/libzim.git" git_dir = "libzim" - release_git_ref = "3.0.0" Builder = MesonBuilder @@ -291,7 +281,6 @@ class ZimTools(Dependency): class Source(GitClone): git_remote = "https://github.com/openzim/zim-tools.git" git_dir = "zim-tools" - release_git_ref = "0.0.1" class Builder(MesonBuilder): @property @@ -343,7 +332,6 @@ class Kiwixlib(Dependency): class Source(GitClone): git_remote = "https://github.com/kiwix/kiwix-lib.git" git_dir = "kiwix-lib" - release_git_ref = "1.0.2" class Builder(MesonBuilder): @property @@ -367,7 +355,6 @@ class KiwixTools(Dependency): class Source(GitClone): git_remote = "https://github.com/kiwix/kiwix-tools.git" git_dir = "kiwix-tools" - release_git_ref = "0.3.0" class Builder(MesonBuilder): @property @@ -379,7 +366,6 @@ class KiwixTools(Dependency): class Gradle(Dependency): name = "Gradle" - version = "3.4" class Source(ReleaseDownload): archive = Remotefile('gradle-4.1-bin.zip', diff --git a/dependency_utils.py b/dependency_utils.py index 5326bd4..53a18aa 100644 --- a/dependency_utils.py +++ b/dependency_utils.py @@ -3,6 +3,7 @@ import os import shutil from utils import pj, Context, SkipCommand, extract_archive, Defaultdict, StopBuild +from dependency_versions import main_project_versions, base_deps_versions SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -20,7 +21,6 @@ class Dependency(metaclass=_MetaDependency): all_deps = {} dependencies = [] force_native_build = False - version = None def __init__(self, buildEnv): self.buildEnv = buildEnv @@ -28,6 +28,10 @@ class Dependency(metaclass=_MetaDependency): self.builder = self.Builder(self) self.skip = False + @property + def version(self): + return base_deps_versions.get(self.name, None) + @property def full_name(self): if self.version: @@ -124,7 +128,10 @@ class ReleaseDownload(Source): class GitClone(Source): base_git_ref = "master" - release_git_ref = "master" + + @property + def release_git_ref(self): + return main_project_versions.get(self.name, "master") @property def source_dir(self): diff --git a/dependency_versions.py b/dependency_versions.py new file mode 100644 index 0000000..4283429 --- /dev/null +++ b/dependency_versions.py @@ -0,0 +1,21 @@ + +main_project_versions = { + 'kiwix-lib': '1.0.2', + 'kiwix-tools': '0.3.0', + 'libzim': '3.0.0', + 'zim-tools': '0.0.1', + 'zimwriterfs': '1.1', +} + +base_deps_versions = { + 'zlib' : '1.2.8', + 'lzma' : '5.0.4', + 'uuid' : '1.43.4', + 'xapian-core' : '1.4.5', + 'ctpp2' : '2.8.3', + 'pugixml' : '1.2', + 'libmicrohttpd' : '0.9.46', + 'gumbo' : '0.10.1', + 'icu4c' : '58.2', + 'Gradle' : '3.4', +} From f1c15a5d3d3c7362fb23a51589b20067168e78f7 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Fri, 16 Mar 2018 10:43:56 +0100 Subject: [PATCH 02/12] Reuse the versions in dependency_versions.py instead of redefine them. --- travis/compile_all.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/travis/compile_all.py b/travis/compile_all.py index 0e4893f..f8c7744 100755 --- a/travis/compile_all.py +++ b/travis/compile_all.py @@ -9,6 +9,9 @@ import tarfile import subprocess import re +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import dependency_versions + PLATFORM = environ['PLATFORM'] def home(): @@ -22,14 +25,6 @@ DIST_KIWIX_ARCHIVES_DIR = home()/'DIST_KIWIX_ARCHIVES' DIST_ZIM_ARCHIVES_DIR = home()/'DIST_ZIM_ARCHIVES' SSH_KEY = Path(environ['TRAVIS_BUILD_DIR'])/'travis'/'travisci_builder_id_key' -VERSIONS = { - 'kiwix-lib': '1.0.2', - 'kiwix-tools': '0.3.0', - 'libzim': '3.0.0', - 'zim-tools': '0.0.1', - 'zimwriterfs': '1.1' -} - # We have build everything. Now create archives for public deployement. BINARIES = { 'kiwix-tools': ('kiwix-install', 'kiwix-manage', 'kiwix-read', 'kiwix-search', 'kiwix-serve'), @@ -74,7 +69,7 @@ def make_archive(project, platform): if platform == "win32": file_to_archives = ['{}.exe'.format(f) for f in file_to_archives] if make_release: - postfix = VERSIONS[project] + postfix = dependency_versions.main_project_versions[project] if project in ('kiwix-lib', 'kiwix-tools'): archive_dir = RELEASE_KIWIX_ARCHIVES_DIR/project else: @@ -171,10 +166,6 @@ for target in TARGETS: # We have build everything. Now create archives for public deployement. -kiwix_tools_postfix = VERSIONS['kiwix-tools'] if make_release else _date -zim_tools_postfix = VERSIONS['zim-tools'] if make_release else _date -zimwriterfs_postfix = VERSIONS['zimwriterfs'] if make_release else _date - if make_release and PLATFORM == 'native_dyn': for target in TARGETS: if target in ('kiwix-lib', 'kiwix-tools'): @@ -183,10 +174,14 @@ if make_release and PLATFORM == 'native_dyn': out_dir = DIST_ZIM_ARCHIVES_DIR if target in ('kiwix-lib', 'kiwix-tools', 'libzim', 'zim-tools'): - shutil.copy(str(BASE_DIR/target/'meson-dist'/'{}-{}.tar.xz'.format(target, VERSIONS[target])), + shutil.copy(str(BASE_DIR/target/'meson-dist'/'{}-{}.tar.xz'.format( + target, + dependency_versions.main_project_versions[target])), str(out_dir)) if target in ('zimwriterfs',): - shutil.copy(str(BASE_DIR/target/'{}-{}.tar.gz'.format(target, VERSIONS[target])), + shutil.copy(str(BASE_DIR/target/'{}-{}.tar.gz'.format( + target, + dependency_versions.main_project_versions[target])), str(out_dir)) elif PLATFORM == 'native_static': for target in ('kiwix-tools', 'zim-tools', 'zimwriterfs'): From e0171379c79913bb39861167dd879208fb605629 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 19 Mar 2018 17:50:17 +0100 Subject: [PATCH 03/12] Add an intermediate target that build all base dependencies. This will allow travis to build all base dependencies all by once. --- dependencies.py | 25 +++++++++++++++++++++++++ dependency_utils.py | 13 +++++++++++++ 2 files changed, 38 insertions(+) diff --git a/dependencies.py b/dependencies.py index 66d9f05..6ada955 100644 --- a/dependencies.py +++ b/dependencies.py @@ -6,10 +6,12 @@ from dependency_utils import ( ReleaseDownload, GitClone, SvnClone, + NoopSource, MakeBuilder, CMakeBuilder, MesonBuilder, GradleBuilder, + NoopBuilder, Builder as BaseBuilder) from utils import Remotefile, pj, SkipCommand, copy_tree, add_execution_right @@ -386,6 +388,29 @@ class Gradle(Dependency): pj(self.buildEnv.install_dir, "lib")) +class AllBaseDependencies(Dependency): + name = "alldependencies" + + @property + def dependencies(self): + base_deps = ['zlib', 'lzma', 'xapian-core', 'gumbo', 'pugixml', 'libmicrohttpd'] + if self.buildEnv.platform_info.build != 'native': + base_deps += ["icu4c_cross-compile"] + else: + base_deps += ["icu4c"] + if ( self.buildEnv.platform_info.build != 'android' + and self.buildEnv.distname != 'Darwin'): + base_deps += ['ctpp2c', 'ctpp2'] + if self.buildEnv.platform_info.build == 'android': + base_deps += ['Gradle'] + + return base_deps + + + Source = NoopSource + Builder = NoopBuilder + + class KiwixAndroid(Dependency): name = "kiwix-android" dependencies = ["Gradle", "kiwix-lib"] diff --git a/dependency_utils.py b/dependency_utils.py index 53a18aa..2670a1a 100644 --- a/dependency_utils.py +++ b/dependency_utils.py @@ -98,6 +98,11 @@ class Source: return self.target.command(*args, **kwargs) +class NoopSource(Source): + def prepare(self): + pass + + class ReleaseDownload(Source): archive_top_dir = None @@ -243,6 +248,14 @@ class Builder: self.command('make_dist', self._make_dist) +class NoopBuilder(Builder): + def build(self): + pass + + def make_dist(self): + pass + + class MakeBuilder(Builder): configure_option = "" dynamic_configure_option = "--enable-shared --disable-static" From df512467532f38e12396e26e0f561e94cc4f8de4 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 19 Mar 2018 17:53:02 +0100 Subject: [PATCH 04/12] Move creation of dependencies archive into a separated function. --- travis/compile_all.py | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/travis/compile_all.py b/travis/compile_all.py index f8c7744..5a3fd17 100755 --- a/travis/compile_all.py +++ b/travis/compile_all.py @@ -89,6 +89,24 @@ def make_archive(project, platform): arch.add(str(base_bin_dir/f), arcname=str(f)) +def make_deps_archive(target): + (BASE_DIR/'.install_packages_ok').unlink() + + archive_name = "deps_{}_{}.tar.gz".format(PLATFORM, target) + files_to_archive = [BASE_DIR/'INSTALL'] + files_to_archive += BASE_DIR.glob('**/android-ndk*') + if (BASE_DIR/'meson_cross_file.txt').exists(): + files_to_archive.append(BASE_DIR/'meson_cross_file.txt') + + manifest_file = BASE_DIR/'manifest.txt' + write_manifest(manifest_file, archive_name, target, PLATFORM) + files_to_archive.append(manifest_file) + with tarfile.open(str(BASE_DIR/archive_name), 'w:gz') as tar: + for name in files_to_archive: + tar.add(str(name), arcname=str(name.relative_to(BASE_DIR))) + return BASE_DIR/archive_name + + def scp(what, where): command = ['scp', '-i', str(SSH_KEY), what, where] subprocess.check_call(command) @@ -138,21 +156,8 @@ for target in TARGETS: run_kiwix_build(target, platform=PLATFORM, build_deps_only=True) - (BASE_DIR/'.install_packages_ok').unlink() - - archive_name = "deps_{}_{}.tar.gz".format(PLATFORM, target) - files_to_archive = [BASE_DIR/'INSTALL'] - files_to_archive += BASE_DIR.glob('**/android-ndk*') - if (BASE_DIR/'meson_cross_file.txt').exists(): - files_to_archive.append(BASE_DIR/'meson_cross_file.txt') - - manifest_file = BASE_DIR/'manifest.txt' - write_manifest(manifest_file, archive_name, target, PLATFORM) - files_to_archive.append(manifest_file) - with tarfile.open(str(BASE_DIR/archive_name), 'w:gz') as tar: - for name in files_to_archive: - tar.add(str(name), arcname=str(name.relative_to(BASE_DIR))) - scp(str(BASE_DIR/archive_name), 'nightlybot@download.kiwix.org:/var/www/tmp.kiwix.org/ci/') + archive = make_deps_archive(target) + scp(str(archive), 'nightlybot@download.kiwix.org:/var/www/tmp.kiwix.org/ci/') run_kiwix_build(target, platform=PLATFORM, From 6cc4c43ffa17673f594134d73be20cf61975f8ac Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 19 Mar 2018 17:54:06 +0100 Subject: [PATCH 05/12] scp convert its arguments to str itself. --- travis/compile_all.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/travis/compile_all.py b/travis/compile_all.py index 5a3fd17..6d603b5 100755 --- a/travis/compile_all.py +++ b/travis/compile_all.py @@ -108,7 +108,7 @@ def make_deps_archive(target): def scp(what, where): - command = ['scp', '-i', str(SSH_KEY), what, where] + command = ['scp', '-i', str(SSH_KEY), str(what), str(where)] subprocess.check_call(command) @@ -157,7 +157,7 @@ for target in TARGETS: platform=PLATFORM, build_deps_only=True) archive = make_deps_archive(target) - scp(str(archive), 'nightlybot@download.kiwix.org:/var/www/tmp.kiwix.org/ci/') + scp(archive, 'nightlybot@download.kiwix.org:/var/www/tmp.kiwix.org/ci/') run_kiwix_build(target, platform=PLATFORM, From e95456dea097fd3203e420b35db7bd814bc73d52 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 19 Mar 2018 20:22:00 +0100 Subject: [PATCH 06/12] Remove travis_wait to get full log. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ebd8fc0..de4569b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ cache: - $HOME/.gradle/wrapper/ - $HOME/.android/build-cache install: travis/install_extra_deps.sh -script: travis_wait 30 travis/compile_all.py +script: travis/compile_all.py deploy: - provider: script skip_cleanup: true From 4f2cc6bf928d4a6b1724396fb054263535908e7f Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 19 Mar 2018 18:00:52 +0100 Subject: [PATCH 07/12] Make travis compile and store all compiled version of the base deps. This should greatly improve travis speed. --- dependency_versions.py | 6 +++++ travis/compile_all.py | 50 ++++++++++++++++++++++++++++++++++++++---- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/dependency_versions.py b/dependency_versions.py index 4283429..b0cb2e3 100644 --- a/dependency_versions.py +++ b/dependency_versions.py @@ -7,6 +7,12 @@ main_project_versions = { 'zimwriterfs': '1.1', } + +# This is the "version" of the whole base_deps_versions dict. +# Change this when you change base_deps_versions. +base_deps_meta_version = '0' + + base_deps_versions = { 'zlib' : '1.2.8', 'lzma' : '5.0.4', diff --git a/travis/compile_all.py b/travis/compile_all.py index 6d603b5..1f068e5 100755 --- a/travis/compile_all.py +++ b/travis/compile_all.py @@ -8,6 +8,8 @@ from datetime import date import tarfile import subprocess import re +from urllib.request import urlretrieve +from urllib.error import URLError sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import dependency_versions @@ -18,6 +20,9 @@ def home(): return Path(os.path.expanduser('~')) BASE_DIR = home()/"BUILD_{}".format(PLATFORM) +SOURCE_DIR = home()/"SOURCE" +ARCHIVE_DIR = home()/"ARCHIVE" +TOOLCHAINS_DIR = home()/"TOOLCHAINS" NIGHTLY_ARCHIVES_DIR = home()/'NIGHTLY_ARCHIVES' RELEASE_KIWIX_ARCHIVES_DIR = home()/'RELEASE_KIWIX_ARCHIVES' RELEASE_ZIM_ARCHIVES_DIR = home()/'RELEASE_ZIM_ARCHIVES' @@ -61,6 +66,8 @@ def run_kiwix_build(target, platform, build_deps_only=False, make_release=False, command.append('--make-release') if make_dist: command.append('--make-dist') + print("--- Build {} (deps={}, release={}, dist={}) ---".format( + target, build_deps_only, make_release, make_dist), flush=True) subprocess.check_call(command, cwd=str(home())) @@ -89,7 +96,7 @@ def make_archive(project, platform): arch.add(str(base_bin_dir/f), arcname=str(f)) -def make_deps_archive(target): +def make_deps_archive(target, full=False): (BASE_DIR/'.install_packages_ok').unlink() archive_name = "deps_{}_{}.tar.gz".format(PLATFORM, target) @@ -101,10 +108,24 @@ def make_deps_archive(target): manifest_file = BASE_DIR/'manifest.txt' write_manifest(manifest_file, archive_name, target, PLATFORM) files_to_archive.append(manifest_file) - with tarfile.open(str(BASE_DIR/archive_name), 'w:gz') as tar: + + relative_path = BASE_DIR + if full: + files_to_archive += [ARCHIVE_DIR] + files_to_archive += BASE_DIR.glob('*/.*_ok') + files_to_archive += SOURCE_DIR.glob('*/.*_ok') + files_to_archive += [SOURCE_DIR/'pugixml-{}'.format( + dependency_versions.base_deps_versions['pugixml'])] + files_to_archive += [BASE_DIR/'pugixml-{}'.format( + dependency_versions.base_deps_versions['pugixml'])] + if (TOOLCHAINS_DIR).exists(): + files_to_archive.append(TOOLCHAINS_DIR) + relative_path = home() + + with tarfile.open(str(relative_path/archive_name), 'w:gz') as tar: for name in files_to_archive: - tar.add(str(name), arcname=str(name.relative_to(BASE_DIR))) - return BASE_DIR/archive_name + tar.add(str(name), arcname=str(name.relative_to(relative_path))) + return relative_path/archive_name def scp(what, where): @@ -124,6 +145,27 @@ for p in (NIGHTLY_ARCHIVES_DIR, make_release = re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", environ.get('TRAVIS_TAG', '')) is not None +# The first thing we need to do is to (potentially) download already compiled base dependencies. +BASE_DEP_VERSION = dependency_versions.base_deps_meta_version +base_dep_archive_name = "base_deps_{}_{}.tar.gz".format(PLATFORM, BASE_DEP_VERSION) + +print("--- Getting archive {} ---".format(base_dep_archive_name), flush=True) +try: + local_filename, headers = urlretrieve( + 'http://tmp.kiwix.org/ci/{}'.format(base_dep_archive_name)) + with tarfile.open(local_filename) as f: + f.extractall(str(home())) +except URLError: + print("--- Cannot get archive. Build dependencies ---", flush=True) + run_kiwix_build('alldependencies', platform=PLATFORM) + archive = make_deps_archive('alldependencies', full=True) + destination = 'nightlybot@download.kiwix.org:/var/www/tmp.kiwix.org/ci/{}' + destination = destination.format(base_dep_archive_name) + scp(archive, destination) + + + + # A basic compilation to be sure everything is working (for a PR) if environ['TRAVIS_EVENT_TYPE'] != 'cron' and not make_release: if PLATFORM.startswith('android'): From d93fc9dc9c156ac5934b020c8194361d9a608c47 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 19 Mar 2018 18:39:03 +0100 Subject: [PATCH 08/12] Also cache pip in travis. --- .travis.yml | 1 + travis/install_extra_deps.sh | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index de4569b..ba7956f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,7 @@ before_cache: cache: ccache: true directories: + - $HOME/.cache/pip - $HOME/.gradle/caches/ - $HOME/.gradle/wrapper/ - $HOME/.android/build-cache diff --git a/travis/install_extra_deps.sh b/travis/install_extra_deps.sh index 987c2d0..1d7c544 100755 --- a/travis/install_extra_deps.sh +++ b/travis/install_extra_deps.sh @@ -6,8 +6,8 @@ orig_dir=$(pwd) sudo apt-get update -qq sudo apt-get install -qq python3-pip zlib1g-dev libjpeg-dev -pip3 install --user meson==0.43.0 -pip3 install --user pillow +pip3 install --user --upgrade pip wheel +pip3 install --user pillow meson==0.43.0 # ninja git clone git://github.com/ninja-build/ninja.git From c54439831daaf8261914213378c0162090c8f47b Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 19 Mar 2018 20:10:23 +0100 Subject: [PATCH 09/12] Use already compiled ninja binary instead of building one. --- travis/install_extra_deps.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/travis/install_extra_deps.sh b/travis/install_extra_deps.sh index 1d7c544..997f11a 100755 --- a/travis/install_extra_deps.sh +++ b/travis/install_extra_deps.sh @@ -10,10 +10,8 @@ pip3 install --user --upgrade pip wheel pip3 install --user pillow meson==0.43.0 # ninja -git clone git://github.com/ninja-build/ninja.git -cd ninja -git checkout release -./configure.py --bootstrap +wget https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip +unzip ninja-linux.zip ninja sudo cp ninja /bin cd $orig_dir From 0d18fbfd3dcb65c1e4fa293e2ba2a74492275b2e Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 20 Mar 2018 13:36:05 +0100 Subject: [PATCH 10/12] Add a longer timeout for meson test. --- dependencies.py | 3 ++- dependency_utils.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dependencies.py b/dependencies.py index 6ada955..7eb0e57 100644 --- a/dependencies.py +++ b/dependencies.py @@ -273,7 +273,8 @@ class Libzim(Dependency): git_remote = "https://github.com/openzim/libzim.git" git_dir = "libzim" - Builder = MesonBuilder + class Builder(MesonBuilder): + test_option = "-t 8" class ZimTools(Dependency): diff --git a/dependency_utils.py b/dependency_utils.py index 2670a1a..d218974 100644 --- a/dependency_utils.py +++ b/dependency_utils.py @@ -351,6 +351,7 @@ class CMakeBuilder(MakeBuilder): class MesonBuilder(Builder): configure_option = "" + test_option = "" @property def library_type(self): @@ -392,7 +393,7 @@ class MesonBuilder(Builder): and not self.buildEnv.platform_info.static) ): raise SkipCommand() - command = "{} --verbose".format(self.buildEnv.mesontest_command) + command = "{} --verbose {}".format(self.buildEnv.mesontest_command, self.test_option) self.buildEnv.run_command(command, self.build_path, context) def _install(self, context): From a7f1da3561322ff70ad99c80bf0e8c9bb4e78a9b Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 20 Mar 2018 14:05:30 +0100 Subject: [PATCH 11/12] Upload build log on failure. --- .travis.yml | 1 + travis/upload_all_log.sh | 13 +++++++++++++ 2 files changed, 14 insertions(+) create mode 100755 travis/upload_all_log.sh diff --git a/.travis.yml b/.travis.yml index ba7956f..0f5f38f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,7 @@ cache: - $HOME/.android/build-cache install: travis/install_extra_deps.sh script: travis/compile_all.py +after_failure: travis/upload_all_log.sh deploy: - provider: script skip_cleanup: true diff --git a/travis/upload_all_log.sh b/travis/upload_all_log.sh new file mode 100755 index 0000000..87c4838 --- /dev/null +++ b/travis/upload_all_log.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -e + +SSH_KEY=$(pwd)/travis/travisci_builder_id_key + +cd $HOME + +tar -czf fail_log_${PLATFORM}.tar.gz BUILD_${PLATFORM} + +scp -vrp -i ${SSH_KEY} \ + fail_log_${PLATFORM}.tar.gz \ + nightlybot@download.kiwix.org:/var/www/tmp.kiwix.org/ci From 752a5d58ce614a12153a6fb9662bc8376dfd21aa Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 20 Mar 2018 10:12:51 +0100 Subject: [PATCH 12/12] Do not do sudo command. - Install all packages using travis apt plugin - Copy ninja binary in $HOME/bin. We still need sudo required because travis image has more memory in this case and this is needed to test big cluster in libzim. --- .travis.yml | 13 +++++++++++++ travis/install_extra_deps.sh | 4 +--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0f5f38f..b0328b1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -76,3 +76,16 @@ addons: - ubuntu-toolchain-r-test packages: - g++-5 + - cmake + - python3-pip + - zlib1g-dev + - libjpeg-dev + - libmagic-dev + - ctpp2-utils + - libctpp2-dev + - libmicrohttpd-dev + - g++-mingw-w64-i686 + - gcc-mingw-w64-i686 + - gcc-mingw-w64-base + - mingw-w64-tools + - default-jdk diff --git a/travis/install_extra_deps.sh b/travis/install_extra_deps.sh index 997f11a..107021a 100755 --- a/travis/install_extra_deps.sh +++ b/travis/install_extra_deps.sh @@ -4,15 +4,13 @@ set -e orig_dir=$(pwd) -sudo apt-get update -qq -sudo apt-get install -qq python3-pip zlib1g-dev libjpeg-dev pip3 install --user --upgrade pip wheel pip3 install --user pillow meson==0.43.0 # ninja wget https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip unzip ninja-linux.zip ninja -sudo cp ninja /bin +cp ninja $HOME/bin cd $orig_dir