Merge pull request #130 from kiwix/faster_build

Faster build

.travis.yml

@@ -17,11 +17,13 @@ before_cache:
 cache:
   ccache: true
   directories:
+    - $HOME/.cache/pip
     - $HOME/.gradle/caches/
     - $HOME/.gradle/wrapper/
     - $HOME/.android/build-cache
 install: travis/install_extra_deps.sh
-script: travis_wait 30 travis/compile_all.py
+script: travis/compile_all.py
+after_failure: travis/upload_all_log.sh
 deploy:
   - provider: script
     skip_cleanup: true
@@ -74,3 +76,16 @@ addons:
       - ubuntu-toolchain-r-test
     packages:
       - g++-5
+      - cmake
+      - python3-pip
+      - zlib1g-dev
+      - libjpeg-dev
+      - libmagic-dev
+      - ctpp2-utils
+      - libctpp2-dev
+      - libmicrohttpd-dev
+      - g++-mingw-w64-i686
+      - gcc-mingw-w64-i686
+      - gcc-mingw-w64-base
+      - mingw-w64-tools
+      - default-jdk

dependencies.py

@@ -6,10 +6,12 @@ from dependency_utils import (
     ReleaseDownload,
     GitClone,
     SvnClone,
+    NoopSource,
     MakeBuilder,
     CMakeBuilder,
     MesonBuilder,
     GradleBuilder,
+    NoopBuilder,
     Builder as BaseBuilder)
 
 from utils import Remotefile, pj, SkipCommand, copy_tree, add_execution_right
@@ -31,7 +33,6 @@ from utils import Remotefile, pj, SkipCommand, copy_tree, add_execution_right
 
 class zlib(Dependency):
     name = 'zlib'
-    version = '1.2.8'
 
     class Source(ReleaseDownload):
         archive = Remotefile('zlib-1.2.8.tar.gz',
@@ -78,7 +79,6 @@ class zlib(Dependency):
 
 class lzma(Dependency):
     name = 'lzma'
-    version = '5.0.4'
 
     class Source(ReleaseDownload):
         archive = Remotefile('xz-5.0.4.tar.bz2',
@@ -91,7 +91,6 @@ class lzma(Dependency):
 
 class UUID(Dependency):
     name = 'uuid'
-    version = "1.43.4"
 
     class Source(ReleaseDownload):
         archive = Remotefile('e2fsprogs-libs-1.43.4.tar.gz',
@@ -110,7 +109,6 @@ class UUID(Dependency):
 
 class Xapian(Dependency):
     name = "xapian-core"
-    version = "1.4.5"
 
     class Source(ReleaseDownload):
         archive = Remotefile('xapian-core-1.4.5.tar.xz',
@@ -132,7 +130,6 @@ class Xapian(Dependency):
 
 class CTPP2(Dependency):
     name = "ctpp2"
-    version = "2.8.3"
 
     class Source(ReleaseDownload):
         name = "ctpp2"
@@ -175,7 +172,6 @@ class CTPP2C(CTPP2):
 
 class Pugixml(Dependency):
     name = "pugixml"
-    version = "1.2"
 
     class Source(ReleaseDownload):
         archive = Remotefile('pugixml-1.2.tar.gz',
@@ -187,7 +183,6 @@ class Pugixml(Dependency):
 
 class MicroHttpd(Dependency):
     name = "libmicrohttpd"
-    version = "0.9.46"
 
     class Source(ReleaseDownload):
         archive = Remotefile('libmicrohttpd-0.9.46.tar.gz',
@@ -200,7 +195,6 @@ class MicroHttpd(Dependency):
 
 class Gumbo(Dependency):
     name = "gumbo"
-    version = "0.10.1"
 
     class Source(ReleaseDownload):
         archive = Remotefile('gumbo-0.10.1.tar.gz',
@@ -217,7 +211,6 @@ class Gumbo(Dependency):
 
 class Icu(Dependency):
     name = "icu4c"
-    version = "58.2"
 
     class Source(SvnClone):
         name = "icu4c"
@@ -279,9 +272,9 @@ class Libzim(Dependency):
     class Source(GitClone):
         git_remote = "https://github.com/openzim/libzim.git"
         git_dir = "libzim"
-        release_git_ref = "3.0.0"
 
-    Builder = MesonBuilder
+    class Builder(MesonBuilder):
+        test_option = "-t 8"
 
 
 class ZimTools(Dependency):
@@ -291,7 +284,6 @@ class ZimTools(Dependency):
     class Source(GitClone):
         git_remote = "https://github.com/openzim/zim-tools.git"
         git_dir = "zim-tools"
-        release_git_ref = "0.0.1"
 
     class Builder(MesonBuilder):
         @property
@@ -343,7 +335,6 @@ class Kiwixlib(Dependency):
     class Source(GitClone):
         git_remote = "https://github.com/kiwix/kiwix-lib.git"
         git_dir = "kiwix-lib"
-        release_git_ref = "1.0.2"
 
     class Builder(MesonBuilder):
         @property
@@ -367,7 +358,6 @@ class KiwixTools(Dependency):
     class Source(GitClone):
         git_remote = "https://github.com/kiwix/kiwix-tools.git"
         git_dir = "kiwix-tools"
-        release_git_ref = "0.3.0"
 
     class Builder(MesonBuilder):
         @property
@@ -379,7 +369,6 @@ class KiwixTools(Dependency):
 
 class Gradle(Dependency):
     name = "Gradle"
-    version = "3.4"
 
     class Source(ReleaseDownload):
         archive = Remotefile('gradle-4.1-bin.zip',
@@ -400,6 +389,29 @@ class Gradle(Dependency):
                 pj(self.buildEnv.install_dir, "lib"))
 
 
+class AllBaseDependencies(Dependency):
+    name = "alldependencies"
+
+    @property
+    def dependencies(self):
+        base_deps = ['zlib', 'lzma', 'xapian-core', 'gumbo', 'pugixml', 'libmicrohttpd']
+        if self.buildEnv.platform_info.build != 'native':
+            base_deps += ["icu4c_cross-compile"]
+        else:
+            base_deps += ["icu4c"]
+        if ( self.buildEnv.platform_info.build != 'android'
+           and self.buildEnv.distname != 'Darwin'):
+            base_deps += ['ctpp2c', 'ctpp2']
+        if self.buildEnv.platform_info.build == 'android':
+            base_deps += ['Gradle']
+
+        return base_deps
+
+
+    Source = NoopSource
+    Builder = NoopBuilder
+
+
 class KiwixAndroid(Dependency):
     name = "kiwix-android"
     dependencies = ["Gradle", "kiwix-lib"]

dependency_utils.py

@@ -3,6 +3,7 @@ import os
 import shutil
 
 from utils import pj, Context, SkipCommand, extract_archive, Defaultdict, StopBuild
+from dependency_versions import main_project_versions, base_deps_versions
 
 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
 
@@ -20,7 +21,6 @@ class Dependency(metaclass=_MetaDependency):
     all_deps = {}
     dependencies = []
     force_native_build = False
-    version = None
 
     def __init__(self, buildEnv):
         self.buildEnv = buildEnv
@@ -28,6 +28,10 @@ class Dependency(metaclass=_MetaDependency):
         self.builder = self.Builder(self)
         self.skip = False
 
+    @property
+    def version(self):
+        return base_deps_versions.get(self.name, None)
+
     @property
     def full_name(self):
         if self.version:
@@ -94,6 +98,11 @@ class Source:
         return self.target.command(*args, **kwargs)
 
 
+class NoopSource(Source):
+    def prepare(self):
+        pass
+
+
 class ReleaseDownload(Source):
     archive_top_dir = None
 
@@ -124,7 +133,10 @@ class ReleaseDownload(Source):
 
 class GitClone(Source):
     base_git_ref = "master"
-    release_git_ref = "master"
+
+    @property
+    def release_git_ref(self):
+        return main_project_versions.get(self.name, "master")
 
     @property
     def source_dir(self):
@@ -236,6 +248,14 @@ class Builder:
         self.command('make_dist', self._make_dist)
 
 
+class NoopBuilder(Builder):
+    def build(self):
+        pass
+
+    def make_dist(self):
+        pass
+
+
 class MakeBuilder(Builder):
     configure_option = ""
     dynamic_configure_option = "--enable-shared --disable-static"
@@ -331,6 +351,7 @@ class CMakeBuilder(MakeBuilder):
 
 class MesonBuilder(Builder):
     configure_option = ""
+    test_option = ""
 
     @property
     def library_type(self):
@@ -372,7 +393,7 @@ class MesonBuilder(Builder):
                  and not self.buildEnv.platform_info.static)
            ):
             raise SkipCommand()
-        command = "{} --verbose".format(self.buildEnv.mesontest_command)
+        command = "{} --verbose {}".format(self.buildEnv.mesontest_command, self.test_option)
         self.buildEnv.run_command(command, self.build_path, context)
 
     def _install(self, context):

dependency_versions.py

@@ -0,0 +1,27 @@
+
+main_project_versions = {
+    'kiwix-lib': '1.0.2',
+    'kiwix-tools': '0.3.0',
+    'libzim': '3.0.0',
+    'zim-tools': '0.0.1',
+    'zimwriterfs': '1.1',
+}
+
+
+# This is the "version" of the whole base_deps_versions dict.
+# Change this when you change base_deps_versions.
+base_deps_meta_version = '0'
+
+
+base_deps_versions = {
+  'zlib' : '1.2.8',
+  'lzma' : '5.0.4',
+  'uuid' : '1.43.4',
+  'xapian-core' : '1.4.5',
+  'ctpp2' : '2.8.3',
+  'pugixml' : '1.2',
+  'libmicrohttpd' : '0.9.46',
+  'gumbo' : '0.10.1',
+  'icu4c' : '58.2',
+  'Gradle' : '3.4',
+}

travis/compile_all.py

@@ -8,6 +8,11 @@ from datetime import date
 import tarfile
 import subprocess
 import re
+from urllib.request import urlretrieve
+from urllib.error import URLError
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+import dependency_versions
 
 PLATFORM = environ['PLATFORM']
 
@@ -15,6 +20,9 @@ def home():
     return Path(os.path.expanduser('~'))
 
 BASE_DIR = home()/"BUILD_{}".format(PLATFORM)
+SOURCE_DIR = home()/"SOURCE"
+ARCHIVE_DIR = home()/"ARCHIVE"
+TOOLCHAINS_DIR = home()/"TOOLCHAINS"
 NIGHTLY_ARCHIVES_DIR = home()/'NIGHTLY_ARCHIVES'
 RELEASE_KIWIX_ARCHIVES_DIR = home()/'RELEASE_KIWIX_ARCHIVES'
 RELEASE_ZIM_ARCHIVES_DIR = home()/'RELEASE_ZIM_ARCHIVES'
@@ -22,14 +30,6 @@ DIST_KIWIX_ARCHIVES_DIR = home()/'DIST_KIWIX_ARCHIVES'
 DIST_ZIM_ARCHIVES_DIR = home()/'DIST_ZIM_ARCHIVES'
 SSH_KEY = Path(environ['TRAVIS_BUILD_DIR'])/'travis'/'travisci_builder_id_key'
 
-VERSIONS = {
-    'kiwix-lib': '1.0.2',
-    'kiwix-tools': '0.3.0',
-    'libzim': '3.0.0',
-    'zim-tools': '0.0.1',
-    'zimwriterfs': '1.1'
-}
-
 # We have build everything. Now create archives for public deployement.
 BINARIES = {
     'kiwix-tools': ('kiwix-install', 'kiwix-manage', 'kiwix-read', 'kiwix-search', 'kiwix-serve'),
@@ -66,6 +66,8 @@ def run_kiwix_build(target, platform, build_deps_only=False, make_release=False,
         command.append('--make-release')
     if make_dist:
         command.append('--make-dist')
+    print("--- Build {} (deps={}, release={}, dist={}) ---".format(
+        target, build_deps_only, make_release, make_dist), flush=True)
     subprocess.check_call(command, cwd=str(home()))
 
 
@@ -74,7 +76,7 @@ def make_archive(project, platform):
     if platform == "win32":
         file_to_archives = ['{}.exe'.format(f) for f in file_to_archives]
     if make_release:
-        postfix = VERSIONS[project]
+        postfix = dependency_versions.main_project_versions[project]
         if project in ('kiwix-lib', 'kiwix-tools'):
             archive_dir = RELEASE_KIWIX_ARCHIVES_DIR/project
         else:
@@ -94,8 +96,40 @@ def make_archive(project, platform):
             arch.add(str(base_bin_dir/f), arcname=str(f))
 
 
+def make_deps_archive(target, full=False):
+    (BASE_DIR/'.install_packages_ok').unlink()
+
+    archive_name = "deps_{}_{}.tar.gz".format(PLATFORM, target)
+    files_to_archive = [BASE_DIR/'INSTALL']
+    files_to_archive += BASE_DIR.glob('**/android-ndk*')
+    if (BASE_DIR/'meson_cross_file.txt').exists():
+        files_to_archive.append(BASE_DIR/'meson_cross_file.txt')
+
+    manifest_file = BASE_DIR/'manifest.txt'
+    write_manifest(manifest_file, archive_name, target, PLATFORM)
+    files_to_archive.append(manifest_file)
+
+    relative_path = BASE_DIR
+    if full:
+        files_to_archive += [ARCHIVE_DIR]
+        files_to_archive += BASE_DIR.glob('*/.*_ok')
+        files_to_archive += SOURCE_DIR.glob('*/.*_ok')
+        files_to_archive += [SOURCE_DIR/'pugixml-{}'.format(
+            dependency_versions.base_deps_versions['pugixml'])]
+        files_to_archive += [BASE_DIR/'pugixml-{}'.format(
+            dependency_versions.base_deps_versions['pugixml'])]
+        if (TOOLCHAINS_DIR).exists():
+            files_to_archive.append(TOOLCHAINS_DIR)
+        relative_path = home()
+
+    with tarfile.open(str(relative_path/archive_name), 'w:gz') as tar:
+        for name in files_to_archive:
+            tar.add(str(name), arcname=str(name.relative_to(relative_path)))
+    return relative_path/archive_name
+
+
 def scp(what, where):
-    command = ['scp', '-i', str(SSH_KEY), what, where]
+    command = ['scp', '-i', str(SSH_KEY), str(what), str(where)]
     subprocess.check_call(command)
 
 
@@ -111,6 +145,27 @@ for p in (NIGHTLY_ARCHIVES_DIR,
 
 make_release = re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", environ.get('TRAVIS_TAG', '')) is not None
 
+# The first thing we need to do is to (potentially) download already compiled base dependencies.
+BASE_DEP_VERSION = dependency_versions.base_deps_meta_version
+base_dep_archive_name = "base_deps_{}_{}.tar.gz".format(PLATFORM, BASE_DEP_VERSION)
+
+print("--- Getting archive {} ---".format(base_dep_archive_name), flush=True)
+try:
+    local_filename, headers = urlretrieve(
+        'http://tmp.kiwix.org/ci/{}'.format(base_dep_archive_name))
+    with tarfile.open(local_filename) as f:
+        f.extractall(str(home()))
+except URLError:
+    print("--- Cannot get archive. Build dependencies ---", flush=True)
+    run_kiwix_build('alldependencies', platform=PLATFORM)
+    archive = make_deps_archive('alldependencies', full=True)
+    destination = 'nightlybot@download.kiwix.org:/var/www/tmp.kiwix.org/ci/{}'
+    destination = destination.format(base_dep_archive_name)
+    scp(archive, destination)
+
+
+
+
 # A basic compilation to be sure everything is working (for a PR)
 if environ['TRAVIS_EVENT_TYPE'] != 'cron' and not make_release:
     if PLATFORM.startswith('android'):
@@ -143,21 +198,8 @@ for target in TARGETS:
         run_kiwix_build(target,
                         platform=PLATFORM,
                         build_deps_only=True)
-        (BASE_DIR/'.install_packages_ok').unlink()
+        archive = make_deps_archive(target)
-
+        scp(archive, 'nightlybot@download.kiwix.org:/var/www/tmp.kiwix.org/ci/')
-        archive_name = "deps_{}_{}.tar.gz".format(PLATFORM, target)
-        files_to_archive = [BASE_DIR/'INSTALL']
-        files_to_archive += BASE_DIR.glob('**/android-ndk*')
-        if (BASE_DIR/'meson_cross_file.txt').exists():
-            files_to_archive.append(BASE_DIR/'meson_cross_file.txt')
-
-        manifest_file = BASE_DIR/'manifest.txt'
-        write_manifest(manifest_file, archive_name, target, PLATFORM)
-        files_to_archive.append(manifest_file)
-        with tarfile.open(str(BASE_DIR/archive_name), 'w:gz') as tar:
-            for name in files_to_archive:
-                tar.add(str(name), arcname=str(name.relative_to(BASE_DIR)))
-        scp(str(BASE_DIR/archive_name), 'nightlybot@download.kiwix.org:/var/www/tmp.kiwix.org/ci/')
 
     run_kiwix_build(target,
                     platform=PLATFORM,
@@ -171,10 +213,6 @@ for target in TARGETS:
 
 
 # We have build everything. Now create archives for public deployement.
-kiwix_tools_postfix = VERSIONS['kiwix-tools'] if make_release else _date
-zim_tools_postfix = VERSIONS['zim-tools'] if make_release else _date
-zimwriterfs_postfix = VERSIONS['zimwriterfs'] if make_release else _date
-
 if make_release and PLATFORM == 'native_dyn':
     for target in TARGETS:
         if target in ('kiwix-lib', 'kiwix-tools'):
@@ -183,10 +221,14 @@ if make_release and PLATFORM == 'native_dyn':
             out_dir = DIST_ZIM_ARCHIVES_DIR
 
         if target in ('kiwix-lib', 'kiwix-tools', 'libzim', 'zim-tools'):
-            shutil.copy(str(BASE_DIR/target/'meson-dist'/'{}-{}.tar.xz'.format(target, VERSIONS[target])),
+            shutil.copy(str(BASE_DIR/target/'meson-dist'/'{}-{}.tar.xz'.format(
+                            target,
+                            dependency_versions.main_project_versions[target])),
                         str(out_dir))
         if target in ('zimwriterfs',):
-            shutil.copy(str(BASE_DIR/target/'{}-{}.tar.gz'.format(target, VERSIONS[target])),
+            shutil.copy(str(BASE_DIR/target/'{}-{}.tar.gz'.format(
+                            target,
+                            dependency_versions.main_project_versions[target])),
                         str(out_dir))
 elif PLATFORM == 'native_static':
     for target in ('kiwix-tools', 'zim-tools', 'zimwriterfs'):

travis/install_extra_deps.sh

@@ -4,17 +4,13 @@ set -e
 
 orig_dir=$(pwd)
 
-sudo apt-get update -qq
+pip3 install --user --upgrade pip wheel
-sudo apt-get install -qq python3-pip zlib1g-dev libjpeg-dev
+pip3 install --user pillow meson==0.43.0
-pip3 install --user meson==0.43.0
-pip3 install --user pillow
 
 # ninja
-git clone git://github.com/ninja-build/ninja.git
+wget https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip
-cd ninja
+unzip ninja-linux.zip ninja
-git checkout release
+cp ninja $HOME/bin
-./configure.py --bootstrap
-sudo cp ninja /bin
 
 cd $orig_dir
 

travis/upload_all_log.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -e
+
+SSH_KEY=$(pwd)/travis/travisci_builder_id_key
+
+cd $HOME
+
+tar -czf fail_log_${PLATFORM}.tar.gz BUILD_${PLATFORM}
+
+scp -vrp -i ${SSH_KEY} \
+  fail_log_${PLATFORM}.tar.gz \
+  nightlybot@download.kiwix.org:/var/www/tmp.kiwix.org/ci