From 4f2cc6bf928d4a6b1724396fb054263535908e7f Mon Sep 17 00:00:00 2001
From: Matthieu Gautier <mgautier@kymeria.fr>
Date: Mon, 19 Mar 2018 18:00:52 +0100
Subject: [PATCH] Make travis compile and store all compiled version of the
 base deps.

This should greatly improve travis speed.
---
 dependency_versions.py |  6 +++++
 travis/compile_all.py  | 50 ++++++++++++++++++++++++++++++++++++++----
 2 files changed, 52 insertions(+), 4 deletions(-)

diff --git a/dependency_versions.py b/dependency_versions.py
index 4283429..b0cb2e3 100644
--- a/dependency_versions.py
+++ b/dependency_versions.py
@@ -7,6 +7,12 @@ main_project_versions = {
     'zimwriterfs': '1.1',
 }
 
+
+# This is the "version" of the whole base_deps_versions dict.
+# Change this when you change base_deps_versions.
+base_deps_meta_version = '0'
+
+
 base_deps_versions = {
   'zlib' : '1.2.8',
   'lzma' : '5.0.4',
diff --git a/travis/compile_all.py b/travis/compile_all.py
index 6d603b5..1f068e5 100755
--- a/travis/compile_all.py
+++ b/travis/compile_all.py
@@ -8,6 +8,8 @@ from datetime import date
 import tarfile
 import subprocess
 import re
+from urllib.request import urlretrieve
+from urllib.error import URLError
 
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 import dependency_versions
@@ -18,6 +20,9 @@ def home():
     return Path(os.path.expanduser('~'))
 
 BASE_DIR = home()/"BUILD_{}".format(PLATFORM)
+SOURCE_DIR = home()/"SOURCE"
+ARCHIVE_DIR = home()/"ARCHIVE"
+TOOLCHAINS_DIR = home()/"TOOLCHAINS"
 NIGHTLY_ARCHIVES_DIR = home()/'NIGHTLY_ARCHIVES'
 RELEASE_KIWIX_ARCHIVES_DIR = home()/'RELEASE_KIWIX_ARCHIVES'
 RELEASE_ZIM_ARCHIVES_DIR = home()/'RELEASE_ZIM_ARCHIVES'
@@ -61,6 +66,8 @@ def run_kiwix_build(target, platform, build_deps_only=False, make_release=False,
         command.append('--make-release')
     if make_dist:
         command.append('--make-dist')
+    print("--- Build {} (deps={}, release={}, dist={}) ---".format(
+        target, build_deps_only, make_release, make_dist), flush=True)
     subprocess.check_call(command, cwd=str(home()))
 
 
@@ -89,7 +96,7 @@ def make_archive(project, platform):
             arch.add(str(base_bin_dir/f), arcname=str(f))
 
 
-def make_deps_archive(target):
+def make_deps_archive(target, full=False):
     (BASE_DIR/'.install_packages_ok').unlink()
 
     archive_name = "deps_{}_{}.tar.gz".format(PLATFORM, target)
@@ -101,10 +108,24 @@ def make_deps_archive(target):
     manifest_file = BASE_DIR/'manifest.txt'
     write_manifest(manifest_file, archive_name, target, PLATFORM)
     files_to_archive.append(manifest_file)
-    with tarfile.open(str(BASE_DIR/archive_name), 'w:gz') as tar:
+
+    relative_path = BASE_DIR
+    if full:
+        files_to_archive += [ARCHIVE_DIR]
+        files_to_archive += BASE_DIR.glob('*/.*_ok')
+        files_to_archive += SOURCE_DIR.glob('*/.*_ok')
+        files_to_archive += [SOURCE_DIR/'pugixml-{}'.format(
+            dependency_versions.base_deps_versions['pugixml'])]
+        files_to_archive += [BASE_DIR/'pugixml-{}'.format(
+            dependency_versions.base_deps_versions['pugixml'])]
+        if (TOOLCHAINS_DIR).exists():
+            files_to_archive.append(TOOLCHAINS_DIR)
+        relative_path = home()
+
+    with tarfile.open(str(relative_path/archive_name), 'w:gz') as tar:
         for name in files_to_archive:
-            tar.add(str(name), arcname=str(name.relative_to(BASE_DIR)))
-    return BASE_DIR/archive_name
+            tar.add(str(name), arcname=str(name.relative_to(relative_path)))
+    return relative_path/archive_name
 
 
 def scp(what, where):
@@ -124,6 +145,27 @@ for p in (NIGHTLY_ARCHIVES_DIR,
 
 make_release = re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", environ.get('TRAVIS_TAG', '')) is not None
 
+# The first thing we need to do is to (potentially) download already compiled base dependencies.
+BASE_DEP_VERSION = dependency_versions.base_deps_meta_version
+base_dep_archive_name = "base_deps_{}_{}.tar.gz".format(PLATFORM, BASE_DEP_VERSION)
+
+print("--- Getting archive {} ---".format(base_dep_archive_name), flush=True)
+try:
+    local_filename, headers = urlretrieve(
+        'http://tmp.kiwix.org/ci/{}'.format(base_dep_archive_name))
+    with tarfile.open(local_filename) as f:
+        f.extractall(str(home()))
+except URLError:
+    print("--- Cannot get archive. Build dependencies ---", flush=True)
+    run_kiwix_build('alldependencies', platform=PLATFORM)
+    archive = make_deps_archive('alldependencies', full=True)
+    destination = 'nightlybot@download.kiwix.org:/var/www/tmp.kiwix.org/ci/{}'
+    destination = destination.format(base_dep_archive_name)
+    scp(archive, destination)
+
+
+
+
 # A basic compilation to be sure everything is working (for a PR)
 if environ['TRAVIS_EVENT_TYPE'] != 'cron' and not make_release:
     if PLATFORM.startswith('android'):