smallsolar
/
kiwix-build
mirror of https://github.com/kiwix/kiwix-build.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #130 from kiwix/faster_build 

Faster build
This commit is contained in:
Matthieu Gautier 2018-03-20 16:47:13 +01:00 committed by GitHub
parent e8c220c17d 752a5d58ce
commit c04bfaba14
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 185 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
.travis.yml
View File

@ -17,11 +17,13 @@ before_cache:
cache:
ccache: true
directories:
- $HOME/.cache/pip
- $HOME/.gradle/caches/
- $HOME/.gradle/wrapper/
- $HOME/.android/build-cache
install: travis/install_extra_deps.sh
script: travis_wait 30 travis/compile_all.py
script: travis/compile_all.py
after_failure: travis/upload_all_log.sh
deploy:
- provider: script
skip_cleanup: true
@ -74,3 +76,16 @@ addons:
- ubuntu-toolchain-r-test
packages:
- g++-5
- cmake
- python3-pip
- zlib1g-dev
- libjpeg-dev
- libmagic-dev
- ctpp2-utils
- libctpp2-dev
- libmicrohttpd-dev
- g++-mingw-w64-i686
- gcc-mingw-w64-i686
- gcc-mingw-w64-base
- mingw-w64-tools
- default-jdk

42
dependencies.py
View File

@ -6,10 +6,12 @@ from dependency_utils import (
ReleaseDownload,
GitClone,
SvnClone,
NoopSource,
MakeBuilder,
CMakeBuilder,
MesonBuilder,
GradleBuilder,
NoopBuilder,
Builder as BaseBuilder)
from utils import Remotefile, pj, SkipCommand, copy_tree, add_execution_right
@ -31,7 +33,6 @@ from utils import Remotefile, pj, SkipCommand, copy_tree, add_execution_right
class zlib(Dependency):
name = 'zlib'
version = '1.2.8'
class Source(ReleaseDownload):
archive = Remotefile('zlib-1.2.8.tar.gz',
@ -78,7 +79,6 @@ class zlib(Dependency):
class lzma(Dependency):
name = 'lzma'
version = '5.0.4'
class Source(ReleaseDownload):
archive = Remotefile('xz-5.0.4.tar.bz2',
@ -91,7 +91,6 @@ class lzma(Dependency):
class UUID(Dependency):
name = 'uuid'
version = "1.43.4"
class Source(ReleaseDownload):
archive = Remotefile('e2fsprogs-libs-1.43.4.tar.gz',
@ -110,7 +109,6 @@ class UUID(Dependency):
class Xapian(Dependency):
name = "xapian-core"
version = "1.4.5"
class Source(ReleaseDownload):
archive = Remotefile('xapian-core-1.4.5.tar.xz',
@ -132,7 +130,6 @@ class Xapian(Dependency):
class CTPP2(Dependency):
name = "ctpp2"
version = "2.8.3"
class Source(ReleaseDownload):
name = "ctpp2"
@ -175,7 +172,6 @@ class CTPP2C(CTPP2):
class Pugixml(Dependency):
name = "pugixml"
version = "1.2"
class Source(ReleaseDownload):
archive = Remotefile('pugixml-1.2.tar.gz',
@ -187,7 +183,6 @@ class Pugixml(Dependency):
class MicroHttpd(Dependency):
name = "libmicrohttpd"
version = "0.9.46"
class Source(ReleaseDownload):
archive = Remotefile('libmicrohttpd-0.9.46.tar.gz',
@ -200,7 +195,6 @@ class MicroHttpd(Dependency):
class Gumbo(Dependency):
name = "gumbo"
version = "0.10.1"
class Source(ReleaseDownload):
archive = Remotefile('gumbo-0.10.1.tar.gz',
@ -217,7 +211,6 @@ class Gumbo(Dependency):
class Icu(Dependency):
name = "icu4c"
version = "58.2"
class Source(SvnClone):
name = "icu4c"
@ -279,9 +272,9 @@ class Libzim(Dependency):
class Source(GitClone):
git_remote = "https://github.com/openzim/libzim.git"
git_dir = "libzim"
release_git_ref = "3.0.0"
Builder = MesonBuilder
class Builder(MesonBuilder):
test_option = "-t 8"
class ZimTools(Dependency):
@ -291,7 +284,6 @@ class ZimTools(Dependency):
class Source(GitClone):
git_remote = "https://github.com/openzim/zim-tools.git"
git_dir = "zim-tools"
release_git_ref = "0.0.1"
class Builder(MesonBuilder):
@property
@ -343,7 +335,6 @@ class Kiwixlib(Dependency):
class Source(GitClone):
git_remote = "https://github.com/kiwix/kiwix-lib.git"
git_dir = "kiwix-lib"
release_git_ref = "1.0.2"
class Builder(MesonBuilder):
@property
@ -367,7 +358,6 @@ class KiwixTools(Dependency):
class Source(GitClone):
git_remote = "https://github.com/kiwix/kiwix-tools.git"
git_dir = "kiwix-tools"
release_git_ref = "0.3.0"
class Builder(MesonBuilder):
@property
@ -379,7 +369,6 @@ class KiwixTools(Dependency):
class Gradle(Dependency):
name = "Gradle"
version = "3.4"
class Source(ReleaseDownload):
archive = Remotefile('gradle-4.1-bin.zip',
@ -400,6 +389,29 @@ class Gradle(Dependency):
pj(self.buildEnv.install_dir, "lib"))
class AllBaseDependencies(Dependency):
name = "alldependencies"
@property
def dependencies(self):
base_deps = ['zlib', 'lzma', 'xapian-core', 'gumbo', 'pugixml', 'libmicrohttpd']
if self.buildEnv.platform_info.build != 'native':
base_deps += ["icu4c_cross-compile"]
else:
base_deps += ["icu4c"]
if ( self.buildEnv.platform_info.build != 'android'
and self.buildEnv.distname != 'Darwin'):
base_deps += ['ctpp2c', 'ctpp2']
if self.buildEnv.platform_info.build == 'android':
base_deps += ['Gradle']
return base_deps
Source = NoopSource
Builder = NoopBuilder
class KiwixAndroid(Dependency):
name = "kiwix-android"
dependencies = ["Gradle", "kiwix-lib"]

27
dependency_utils.py
View File

@ -3,6 +3,7 @@ import os
import shutil
from utils import pj, Context, SkipCommand, extract_archive, Defaultdict, StopBuild
from dependency_versions import main_project_versions, base_deps_versions
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
@ -20,7 +21,6 @@ class Dependency(metaclass=_MetaDependency):
all_deps = {}
dependencies = []
force_native_build = False
version = None
def __init__(self, buildEnv):
self.buildEnv = buildEnv
@ -28,6 +28,10 @@ class Dependency(metaclass=_MetaDependency):
self.builder = self.Builder(self)
self.skip = False
@property
def version(self):
return base_deps_versions.get(self.name, None)
@property
def full_name(self):
if self.version:
@ -94,6 +98,11 @@ class Source:
return self.target.command(*args, **kwargs)
class NoopSource(Source):
def prepare(self):
pass
class ReleaseDownload(Source):
archive_top_dir = None
@ -124,7 +133,10 @@ class ReleaseDownload(Source):
class GitClone(Source):
base_git_ref = "master"
release_git_ref = "master"
@property
def release_git_ref(self):
return main_project_versions.get(self.name, "master")
@property
def source_dir(self):
@ -236,6 +248,14 @@ class Builder:
self.command('make_dist', self._make_dist)
class NoopBuilder(Builder):
def build(self):
pass
def make_dist(self):
pass
class MakeBuilder(Builder):
configure_option = ""
dynamic_configure_option = "--enable-shared --disable-static"
@ -331,6 +351,7 @@ class CMakeBuilder(MakeBuilder):
class MesonBuilder(Builder):
configure_option = ""
test_option = ""
@property
def library_type(self):
@ -372,7 +393,7 @@ class MesonBuilder(Builder):
and not self.buildEnv.platform_info.static)
):
raise SkipCommand()
command = "{} --verbose".format(self.buildEnv.mesontest_command)
command = "{} --verbose {}".format(self.buildEnv.mesontest_command, self.test_option)
self.buildEnv.run_command(command, self.build_path, context)
def _install(self, context):

27
dependency_versions.py Normal file
View File

@ -0,0 +1,27 @@
main_project_versions = {
'kiwix-lib': '1.0.2',
'kiwix-tools': '0.3.0',
'libzim': '3.0.0',
'zim-tools': '0.0.1',
'zimwriterfs': '1.1',
}
# This is the "version" of the whole base_deps_versions dict.
# Change this when you change base_deps_versions.
base_deps_meta_version = '0'
base_deps_versions = {
'zlib' : '1.2.8',
'lzma' : '5.0.4',
'uuid' : '1.43.4',
'xapian-core' : '1.4.5',
'ctpp2' : '2.8.3',
'pugixml' : '1.2',
'libmicrohttpd' : '0.9.46',
'gumbo' : '0.10.1',
'icu4c' : '58.2',
'Gradle' : '3.4',
}

104
travis/compile_all.py
View File

@ -8,6 +8,11 @@ from datetime import date
import tarfile
import subprocess
import re
from urllib.request import urlretrieve
from urllib.error import URLError
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import dependency_versions
PLATFORM = environ['PLATFORM']
@ -15,6 +20,9 @@ def home():
return Path(os.path.expanduser('~'))
BASE_DIR = home()/"BUILD_{}".format(PLATFORM)
SOURCE_DIR = home()/"SOURCE"
ARCHIVE_DIR = home()/"ARCHIVE"
TOOLCHAINS_DIR = home()/"TOOLCHAINS"
NIGHTLY_ARCHIVES_DIR = home()/'NIGHTLY_ARCHIVES'
RELEASE_KIWIX_ARCHIVES_DIR = home()/'RELEASE_KIWIX_ARCHIVES'
RELEASE_ZIM_ARCHIVES_DIR = home()/'RELEASE_ZIM_ARCHIVES'
@ -22,14 +30,6 @@ DIST_KIWIX_ARCHIVES_DIR = home()/'DIST_KIWIX_ARCHIVES'
DIST_ZIM_ARCHIVES_DIR = home()/'DIST_ZIM_ARCHIVES'
SSH_KEY = Path(environ['TRAVIS_BUILD_DIR'])/'travis'/'travisci_builder_id_key'
VERSIONS = {
'kiwix-lib': '1.0.2',
'kiwix-tools': '0.3.0',
'libzim': '3.0.0',
'zim-tools': '0.0.1',
'zimwriterfs': '1.1'
}
# We have build everything. Now create archives for public deployement.
BINARIES = {
'kiwix-tools': ('kiwix-install', 'kiwix-manage', 'kiwix-read', 'kiwix-search', 'kiwix-serve'),
@ -66,6 +66,8 @@ def run_kiwix_build(target, platform, build_deps_only=False, make_release=False,
command.append('--make-release')
if make_dist:
command.append('--make-dist')
print("--- Build {} (deps={}, release={}, dist={}) ---".format(
target, build_deps_only, make_release, make_dist), flush=True)
subprocess.check_call(command, cwd=str(home()))
@ -74,7 +76,7 @@ def make_archive(project, platform):
if platform == "win32":
file_to_archives = ['{}.exe'.format(f) for f in file_to_archives]
if make_release:
postfix = VERSIONS[project]
postfix = dependency_versions.main_project_versions[project]
if project in ('kiwix-lib', 'kiwix-tools'):
archive_dir = RELEASE_KIWIX_ARCHIVES_DIR/project
else:
@ -94,8 +96,40 @@ def make_archive(project, platform):
arch.add(str(base_bin_dir/f), arcname=str(f))
def make_deps_archive(target, full=False):
(BASE_DIR/'.install_packages_ok').unlink()
archive_name = "deps_{}_{}.tar.gz".format(PLATFORM, target)
files_to_archive = [BASE_DIR/'INSTALL']
files_to_archive += BASE_DIR.glob('**/android-ndk*')
if (BASE_DIR/'meson_cross_file.txt').exists():
files_to_archive.append(BASE_DIR/'meson_cross_file.txt')
manifest_file = BASE_DIR/'manifest.txt'
write_manifest(manifest_file, archive_name, target, PLATFORM)
files_to_archive.append(manifest_file)
relative_path = BASE_DIR
if full:
files_to_archive += [ARCHIVE_DIR]
files_to_archive += BASE_DIR.glob('*/.*_ok')
files_to_archive += SOURCE_DIR.glob('*/.*_ok')
files_to_archive += [SOURCE_DIR/'pugixml-{}'.format(
dependency_versions.base_deps_versions['pugixml'])]
files_to_archive += [BASE_DIR/'pugixml-{}'.format(
dependency_versions.base_deps_versions['pugixml'])]
if (TOOLCHAINS_DIR).exists():
files_to_archive.append(TOOLCHAINS_DIR)
relative_path = home()
with tarfile.open(str(relative_path/archive_name), 'w:gz') as tar:
for name in files_to_archive:
tar.add(str(name), arcname=str(name.relative_to(relative_path)))
return relative_path/archive_name
def scp(what, where):
command = ['scp', '-i', str(SSH_KEY), what, where]
command = ['scp', '-i', str(SSH_KEY), str(what), str(where)]
subprocess.check_call(command)
@ -111,6 +145,27 @@ for p in (NIGHTLY_ARCHIVES_DIR,
make_release = re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", environ.get('TRAVIS_TAG', '')) is not None
# The first thing we need to do is to (potentially) download already compiled base dependencies.
BASE_DEP_VERSION = dependency_versions.base_deps_meta_version
base_dep_archive_name = "base_deps_{}_{}.tar.gz".format(PLATFORM, BASE_DEP_VERSION)
print("--- Getting archive {} ---".format(base_dep_archive_name), flush=True)
try:
local_filename, headers = urlretrieve(
'http://tmp.kiwix.org/ci/{}'.format(base_dep_archive_name))
with tarfile.open(local_filename) as f:
f.extractall(str(home()))
except URLError:
print("--- Cannot get archive. Build dependencies ---", flush=True)
run_kiwix_build('alldependencies', platform=PLATFORM)
archive = make_deps_archive('alldependencies', full=True)
destination = 'nightlybot@download.kiwix.org:/var/www/tmp.kiwix.org/ci/{}'
destination = destination.format(base_dep_archive_name)
scp(archive, destination)
# A basic compilation to be sure everything is working (for a PR)
if environ['TRAVIS_EVENT_TYPE'] != 'cron' and not make_release:
if PLATFORM.startswith('android'):
@ -143,21 +198,8 @@ for target in TARGETS:
run_kiwix_build(target,
platform=PLATFORM,
build_deps_only=True)
(BASE_DIR/'.install_packages_ok').unlink()
archive_name = "deps_{}_{}.tar.gz".format(PLATFORM, target)
files_to_archive = [BASE_DIR/'INSTALL']
files_to_archive += BASE_DIR.glob('**/android-ndk*')
if (BASE_DIR/'meson_cross_file.txt').exists():
files_to_archive.append(BASE_DIR/'meson_cross_file.txt')
manifest_file = BASE_DIR/'manifest.txt'
write_manifest(manifest_file, archive_name, target, PLATFORM)
files_to_archive.append(manifest_file)
with tarfile.open(str(BASE_DIR/archive_name), 'w:gz') as tar:
for name in files_to_archive:
tar.add(str(name), arcname=str(name.relative_to(BASE_DIR)))
scp(str(BASE_DIR/archive_name), 'nightlybot@download.kiwix.org:/var/www/tmp.kiwix.org/ci/')
archive = make_deps_archive(target)
scp(archive, 'nightlybot@download.kiwix.org:/var/www/tmp.kiwix.org/ci/')
run_kiwix_build(target,
platform=PLATFORM,
@ -171,10 +213,6 @@ for target in TARGETS:
# We have build everything. Now create archives for public deployement.
kiwix_tools_postfix = VERSIONS['kiwix-tools'] if make_release else _date
zim_tools_postfix = VERSIONS['zim-tools'] if make_release else _date
zimwriterfs_postfix = VERSIONS['zimwriterfs'] if make_release else _date
if make_release and PLATFORM == 'native_dyn':
for target in TARGETS:
if target in ('kiwix-lib', 'kiwix-tools'):
@ -183,10 +221,14 @@ if make_release and PLATFORM == 'native_dyn':
out_dir = DIST_ZIM_ARCHIVES_DIR
if target in ('kiwix-lib', 'kiwix-tools', 'libzim', 'zim-tools'):
shutil.copy(str(BASE_DIR/target/'meson-dist'/'{}-{}.tar.xz'.format(target, VERSIONS[target])),
shutil.copy(str(BASE_DIR/target/'meson-dist'/'{}-{}.tar.xz'.format(
target,
dependency_versions.main_project_versions[target])),
str(out_dir))
if target in ('zimwriterfs',):
shutil.copy(str(BASE_DIR/target/'{}-{}.tar.gz'.format(target, VERSIONS[target])),
shutil.copy(str(BASE_DIR/target/'{}-{}.tar.gz'.format(
target,
dependency_versions.main_project_versions[target])),
str(out_dir))
elif PLATFORM == 'native_static':
for target in ('kiwix-tools', 'zim-tools', 'zimwriterfs'):

14
travis/install_extra_deps.sh
View File

@ -4,17 +4,13 @@ set -e
orig_dir=$(pwd)
sudo apt-get update -qq
sudo apt-get install -qq python3-pip zlib1g-dev libjpeg-dev
pip3 install --user meson==0.43.0
pip3 install --user pillow
pip3 install --user --upgrade pip wheel
pip3 install --user pillow meson==0.43.0
# ninja
git clone git://github.com/ninja-build/ninja.git
cd ninja
git checkout release
./configure.py --bootstrap
sudo cp ninja /bin
wget https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip
unzip ninja-linux.zip ninja
cp ninja $HOME/bin
cd $orig_dir

13
travis/upload_all_log.sh Executable file
View File

@ -0,0 +1,13 @@
#!/usr/bin/env bash
set -e
SSH_KEY=$(pwd)/travis/travisci_builder_id_key
cd $HOME
tar -czf fail_log_${PLATFORM}.tar.gz BUILD_${PLATFORM}
scp -vrp -i ${SSH_KEY} \
fail_log_${PLATFORM}.tar.gz \
nightlybot@download.kiwix.org:/var/www/tmp.kiwix.org/ci