From a49fb159649c67826d4bfe0889df459ad1ceb94c Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 7 Mar 2017 20:51:11 +0100 Subject: [PATCH 1/5] BuildEnv.run_command now take a str as input instead of a file object. We can get a str from a file object. The contrary is more difficult. --- dependency_utils.py | 2 +- kiwix-build.py | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/dependency_utils.py b/dependency_utils.py index 8182553..6959ac1 100644 --- a/dependency_utils.py +++ b/dependency_utils.py @@ -110,7 +110,7 @@ class ReleaseDownload(Source): context.force_native_build = True for p in self.patches: with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input: - self.buildEnv.run_command("patch -p1", self.extract_path, context, input=patch_input) + self.buildEnv.run_command("patch -p1", self.extract_path, context, input=patch_input.read()) def prepare(self): self.command('download', self._download) diff --git a/kiwix-build.py b/kiwix-build.py index 49aeaa8..f9b6a38 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -397,8 +397,13 @@ class BuildEnv: kwargs = dict() if input: - kwargs['stdin'] = input - return subprocess.check_call(command, shell=True, cwd=cwd, env=env, stdout=log or sys.stdout, stderr=subprocess.STDOUT, **kwargs) + kwargs['stdin'] = subprocess.PIPE + process = subprocess.Popen(command, shell=True, cwd=cwd, env=env, stdout=log or sys.stdout, stderr=subprocess.STDOUT, **kwargs) + if input: + process.communicate(input.encode()) + retcode = process.wait() + if retcode: + raise subprocess.CalledProcessError(retcode, command) finally: if log: log.close() From 56862b09d614a877e284310b8a7357a04ea20f70 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 7 Mar 2017 22:27:34 +0100 Subject: [PATCH 2/5] Add (and use) the utility function `add_execution_right`. --- kiwix-build.py | 9 ++++----- utils.py | 6 +++++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/kiwix-build.py b/kiwix-build.py index f9b6a38..27341da 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import os, sys, stat +import os, sys import argparse import ssl import urllib.request @@ -13,6 +13,7 @@ from dependency_utils import ReleaseDownload, Builder from utils import ( pj, remove_duplicates, + add_execution_right, get_sha256, StopBuild, SkipCommand, @@ -667,8 +668,7 @@ class android_ndk(Toolchain): def _build_platform(self, context): context.try_skip(self.build_path) script = pj(self.source_path, 'build/tools/make_standalone_toolchain.py') - current_permissions = stat.S_IMODE(os.lstat(script).st_mode) - os.chmod(script, current_permissions | stat.S_IXUSR) + add_execution_right(script) command = '{script} --arch={arch} --api={api} --install-dir={install_dir} --force' command = command.format( script=script, @@ -692,8 +692,7 @@ class android_ndk(Toolchain): file_path = pj(root, file_) if os.path.islink(file_path): continue - current_permissions = stat.S_IMODE(os.lstat(file_path).st_mode) - os.chmod(file_path, current_permissions | stat.S_IXUSR) + add_execution_right(file_path) def build(self): self.command('build_platform', self._build_platform) diff --git a/utils.py b/utils.py index e3ef32c..7dcdbc5 100644 --- a/utils.py +++ b/utils.py @@ -2,7 +2,7 @@ import os.path import hashlib import tarfile, zipfile import tempfile -import os +import os, stat from collections import namedtuple, defaultdict pj = os.path.join @@ -32,6 +32,10 @@ def get_sha256(path): return sha256.hexdigest() +def add_execution_right(file_path): + current_permissions = stat.S_IMODE(os.lstat(file_path).st_mode) + os.chmod(file_path, current_permissions | stat.S_IXUSR) + class SkipCommand(Exception): pass From 86c1547f8863044dfbf4c58091af4a3436a0ea4c Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 8 Mar 2017 10:31:00 +0100 Subject: [PATCH 3/5] Allow kiwix-build to build android APK. Previous script (from kiwix repository) created a APK with all architecture embeded. Now, we generated a APK per architecture. It simplify the build process and generate smaller APKs. --- .gitignore | 1 + dependencies.py | 47 ++++++++++++++++++++++++++++++++--- dependency_utils.py | 25 +++++++++++++++++++ kiwix-build.py | 58 ++++++++++++++++++++++++++++++++++++++++--- travis/compile_all.sh | 4 +-- utils.py | 13 ++++++++++ 6 files changed, 140 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 173fd8a..32bdd91 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ ARCHIVE BUILD_* LOGS SOURCE +TOOLCHAINS diff --git a/dependencies.py b/dependencies.py index 6c55f82..c75aac2 100644 --- a/dependencies.py +++ b/dependencies.py @@ -1,4 +1,4 @@ -import shutil +import shutil,os from dependency_utils import ( Dependency, @@ -6,9 +6,11 @@ from dependency_utils import ( GitClone, MakeBuilder, CMakeBuilder, - MesonBuilder) + MesonBuilder, + GradleBuilder, + Builder as BaseBuilder) -from utils import Remotefile, pj, SkipCommand +from utils import Remotefile, pj, SkipCommand, copy_tree, add_execution_right # ************************************* # Missing dependencies @@ -318,3 +320,42 @@ class KiwixTools(Dependency): if self.buildEnv.platform_info.static: return "-Dstatic-linkage=true" return "" + + +class Gradle(Dependency): + name = "Gradle" + version = "3.4" + + class Source(ReleaseDownload): + archive = Remotefile('gradle-3.4-bin.zip', + '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205', + 'https://services.gradle.org/distributions/gradle-3.4-bin.zip') + + class Builder(BaseBuilder): + def build(self): + self.command('install', self._install) + + def _install(self, context): + copy_tree( + pj(self.source_path, "bin"), + pj(self.buildEnv.install_dir, "bin"), + post_copy_function = add_execution_right) + copy_tree( + pj(self.source_path, "lib"), + pj(self.buildEnv.install_dir, "lib")) + + +class KiwixAndroid(Dependency): + name = "kiwix-android" + dependencies = ["Gradle", "kiwix-lib"] + + class Source(GitClone): + git_remote = "https://github.com/kiwix/kiwix-android" + git_dir = "kiwix-android" + + class Builder(GradleBuilder): + def _configure(self, context): + if not os.path.exists(self.build_path): + shutil.copytree(self.source_path, self.build_path) + shutil.rmtree(pj(self.build_path, 'kiwixlib', 'src', 'main')) + shutil.copytree(pj(self.buildEnv.install_dir, 'kiwix-lib'), pj(self.build_path, 'kiwixlib', 'src', 'main')) diff --git a/dependency_utils.py b/dependency_utils.py index 6959ac1..9e024ca 100644 --- a/dependency_utils.py +++ b/dependency_utils.py @@ -305,3 +305,28 @@ class MesonBuilder(Builder): def _install(self, context): command = "{} -v install".format(self.buildEnv.ninja_command) self.buildEnv.run_command(command, self.build_path, context) + + +class GradleBuilder(Builder): + def build(self): + self.command('configure', self._configure) + if hasattr(self, '_pre_compile_script'): + self.command('pre_compile_script', self._pre_compile_script) + self.command('compile', self._compile) + self.command('install', self._install) + + def _configure(self, context): + # We don't have a lot to configure by itself + context.try_skip(self.build_path) + if os.path.exists(self.build_path): + shutil.rmtree(self.build_path) + shutil.copytree(self.source_path, self.build_path) + + def _compile(self, context): + command = "gradle clean assemble --info" + self.buildEnv.run_command(command, self.build_path, context) + command = "gradle build --info" + self.buildEnv.run_command(command, self.build_path, context) + + def _install(self, context): + pass diff --git a/kiwix-build.py b/kiwix-build.py index 27341da..27d1ce7 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import os, sys +import os, sys, shutil import argparse import ssl import urllib.request @@ -39,7 +39,7 @@ CROSS_ENV = { } }, 'fedora_android': { - 'toolchain_names': ['android_ndk'], + 'toolchain_names': ['android_ndk', 'android_sdk'], 'extra_libs': [], 'extra_cflags': [], 'host_machine': { @@ -62,7 +62,7 @@ CROSS_ENV = { } }, 'debian_android': { - 'toolchain_names': ['android_ndk'], + 'toolchain_names': ['android_ndk', 'android_sdk'], 'extra_libs': [], 'extra_cflags': [], 'host_machine': { @@ -190,11 +190,13 @@ class BuildEnv: build_dir = "BUILD_{}".format(options.target_platform) self.build_dir = pj(options.working_dir, build_dir) self.archive_dir = pj(options.working_dir, "ARCHIVE") + self.toolchain_dir = pj(options.working_dir, "TOOLCHAINS") self.log_dir = pj(self.build_dir, 'LOGS') self.install_dir = pj(self.build_dir, "INSTALL") for d in (self.source_dir, self.build_dir, self.archive_dir, + self.toolchain_dir, self.log_dir, self.install_dir): os.makedirs(d, exist_ok=True) @@ -717,6 +719,56 @@ class android_ndk(Toolchain): env['NDK_DEBUG'] = '0' +class android_sdk(Toolchain): + name = 'android-sdk' + version = 'r25.2.3' + + class Source(ReleaseDownload): + archive = Remotefile('tools_r25.2.3-linux.zip', + '1b35bcb94e9a686dff6460c8bca903aa0281c6696001067f34ec00093145b560', + 'https://dl.google.com/android/repository/tools_r25.2.3-linux.zip') + + class Builder(Builder): + + @property + def install_path(self): + return pj(self.buildEnv.toolchain_dir, self.target.full_name) + + def _build_platform(self, context): + context.try_skip(self.install_path) + tools_dir = pj(self.install_path, 'tools') + shutil.copytree(self.source_path, tools_dir) + script = pj(tools_dir, 'android') + command = '{script} --verbose update sdk -a --no-ui --filter {packages}' + command = command.format( + script=script, + packages = ','.join(str(i) for i in [1,2,8,34,162]) + ) + # packages correspond to : + # - 1 : Android SDK Tools, revision 25.2.5 + # - 2 : Android SDK Platform-tools, revision 25.0.3 + # - 8 : Android SDK Build-tools, revision 24.0.1 + # - 34 : SDK Platform Android 7.0, API 24, revision 2 + # - 162 : Android Support Repository, revision 44 + self.buildEnv.run_command(command, self.install_path, context, input="y\n") + + def _fix_licenses(self, context): + context.try_skip(self.install_path) + os.makedirs(pj(self.install_path, 'licenses'), exist_ok=True) + with open(pj(self.install_path, 'licenses', 'android-sdk-license'), 'w') as f: + f.write("\n8933bad161af4178b1185d1a37fbf41ea5269c55") + + def build(self): + self.command('build_platform', self._build_platform) + self.command('fix_licenses', self._fix_licenses) + + def get_bin_dir(self): + return [] + + def set_env(self, env): + env['ANDROID_HOME'] = self.builder.install_path + + class Builder: def __init__(self, options): self.options = options diff --git a/travis/compile_all.sh b/travis/compile_all.sh index ac8bd59..506824d 100755 --- a/travis/compile_all.sh +++ b/travis/compile_all.sh @@ -14,7 +14,7 @@ if [[ "$TRAVIS_EVENT_TYPE" = "cron" ]] then if [[ ${PLATFORM} = android* ]] then - TARGETS="libzim kiwix-lib" + TARGETS="libzim kiwix-lib kiwix-android" else TARGETS="libzim kiwix-lib kiwix-tools" fi @@ -77,7 +77,7 @@ else # No a cron job, we just have to build to be sure nothing is broken. if [[ ${PLATFORM} = android* ]] then - TARGET=kiwix-lib + TARGET=kiwix-android else TARGET=kiwix-tools fi diff --git a/utils.py b/utils.py index 7dcdbc5..f41a400 100644 --- a/utils.py +++ b/utils.py @@ -2,6 +2,7 @@ import os.path import hashlib import tarfile, zipfile import tempfile +import shutil import os, stat from collections import namedtuple, defaultdict @@ -36,6 +37,18 @@ def add_execution_right(file_path): current_permissions = stat.S_IMODE(os.lstat(file_path).st_mode) os.chmod(file_path, current_permissions | stat.S_IXUSR) +def copy_tree(src, dst, post_copy_function=None): + os.makedirs(dst, exist_ok=True) + for root, dirs, files in os.walk(src): + r = os.path.relpath(root, src) + dstdir = pj(dst, r) + os.makedirs(dstdir, exist_ok=True) + for f in files: + dstfile = pj(dstdir, f) + shutil.copy2(pj(root, f), dstfile) + if post_copy_function is not None: + post_copy_function(dstfile) + class SkipCommand(Exception): pass From 3beb6d81f29386e665552b9d184bc9d21d8926e1 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 24 Apr 2017 15:26:28 +0200 Subject: [PATCH 4/5] Add a option to kiwix-build.py to clean intermediate directories. Travis fails if we try to build a lot of targets in the same job because of quota limit. By removing intermediate build files, we limit our disk usage. --- kiwix-build.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/kiwix-build.py b/kiwix-build.py index 27d1ce7..0489fe3 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -213,6 +213,16 @@ class BuildEnv: self.libprefix = options.libprefix or self._detect_libdir() self.targetsDict = targetsDict + def clean_intermediate_directories(self): + for subdir in os.listdir(self.build_dir): + subpath = pj(self.build_dir, subdir) + if subpath == self.install_dir: + continue + if os.path.isdir(subpath): + shutil.rmtree(subpath) + else: + os.remove(subpath) + def detect_platform(self): _platform = platform.system() self.distname = _platform @@ -837,6 +847,12 @@ class Builder: self.prepare_sources() print("[BUILD]") self.build() + # No error, clean intermediate file at end of build if needed. + print("[CLEAN]") + if self.buildEnv.options.clean_at_end: + self.buildEnv.clean_intermediate_directories() + else: + print("SKIP") except StopBuild: sys.exit("Stopping build due to errors") @@ -857,6 +873,8 @@ def parse_args(): help="Skip the source download part") parser.add_argument('--build-deps-only', action='store_true', help=("Build only the dependencies of the specified targets.")) + parser.add_argument('--clean-at-end', action='store_true', + help="Clean all intermediate files after the (successfull) build") return parser.parse_args() From c15ddde6eba62db8a20c502fa44acfa39b96ea33 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 24 Apr 2017 18:21:25 +0200 Subject: [PATCH 5/5] Make travisCI publish nightly kiwix-android APKs. --- travis/compile_all.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/travis/compile_all.sh b/travis/compile_all.sh index 506824d..a7c5cc4 100755 --- a/travis/compile_all.sh +++ b/travis/compile_all.sh @@ -71,6 +71,11 @@ EOF tar -czf "${NIGHTLY_ARCHIVES_DIR}/$ARCHIVE_NAME" $FILES_LIST ) ;; + android_*) + APK_NAME="kiwix-${PLATFORM}" + cp ${BASE_DIR}/kiwix-android/app/build/outputs/apk/app-kiwix-debug.apk ${NIGHTLY_ARCHIVES_DIR}/${APK_NAME}-debug.apk + cp ${BASE_DIR}/kiwix-android/app/build/outputs/apk/app-kiwix-release-unsigned.apk ${NIGHTLY_ARCHIVES_DIR}/${APK_NAME}-release-unsigned.apk + ;; esac else