From c8f5acb41f139ae421c63a787c5a6a01b87538e2 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 26 Apr 2017 18:11:05 +0200 Subject: [PATCH 1/4] Add better progress information output for long operation. Checking sha256sum and downloading file can be pretty long. Let's print few information to the user. --- kiwix-build.py | 12 ++++++++++++ utils.py | 14 +++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/kiwix-build.py b/kiwix-build.py index 36865b0..c4228bd 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -15,6 +15,7 @@ from utils import ( remove_duplicates, add_execution_right, get_sha256, + print_status, StopBuild, SkipCommand, Defaultdict, @@ -440,11 +441,22 @@ class BuildEnv: context = None batch_size = 1024 * 8 extra_args = {'context':context} if sys.version_info >= (3, 4, 3) else {} + progress_chars = "/-\|" with urllib.request.urlopen(file_url, **extra_args) as resource, open(file_path, 'wb') as file: + tsize = resource.getheader('Content-Length', None) + if tsize is not None: + tsize = int(tsize) + current = 0 while True: batch = resource.read(batch_size) if not batch: break + if tsize: + current += batch_size + print_status("{:.2%}".format(current/tsize)) + else: + print_status(progress_chars[current]) + current = (current+1)%4 file.write(batch) if not what.sha256: diff --git a/utils.py b/utils.py index f41a400..2d22c6c 100644 --- a/utils.py +++ b/utils.py @@ -27,11 +27,23 @@ def remove_duplicates(iterable, key_function=None): def get_sha256(path): + progress_chars = "/-\|" + current = 0 + batch_size = 1024 * 8 sha256 = hashlib.sha256() with open(path, 'br') as f: - sha256.update(f.read()) + while True: + batch = f.read(batch_size) + if not batch: + break + sha256.update(batch) + print_status(progress_chars[current]) + current = (current+1)%4 return sha256.hexdigest() +def print_status(status): + text = "{}\033[{}D".format(status, len(status)) + print(text, end="") def add_execution_right(file_path): current_permissions = stat.S_IMODE(os.lstat(file_path).st_mode) From 39c2e67b807926f1f8403d9f6f99d35b81915f93 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 2 May 2017 14:26:45 +0200 Subject: [PATCH 2/4] Make GradleBuilder configurable. A dependency (future kiwix-android-custom app) may want to use the gradle builder for something else than `build` with the default options. --- dependency_utils.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/dependency_utils.py b/dependency_utils.py index 9e024ca..b26cd35 100644 --- a/dependency_utils.py +++ b/dependency_utils.py @@ -308,12 +308,14 @@ class MesonBuilder(Builder): class GradleBuilder(Builder): + gradle_target = "build" + gradle_option = "-i" + def build(self): self.command('configure', self._configure) if hasattr(self, '_pre_compile_script'): self.command('pre_compile_script', self._pre_compile_script) self.command('compile', self._compile) - self.command('install', self._install) def _configure(self, context): # We don't have a lot to configure by itself @@ -323,10 +325,8 @@ class GradleBuilder(Builder): shutil.copytree(self.source_path, self.build_path) def _compile(self, context): - command = "gradle clean assemble --info" + command = "gradle {gradle_target} {gradle_option}" + command = command.format( + gradle_target=self.gradle_target, + gradle_option=self.gradle_option) self.buildEnv.run_command(command, self.build_path, context) - command = "gradle build --info" - self.buildEnv.run_command(command, self.build_path, context) - - def _install(self, context): - pass From a86d0b143b53c8a0c7b7f094c2bb2e456b81af16 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 2 May 2017 14:28:30 +0200 Subject: [PATCH 3/4] Do not fail if the `kiwixlib/src/main` doesn't exist when deleting him. When deleting the directory, this is not a problem if the directory doesn't exist. --- dependencies.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dependencies.py b/dependencies.py index c75aac2..7cb58f8 100644 --- a/dependencies.py +++ b/dependencies.py @@ -357,5 +357,8 @@ class KiwixAndroid(Dependency): def _configure(self, context): if not os.path.exists(self.build_path): shutil.copytree(self.source_path, self.build_path) - shutil.rmtree(pj(self.build_path, 'kiwixlib', 'src', 'main')) + try: + shutil.rmtree(pj(self.build_path, 'kiwixlib', 'src', 'main')) + except FileNotFoundError: + pass shutil.copytree(pj(self.buildEnv.install_dir, 'kiwix-lib'), pj(self.build_path, 'kiwixlib', 'src', 'main')) From f044fd15bf16b62a04cb67b052da098dd2b7c580 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 2 May 2017 17:29:28 +0200 Subject: [PATCH 4/4] Add support to build kiwix android custom app. The custom app to build must be specified as an option when running kiwix-build. In the same way, the zim file to download must be specified in the `info.json` (`zim_url` key) or given as option. --- dependencies.py | 78 ++++++++++++++++++++++++++++++++++++++++++++++++- kiwix-build.py | 15 +++++++++- 2 files changed, 91 insertions(+), 2 deletions(-) diff --git a/dependencies.py b/dependencies.py index 7cb58f8..668ac2e 100644 --- a/dependencies.py +++ b/dependencies.py @@ -1,4 +1,5 @@ -import shutil,os +import shutil, os, json +from urllib.parse import urlparse from dependency_utils import ( Dependency, @@ -362,3 +363,78 @@ class KiwixAndroid(Dependency): except FileNotFoundError: pass shutil.copytree(pj(self.buildEnv.install_dir, 'kiwix-lib'), pj(self.build_path, 'kiwixlib', 'src', 'main')) + + +class KiwixCustomApp(Dependency): + name = "kiwix-android-custom" + dependencies = ["kiwix-android", "kiwix-lib"] + + def __init__(self, buildEnv): + super().__init__(buildEnv) + self.custom_name = buildEnv.options.android_custom_app + + class Source(GitClone): + git_remote = "https://github.com/kiwix/kiwix-android-custom" + git_dir = "kiwix-android-custom" + + class Builder(GradleBuilder): + @property + def gradle_target(self): + return "assemble{}".format(self.target.custom_name) + + @property + def gradle_option(self): + return "-i -P customDir={}".format(pj(self.build_path, 'custom')) + + @property + def build_path(self): + return pj(self.buildEnv.build_dir, "{}_{}".format(self.target.full_name, self.target.custom_name)) + + @property + def custom_build_path(self): + return pj(self.build_path, 'custom', self.target.custom_name) + + def build(self): + self.command('configure', self._configure) + self.command('download_zim', self._download_zim) + self.command('compile', self._compile) + + def _download_zim(self, context): + zim_url = self.buildEnv.options.zim_file_url + if zim_url is None: + raise SkipCommand() + with open(pj(self.source_path, self.target.custom_name, 'info.json')) as f: + app_info = json.load(f) + zim_url = app_info.get('zim_url', zim_url) + out_filename = urlparse(zim_url).path + out_filename = os.path.basename(out_filename) + zim_file = Remotefile(out_filename, '', zim_url) + self.buildEnv.download(zim_file) + shutil.copy(pj(self.buildEnv.archive_dir, out_filename), + pj(self.custom_build_path, app_info['zim_file'])) + + def _configure(self, context): + # Copy kiwix-android in build dir. + kiwix_android_dep = self.buildEnv.targetsDict['kiwix-android'] + if not os.path.exists(self.build_path): + shutil.copytree(kiwix_android_dep.source_path, self.build_path) + + # Copy kiwix-lib application in build dir + try: + shutil.rmtree(pj(self.build_path, 'kiwixlib', 'src', 'main')) + except FileNotFoundError: + pass + shutil.copytree(pj(self.buildEnv.install_dir, 'kiwix-lib'), pj(self.build_path, 'kiwixlib', 'src', 'main')) + + # Generate custom directory + try: + shutil.rmtree(pj(self.build_path, 'custom')) + except FileNotFoundError: + pass + os.makedirs(pj(self.build_path, 'custom')) + command = "./gen-custom-android-directory.py {custom_name} --output-dir {custom_dir}" + command = command.format( + custom_name=self.target.custom_name, + custom_dir=pj(self.build_path, 'custom', self.target.custom_name) + ) + self.buildEnv.run_command(command, self.source_path, context) diff --git a/kiwix-build.py b/kiwix-build.py index c4228bd..7bf8d38 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -933,7 +933,20 @@ def parse_args(): parser.add_argument('--clean-at-end', action='store_true', help="Clean all intermediate files after the (successfull) build") - return parser.parse_args() + subgroup = parser.add_argument_group('custom app', + description="Android custom app specific options") + subgroup.add_argument('--android-custom-app', + help="The custom android app to build") + subgroup.add_argument('--zim-file-url', + help="The url of the zim file to download") + options = parser.parse_args() + + if options.targets == 'kiwix-android-custom': + if not options.android_custom_app or not options.zim_file_url: + print("You need to specify ANDROID_CUSTOM_APP and ZIM_FILE_URL if " + "want to build a kiwix-android-custom target") + sys.exit(1) + return options if __name__ == "__main__":