From 81796e7125a8bad2aefcba13ca7622bce2672b5f Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Fri, 7 Apr 2017 15:48:35 +0200 Subject: [PATCH 1/4] Add a option to skip the source installation. It make the kiwix-build.py script run a bit father. --- kiwix-build.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/kiwix-build.py b/kiwix-build.py index c018fd9..2f5fe60 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -710,6 +710,7 @@ class android_ndk(Toolchain): class Builder: def __init__(self, options): + self.options = options self.targets = OrderedDict() self.buildEnv = buildEnv = BuildEnv(options, self.targets) @@ -738,6 +739,10 @@ class Builder: yield targetName def prepare_sources(self): + if self.options.skip_source_prepare: + print("SKIP") + return + toolchain_sources = (tlc.source for tlc in self.buildEnv.toolchains if tlc.source) for toolchain_source in toolchain_sources: print("prepare sources for toolchain {} :".format(toolchain_source.name)) @@ -784,6 +789,8 @@ def parse_args(): " log files per commands")) parser.add_argument('--no-cert-check', action='store_true', help="Skip SSL certificate verification during download") + parser.add_argument('--skip-source-prepare', action='store_true', + help="Skip the source download part") return parser.parse_args() From 308cfa69863983e5eb3f74c5e047eacd7b5affeb Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Sat, 8 Apr 2017 17:47:11 +0200 Subject: [PATCH 2/4] Use dependency names as reference instead of class names. Class names are implementation names. Using the dependency names is better as it surprise less the user. --- .travis.yml | 12 ++++++------ dependencies.py | 38 ++++++++++++++++++++++---------------- dependency_utils.py | 3 ++- kiwix-build.py | 2 +- 4 files changed, 31 insertions(+), 24 deletions(-) diff --git a/.travis.yml b/.travis.yml index 78778a4..ba40ef4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,12 +18,12 @@ env: - BUILD_OPTION="--target-platform=native_static" ARCHIVE_TYPE="--tar" DEPLOY=true - BUILD_OPTION="--target-platform=win32_dyn" - BUILD_OPTION="--target-platform=win32_static" ARCHIVE_TYPE="--zip" DEPLOY=true - - BUILD_OPTION="--target-platform=android_arm Kiwixlib" - - BUILD_OPTION="--target-platform=android_arm64 Kiwixlib" - - BUILD_OPTION="--target-platform=android_mips Kiwixlib" - - BUILD_OPTION="--target-platform=android_mips64 Kiwixlib" - - BUILD_OPTION="--target-platform=android_x86 Kiwixlib" - - BUILD_OPTION="--target-platform=android_x86_64 Kiwixlib" + - BUILD_OPTION="--target-platform=android_arm kiwix-lib" + - BUILD_OPTION="--target-platform=android_arm64 kiwix-lib" + - BUILD_OPTION="--target-platform=android_mips kiwix-lib" + - BUILD_OPTION="--target-platform=android_mips64 kiwix-lib" + - BUILD_OPTION="--target-platform=android_x86 kiwix-lib" + - BUILD_OPTION="--target-platform=android_x86_64 kiwix-lib" notifications: irc: channels: diff --git a/dependencies.py b/dependencies.py index f0ec13f..ad02524 100644 --- a/dependencies.py +++ b/dependencies.py @@ -123,7 +123,7 @@ class Xapian(Dependency): deps = ['zlib', 'lzma'] if self.buildEnv.platform_info.build == 'win32': return deps - return deps + ['UUID'] + return deps + ['uuid'] class CTPP2(Dependency): @@ -175,6 +175,12 @@ class Icu(Dependency): version = "58_2" class Source(ReleaseDownload): + name = "icu4c" + + @property + def source_dir(self): + return "{}-{}".format(self.name, self.target.version) + archive = Remotefile('icu4c-58_2-src.tgz', '2b0a4410153a9b20de0e20c7d8b66049a72aef244b53683d0d7521371683da0c', 'https://freefr.dl.sourceforge.net/project/icu/ICU4C/58.2/icu4c-58_2-src.tgz') @@ -201,10 +207,10 @@ class Icu(Dependency): class Icu_native(Icu): + name = "icu4c_native" force_native_build = True class Builder(Icu.Builder): - name = "icu_native" @property def build_path(self): return super().build_path+"_native" @@ -214,14 +220,14 @@ class Icu_native(Icu): class Icu_cross_compile(Icu): - dependencies = ['Icu_native'] + name = "icu4c_cross-compile" + dependencies = ['icu4c_native'] class Builder(Icu.Builder): - name = "icu_cross-compile" @property def configure_option(self): - Icu_native = self.buildEnv.targetsDict['Icu_native'] - return super().configure_option + " --with-cross-build=" + Icu_native.builder.build_path + icu_native_dep = self.buildEnv.targetsDict['icu4c_native'] + return super().configure_option + " --with-cross-build=" + icu_native_dep.builder.build_path class OpenzimSource(GitClone): @@ -234,8 +240,8 @@ class OpenzimSource(GitClone): self.buildEnv.run_command(command, pj(self.git_path, 'zimwriterfs'), context) -class Zimlib(Dependency): - name = "zimlib" +class Libzim(Dependency): + name = "libzim" Source = OpenzimSource @@ -249,11 +255,11 @@ class Zimwriterfs(Dependency): @property def dependencies(self): - base_dependencies = ['Zimlib', 'zlib', 'lzma', 'Xapian'] + base_dependencies = ['libzim', 'zlib', 'lzma', 'xapian-core'] if self.buildEnv.platform_info.build != 'native': - return base_dependencies + ["Icu_cross_compile"] + return base_dependencies + ["icu4c_cross-compile"] else: - return base_dependencies + ["Icu"] + return base_dependencies + ["icu4c"] Source = OpenzimSource @@ -266,13 +272,13 @@ class Kiwixlib(Dependency): @property def dependencies(self): - base_dependencies = ["Xapian", "Pugixml", "Zimlib", "zlib", "lzma"] + base_dependencies = ["xapian-core", "pugixml", "libzim", "zlib", "lzma"] if self.buildEnv.platform_info.build != 'android': - base_dependencies += ['CTPP2'] + base_dependencies += ['ctpp2'] if self.buildEnv.platform_info.build != 'native': - return base_dependencies + ["Icu_cross_compile"] + return base_dependencies + ["icu4c_cross-compile"] else: - return base_dependencies + ["Icu"] + return base_dependencies + ["icu4c"] class Source(GitClone): git_remote = "https://github.com/kiwix/kiwix-lib.git" @@ -295,7 +301,7 @@ class Kiwixlib(Dependency): class KiwixTools(Dependency): name = "kiwix-tools" - dependencies = ["Kiwixlib", "MicroHttpd", "zlib"] + dependencies = ["kiwix-lib", "libmicrohttpd", "zlib"] class Source(GitClone): git_remote = "https://github.com/kiwix/kiwix-tools.git" diff --git a/dependency_utils.py b/dependency_utils.py index bf78d0c..aff8281 100644 --- a/dependency_utils.py +++ b/dependency_utils.py @@ -11,7 +11,8 @@ class _MetaDependency(type): def __new__(cls, name, bases, dct): _class = type.__new__(cls, name, bases, dct) if name != 'Dependency': - Dependency.all_deps[name] = _class + dep_name = dct['name'] + Dependency.all_deps[dep_name] = _class return _class diff --git a/kiwix-build.py b/kiwix-build.py index 2f5fe60..6bac18a 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -780,7 +780,7 @@ class Builder: def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument('targets', default='KiwixTools', nargs='?') + parser.add_argument('targets', default='kiwix-tools', nargs='?') parser.add_argument('--working-dir', default=".") parser.add_argument('--libprefix', default=None) parser.add_argument('--target-platform', default="native_dyn", choices=BuildEnv.target_platforms) From 82625ed62e6a8c090782263b38107738e7a369c8 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Sat, 8 Apr 2017 17:49:27 +0200 Subject: [PATCH 3/4] Update url of libzim and zimwriterfs. The openzim repository has been split into different smaller repositories. Especially libzim and zimwriterfs. --- dependencies.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/dependencies.py b/dependencies.py index ad02524..1befa3d 100644 --- a/dependencies.py +++ b/dependencies.py @@ -230,23 +230,14 @@ class Icu_cross_compile(Icu): return super().configure_option + " --with-cross-build=" + icu_native_dep.builder.build_path -class OpenzimSource(GitClone): - git_remote = "https://gerrit.wikimedia.org/r/p/openzim.git" - git_dir = "openzim" - - def _post_prepare_script(self, context): - context.try_skip(self.git_path) - command = "./autogen.sh" - self.buildEnv.run_command(command, pj(self.git_path, 'zimwriterfs'), context) - - class Libzim(Dependency): name = "libzim" - Source = OpenzimSource + class Source(GitClone): + git_remote = "https://github.com/openzim/libzim.git" + git_dir = "libzim" - class Builder(MesonBuilder): - subsource_dir = "zimlib" + Builder = MesonBuilder class Zimwriterfs(Dependency): @@ -261,10 +252,16 @@ class Zimwriterfs(Dependency): else: return base_dependencies + ["icu4c"] - Source = OpenzimSource + class Source(GitClone): + git_remote = "https://github.com/openzim/zimwriterfs.git" + git_dir = "zimwriterfs" - class Builder(MakeBuilder): - subsource_dir = "zimwriterfs" + def _post_prepare_script(self, context): + context.try_skip(self.git_path) + command = "./autogen.sh" + self.buildEnv.run_command(command, self.git_path, context) + + Builder = MakeBuilder class Kiwixlib(Dependency): From e816979d5119ecf082fa5c96f745ddc03c191661 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Sat, 8 Apr 2017 17:54:16 +0200 Subject: [PATCH 4/4] Limit the choice of the targets argument to target we know about. By limiting the choices in the argument parser, we catch argument error at parsing and we can provide useful output (the names of dependencies we can handle) to user in the help or in the error message. --- kiwix-build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kiwix-build.py b/kiwix-build.py index 6bac18a..5c87a29 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -780,7 +780,8 @@ class Builder: def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument('targets', default='kiwix-tools', nargs='?') + parser.add_argument('targets', default='kiwix-tools', nargs='?', + choices=Dependency.all_deps.keys()) parser.add_argument('--working-dir', default=".") parser.add_argument('--libprefix', default=None) parser.add_argument('--target-platform', default="native_dyn", choices=BuildEnv.target_platforms)