From f942c5951fc3e315be112f8f01f364908a9318ef Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 3 May 2017 10:52:47 +0200 Subject: [PATCH 1/7] Move `_patch` method in `Source` class. The patch method is not specific to a `RealeaseDownload` source. --- dependency_utils.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/dependency_utils.py b/dependency_utils.py index b26cd35..36e4932 100644 --- a/dependency_utils.py +++ b/dependency_utils.py @@ -82,6 +82,14 @@ class Source: def source_dir(self): return self.target.full_name + def _patch(self, context): + source_path = pj(self.buildEnv.source_dir, self.source_dir) + context.try_skip(source_path) + context.force_native_build = True + for p in self.patches: + with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input: + self.buildEnv.run_command("patch -p1", source_path, context, input=patch_input.read()) + def command(self, *args, **kwargs): return self.target.command(*args, **kwargs) @@ -105,13 +113,6 @@ class ReleaseDownload(Source): topdir=self.archive_top_dir, name=self.source_dir) - def _patch(self, context): - context.try_skip(self.extract_path) - context.force_native_build = True - for p in self.patches: - with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input: - self.buildEnv.run_command("patch -p1", self.extract_path, context, input=patch_input.read()) - def prepare(self): self.command('download', self._download) self.command('extract', self._extract) From c2ad927f809056a4af8cc4e223e6d45ec5faec31 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 3 May 2017 10:54:02 +0200 Subject: [PATCH 2/7] Add a `SvnSource`. This is needed to get source from svn. --- dependency_utils.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/dependency_utils.py b/dependency_utils.py index 36e4932..de6d82c 100644 --- a/dependency_utils.py +++ b/dependency_utils.py @@ -150,6 +150,33 @@ class GitClone(Source): self.command('post_prepare_script', self._post_prepare_script) +class SvnClone(Source): + @property + def source_dir(self): + return self.svn_dir + + @property + def svn_path(self): + return pj(self.buildEnv.source_dir, self.svn_dir) + + def _svn_checkout(self, context): + context.force_native_build = True + if os.path.exists(self.svn_path): + raise SkipCommand() + command = "svn checkout {} {}".format(self.svn_remote, self.svn_dir) + self.buildEnv.run_command(command, self.buildEnv.source_dir, context) + + def _svn_update(self, context): + context.force_native_build = True + self.buildEnv.run_command("svn update", self.svn_path, context) + + def prepare(self): + self.command('svncheckout', self._svn_checkout) + self.command('svnupdate', self._svn_update) + if hasattr(self, 'patches'): + self.command('patch', self._patch) + + class Builder: subsource_dir = None From e50dc846a553f16183255baba5a1c9f207fb963b Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 3 May 2017 10:59:37 +0200 Subject: [PATCH 3/7] =?UTF-8?q?Get=20ICU=C2=A0source=20from=20SVN=20instea?= =?UTF-8?q?d=20of=20Release=20archive.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ICU doesn't provide data sources in their release archives but directly pre-compiled data. As we need to compile ourselves our data to remove unwanted data, we need to get the sources directly from the SVN. --- dependencies.py | 27 +++++---------------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/dependencies.py b/dependencies.py index 668ac2e..911575a 100644 --- a/dependencies.py +++ b/dependencies.py @@ -5,6 +5,7 @@ from dependency_utils import ( Dependency, ReleaseDownload, GitClone, + SvnClone, MakeBuilder, CMakeBuilder, MesonBuilder, @@ -175,34 +176,16 @@ class MicroHttpd(Dependency): class Icu(Dependency): name = "icu4c" - version = "58_2" + version = "58.2" - class Source(ReleaseDownload): + class Source(SvnClone): name = "icu4c" + svn_remote = "http://source.icu-project.org/repos/icu/tags/release-58-2/icu4c" + svn_dir = "icu4c" - @property - def source_dir(self): - return "{}-{}".format(self.name, self.target.version) - - archive = Remotefile('icu4c-58_2-src.tgz', - '2b0a4410153a9b20de0e20c7d8b66049a72aef244b53683d0d7521371683da0c', - 'https://freefr.dl.sourceforge.net/project/icu/ICU4C/58.2/icu4c-58_2-src.tgz') patches = ["icu4c_fix_static_lib_name_mingw.patch", "icu4c_android_elf64_st_info.patch"] - data = Remotefile('icudt56l.dat', - 'e23d85eee008f335fc49e8ef37b1bc2b222db105476111e3d16f0007d371cbca') - def _download_data(self, context): - self.buildEnv.download(self.data) - - def _copy_data(self, context): - context.try_skip(self.extract_path) - shutil.copyfile(pj(self.buildEnv.archive_dir, self.data.name), pj(self.extract_path, 'source', 'data', 'in', self.data.name)) - - def prepare(self): - super().prepare() - self.command("download_data", self._download_data) - self.command("copy_data", self._copy_data) class Builder(MakeBuilder): subsource_dir = "source" From fbb84e9d2709ec0efc0359be89d82b006c9bb0c4 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 3 May 2017 11:05:43 +0200 Subject: [PATCH 4/7] =?UTF-8?q?Patch=20ICU=C2=A0to=20remove=20unwanted=20d?= =?UTF-8?q?ata.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We don't need all the ICU data and we need to reduce binaries' size (Especially for android where we have a limit of 50Mo per APK). By removing unwanted ICU data, we reduce a lot the data size and so the final binaries' size. There is also some "problems" to compile the `ja.txt` collfiles, the "resource compiler" crashes. As we don't need it, we also remove it. --- dependencies.py | 3 +- patches/icu4c_custom_data.patch | 92 +++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 patches/icu4c_custom_data.patch diff --git a/dependencies.py b/dependencies.py index 911575a..afe31fc 100644 --- a/dependencies.py +++ b/dependencies.py @@ -184,7 +184,8 @@ class Icu(Dependency): svn_dir = "icu4c" patches = ["icu4c_fix_static_lib_name_mingw.patch", - "icu4c_android_elf64_st_info.patch"] + "icu4c_android_elf64_st_info.patch", + "icu4c_custom_data.patch"] class Builder(MakeBuilder): diff --git a/patches/icu4c_custom_data.patch b/patches/icu4c_custom_data.patch new file mode 100644 index 0000000..be07819 --- /dev/null +++ b/patches/icu4c_custom_data.patch @@ -0,0 +1,92 @@ +diff -ur icu4c/source/data/coll/colfiles.mk icu4c.patched/source/data/coll/colfiles.mk +--- icu4c/source/data/coll/colfiles.mk 2017-05-02 17:16:35.894345704 +0200 ++++ icu4c.patched/source/data/coll/colfiles.mk 2017-05-02 17:00:55.244124482 +0200 +@@ -56,7 +56,7 @@ + fr.txt fr_CA.txt ga.txt gl.txt gu.txt\ + ha.txt haw.txt he.txt hi.txt hr.txt\ + hsb.txt hu.txt hy.txt id.txt ig.txt\ +- is.txt it.txt ja.txt ka.txt kk.txt\ ++ is.txt it.txt ka.txt kk.txt\ + kl.txt km.txt kn.txt ko.txt kok.txt\ + ky.txt lb.txt lkt.txt ln.txt lo.txt\ + lt.txt lv.txt mk.txt ml.txt mn.txt\ +diff -ur icu4c/source/data/Makefile.in icu4c.patched/source/data/Makefile.in +--- icu4c/source/data/Makefile.in 2017-05-02 17:16:41.485257459 +0200 ++++ icu4c.patched/source/data/Makefile.in 2017-05-02 16:55:36.419069746 +0200 +@@ -277,10 +277,10 @@ + CFU_FILES=$(BUILDDIR)/$(CFU_FILES_SHORT) + + ## UCM files +--include $(UCMSRCDIR)/ucmcore.mk +--include $(UCMSRCDIR)/ucmfiles.mk +--include $(UCMSRCDIR)/ucmebcdic.mk +--include $(UCMSRCDIR)/ucmlocal.mk ++#-include $(UCMSRCDIR)/ucmcore.mk ++#-include $(UCMSRCDIR)/ucmfiles.mk ++#-include $(UCMSRCDIR)/ucmebcdic.mk ++#-include $(UCMSRCDIR)/ucmlocal.mk + ALL_UCM_SOURCE=ibm-37_P100-1995.ucm ibm-1047_P100-1995.ucm $(UCM_SOURCE_CORE) $(UCM_SOURCE_FILES) $(UCM_SOURCE_EBCDIC) $(UCM_SOURCE_LOCAL) + UCM_FILES = $(ALL_UCM_SOURCE:%=$(SRCDATADIR)/%) + CNV_FILES = $(ALL_UCM_SOURCE:%.ucm=$(BUILDDIR)/%.cnv) +@@ -291,24 +291,24 @@ + CNV_FILES_SHORT_SPECIAL=$(UCM_SOURCE_SPECIAL:%.ucm=%.cnv) + + ## RES files +--include $(LOCSRCDIR)/resfiles.mk +--include $(CURRSRCDIR)/resfiles.mk +--include $(LANGSRCDIR)/resfiles.mk +--include $(REGIONSRCDIR)/resfiles.mk +--include $(ZONESRCDIR)/resfiles.mk +--include $(UNITSRCDIR)/resfiles.mk ++#-include $(LOCSRCDIR)/resfiles.mk ++#-include $(CURRSRCDIR)/resfiles.mk ++#-include $(LANGSRCDIR)/resfiles.mk ++#-include $(REGIONSRCDIR)/resfiles.mk ++#-include $(ZONESRCDIR)/resfiles.mk ++#-include $(UNITSRCDIR)/resfiles.mk + -include $(COLSRCDIR)/colfiles.mk +--include $(RBNFSRCDIR)/rbnffiles.mk ++#-include $(RBNFSRCDIR)/rbnffiles.mk + -include $(TRANSLITSRCDIR)/trnsfiles.mk +--include $(LOCSRCDIR)/reslocal.mk +--include $(CURRSRCDIR)/reslocal.mk +--include $(LANGSRCDIR)/reslocal.mk +--include $(REGIONSRCDIR)/reslocal.mk +--include $(ZONESRCDIR)/reslocal.mk +--include $(UNITSRCDIR)/reslocal.mk ++#-include $(LOCSRCDIR)/reslocal.mk ++#-include $(CURRSRCDIR)/reslocal.mk ++#-include $(LANGSRCDIR)/reslocal.mk ++#-include $(REGIONSRCDIR)/reslocal.mk ++#-include $(ZONESRCDIR)/reslocal.mk ++#-include $(UNITSRCDIR)/reslocal.mk + -include $(COLSRCDIR)/collocal.mk +--include $(BRKSRCDIR)/brslocal.mk +--include $(RBNFSRCDIR)/rbnflocal.mk ++#-include $(BRKSRCDIR)/brslocal.mk ++#-include $(RBNFSRCDIR)/rbnflocal.mk + -include $(TRANSLITSRCDIR)/trnslocal.mk + ifdef GENRB_SOURCE + RES_SRC= root.txt $(GENRB_SOURCE) $(GENRB_ALIAS_SOURCE) $(GENRB_SOURCE_LOCAL) $(GENRB_ALIAS_SOURCE_LOCAL) +@@ -358,8 +358,8 @@ + GENRBOPTS=-k + + ## MISC files +--include $(MISCSRCDIR)/miscfiles.mk +--include $(MISCSRCDIR)/misclocal.mk ++#-include $(MISCSRCDIR)/miscfiles.mk ++#-include $(MISCSRCDIR)/misclocal.mk + MSC_SOURCE= $(MISC_SOURCE) $(MISC_SOURCE_LOCAL) + MSC_SRC_FILES=$(MSC_SOURCE:%=$(MISCSRCDIR)/%) + +@@ -436,8 +436,8 @@ + TRANSLIT_FILES_SHORT = $(TRANSLIT_SRC:%.txt=$(TRANSLIT_TREE)/%.res) + + ## SPP files +--include $(SPREPSRCDIR)/sprepfiles.mk +--include $(SPREPSRCDIR)/spreplocal.mk ++#-include $(SPREPSRCDIR)/sprepfiles.mk ++#-include $(SPREPSRCDIR)/spreplocal.mk + ALL_SPREP_SOURCE=$(SPREP_SOURCE) $(SPREP_SOURCE_LOCAL) + SPREP_FILES = $(ALL_SPREP_SOURCE:%.txt=$(BUILDDIR)/%.spp) + SPREP_FILES_SHORT = $(ALL_SPREP_SOURCE:%.txt=%.spp) From cfa7f0cb42672b651a7b34715f9dc4bf11536599 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 3 May 2017 11:09:12 +0200 Subject: [PATCH 5/7] =?UTF-8?q?Use=20ICU=C2=A0data=20as=20external=20archi?= =?UTF-8?q?ve=20on=20android=20platform.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On android platform, we load the ICU data as external data, not embeded in the kiwix-lib. So we need to generate correctly the dat file and use it in kiwix-android. --- dependencies.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/dependencies.py b/dependencies.py index afe31fc..8bf08dc 100644 --- a/dependencies.py +++ b/dependencies.py @@ -190,7 +190,13 @@ class Icu(Dependency): class Builder(MakeBuilder): subsource_dir = "source" - configure_option = "--disable-samples --disable-tests --disable-extras --disable-dyload" + + @property + def configure_option(self): + options = "--disable-samples --disable-tests --disable-extras --disable-dyload" + if self.buildEnv.platform_info.build == 'android': + options += " --with-data-packaging=archive" + return options class Icu_native(Icu): @@ -347,6 +353,7 @@ class KiwixAndroid(Dependency): except FileNotFoundError: pass shutil.copytree(pj(self.buildEnv.install_dir, 'kiwix-lib'), pj(self.build_path, 'kiwixlib', 'src', 'main')) + shutil.copy2(pj(self.buildEnv.install_dir, 'share', 'icu', '58.2', 'icudt58l.dat'), pj(self.build_path, 'app', 'src', 'main', 'assets', 'icudt.dat')) class KiwixCustomApp(Dependency): @@ -409,6 +416,7 @@ class KiwixCustomApp(Dependency): except FileNotFoundError: pass shutil.copytree(pj(self.buildEnv.install_dir, 'kiwix-lib'), pj(self.build_path, 'kiwixlib', 'src', 'main')) + shutil.copy2(pj(self.buildEnv.install_dir, 'share', 'icu', '58.2', 'icudt58l.dat'), pj(self.build_path, 'app', 'src', 'main', 'assets', 'icudt.dat')) # Generate custom directory try: From 317bc207fbed7dcd64056c11766f74720a2ab405 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 3 May 2017 18:56:43 +0200 Subject: [PATCH 6/7] Do not force specific arguments when compiling for win32. Those option should be set by build system, not by kiwix-build. Moreover, this impact other libraries (as icu) and generate pretty big library binaries. --- kiwix-build.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/kiwix-build.py b/kiwix-build.py index 2d68769..7f6bdb2 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -614,8 +614,6 @@ class mingw32_toolchain(Toolchain): env[k] = v env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') - env['CFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS'] - env['CXXFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS'] env['LIBS'] = " ".join(self.buildEnv.cross_config['extra_libs']) + " " +env['LIBS'] From 0e5fb294f68ab2a1e78df45f948ff4214b8c3424 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 3 May 2017 18:57:38 +0200 Subject: [PATCH 7/7] Better argument parse for better help text. (And one clean line :) ) --- kiwix-build.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/kiwix-build.py b/kiwix-build.py index 7f6bdb2..1ef4558 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -255,7 +255,7 @@ class BuildEnv: self.distname = 'debian' def setup_build(self, target_platform): - self.platform_info = platform_info = self.target_platforms[target_platform] + self.platform_info = self.target_platforms[target_platform] self.cross_config = self.platform_info.get_cross_config(self.distname) def setup_toolchains(self): @@ -915,7 +915,7 @@ class Builder: def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument('targets', default='kiwix-tools', nargs='?', + parser.add_argument('targets', default='kiwix-tools', nargs='?', metavar='TARGET', choices=Dependency.all_deps.keys()) parser.add_argument('--working-dir', default=".") parser.add_argument('--libprefix', default=None) @@ -925,15 +925,15 @@ def parse_args(): " log files per commands")) parser.add_argument('--hide-progress', action='store_false', dest='show_progress', help="Hide intermediate progress information.") - parser.add_argument('--no-cert-check', action='store_true', - help="Skip SSL certificate verification during download") parser.add_argument('--skip-source-prepare', action='store_true', help="Skip the source download part") parser.add_argument('--build-deps-only', action='store_true', help=("Build only the dependencies of the specified targets.")) - parser.add_argument('--clean-at-end', action='store_true', - help="Clean all intermediate files after the (successfull) build") - + subgroup = parser.add_argument_group('advanced') + subgroup.add_argument('--no-cert-check', action='store_true', + help="Skip SSL certificate verification during download") + subgroup.add_argument('--clean-at-end', action='store_true', + help="Clean all intermediate files after the (successfull) build") subgroup = parser.add_argument_group('custom app', description="Android custom app specific options") subgroup.add_argument('--android-custom-app',