Merge pull request #44 from kiwix/icu_custom_data

Icu custom data

dependencies.py

@@ -5,6 +5,7 @@ from dependency_utils import (
     Dependency,
     ReleaseDownload,
     GitClone,
+    SvnClone,
     MakeBuilder,
     CMakeBuilder,
     MesonBuilder,
@@ -175,38 +176,27 @@ class MicroHttpd(Dependency):
 
 class Icu(Dependency):
     name = "icu4c"
-    version = "58_2"
+    version = "58.2"
 
-    class Source(ReleaseDownload):
+    class Source(SvnClone):
         name = "icu4c"
+        svn_remote = "http://source.icu-project.org/repos/icu/tags/release-58-2/icu4c"
+        svn_dir = "icu4c"
 
-        @property
-        def source_dir(self):
-            return "{}-{}".format(self.name, self.target.version)
-
-        archive = Remotefile('icu4c-58_2-src.tgz',
-                             '2b0a4410153a9b20de0e20c7d8b66049a72aef244b53683d0d7521371683da0c',
-                             'https://freefr.dl.sourceforge.net/project/icu/ICU4C/58.2/icu4c-58_2-src.tgz')
         patches = ["icu4c_fix_static_lib_name_mingw.patch",
-                   "icu4c_android_elf64_st_info.patch"]
+                   "icu4c_android_elf64_st_info.patch",
-        data = Remotefile('icudt56l.dat',
+                   "icu4c_custom_data.patch"]
-                          'e23d85eee008f335fc49e8ef37b1bc2b222db105476111e3d16f0007d371cbca')
 
-        def _download_data(self, context):
-            self.buildEnv.download(self.data)
-
-        def _copy_data(self, context):
-            context.try_skip(self.extract_path)
-            shutil.copyfile(pj(self.buildEnv.archive_dir, self.data.name), pj(self.extract_path, 'source', 'data', 'in', self.data.name))
-
-        def prepare(self):
-            super().prepare()
-            self.command("download_data", self._download_data)
-            self.command("copy_data", self._copy_data)
 
     class Builder(MakeBuilder):
         subsource_dir = "source"
-        configure_option = "--disable-samples --disable-tests --disable-extras --disable-dyload"
+
+        @property
+        def configure_option(self):
+            options = "--disable-samples --disable-tests --disable-extras --disable-dyload"
+            if self.buildEnv.platform_info.build == 'android':
+                options += " --with-data-packaging=archive"
+            return options
 
 
 class Icu_native(Icu):
@@ -363,6 +353,7 @@ class KiwixAndroid(Dependency):
             except FileNotFoundError:
                 pass
             shutil.copytree(pj(self.buildEnv.install_dir, 'kiwix-lib'), pj(self.build_path, 'kiwixlib', 'src', 'main'))
+            shutil.copy2(pj(self.buildEnv.install_dir, 'share', 'icu', '58.2', 'icudt58l.dat'), pj(self.build_path, 'app', 'src', 'main', 'assets', 'icudt.dat'))
 
 
 class KiwixCustomApp(Dependency):
@@ -425,6 +416,7 @@ class KiwixCustomApp(Dependency):
             except FileNotFoundError:
                 pass
             shutil.copytree(pj(self.buildEnv.install_dir, 'kiwix-lib'), pj(self.build_path, 'kiwixlib', 'src', 'main'))
+            shutil.copy2(pj(self.buildEnv.install_dir, 'share', 'icu', '58.2', 'icudt58l.dat'), pj(self.build_path, 'app', 'src', 'main', 'assets', 'icudt.dat'))
 
             # Generate custom directory
             try:

dependency_utils.py

@@ -82,6 +82,14 @@ class Source:
     def source_dir(self):
         return self.target.full_name
 
+    def _patch(self, context):
+        source_path = pj(self.buildEnv.source_dir, self.source_dir)
+        context.try_skip(source_path)
+        context.force_native_build = True
+        for p in self.patches:
+            with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input:
+                self.buildEnv.run_command("patch -p1", source_path, context, input=patch_input.read())
+
     def command(self, *args, **kwargs):
         return self.target.command(*args, **kwargs)
 
@@ -105,13 +113,6 @@ class ReleaseDownload(Source):
                         topdir=self.archive_top_dir,
                         name=self.source_dir)
 
-    def _patch(self, context):
-        context.try_skip(self.extract_path)
-        context.force_native_build = True
-        for p in self.patches:
-            with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input:
-                self.buildEnv.run_command("patch -p1", self.extract_path, context, input=patch_input.read())
-
     def prepare(self):
         self.command('download', self._download)
         self.command('extract', self._extract)
@@ -149,6 +150,33 @@ class GitClone(Source):
             self.command('post_prepare_script', self._post_prepare_script)
 
 
+class SvnClone(Source):
+    @property
+    def source_dir(self):
+        return self.svn_dir
+
+    @property
+    def svn_path(self):
+        return pj(self.buildEnv.source_dir, self.svn_dir)
+
+    def _svn_checkout(self, context):
+        context.force_native_build = True
+        if os.path.exists(self.svn_path):
+            raise SkipCommand()
+        command = "svn checkout {} {}".format(self.svn_remote, self.svn_dir)
+        self.buildEnv.run_command(command, self.buildEnv.source_dir, context)
+
+    def _svn_update(self, context):
+        context.force_native_build = True
+        self.buildEnv.run_command("svn update", self.svn_path, context)
+
+    def prepare(self):
+        self.command('svncheckout', self._svn_checkout)
+        self.command('svnupdate', self._svn_update)
+        if hasattr(self, 'patches'):
+            self.command('patch', self._patch)
+
+
 class Builder:
     subsource_dir = None
 

kiwix-build.py

@@ -255,7 +255,7 @@ class BuildEnv:
                 self.distname = 'debian'
 
     def setup_build(self, target_platform):
-        self.platform_info = platform_info = self.target_platforms[target_platform]
+        self.platform_info = self.target_platforms[target_platform]
         self.cross_config = self.platform_info.get_cross_config(self.distname)
 
     def setup_toolchains(self):
@@ -614,8 +614,6 @@ class mingw32_toolchain(Toolchain):
             env[k] = v
 
         env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
-        env['CFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS']
-        env['CXXFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
         env['LIBS'] = " ".join(self.buildEnv.cross_config['extra_libs']) + " " +env['LIBS']
 
 
@@ -917,7 +915,7 @@ class Builder:
 
 def parse_args():
     parser = argparse.ArgumentParser()
-    parser.add_argument('targets', default='kiwix-tools', nargs='?',
+    parser.add_argument('targets', default='kiwix-tools', nargs='?', metavar='TARGET',
                         choices=Dependency.all_deps.keys())
     parser.add_argument('--working-dir', default=".")
     parser.add_argument('--libprefix', default=None)
@@ -927,15 +925,15 @@ def parse_args():
                               " log files per commands"))
     parser.add_argument('--hide-progress', action='store_false', dest='show_progress',
                         help="Hide intermediate progress information.")
-    parser.add_argument('--no-cert-check', action='store_true',
-                        help="Skip SSL certificate verification during download")
     parser.add_argument('--skip-source-prepare', action='store_true',
                         help="Skip the source download part")
     parser.add_argument('--build-deps-only', action='store_true',
                         help=("Build only the dependencies of the specified targets."))
-    parser.add_argument('--clean-at-end', action='store_true',
+    subgroup = parser.add_argument_group('advanced')
-                        help="Clean all intermediate files after the (successfull) build")
+    subgroup.add_argument('--no-cert-check', action='store_true',
-
+                          help="Skip SSL certificate verification during download")
+    subgroup.add_argument('--clean-at-end', action='store_true',
+                          help="Clean all intermediate files after the (successfull) build")
     subgroup = parser.add_argument_group('custom app',
                                          description="Android custom app specific options")
     subgroup.add_argument('--android-custom-app',

patches/icu4c_custom_data.patch

@@ -0,0 +1,92 @@
+diff -ur icu4c/source/data/coll/colfiles.mk icu4c.patched/source/data/coll/colfiles.mk
+--- icu4c/source/data/coll/colfiles.mk	2017-05-02 17:16:35.894345704 +0200
++++ icu4c.patched/source/data/coll/colfiles.mk	2017-05-02 17:00:55.244124482 +0200
+@@ -56,7 +56,7 @@
+  fr.txt fr_CA.txt ga.txt gl.txt gu.txt\
+  ha.txt haw.txt he.txt hi.txt hr.txt\
+  hsb.txt hu.txt hy.txt id.txt ig.txt\
+- is.txt it.txt ja.txt ka.txt kk.txt\
++ is.txt it.txt ka.txt kk.txt\
+  kl.txt km.txt kn.txt ko.txt kok.txt\
+  ky.txt lb.txt lkt.txt ln.txt lo.txt\
+  lt.txt lv.txt mk.txt ml.txt mn.txt\
+diff -ur icu4c/source/data/Makefile.in icu4c.patched/source/data/Makefile.in
+--- icu4c/source/data/Makefile.in	2017-05-02 17:16:41.485257459 +0200
++++ icu4c.patched/source/data/Makefile.in	2017-05-02 16:55:36.419069746 +0200
+@@ -277,10 +277,10 @@
+ CFU_FILES=$(BUILDDIR)/$(CFU_FILES_SHORT)
+ 
+ ## UCM files
+--include $(UCMSRCDIR)/ucmcore.mk
+--include $(UCMSRCDIR)/ucmfiles.mk
+--include $(UCMSRCDIR)/ucmebcdic.mk
+--include $(UCMSRCDIR)/ucmlocal.mk
++#-include $(UCMSRCDIR)/ucmcore.mk
++#-include $(UCMSRCDIR)/ucmfiles.mk
++#-include $(UCMSRCDIR)/ucmebcdic.mk
++#-include $(UCMSRCDIR)/ucmlocal.mk
+ ALL_UCM_SOURCE=ibm-37_P100-1995.ucm ibm-1047_P100-1995.ucm $(UCM_SOURCE_CORE) $(UCM_SOURCE_FILES) $(UCM_SOURCE_EBCDIC) $(UCM_SOURCE_LOCAL)
+ UCM_FILES = $(ALL_UCM_SOURCE:%=$(SRCDATADIR)/%)
+ CNV_FILES = $(ALL_UCM_SOURCE:%.ucm=$(BUILDDIR)/%.cnv)
+@@ -291,24 +291,24 @@
+ CNV_FILES_SHORT_SPECIAL=$(UCM_SOURCE_SPECIAL:%.ucm=%.cnv) 
+ 
+ ## RES files
+--include $(LOCSRCDIR)/resfiles.mk
+--include $(CURRSRCDIR)/resfiles.mk
+--include $(LANGSRCDIR)/resfiles.mk
+--include $(REGIONSRCDIR)/resfiles.mk
+--include $(ZONESRCDIR)/resfiles.mk
+--include $(UNITSRCDIR)/resfiles.mk
++#-include $(LOCSRCDIR)/resfiles.mk
++#-include $(CURRSRCDIR)/resfiles.mk
++#-include $(LANGSRCDIR)/resfiles.mk
++#-include $(REGIONSRCDIR)/resfiles.mk
++#-include $(ZONESRCDIR)/resfiles.mk
++#-include $(UNITSRCDIR)/resfiles.mk
+ -include $(COLSRCDIR)/colfiles.mk
+--include $(RBNFSRCDIR)/rbnffiles.mk
++#-include $(RBNFSRCDIR)/rbnffiles.mk
+ -include $(TRANSLITSRCDIR)/trnsfiles.mk
+--include $(LOCSRCDIR)/reslocal.mk
+--include $(CURRSRCDIR)/reslocal.mk
+--include $(LANGSRCDIR)/reslocal.mk
+--include $(REGIONSRCDIR)/reslocal.mk
+--include $(ZONESRCDIR)/reslocal.mk
+--include $(UNITSRCDIR)/reslocal.mk
++#-include $(LOCSRCDIR)/reslocal.mk
++#-include $(CURRSRCDIR)/reslocal.mk
++#-include $(LANGSRCDIR)/reslocal.mk
++#-include $(REGIONSRCDIR)/reslocal.mk
++#-include $(ZONESRCDIR)/reslocal.mk
++#-include $(UNITSRCDIR)/reslocal.mk
+ -include $(COLSRCDIR)/collocal.mk
+--include $(BRKSRCDIR)/brslocal.mk
+--include $(RBNFSRCDIR)/rbnflocal.mk
++#-include $(BRKSRCDIR)/brslocal.mk
++#-include $(RBNFSRCDIR)/rbnflocal.mk
+ -include $(TRANSLITSRCDIR)/trnslocal.mk
+ ifdef GENRB_SOURCE
+ RES_SRC= root.txt $(GENRB_SOURCE) $(GENRB_ALIAS_SOURCE) $(GENRB_SOURCE_LOCAL) $(GENRB_ALIAS_SOURCE_LOCAL)
+@@ -358,8 +358,8 @@
+ GENRBOPTS=-k
+ 
+ ## MISC files
+--include $(MISCSRCDIR)/miscfiles.mk
+--include $(MISCSRCDIR)/misclocal.mk
++#-include $(MISCSRCDIR)/miscfiles.mk
++#-include $(MISCSRCDIR)/misclocal.mk
+ MSC_SOURCE= $(MISC_SOURCE) $(MISC_SOURCE_LOCAL)
+ MSC_SRC_FILES=$(MSC_SOURCE:%=$(MISCSRCDIR)/%)
+ 
+@@ -436,8 +436,8 @@
+ TRANSLIT_FILES_SHORT = $(TRANSLIT_SRC:%.txt=$(TRANSLIT_TREE)/%.res)
+ 
+ ## SPP files
+--include $(SPREPSRCDIR)/sprepfiles.mk
+--include $(SPREPSRCDIR)/spreplocal.mk
++#-include $(SPREPSRCDIR)/sprepfiles.mk
++#-include $(SPREPSRCDIR)/spreplocal.mk
+ ALL_SPREP_SOURCE=$(SPREP_SOURCE) $(SPREP_SOURCE_LOCAL)
+ SPREP_FILES = $(ALL_SPREP_SOURCE:%.txt=$(BUILDDIR)/%.spp)
+ SPREP_FILES_SHORT = $(ALL_SPREP_SOURCE:%.txt=%.spp)