Merge pull request #642 from kiwix/update_icu

Update icu4c to version 71.1

.github/scripts/common.py

@@ -93,6 +93,10 @@ EXPORT_FILES = {
             "lib/libzim.dylib",
             "lib/*/libzim.pc",
             "include/zim/**/*.h",
+            "share/icu/{}/icudt{}l.dat".format(
+                base_deps_versions["icu4c"],
+                major_version(base_deps_versions["icu4c"])
+            )
         ),
     ),
     "libkiwix": (
@@ -110,7 +114,11 @@ EXPORT_FILES = {
             ),
             "lib/libkiwix.dylib",
             "lib/*/libkiwix.pc",
-            "include/kiwix/**/*.h"
+            "include/kiwix/**/*.h",
+            "share/icu/{}/icudt{}l.dat".format(
+                base_deps_versions["icu4c"],
+                major_version(base_deps_versions["icu4c"])
+            )
         ),
     ),
 }

MANIFEST.in

@@ -1,2 +1,3 @@
 include kiwixbuild/templates/*_cross_file.txt
 include kiwixbuild/patches/*.patch
+include kiwixbuild/dependencies/icu4c_data_filter.json

kiwixbuild/dependencies/base.py

@@ -110,22 +110,28 @@ class NoopSource(Source):
 class ReleaseDownload(Source):
     archive_top_dir = None
 
+    @property
+    def archives(self):
+        return (self.archive, )
+
     @property
     def extract_path(self):
         return pj(neutralEnv('source_dir'), self.source_dir)
 
     def _download(self, context):
         context.try_skip(neutralEnv('archive_dir'), self.full_name)
-        neutralEnv('download')(self.archive)
+        for archive in self.archives:
+            neutralEnv('download')(archive)
 
     def _extract(self, context):
         context.try_skip(self.extract_path)
         if os.path.exists(self.extract_path):
             shutil.rmtree(self.extract_path)
-        extract_archive(pj(neutralEnv('archive_dir'), self.archive.name),
+        for archive in self.archives:
-                        neutralEnv('source_dir'),
+            extract_archive(pj(neutralEnv('archive_dir'), archive.name),
-                        topdir=self.archive_top_dir,
+                            neutralEnv('source_dir'),
-                        name=self.source_dir)
+                            topdir=self.archive_top_dir,
+                            name=self.source_dir)
 
     def prepare(self):
         self.command('download', self._download)
@@ -277,6 +283,8 @@ class Builder:
         if hasattr(self, '_pre_build_script'):
             self.command('pre_build_script', self._pre_build_script)
         self.command('configure', self._configure)
+        if hasattr(self, '_post_configure_script'):
+            self.command('post_configure_script', self._post_configure_script)
         self.command('compile', self._compile)
         if hasattr(self, '_test'):
             self.command('test', self._test)

kiwixbuild/dependencies/icu4c.py

@@ -4,21 +4,49 @@ from .base import (
     MakeBuilder
 )
 
-from kiwixbuild.utils import SkipCommand, Remotefile
+from kiwixbuild.utils import pj, SkipCommand, Remotefile, extract_archive
-from kiwixbuild._global import get_target_step
+from kiwixbuild._global import get_target_step, neutralEnv
+import os, shutil
+import fileinput
 
 class Icu(Dependency):
     name = "icu4c"
 
     class Source(ReleaseDownload):
-        archive = Remotefile('icu4c_svn_58-2.tar.gz',
+        archive_src = Remotefile('icu4c-73_2-src.tgz',
-                             'fd8fcc1f1b8b2b71b879e88844480ebec107189c21076c81573f71dca5686a0d')
+                                 '818a80712ed3caacd9b652305e01afc7fa167e6f2e94996da44b90c2ab604ce1',
-        patches = ["icu4c_fix_static_lib_name_mingw.patch",
+                                 'https://github.com/unicode-org/icu/releases/download/release-73-2/icu4c-73_2-src.tgz')
-                   "icu4c_android_elf64_st_info.patch",
+        archive_data = Remotefile('icu4c-73_2-data.zip',
-                   "icu4c_custom_data.patch",
+                                  'ca1ee076163b438461e484421a7679fc33a64cd0a54f9d4b401893fa1eb42701',
-                   "icu4c_noxlocale.patch",
+                                  'https://github.com/unicode-org/icu/releases/download/release-73-2/icu4c-73_2-data.zip')
+
+
+        archives = [archive_src, archive_data]
+
+        def _extract(self, context):
+            context.try_skip(self.extract_path)
+            if os.path.exists(self.extract_path):
+                shutil.rmtree(self.extract_path)
+            extract_archive(
+                pj(neutralEnv('archive_dir'), self.archive_src.name),
+                neutralEnv('source_dir'),
+                topdir=None,
+                name=self.source_dir)
+            shutil.rmtree(pj(neutralEnv('source_dir'), self.source_dir, 'source', 'data'))
+            extract_archive(
+                pj(neutralEnv('archive_dir'), self.archive_data.name),
+                pj(neutralEnv('source_dir'), self.source_dir, 'source'),
+                topdir='data',
+                name='data'
+            )
+
+        patches = [
+                   "icu4c_fix_static_lib_name_mingw.patch",
+        #           "icu4c_android_elf64_st_info.patch",
+        #           "icu4c_custom_data.patch",
+        #           "icu4c_noxlocale.patch",
                    "icu4c_rpath.patch",
-                   "icu4c_build_config.patch",
+        #           "icu4c_build_config.patch",
                    "icu4c_wasm.patch"
                   ]
 
@@ -47,3 +75,16 @@ class Icu(Dependency):
             if platformInfo.build in ('android', 'wasm'):
                 options += " --with-data-packaging=archive"
             return options
+
+        def set_env(self, env):
+            env['ICU_DATA_FILTER_FILE'] = pj(os.path.dirname(os.path.realpath(__file__)), "icu4c_data_filter.json")
+
+        def _post_configure_script(self, context):
+            if self.buildEnv.platformInfo.build != "wasm":
+                context.skip()
+            context.try_skip(self.build_path)
+            for line in fileinput.input(pj(self.build_path, 'Makefile'), inplace=True):
+                if line == "#DATASUBDIR = data\n":
+                    print("DATASUBDIR = data")
+                else:
+                    print(line, end="")

kiwixbuild/dependencies/icu4c_data_filter.json

@@ -0,0 +1,14 @@
+{
+    "strategy": "additive",
+    "featureFilters": {
+        "lang_tree": "include",
+        "locales_tree": "include",
+        "translit": "include",
+        "misc": {
+            "includelist": [
+               "likelySubtags",
+               "metadata"
+            ]
+        }
+    }
+}

kiwixbuild/utils.py

@@ -189,6 +189,9 @@ class Context:
         self.autoskip_file = None
         self.no_skip = False
 
+    def skip(self, msg=""):
+        raise SkipCommand(msg)
+
     def try_skip(self, path, extra_name=""):
         if self.no_skip:
             return

kiwixbuild/versions.py

@@ -39,7 +39,7 @@ release_versions = {
 
 # This is the "version" of the whole base_deps_versions dict.
 # Change this when you change base_deps_versions.
-base_deps_meta_version = '93'
+base_deps_meta_version = '94'
 
 base_deps_versions = {
   'zlib' : '1.2.12',
@@ -52,7 +52,7 @@ base_deps_versions = {
   'pugixml' : '1.2',
   'libmicrohttpd' : '0.9.76',
   'gumbo' : '0.10.1',
-  'icu4c' : '58.2',
+  'icu4c' : '73.2',
   'libaria2' : '1.36.0',
   'libmagic' : '5.44',
   'android-ndk' : 'r21e',