Merge pull request #548 from kiwix/wasm

.github/scripts/build_projects.py

@@ -6,9 +6,11 @@ from common import (
     make_archive,
     create_desktop_image,
     fix_macos_rpath,
+    upload_archive,
     OS_NAME,
     PLATFORM_TARGET,
     DESKTOP,
+    DEV_BRANCH,
 )
 
 if (PLATFORM_TARGET.startswith("android_")
@@ -32,14 +34,18 @@ elif PLATFORM_TARGET in ("win32_static", "armhf_static", "armhf_dyn", "i586_stat
     TARGETS = ("kiwix-tools",)
 elif PLATFORM_TARGET == "flatpak":
     TARGETS = ("kiwix-desktop",)
+elif PLATFORM_TARGET == "wasm":
+    TARGETS = ("libzim", )
 else:
     TARGETS = ("libzim", "zim-tools", "libkiwix", "kiwix-tools")
 
 for target in TARGETS:
     run_kiwix_build(target, platform=PLATFORM_TARGET)
     if target == "kiwix-desktop":
-        create_desktop_image(make_release=False)
+        archive = create_desktop_image(make_release=False)
     else:
         if PLATFORM_TARGET == "native_mixed" and OS_NAME == "osx":
             fix_macos_rpath(target)
-        make_archive(target, make_release=False)
+        archive = make_archive(target, make_release=False)
+    if archive:
+        upload_archive(archive, target, make_release=False, dev_branch=DEV_BRANCH)

.github/scripts/build_release_nightly.py

@@ -38,6 +38,8 @@ elif PLATFORM_TARGET in ("win32_static", "armhf_static", "i586_static"):
     TARGETS = ("kiwix-tools",)
 elif PLATFORM_TARGET == "flatpak":
     TARGETS = ("kiwix-desktop",)
+elif PLATFORM_TARGET == "wasm":
+    TARGETS = ("libzim", )
 else:
     TARGETS = ("libzim", "zim-tools", "libkiwix", "kiwix-tools")
 

.github/scripts/common.py

@@ -39,6 +39,11 @@ _ref = _environ.get("GITHUB_REF", "").split("/")[-1]
 MAKE_RELEASE = re.fullmatch(r"r_[0-9]+", _ref) is not None
 MAKE_RELEASE = MAKE_RELEASE and (_environ.get('GITHUB_EVENT_NAME') != 'schedule')
 
+if not MAKE_RELEASE and _ref != "master":
+    DEV_BRANCH = _ref
+else:
+    DEV_BRANCH = None
+
 RELEASE_OS_NAME = "macos" if OS_NAME == "osx" else "linux"
 
 PLATFORM_TO_RELEASE = {
@@ -51,13 +56,7 @@ PLATFORM_TO_RELEASE = {
     "android_arm64": "android-arm64",
     "android_x86": "android-x86",
     "android_x86_64": "android-x86_64",
-}
+    "wasm": "wasm-emscripten",
-
-LIB_PREFIX = {
-    "android_arm": "arm-linux-androideabi",
-    "android_arm64": "aarch64-linux-android",
-    "android_x86": "i686-linux-android",
-    "android_x86_64": "x86_64-linux-android",
 }
 
 FLATPAK_HTTP_GIT_REMOTE = "https://github.com/flathub/org.kiwix.desktop.git"
@@ -94,15 +93,12 @@ EXPORT_FILES = {
     "libzim": (
         INSTALL_DIR,
         (
-            "lib/{libprefix}/libzim.so".format(
+            "lib/*/libzim.a",
-                libprefix=LIB_PREFIX.get(PLATFORM_TARGET, "x86_64-linux-gnu"),
+            "lib/*/libzim.so",
-            ),
+            "lib/*/libzim.so.{version}".format(
-            "lib/{libprefix}/libzim.so.{version}".format(
-                libprefix=LIB_PREFIX.get(PLATFORM_TARGET, "x86_64-linux-gnu"),
                 version=main_project_versions["libzim"]
             ),
-            "lib/{libprefix}/libzim.so.{version}".format(
+            "lib/*/libzim.so.{version}".format(
-                libprefix=LIB_PREFIX.get(PLATFORM_TARGET, "x86_64-linux-gnu"),
                 version=main_project_versions["libzim"][0]
             ),
             "lib/libzim.{}.dylib".format(
@@ -115,15 +111,11 @@ EXPORT_FILES = {
     "libkiwix": (
         INSTALL_DIR,
         (
-            "lib/{libprefix}/libkiwix.so".format(
+            "lib/*/libkiwix.so",
-                libprefix=LIB_PREFIX.get(PLATFORM_TARGET, "x86_64-linux-gnu"),
+            "lib/*/libkiwix.so.{version}".format(
-            ),
-            "lib/{libprefix}/libkiwix.so.{version}".format(
-                libprefix=LIB_PREFIX.get(PLATFORM_TARGET, "x86_64-linux-gnu"),
                 version=main_project_versions["libkiwix"]
             ),
-            "lib/{libprefix}/libkiwix.so.{version}".format(
+            "lib/*/libkiwix.so.{version}".format(
-                libprefix=LIB_PREFIX.get(PLATFORM_TARGET, "x86_64-linux-gnu"),
                 version=main_project_versions["libkiwix"][0]
             ),
             "include/kiwix/**/*.h"
@@ -240,7 +232,7 @@ def upload(file_to_upload, host, dest_path):
     subprocess.check_call(command)
 
 
-def upload_archive(archive, project, make_release):
+def upload_archive(archive, project, make_release, dev_branch=None):
     if not archive.exists():
         print_message("No archive {} to upload!", archive)
         return
@@ -257,6 +249,9 @@ def upload_archive(archive, project, make_release):
     else:
         dest_path = dest_path + "nightly/" + DATE
 
+    if dev_branch:
+        dest_path = "/data/tmp/ci/" + dev_branch
+    else:
         # Make the archive read only. This way, scp will preserve rights.
         # If somehow we try to upload twice the same archive, scp will fails.
         archive.chmod(0o444)
@@ -280,6 +275,7 @@ def make_deps_archive(target=None, name=None, full=False):
         if (base_dir / "meson_cross_file.txt").exists():
             files_to_archive.append(base_dir / "meson_cross_file.txt")
     files_to_archive += HOME.glob("BUILD_*/android-ndk*")
+    files_to_archive += HOME.glob("BUILD_*/emsdk*")
     if (BASE_DIR / "meson_cross_file.txt").exists():
         files_to_archive.append(BASE_DIR / "meson_cross_file.txt")
 

.github/scripts/compile_all_deps.py

@@ -14,6 +14,8 @@ from common import (
 
 if PLATFORM_TARGET.startswith("android_") or PLATFORM_TARGET.startswith("iOS"):
     TARGETS = ("libzim", "libkiwix")
+elif PLATFORM_TARGET == "wasm":
+    TARGETS = ("libzim", )
 elif PLATFORM_TARGET.startswith("native_"):
     if OS_NAME == "osx":
         TARGETS = ("libzim", "zim-tools", "libkiwix")

.github/workflows/ci.yml

@@ -38,6 +38,7 @@ jobs:
           - native_dyn
           - native_mixed
           - native_desktop
+          - wasm
           - armhf_static
           - armhf_dyn
           - i586_static

kiwixbuild/buildenv.py

@@ -148,3 +148,21 @@ class BuildEnv:
         if cross_path:
             env['PATH'] = ':'.join(self.platformInfo.get_bin_dir() + [env['PATH']])
         return env
+
+
+    @property
+    def configure_wrapper(self):
+        wrapper = getattr(self.platformInfo, "configure_wrapper", "")
+        if wrapper:
+            return "{} ".format(wrapper)
+        else:
+            return ""
+
+    @property
+    def make_wrapper(self):
+        wrapper = getattr(self.platformInfo, "make_wrapper", "")
+        if wrapper:
+            return "{} ".format(wrapper)
+        else:
+            return ""
+

kiwixbuild/dependencies/__init__.py

@@ -6,6 +6,7 @@ from . import (
     aria2,
     armhf,
     docoptcpp,
+    emsdk,
     flatpak,
     gumbo,
     icu4c,

kiwixbuild/dependencies/all_dependencies.py

@@ -12,6 +12,9 @@ class AllBaseDependencies(Dependency):
     class Builder(NoopBuilder):
         @classmethod
         def get_dependencies(cls, platformInfo, allDeps):
+            if platformInfo.build == "wasm":
+                return ['zlib', 'lzma', 'zstd', 'icu4c', 'xapian-core']
+
             base_deps = ['zlib', 'lzma', 'zstd', 'xapian-core', 'pugixml', 'libcurl', 'icu4c', 'mustache', 'libmicrohttpd', 'zim-testing-suite']
             # Add specific dependencies depending of the platform
             if platformInfo.build not in ('android', 'iOS'):

kiwixbuild/dependencies/base.py

@@ -70,8 +70,9 @@ class Source:
     def _patch(self, context):
         context.try_skip(self.source_path)
         for p in self.patches:
-            with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input:
+            patch_file_path = pj(SCRIPT_DIR, 'patches', p)
-                run_command("patch -p1", self.source_path, context, input=patch_input.read())
+            patch_command = "patch -p1 -i {patch}".format(patch=patch_file_path)
+            run_command(patch_command, self.source_path, context)
 
     def command(self, name, function, *args):
         print("  {} {} : ".format(name, self.name), end="", flush=True)
@@ -339,7 +340,7 @@ class MakeBuilder(Builder):
 
     @property
     def make_install_target(self):
-        if self.buildEnv.platformInfo.build == 'iOS':
+        if self.buildEnv.platformInfo.build in ('iOS', "wasm"):
             return 'install'
         return 'install-strip'
 
@@ -368,8 +369,9 @@ class MakeBuilder(Builder):
 
     def _configure(self, context):
         context.try_skip(self.build_path)
-        command = "{configure_script} {configure_option}"
+        command = "{configure_wrapper}{configure_script} {configure_option}"
         command = command.format(
+            configure_wrapper=self.buildEnv.configure_wrapper,
             configure_script=pj(self.source_path, self.configure_script),
             configure_option=self.all_configure_option
         )
@@ -379,7 +381,8 @@ class MakeBuilder(Builder):
 
     def _compile(self, context):
         context.try_skip(self.build_path)
-        command = "make -j4 {make_target} {make_option}".format(
+        command = "{make_wrapper}make -j4 {make_target} {make_option}".format(
+            make_wrapper=self.buildEnv.make_wrapper,
             make_target=self.make_target,
             make_option=self.make_option
         )
@@ -388,7 +391,8 @@ class MakeBuilder(Builder):
 
     def _install(self, context):
         context.try_skip(self.build_path)
-        command = "make {make_install_target} {make_option}".format(
+        command = "{make_wrapper}make {make_install_target} {make_option}".format(
+            make_wrapper=self.buildEnv.make_wrapper,
             make_install_target=self.make_install_target,
             make_option=self.make_option
         )
@@ -397,7 +401,9 @@ class MakeBuilder(Builder):
 
     def _make_dist(self, context):
         context.try_skip(self.build_path)
-        command = "make dist"
+        command = "{make_wrapper}make dist".format(
+            make_wrapper=self.buildEnv.make_wrapper
+        )
         env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
         run_command(command, self.build_path, context, env=env)
 

kiwixbuild/dependencies/emsdk.py

@@ -0,0 +1,47 @@
+import os
+
+from .base import Dependency, ReleaseDownload, Builder
+from kiwixbuild.utils import Remotefile, run_command, copy_tree
+
+pj = os.path.join
+
+class emsdk(Dependency):
+    dont_skip = True
+    neutral = False
+    name = 'emsdk'
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('emsdk-3.1.24.tar.gz',
+                             '1aa5365ccb2147701cc9d1e59a5a49577c1d6aea55da7c450df2d5ffa48b8a58',
+                             'https://codeload.github.com/emscripten-core/emsdk/tar.gz/refs/tags/3.1.24')
+
+        @property
+        def source_dir(self):
+            return self.target.full_name()
+
+
+    class Builder(Builder):
+        @property
+        def install_path(self):
+            return self.build_path
+
+        def _copy_source(self, context):
+            context.try_skip(self.build_path)
+            copy_tree(self.source_path, self.build_path)
+
+        def _install(self, context):
+            context.try_skip(self.build_path)
+            command = "./emsdk install 3.1.24"
+            run_command(command, self.build_path, context)
+
+        def _activate(self, context):
+            context.try_skip(self.build_path)
+            command = "./emsdk activate 3.1.24"
+            run_command(command, self.build_path, context)
+
+
+        def build(self):
+            self.command('copy_source', self._copy_source)
+            self.command('install', self._install)
+            self.command('activate', self._activate)
+

kiwixbuild/dependencies/icu4c.py

@@ -18,7 +18,9 @@ class Icu(Dependency):
                    "icu4c_custom_data.patch",
                    "icu4c_noxlocale.patch",
                    "icu4c_rpath.patch",
-                   "icu4c_build_config.patch"]
+                   "icu4c_build_config.patch",
+                   "icu4c_wasm.patch"
+                  ]
 
 
     class Builder(MakeBuilder):
@@ -42,6 +44,6 @@ class Icu(Dependency):
                     'native_static' if platformInfo.static else 'native_dyn')
                 options += " --with-cross-build={} --disable-tools".format(
                     icu_native_builder.build_path)
-            if platformInfo.build == 'android':
+            if platformInfo.build in ('android', 'wasm'):
                 options += " --with-data-packaging=archive"
             return options

kiwixbuild/dependencies/libzim.py

@@ -19,7 +19,7 @@ class Libzim(Dependency):
         @classmethod
         def get_dependencies(cls, platformInfo, allDeps):
             deps = ['lzma', 'zstd', 'xapian-core', 'icu4c']
-            if platformInfo.build != 'flatpak':
+            if platformInfo.name not in ('flatpak', 'wasm'):
                 deps.append('zim-testing-suite')
             return deps
 
@@ -37,7 +37,9 @@ class Libzim(Dependency):
             if platformInfo.name == "flatpak":
                 config_options.append("--wrap-mode=nodownload")
                 config_options.append("-Dtest_data_dir=none")
-            else:
+            if platformInfo.name == "wasm":
+                config_options.append("-Dexamples=false")
+            if platformInfo.name not in ("flatpak", "wasm"):
                 zim_testing_suite = get_target_step('zim-testing-suite', 'source')
                 config_options.append('-Dtest_data_dir={}'.format(zim_testing_suite.source_path))
             return " ".join(config_options)

kiwixbuild/dependencies/lzma.py

@@ -9,11 +9,20 @@ class lzma(Dependency):
     name = 'lzma'
 
     class Source(ReleaseDownload):
-        archive = Remotefile('xz-5.2.4.tar.gz',
+        archive = Remotefile('xz-5.2.6.tar.gz',
-                             'b512f3b726d3b37b6dc4c8570e137b9311e7552e8ccbab4d39d47ce5f4177145'
+                             'a2105abee17bcd2ebd15ced31b4f5eda6e17efd6b10f921a01cda4a44c91b3a0',
+                             'https://altushost-swe.dl.sourceforge.net/project/lzmautils/xz-5.2.6.tar.gz'
                             )
 
     class Builder(MakeBuilder):
         @property
         def configure_option(self):
-            return "--disable-xz --disable-xzdec"
+            return ("--disable-xz "
+                    "--disable-xzdec "
+                    "--disable-lzmadec "
+                    "--disable-lzmainfo "
+                    "--disable-lzma-links "
+                    "--disable-scripts "
+                    "--disable-doc "
+#                    "--disable-symbol-versions "
+                  )

kiwixbuild/dependencies/xapian.py

@@ -24,7 +24,7 @@ class Xapian(Dependency):
         @classmethod
         def get_dependencies(cls, platformInfo, allDeps):
             deps = ['zlib', 'lzma']
-            if (platformInfo.build == 'win32'
+            if (platformInfo.build in ('win32', 'wasm')
              or neutralEnv('distname') == 'Darwin'):
                 return deps
             return deps + ['uuid']

kiwixbuild/platforms/__init__.py

@@ -9,5 +9,6 @@ from . import (
     ios,
     native,
     neutral,
-    win32
+    win32,
+    wasm
 )

kiwixbuild/platforms/wasm.py

@@ -0,0 +1,100 @@
+from .base import PlatformInfo
+
+from kiwixbuild.utils import pj
+from kiwixbuild._global import get_target_step
+
+
+class WasmPlatformInfo(PlatformInfo):
+    name = 'wasm'
+    static = True
+    build = 'wasm'
+    arch_full = 'wasm64-emscripten'
+    #arch_full = 'wasm64-linux'
+    toolchain_names = ['emsdk']
+    compatible_hosts = ['fedora', 'debian']
+    exe_wrapper_def = ""
+
+    def get_cross_config(self):
+        return {
+            'binaries': self.binaries,
+            'exe_wrapper_def': '',
+            'root_path': self.root_path,
+            'extra_libs': [],
+            'extra_cflags': [],
+            'host_machine': {
+                'system': 'emscripten',
+                'lsystem': 'emscripten',
+                'cpu_family': 'wasm64',
+                'cpu': 'wasm64',
+                'endian': 'little',
+                'abi': ''
+            }
+        }
+
+    @property
+    def wasm_sdk(self):
+        return get_target_step('emsdk', self.name)
+
+    @property
+    def install_path(self):
+        return self.wasm_sdk.install_path
+
+    @property
+    def root_path(self):
+        return self.install_path
+
+    @property
+    def binaries(self):
+        binaries = (('CC', 'emcc'),
+                    ('CXX', 'em++'),
+                    ('AR', 'emar'),
+                    ('STRIP', 'emstrip'),
+                    ('WINDRES', 'windres'),
+                    ('RANLIB', 'emranlib'),
+                    ('LD', 'wasm-ld'))
+        binaries = {k:pj(self.install_path, 'upstream', 'emscripten', v)
+                    for k,v in binaries}
+        binaries['PKGCONFIG'] = 'pkg-config'
+        return binaries
+
+    @property
+    def configure_option(self):
+        #return ""
+        return '--host={}'.format(self.arch_full)
+
+    @property
+    def configure_wrapper(self):
+        return "emconfigure"
+
+    @property
+    def make_wrapper(self):
+        return "emmake"
+
+    def get_bin_dir(self):
+        return [pj(self.install_path, 'bin')]
+
+    def get_env(self):
+        env = super().get_env()
+        env['PATH'] = ':'.join([
+            env['PATH'],
+            self.install_path,
+            pj(self.install_path, 'upstream', 'emscripten'),
+            pj(self.install_path, 'node', '14.18.2_64bit', 'bin')
+        ])
+        env['EMSDK'] = self.install_path
+        env['EMSDK_NODE'] = pj(self.install_path, 'node', '14.18.2_64bit', 'bin', 'node')
+        return env
+
+    def set_comp_flags(self, env):
+        super().set_comp_flags(env)
+        env['CFLAGS'] = " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS']
+        env['CXXFLAGS'] = " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
+
+    def set_compiler(self, env):
+        for k, v in self.binaries.items():
+            env[k] = v
+
+    def finalize_setup(self):
+        super().finalize_setup()
+        self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
+        self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')

kiwixbuild/versions.py

@@ -39,11 +39,11 @@ release_versions = {
 
 # This is the "version" of the whole base_deps_versions dict.
 # Change this when you change base_deps_versions.
-base_deps_meta_version = '78'
+base_deps_meta_version = '79'
 
 base_deps_versions = {
   'zlib' : '1.2.12',
-  'lzma' : '5.2.4',
+  'lzma' : '5.2.6',
   'zstd' : '1.5.2',
   'docoptcpp' : '0.6.2',
   'uuid' : '1.43.4',