smallsolar
/
kiwix-build
mirror of https://github.com/kiwix/kiwix-build.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #548 from kiwix/wasm

This commit is contained in:
Matthieu Gautier 2022-11-22 13:49:41 +01:00 committed by GitHub
parent 3c445389be a44531bb8a
commit 75ca8dcf2a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 3476 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
.github/scripts/build_projects.py vendored
View File

@ -6,9 +6,11 @@ from common import (
make_archive,
create_desktop_image,
fix_macos_rpath,
upload_archive,
OS_NAME,
PLATFORM_TARGET,
DESKTOP,
DEV_BRANCH,
)
if (PLATFORM_TARGET.startswith("android_")
@ -32,14 +34,18 @@ elif PLATFORM_TARGET in ("win32_static", "armhf_static", "armhf_dyn", "i586_stat
TARGETS = ("kiwix-tools",)
elif PLATFORM_TARGET == "flatpak":
TARGETS = ("kiwix-desktop",)
elif PLATFORM_TARGET == "wasm":
TARGETS = ("libzim", )
else:
TARGETS = ("libzim", "zim-tools", "libkiwix", "kiwix-tools")
for target in TARGETS:
run_kiwix_build(target, platform=PLATFORM_TARGET)
if target == "kiwix-desktop":
create_desktop_image(make_release=False)
archive = create_desktop_image(make_release=False)
else:
if PLATFORM_TARGET == "native_mixed" and OS_NAME == "osx":
fix_macos_rpath(target)
make_archive(target, make_release=False)
archive = make_archive(target, make_release=False)
if archive:
upload_archive(archive, target, make_release=False, dev_branch=DEV_BRANCH)

2
.github/scripts/build_release_nightly.py vendored
View File

@ -38,6 +38,8 @@ elif PLATFORM_TARGET in ("win32_static", "armhf_static", "i586_static"):
TARGETS = ("kiwix-tools",)
elif PLATFORM_TARGET == "flatpak":
TARGETS = ("kiwix-desktop",)
elif PLATFORM_TARGET == "wasm":
TARGETS = ("libzim", )
else:
TARGETS = ("libzim", "zim-tools", "libkiwix", "kiwix-tools")

46
.github/scripts/common.py vendored
View File

@ -39,6 +39,11 @@ _ref = _environ.get("GITHUB_REF", "").split("/")[-1]
MAKE_RELEASE = re.fullmatch(r"r_[0-9]+", _ref) is not None
MAKE_RELEASE = MAKE_RELEASE and (_environ.get('GITHUB_EVENT_NAME') != 'schedule')
if not MAKE_RELEASE and _ref != "master":
DEV_BRANCH = _ref
else:
DEV_BRANCH = None
RELEASE_OS_NAME = "macos" if OS_NAME == "osx" else "linux"
PLATFORM_TO_RELEASE = {
@ -51,13 +56,7 @@ PLATFORM_TO_RELEASE = {
"android_arm64": "android-arm64",
"android_x86": "android-x86",
"android_x86_64": "android-x86_64",
}
LIB_PREFIX = {
"android_arm": "arm-linux-androideabi",
"android_arm64": "aarch64-linux-android",
"android_x86": "i686-linux-android",
"android_x86_64": "x86_64-linux-android",
"wasm": "wasm-emscripten",
}
FLATPAK_HTTP_GIT_REMOTE = "https://github.com/flathub/org.kiwix.desktop.git"
@ -94,15 +93,12 @@ EXPORT_FILES = {
"libzim": (
INSTALL_DIR,
(
"lib/{libprefix}/libzim.so".format(
libprefix=LIB_PREFIX.get(PLATFORM_TARGET, "x86_64-linux-gnu"),
),
"lib/{libprefix}/libzim.so.{version}".format(
libprefix=LIB_PREFIX.get(PLATFORM_TARGET, "x86_64-linux-gnu"),
"lib/*/libzim.a",
"lib/*/libzim.so",
"lib/*/libzim.so.{version}".format(
version=main_project_versions["libzim"]
),
"lib/{libprefix}/libzim.so.{version}".format(
libprefix=LIB_PREFIX.get(PLATFORM_TARGET, "x86_64-linux-gnu"),
"lib/*/libzim.so.{version}".format(
version=main_project_versions["libzim"][0]
),
"lib/libzim.{}.dylib".format(
@ -115,15 +111,11 @@ EXPORT_FILES = {
"libkiwix": (
INSTALL_DIR,
(
"lib/{libprefix}/libkiwix.so".format(
libprefix=LIB_PREFIX.get(PLATFORM_TARGET, "x86_64-linux-gnu"),
),
"lib/{libprefix}/libkiwix.so.{version}".format(
libprefix=LIB_PREFIX.get(PLATFORM_TARGET, "x86_64-linux-gnu"),
"lib/*/libkiwix.so",
"lib/*/libkiwix.so.{version}".format(
version=main_project_versions["libkiwix"]
),
"lib/{libprefix}/libkiwix.so.{version}".format(
libprefix=LIB_PREFIX.get(PLATFORM_TARGET, "x86_64-linux-gnu"),
"lib/*/libkiwix.so.{version}".format(
version=main_project_versions["libkiwix"][0]
),
"include/kiwix/**/*.h"
@ -240,7 +232,7 @@ def upload(file_to_upload, host, dest_path):
subprocess.check_call(command)
def upload_archive(archive, project, make_release):
def upload_archive(archive, project, make_release, dev_branch=None):
if not archive.exists():
print_message("No archive {} to upload!", archive)
return
@ -257,9 +249,12 @@ def upload_archive(archive, project, make_release):
else:
dest_path = dest_path + "nightly/" + DATE
# Make the archive read only. This way, scp will preserve rights.
# If somehow we try to upload twice the same archive, scp will fails.
archive.chmod(0o444)
if dev_branch:
dest_path = "/data/tmp/ci/" + dev_branch
else:
# Make the archive read only. This way, scp will preserve rights.
# If somehow we try to upload twice the same archive, scp will fails.
archive.chmod(0o444)
upload(archive, host, dest_path)
@ -280,6 +275,7 @@ def make_deps_archive(target=None, name=None, full=False):
if (base_dir / "meson_cross_file.txt").exists():
files_to_archive.append(base_dir / "meson_cross_file.txt")
files_to_archive += HOME.glob("BUILD_*/android-ndk*")
files_to_archive += HOME.glob("BUILD_*/emsdk*")
if (BASE_DIR / "meson_cross_file.txt").exists():
files_to_archive.append(BASE_DIR / "meson_cross_file.txt")

2
.github/scripts/compile_all_deps.py vendored
View File

@ -14,6 +14,8 @@ from common import (
if PLATFORM_TARGET.startswith("android_") or PLATFORM_TARGET.startswith("iOS"):
TARGETS = ("libzim", "libkiwix")
elif PLATFORM_TARGET == "wasm":
TARGETS = ("libzim", )
elif PLATFORM_TARGET.startswith("native_"):
if OS_NAME == "osx":
TARGETS = ("libzim", "zim-tools", "libkiwix")

1
.github/workflows/ci.yml vendored
View File

@ -38,6 +38,7 @@ jobs:
- native_dyn
- native_mixed
- native_desktop
- wasm
- armhf_static
- armhf_dyn
- i586_static

18
kiwixbuild/buildenv.py
View File

@ -148,3 +148,21 @@ class BuildEnv:
if cross_path:
env['PATH'] = ':'.join(self.platformInfo.get_bin_dir() + [env['PATH']])
return env
@property
def configure_wrapper(self):
wrapper = getattr(self.platformInfo, "configure_wrapper", "")
if wrapper:
return "{} ".format(wrapper)
else:
return ""
@property
def make_wrapper(self):
wrapper = getattr(self.platformInfo, "make_wrapper", "")
if wrapper:
return "{} ".format(wrapper)
else:
return ""

1
kiwixbuild/dependencies/__init__.py
View File

@ -6,6 +6,7 @@ from . import (
aria2,
armhf,
docoptcpp,
emsdk,
flatpak,
gumbo,
icu4c,

3
kiwixbuild/dependencies/all_dependencies.py
View File

@ -12,6 +12,9 @@ class AllBaseDependencies(Dependency):
class Builder(NoopBuilder):
@classmethod
def get_dependencies(cls, platformInfo, allDeps):
if platformInfo.build == "wasm":
return ['zlib', 'lzma', 'zstd', 'icu4c', 'xapian-core']
base_deps = ['zlib', 'lzma', 'zstd', 'xapian-core', 'pugixml', 'libcurl', 'icu4c', 'mustache', 'libmicrohttpd', 'zim-testing-suite']
# Add specific dependencies depending of the platform
if platformInfo.build not in ('android', 'iOS'):

20
kiwixbuild/dependencies/base.py
View File

@ -70,8 +70,9 @@ class Source:
def _patch(self, context):
context.try_skip(self.source_path)
for p in self.patches:
with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input:
run_command("patch -p1", self.source_path, context, input=patch_input.read())
patch_file_path = pj(SCRIPT_DIR, 'patches', p)
patch_command = "patch -p1 -i {patch}".format(patch=patch_file_path)
run_command(patch_command, self.source_path, context)
def command(self, name, function, *args):
print(" {} {} : ".format(name, self.name), end="", flush=True)
@ -339,7 +340,7 @@ class MakeBuilder(Builder):
@property
def make_install_target(self):
if self.buildEnv.platformInfo.build == 'iOS':
if self.buildEnv.platformInfo.build in ('iOS', "wasm"):
return 'install'
return 'install-strip'
@ -368,8 +369,9 @@ class MakeBuilder(Builder):
def _configure(self, context):
context.try_skip(self.build_path)
command = "{configure_script} {configure_option}"
command = "{configure_wrapper}{configure_script} {configure_option}"
command = command.format(
configure_wrapper=self.buildEnv.configure_wrapper,
configure_script=pj(self.source_path, self.configure_script),
configure_option=self.all_configure_option
)
@ -379,7 +381,8 @@ class MakeBuilder(Builder):
def _compile(self, context):
context.try_skip(self.build_path)
command = "make -j4 {make_target} {make_option}".format(
command = "{make_wrapper}make -j4 {make_target} {make_option}".format(
make_wrapper=self.buildEnv.make_wrapper,
make_target=self.make_target,
make_option=self.make_option
)
@ -388,7 +391,8 @@ class MakeBuilder(Builder):
def _install(self, context):
context.try_skip(self.build_path)
command = "make {make_install_target} {make_option}".format(
command = "{make_wrapper}make {make_install_target} {make_option}".format(
make_wrapper=self.buildEnv.make_wrapper,
make_install_target=self.make_install_target,
make_option=self.make_option
)
@ -397,7 +401,9 @@ class MakeBuilder(Builder):
def _make_dist(self, context):
context.try_skip(self.build_path)
command = "make dist"
command = "{make_wrapper}make dist".format(
make_wrapper=self.buildEnv.make_wrapper
)
env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
run_command(command, self.build_path, context, env=env)

47
kiwixbuild/dependencies/emsdk.py Normal file
View File

@ -0,0 +1,47 @@
import os
from .base import Dependency, ReleaseDownload, Builder
from kiwixbuild.utils import Remotefile, run_command, copy_tree
pj = os.path.join
class emsdk(Dependency):
dont_skip = True
neutral = False
name = 'emsdk'
class Source(ReleaseDownload):
archive = Remotefile('emsdk-3.1.24.tar.gz',
'1aa5365ccb2147701cc9d1e59a5a49577c1d6aea55da7c450df2d5ffa48b8a58',
'https://codeload.github.com/emscripten-core/emsdk/tar.gz/refs/tags/3.1.24')
@property
def source_dir(self):
return self.target.full_name()
class Builder(Builder):
@property
def install_path(self):
return self.build_path
def _copy_source(self, context):
context.try_skip(self.build_path)
copy_tree(self.source_path, self.build_path)
def _install(self, context):
context.try_skip(self.build_path)
command = "./emsdk install 3.1.24"
run_command(command, self.build_path, context)
def _activate(self, context):
context.try_skip(self.build_path)
command = "./emsdk activate 3.1.24"
run_command(command, self.build_path, context)
def build(self):
self.command('copy_source', self._copy_source)
self.command('install', self._install)
self.command('activate', self._activate)

6
kiwixbuild/dependencies/icu4c.py
View File

@ -18,7 +18,9 @@ class Icu(Dependency):
"icu4c_custom_data.patch",
"icu4c_noxlocale.patch",
"icu4c_rpath.patch",
"icu4c_build_config.patch"]
"icu4c_build_config.patch",
"icu4c_wasm.patch"
]
class Builder(MakeBuilder):
@ -42,6 +44,6 @@ class Icu(Dependency):
'native_static' if platformInfo.static else 'native_dyn')
options += " --with-cross-build={} --disable-tools".format(
icu_native_builder.build_path)
if platformInfo.build == 'android':
if platformInfo.build in ('android', 'wasm'):
options += " --with-data-packaging=archive"
return options

6
kiwixbuild/dependencies/libzim.py
View File

@ -19,7 +19,7 @@ class Libzim(Dependency):
@classmethod
def get_dependencies(cls, platformInfo, allDeps):
deps = ['lzma', 'zstd', 'xapian-core', 'icu4c']
if platformInfo.build != 'flatpak':
if platformInfo.name not in ('flatpak', 'wasm'):
deps.append('zim-testing-suite')
return deps
@ -37,7 +37,9 @@ class Libzim(Dependency):
if platformInfo.name == "flatpak":
config_options.append("--wrap-mode=nodownload")
config_options.append("-Dtest_data_dir=none")
else:
if platformInfo.name == "wasm":
config_options.append("-Dexamples=false")
if platformInfo.name not in ("flatpak", "wasm"):
zim_testing_suite = get_target_step('zim-testing-suite', 'source')
config_options.append('-Dtest_data_dir={}'.format(zim_testing_suite.source_path))
return " ".join(config_options)

15
kiwixbuild/dependencies/lzma.py
View File

@ -9,11 +9,20 @@ class lzma(Dependency):
name = 'lzma'
class Source(ReleaseDownload):
archive = Remotefile('xz-5.2.4.tar.gz',
'b512f3b726d3b37b6dc4c8570e137b9311e7552e8ccbab4d39d47ce5f4177145'
archive = Remotefile('xz-5.2.6.tar.gz',
'a2105abee17bcd2ebd15ced31b4f5eda6e17efd6b10f921a01cda4a44c91b3a0',
'https://altushost-swe.dl.sourceforge.net/project/lzmautils/xz-5.2.6.tar.gz'
)
class Builder(MakeBuilder):
@property
def configure_option(self):
return "--disable-xz --disable-xzdec"
return ("--disable-xz "
"--disable-xzdec "
"--disable-lzmadec "
"--disable-lzmainfo "
"--disable-lzma-links "
"--disable-scripts "
"--disable-doc "
# "--disable-symbol-versions "
)

2
kiwixbuild/dependencies/xapian.py
View File

@ -24,7 +24,7 @@ class Xapian(Dependency):
@classmethod
def get_dependencies(cls, platformInfo, allDeps):
deps = ['zlib', 'lzma']
if (platformInfo.build == 'win32'
if (platformInfo.build in ('win32', 'wasm')
or neutralEnv('distname') == 'Darwin'):
return deps
return deps + ['uuid']

3235
kiwixbuild/patches/icu4c_wasm.patch Normal file
View File

File diff suppressed because it is too large Load Diff

3
kiwixbuild/platforms/__init__.py
View File

@ -9,5 +9,6 @@ from . import (
ios,
native,
neutral,
win32
win32,
wasm
)

100
kiwixbuild/platforms/wasm.py Normal file
View File

@ -0,0 +1,100 @@
from .base import PlatformInfo
from kiwixbuild.utils import pj
from kiwixbuild._global import get_target_step
class WasmPlatformInfo(PlatformInfo):
name = 'wasm'
static = True
build = 'wasm'
arch_full = 'wasm64-emscripten'
#arch_full = 'wasm64-linux'
toolchain_names = ['emsdk']
compatible_hosts = ['fedora', 'debian']
exe_wrapper_def = ""
def get_cross_config(self):
return {
'binaries': self.binaries,
'exe_wrapper_def': '',
'root_path': self.root_path,
'extra_libs': [],
'extra_cflags': [],
'host_machine': {
'system': 'emscripten',
'lsystem': 'emscripten',
'cpu_family': 'wasm64',
'cpu': 'wasm64',
'endian': 'little',
'abi': ''
}
}
@property
def wasm_sdk(self):
return get_target_step('emsdk', self.name)
@property
def install_path(self):
return self.wasm_sdk.install_path
@property
def root_path(self):
return self.install_path
@property
def binaries(self):
binaries = (('CC', 'emcc'),
('CXX', 'em++'),
('AR', 'emar'),
('STRIP', 'emstrip'),
('WINDRES', 'windres'),
('RANLIB', 'emranlib'),
('LD', 'wasm-ld'))
binaries = {k:pj(self.install_path, 'upstream', 'emscripten', v)
for k,v in binaries}
binaries['PKGCONFIG'] = 'pkg-config'
return binaries
@property
def configure_option(self):
#return ""
return '--host={}'.format(self.arch_full)
@property
def configure_wrapper(self):
return "emconfigure"
@property
def make_wrapper(self):
return "emmake"
def get_bin_dir(self):
return [pj(self.install_path, 'bin')]
def get_env(self):
env = super().get_env()
env['PATH'] = ':'.join([
env['PATH'],
self.install_path,
pj(self.install_path, 'upstream', 'emscripten'),
pj(self.install_path, 'node', '14.18.2_64bit', 'bin')
])
env['EMSDK'] = self.install_path
env['EMSDK_NODE'] = pj(self.install_path, 'node', '14.18.2_64bit', 'bin', 'node')
return env
def set_comp_flags(self, env):
super().set_comp_flags(env)
env['CFLAGS'] = " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS']
env['CXXFLAGS'] = " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
def set_compiler(self, env):
for k, v in self.binaries.items():
env[k] = v
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')

4
kiwixbuild/versions.py
View File

@ -39,11 +39,11 @@ release_versions = {
# This is the "version" of the whole base_deps_versions dict.
# Change this when you change base_deps_versions.
base_deps_meta_version = '78'
base_deps_meta_version = '79'
base_deps_versions = {
'zlib' : '1.2.12',
'lzma' : '5.2.4',
'lzma' : '5.2.6',
'zstd' : '1.5.2',
'docoptcpp' : '0.6.2',
'uuid' : '1.43.4',