smallsolar
/
kiwix-build
mirror of https://github.com/kiwix/kiwix-build.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #636 from kiwix/musl 

Add x86-64 musl
This commit is contained in:
Matthieu Gautier 2023-10-24 14:09:07 +02:00 committed by GitHub
parent 442fe3df19 3f908f429c
commit d8757c375d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 130 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/scripts/build_definition.py vendored
View File

@ -53,6 +53,8 @@ BUILD_DEF = """
| | aarch64_musl_static| | | BP | BP | | linux-aarch64-musl |
| | aarch64_musl_mixed | BP | | | | | linux-aarch64-musl |
| | aarch64_musl_dyn | d | | B | B | | |
| | x86-64_musl_static | | | BP | BP | | linux-x86-64-musl |
| | x86-64_musl_mixed | BP | | | | | linux-x86-64-musl |
| | win32_static | d | dB | dBP | dBP | | win-i686 |
| | win32_dyn | d | dB | dB | dB | | |
| | i586_static | | | BP | BP | | linux-i586 |

16
.github/scripts/common.py vendored
View File

@ -27,6 +27,7 @@ HOME = Path(os.path.expanduser("~"))
BASE_DIR = HOME / "BUILD_{}".format(PLATFORM_TARGET)
SOURCE_DIR = HOME / "SOURCE"
ARCHIVE_DIR = HOME / "ARCHIVE"
TOOLCHAIN_DIR = BASE_DIR / "TOOLCHAINS"
INSTALL_DIR = BASE_DIR / "INSTALL"
TMP_DIR = Path("/tmp")
KBUILD_SOURCE_DIR = HOME / "kiwix-build"
@ -278,13 +279,9 @@ def make_deps_archive(target=None, name=None, full=False):
base_dir = HOME / "BUILD_{}".format(PLATFORM_TARGET)
if (base_dir / "meson_cross_file.txt").exists():
files_to_archive.append(base_dir / "meson_cross_file.txt")
# Add ndk/sdk/toolchains to allow project's CI to find them and compile
files_to_archive += HOME.glob("BUILD_*/android-ndk*")
files_to_archive += HOME.glob("BUILD_*/emsdk*")
if PLATFORM_TARGET.startswith("aarch64"):
files_to_archive += SOURCE_DIR.glob("aarch64*/*")
if PLATFORM_TARGET.startswith("armv"):
files_to_archive += SOURCE_DIR.glob("armv*/*")
# Copy any toolchain
files_to_archive += [TOOLCHAIN_DIR]
files_to_archive += HOME.glob("BUILD_neutral/TOOLCHAINS/*")
if (BASE_DIR / "meson_cross_file.txt").exists():
files_to_archive.append(BASE_DIR / "meson_cross_file.txt")
@ -307,11 +304,6 @@ def make_deps_archive(target=None, name=None, full=False):
files_to_archive += SOURCE_DIR.glob("*/.*_ok")
files_to_archive += SOURCE_DIR.glob("zim-testing-suite-*/*")
toolchains_subdirs = HOME.glob("BUILD_*/TOOLCHAINS/*/*")
for subdir in toolchains_subdirs:
if not subdir.match("tools"):
files_to_archive.append(subdir)
archive_file = TMP_DIR / archive_name
with tarfile.open(str(archive_file), "w:xz") as tar:
for name in set(files_to_archive):

2
.github/workflows/ci.yml vendored
View File

@ -27,6 +27,8 @@ jobs:
- aarch64_musl_static
- aarch64_musl_dyn
- aarch64_musl_mixed
- x86-64_musl_static
- x86-64_musl_mixed
- i586_static
- i586_dyn
- android_arm

2
.github/workflows/releaseNigthly.yml vendored
View File

@ -24,6 +24,8 @@ jobs:
- aarch64_mixed
- aarch64_musl_static
- aarch64_musl_mixed
- x86-64_musl_static
- x86-64_musl_mixed
- win32_static
- i586_static
- android_arm

10
kiwixbuild/dependencies/__init__.py
View File

@ -2,13 +2,13 @@
from .base import *
from . import (
all_dependencies,
android_ndk,
tc_android_ndk,
aria2,
armhf,
arm_musl,
tc_armhf,
tc_musl,
docoptcpp,
emsdk,
flatpak,
tc_emsdk,
tc_flatpak,
gumbo,
icu4c,
ios_fat_lib,

14
kiwixbuild/dependencies/arm_musl.py
View File

@ -1,14 +0,0 @@
from .base import Dependency, ReleaseDownload, NoopBuilder
from kiwixbuild.utils import Remotefile
class aarch64_musl_toolchain(Dependency):
dont_skip = True
neutral = True
name = "aarch64_musl"
class Source(ReleaseDownload):
archive = Remotefile('aarch64-linux-musl-cross.tgz',
'c909817856d6ceda86aa510894fa3527eac7989f0ef6e87b5721c58737a06c38',
'https://musl.cc/aarch64-linux-musl-cross.tgz')
Builder = NoopBuilder

24
kiwixbuild/dependencies/base.py
View File

@ -3,7 +3,7 @@ import os
import shutil
import time
from kiwixbuild.utils import pj, Context, SkipCommand, WarningMessage, extract_archive, Defaultdict, StopBuild, run_command, colorize
from kiwixbuild.utils import pj, Context, SkipCommand, WarningMessage, extract_archive, Defaultdict, StopBuild, run_command, colorize, copy_tree
from kiwixbuild.versions import main_project_versions, base_deps_versions
from kiwixbuild._global import neutralEnv, option, get_target_step
@ -323,6 +323,28 @@ class NoopBuilder(Builder):
pass
class TcCopyBuilder(Builder):
src_subdir = None
@property
def build_path(self):
return pj(self.buildEnv.toolchain_dir, self.target.full_name())
def build(self):
self.command('copy', self._copy)
def _copy(self, context):
context.try_skip(self.build_path)
if self.src_subdir:
source_path = pj(self.source_path, self.src_subdir)
else:
source_path = self.source_path
copy_tree(source_path, self.build_path)
def make_dist(self):
pass
class MakeBuilder(Builder):
configure_option_template = "{dep_options} {static_option} {env_option} --prefix {install_dir} --libdir {libdir}"
configure_option = ""

2
kiwixbuild/dependencies/android_ndk.py → kiwixbuild/dependencies/tc_android_ndk.py
View File

@ -25,7 +25,7 @@ class android_ndk(Dependency):
class Builder(Builder):
@property
def install_path(self):
return self.build_path
return pj(self.buildEnv.toolchain_dir, self.target.full_name())
@property
def api(self):

11
kiwixbuild/dependencies/armhf.py → kiwixbuild/dependencies/tc_armhf.py
View File

@ -1,4 +1,4 @@
from .base import Dependency, ReleaseDownload, NoopBuilder
from .base import Dependency, ReleaseDownload, TcCopyBuilder
from kiwixbuild.utils import Remotefile
# The arm toolchains
@ -19,7 +19,9 @@ class armv6_toolchain(Dependency):
'4c371c4c5b55ebd1f3d7dd26b14703632d9ba47423f901bcd9303d83ad444434',
base_url + 'x-tools-armv6-rpi-linux-gnueabihf.tar.xz')
Builder = NoopBuilder
class Builder(TcCopyBuilder):
src_subdir = "armv6-rpi-linux-gnueabihf"
class armv8_toolchain(Dependency):
@ -32,7 +34,8 @@ class armv8_toolchain(Dependency):
'cc28f5c3f6a3e7d9985f98779c4e72224b4eb5a7e4dc2bcdefd90cb241fb94a5',
base_url + 'x-tools-armv8-rpi3-linux-gnueabihf.tar.xz')
Builder = NoopBuilder
class Builder(TcCopyBuilder):
src_subdir = "armv8-rpi3-linux-gnueabihf"
class aarch64_toolchain(Dependency):
dont_skip = True
@ -44,4 +47,4 @@ class aarch64_toolchain(Dependency):
'1b048bb8886ad63d21797cd9129fc37b9ea0dfaac7e3c36f888aa16fbec1d320',
aarch_base_url + 'cross-gcc-6.3.0-pi_64.tar.gz')
Builder = NoopBuilder
Builder = TcCopyBuilder

8
kiwixbuild/dependencies/emsdk.py → kiwixbuild/dependencies/tc_emsdk.py
View File

@ -23,21 +23,21 @@ class emsdk(Dependency):
class Builder(Builder):
@property
def install_path(self):
return self.build_path
return pj(self.buildEnv.toolchain_dir, self.target.full_name())
def _copy_source(self, context):
context.try_skip(self.build_path)
copy_tree(self.source_path, self.build_path)
copy_tree(self.source_path, self.install_path)
def _install(self, context):
context.try_skip(self.build_path)
command = "./emsdk install 3.1.24"
run_command(command, self.build_path, context)
run_command(command, self.install_path, context)
def _activate(self, context):
context.try_skip(self.build_path)
command = "./emsdk activate 3.1.24"
run_command(command, self.build_path, context)
run_command(command, self.install_path, context)
def build(self):

0
kiwixbuild/dependencies/flatpak.py → kiwixbuild/dependencies/tc_flatpak.py
View File

27
kiwixbuild/dependencies/tc_musl.py Normal file
View File

@ -0,0 +1,27 @@
from .base import Dependency, ReleaseDownload, TcCopyBuilder
from kiwixbuild.utils import Remotefile
class aarch64_musl_toolchain(Dependency):
dont_skip = True
neutral = True
name = "aarch64_musl"
class Source(ReleaseDownload):
archive = Remotefile('aarch64-linux-musl-cross.tgz',
'0f18a885b161815520bbb5757a4b4ab40d0898c29bebee58d0cddd6112e59cc6',
'https://more.musl.cc/10/x86_64-linux-musl/aarch64-linux-musl-cross.tgz')
Builder = TcCopyBuilder
class x86_64_musl_toolchain(Dependency):
dont_skip = True
neutral = True
name = "x86-64_musl"
class Source(ReleaseDownload):
archive = Remotefile('x86_64-linux-musl-cross.tgz',
'a3d55de8105739fcfb8b10eaa72cdb5d779319726bacff24149388d7608d1ed8',
'https://more.musl.cc/10/x86_64-linux-musl/x86_64-linux-musl-cross.tgz')
Builder = TcCopyBuilder

2
kiwixbuild/platforms/__init__.py
View File

@ -4,7 +4,7 @@ from .base import *
from . import (
android,
armhf,
arm_musl,
musl,
flatpak,
i586,
ios,

8
kiwixbuild/platforms/armhf.py
View File

@ -30,12 +30,12 @@ class ArmPlatformInfo(PlatformInfo):
return "lib/{}".format(self.arch_full)
@property
def tlc_source(self):
return get_target_step(self.build, 'source')
def toolchain(self):
return get_target_step(self.build, 'neutral')
@property
def root_path(self):
return pj(self.tlc_source.source_path, self.arch_full)
return self.toolchain.build_path
@property
def binaries(self):
@ -150,7 +150,7 @@ class Aarch64(ArmPlatformInfo):
@property
def root_path(self):
return self.tlc_source.source_path
return self.toolchain.build_path
class Aarch64Dyn(Aarch64):
name = 'aarch64_dyn'

58
kiwixbuild/platforms/arm_musl.py → kiwixbuild/platforms/musl.py
View File

@ -4,12 +4,8 @@ from kiwixbuild.utils import pj
from kiwixbuild._global import get_target_step
class Aarch64MuslPlatformInfo(PlatformInfo):
build = 'aarch64_musl'
arch_full = 'aarch64-linux-musl'
toolchain_names = ['aarch64_musl']
class MuslPlatformInfo(PlatformInfo):
compatible_hosts = ['fedora', 'debian']
libdir = "lib/aarch64-linux-musl"
def get_cross_config(self):
return {
@ -21,20 +17,20 @@ class Aarch64MuslPlatformInfo(PlatformInfo):
'host_machine': {
'system': 'linux',
'lsystem': 'linux',
'cpu_family': 'arm',
'cpu': 'armhf',
'cpu_family': self.cpu_family,
'cpu': self.cpu,
'endian': 'little',
'abi': ''
}
}
@property
def tlc_source(self):
return get_target_step(self.build, 'source')
def toolchain(self):
return get_target_step(self.build, 'neutral')
@property
def root_path(self):
return self.tlc_source.source_path
return self.toolchain.build_path
@property
def binaries(self):
@ -57,11 +53,13 @@ class Aarch64MuslPlatformInfo(PlatformInfo):
@property
def exe_wrapper_def(self):
try:
which('qemu-arm')
which(self.qemu)
except subprocess.CalledProcessError:
return ""
except AttributeError:
return ""
else:
return "exe_wrapper = 'qemu-arm'"
return f"exe_wrapper = '{self.qemu}'"
@property
def configure_option(self):
@ -88,6 +86,10 @@ class Aarch64MuslPlatformInfo(PlatformInfo):
def set_comp_flags(self, env):
super().set_comp_flags(env)
env['LD_LIBRARY_PATH'] = ':'.join([
pj(self.root_path, self.arch_full, 'lib'),
env['LD_LIBRARY_PATH']
])
env['CFLAGS'] = " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS']
env['CXXFLAGS'] = " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
@ -101,6 +103,16 @@ class Aarch64MuslPlatformInfo(PlatformInfo):
self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
class Aarch64MuslPlatformInfo(MuslPlatformInfo):
build = 'aarch64_musl'
arch_full = 'aarch64-linux-musl'
toolchain_names = ['aarch64_musl']
libdir = "lib/aarch64-linux-musl"
cpu_family = 'arm'
cpu = 'armhf'
qemu = 'qemu-arm'
class Aarch64MuslDyn(Aarch64MuslPlatformInfo):
name = 'aarch64_musl_dyn'
static = False
@ -112,3 +124,25 @@ class Aarch64MuslStatic(Aarch64MuslPlatformInfo):
class Aarch64MuslMixed(MixedMixin('aarch64_musl_static'), Aarch64MuslPlatformInfo):
name = 'aarch64_musl_mixed'
static = False
class X86_64MuslPlatformInfo(MuslPlatformInfo):
build = 'x86-64_musl'
arch_full = 'x86_64-linux-musl'
toolchain_names = ['x86-64_musl']
libdir = "lib/x86_64-linux-musl"
cpu_family = 'x86_64'
cpu = 'x86_64'
class X86_64MuslDyn(X86_64MuslPlatformInfo):
name = 'x86-64_musl_dyn'
static = False
class X86_64MuslStatic(X86_64MuslPlatformInfo):
name = 'x86-64_musl_static'
static = True
class x86_64MuslMixed(MixedMixin('x86-64_musl_static'), X86_64MuslPlatformInfo):
name = 'x86-64_musl_mixed'
static = False

2
kiwixbuild/utils.py
View File

@ -101,7 +101,7 @@ def copy_tree(src, dst, post_copy_function=None):
os.makedirs(dstdir, exist_ok=True)
for f in files:
dstfile = pj(dstdir, f)
shutil.copy2(pj(root, f), dstfile)
shutil.copy2(pj(root, f), dstfile, follow_symlinks=False)
if post_copy_function is not None:
post_copy_function(dstfile)

2
kiwixbuild/versions.py
View File

@ -39,7 +39,7 @@ release_versions = {
# This is the "version" of the whole base_deps_versions dict.
# Change this when you change base_deps_versions.
base_deps_meta_version = '90'
base_deps_meta_version = '92'
base_deps_versions = {
'zlib' : '1.2.12',