smallsolar
/
kiwix-build
mirror of https://github.com/kiwix/kiwix-build.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #580 from kiwix/arm_build

This commit is contained in:
Matthieu Gautier 2023-03-30 14:16:55 +02:00 committed by GitHub
parent bc3a73f26d c1d0ed4e4f
commit 52335fbd32
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 144 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/scripts/build_projects.py vendored
View File

@ -30,11 +30,11 @@ elif PLATFORM_TARGET.startswith("native_"):
TARGETS = ("libzim", "libkiwix")
else:
TARGETS = ("zim-tools", "kiwix-tools")
elif PLATFORM_TARGET in ("win32_static", "armhf_static", "armhf_dyn", "i586_static"):
elif PLATFORM_TARGET in ("win32_static", "armhf_static", "armhf_dyn", "aarch64_static", "aarch64_dyn", "i586_static"):
TARGETS = ("kiwix-tools",)
elif PLATFORM_TARGET == "flatpak":
TARGETS = ("kiwix-desktop",)
elif PLATFORM_TARGET == "wasm":
elif PLATFORM_TARGET in ("wasm", "armhf_mixed", "aarch64_mixed"):
TARGETS = ("libzim", )
else:
TARGETS = ("libzim", "zim-tools", "libkiwix", "kiwix-tools")

4
.github/scripts/build_release_nightly.py vendored
View File

@ -37,11 +37,11 @@ elif PLATFORM_TARGET.startswith("native_"):
TARGETS = ("libzim", "libkiwix")
else:
TARGETS = ("zim-tools", "kiwix-tools")
elif PLATFORM_TARGET in ("win32_static", "armhf_static", "i586_static"):
elif PLATFORM_TARGET in ("win32_static", "armhf_static", "aarch64_static", "i586_static"):
TARGETS = ("kiwix-tools",)
elif PLATFORM_TARGET == "flatpak":
TARGETS = ("kiwix-desktop",)
elif PLATFORM_TARGET == "wasm":
elif PLATFORM_TARGET in ("wasm", "armhf_mixed", "aarch64_mixed"):
TARGETS = ("libzim", )
else:
TARGETS = ("libzim", "zim-tools", "libkiwix", "kiwix-tools")

15
.github/scripts/common.py vendored
View File

@ -51,6 +51,9 @@ PLATFORM_TO_RELEASE = {
"native_static": "{os}-x86_64".format(os=RELEASE_OS_NAME),
"win32_static": "win-i686",
"armhf_static": "{os}-armhf".format(os=RELEASE_OS_NAME),
"armhf_mixed": "{os}-armhf".format(os=RELEASE_OS_NAME),
"aarch64_static": "{os}-aarch64".format(os=RELEASE_OS_NAME),
"aarch64_mixed": "{os}-aarch64".format(os=RELEASE_OS_NAME),
"i586_static": "{os}-i586".format(os=RELEASE_OS_NAME),
"android_arm": "android-arm",
"android_arm64": "android-arm64",
@ -278,8 +281,9 @@ def make_deps_archive(target=None, name=None, full=False):
print_message("Create archive {}.", archive_name)
files_to_archive = list(filter_install_dir(INSTALL_DIR))
files_to_archive += HOME.glob("BUILD_*/LOGS")
if PLATFORM_TARGET == "native_mixed":
files_to_archive += filter_install_dir(HOME / "BUILD_native_static" / "INSTALL")
if PLATFORM_TARGET.endswith("_mixed"):
static_platform = PLATFORM_TARGET.replace("_mixed", "_static")
files_to_archive += filter_install_dir(HOME / ("BUILD_" + static_platform) / "INSTALL")
if PLATFORM_TARGET.startswith("android_"):
files_to_archive += filter_install_dir(HOME / "BUILD_neutral" / "INSTALL")
base_dir = HOME / "BUILD_{}".format(PLATFORM_TARGET)
@ -298,6 +302,11 @@ def make_deps_archive(target=None, name=None, full=False):
if full:
files_to_archive += ARCHIVE_DIR.glob(".*_ok")
files_to_archive += BASE_DIR.glob("*/.*_ok")
# Add also static build for mixed target
if PLATFORM_TARGET.endswith("_mixed"):
static_platform = PLATFORM_TARGET.replace("_mixed", "_static")
files_to_archive += (HOME / ("BUILD_" + static_platform)).glob("*/.*_ok")
# Native dyn and static is needed for potential cross compilation that use native tools (icu)
files_to_archive += (HOME / "BUILD_native_dyn").glob("*/.*_ok")
files_to_archive += (HOME / "BUILD_native_static").glob("*/.*_ok")
files_to_archive += HOME.glob("BUILD_android*/**/.*_ok")
@ -305,6 +314,8 @@ def make_deps_archive(target=None, name=None, full=False):
files_to_archive += SOURCE_DIR.glob("zim-testing-suite-*/*")
if PLATFORM_TARGET.startswith("armhf"):
files_to_archive += (SOURCE_DIR / "armhf").glob("*")
if PLATFORM_TARGET.startswith("aarch64"):
files_to_archive += (SOURCE_DIR / "aarch64").glob("*")
toolchains_subdirs = HOME.glob("BUILD_*/TOOLCHAINS/*/*")
for subdir in toolchains_subdirs:
if not subdir.match("tools"):

4
.github/workflows/ci.yml vendored
View File

@ -18,6 +18,10 @@ jobs:
- wasm
- armhf_static
- armhf_dyn
- armhf_mixed
- aarch64_static
- aarch64_dyn
- aarch64_mixed
- i586_static
- i586_dyn
- android_arm

12
.github/workflows/releaseNigthly.yml vendored
View File

@ -19,6 +19,9 @@ jobs:
- native_desktop
- wasm
- armhf_static
- armhf_mixed
- aarch64_static
- aarch64_mixed
- win32_static
- i586_static
- android_arm
@ -44,6 +47,15 @@ jobs:
- target: armhf_static
image_variant: bionic
lib_postfix: '/x86_64-linux-gnu'
- target: armhf_mixed
image_variant: bionic
lib_postfix: '/x86_64-linux-gnu'
- target: aarch64_static
image_variant: bionic
lib_postfix: '/x86_64-linux-gnu'
- target: aarch64_mixed
image_variant: bionic
lib_postfix: '/x86_64-linux-gnu'
- target: win32_static
image_variant: f35
lib_postfix: '64'

27
kiwixbuild/dependencies/armhf.py
View File

@ -1,13 +1,36 @@
from .base import Dependency, ReleaseDownload, NoopBuilder
from kiwixbuild.utils import Remotefile
base_url = 'https://master.dl.sourceforge.net/project/raspberry-pi-cross-compilers/'
# This is Gcc 10.3.0 and Raspberry Pi 2 and 3 only !
armhf_base_url = base_url + 'Raspberry%20Pi%20GCC%20Cross-Compiler%20Toolchains/Stretch/GCC%2010.3.0/Raspberry%20Pi%202%2C%203/'
# This is Gcc 10.3.0 and ALL rapsberry Pi arch64
aarch_base_url = base_url + 'Bonus%20Raspberry%20Pi%20GCC%2064-Bit%20Toolchains/Raspberry%20Pi%20GCC%2064-Bit%20Cross-Compiler%20Toolchains/Stretch/GCC%2010.3.0/'
class armhf_toolchain(Dependency):
dont_skip = True
neutral = True
name = 'armhf'
class Source(ReleaseDownload):
archive = Remotefile('raspberrypi-tools.tar.gz',
'e72b35436f2f23f2f7df322d6c318b9be57b21596b5ff0b8936af4ad94e04f2e')
archive = Remotefile('cross-gcc-10.3.0-pi_2-3.tar.gz',
'6aef31703fb7bfd63065dda7fb525f1f86a0509c4358c57631a51025805278b3',
armhf_base_url + 'cross-gcc-10.3.0-pi_2-3.tar.gz')
Builder = NoopBuilder
class aarch64_toolchain(Dependency):
dont_skip = True
neutral = True
name = "aarch64"
class Source(ReleaseDownload):
archive = Remotefile('cross-gcc-10.3.0-pi_64.tar.gz',
'5b3fdb7ee8c496c377ab8b11d7ffd404b4d3041f4fdcfeebcbcb734d45a5f3e9',
aarch_base_url + 'cross-gcc-10.3.0-pi_64.tar.gz')
Builder = NoopBuilder

2
kiwixbuild/dependencies/libkiwix.py
View File

@ -28,7 +28,7 @@ class Libkiwix(Dependency):
return '-Db_bitcode=true'
if platformInfo.name == 'flatpak':
return '--wrap-mode=nodownload'
if platformInfo.name == 'native_mixed' and option('target') == 'libkiwix':
if platformInfo.mixed and option('target') == 'libkiwix':
return "-Dstatic-linkage=true"
return ''

2
kiwixbuild/dependencies/libzim.py
View File

@ -32,7 +32,7 @@ class Libzim(Dependency):
config_options.append("-Dstatic-linkage=true")
if platformInfo.build == 'iOS':
config_options.append("-Db_bitcode=true")
if platformInfo.name == 'native_mixed' and option('target') == 'libzim':
if platformInfo.mixed and option('target') == 'libzim':
config_options.append("-Dstatic-linkage=true")
if platformInfo.name == "flatpak":
config_options.append("--wrap-mode=nodownload")

7
kiwixbuild/dependencies/zlib.py
View File

@ -44,3 +44,10 @@ class zlib(Dependency):
binary_path=pj(self.buildEnv.install_dir, 'bin'),
)
return ""
@property
def make_target(self):
if self.buildEnv.platformInfo.static:
return "static"
else:
return "shared"

48
kiwixbuild/platforms/armhf.py
View File

@ -1,4 +1,4 @@
from .base import PlatformInfo
from .base import PlatformInfo, MixedMixin
from kiwixbuild.utils import pj
from kiwixbuild._global import get_target_step
@ -29,14 +29,11 @@ class ArmhfPlatformInfo(PlatformInfo):
@property
def tlc_source(self):
return get_target_step('armhf', 'source')
return get_target_step(self.build, 'source')
@property
def root_path(self):
return pj(self.tlc_source.source_path,
'raspberrypi-tools',
'arm-bcm2708',
'gcc-linaro-{}-raspbian-x64'.format(self.arch_full))
return self.tlc_source.source_path
@property
def binaries(self):
@ -47,7 +44,9 @@ class ArmhfPlatformInfo(PlatformInfo):
('STRIP', 'strip'),
('WINDRES', 'windres'),
('RANLIB', 'ranlib'),
('LD', 'ld'))
('LD', 'ld'),
('LDSHARED', 'g++ -shared')
)
)
binaries = {k:pj(self.root_path, 'bin', v)
for k,v in binaries}
@ -72,8 +71,13 @@ class ArmhfPlatformInfo(PlatformInfo):
def get_env(self):
env = super().get_env()
env['LD_LIBRARY_PATH'] = ':'.join([
pj(self.root_path, self.arch_full, 'lib64'),
pj(self.root_path, 'lib'),
env['LD_LIBRARY_PATH']
])
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
env['QEMU_LD_PREFIX'] = pj(self.root_path, "arm-linux-gnueabihf", "libc")
env['QEMU_LD_PREFIX'] = pj(self.root_path, self.arch_full, "libc")
env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format(
':'.join([
pj(self.root_path, self.arch_full, "lib"),
@ -83,8 +87,8 @@ class ArmhfPlatformInfo(PlatformInfo):
def set_comp_flags(self, env):
super().set_comp_flags(env)
env['CFLAGS'] = " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS']
env['CXXFLAGS'] = " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
env['CFLAGS'] = " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS']
env['CXXFLAGS'] = " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
def set_compiler(self, env):
for k, v in self.binaries.items():
@ -95,6 +99,7 @@ class ArmhfPlatformInfo(PlatformInfo):
self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
class ArmhfDyn(ArmhfPlatformInfo):
name = 'armhf_dyn'
static = False
@ -102,3 +107,26 @@ class ArmhfDyn(ArmhfPlatformInfo):
class ArmhfStatic(ArmhfPlatformInfo):
name = 'armhf_static'
static = True
class ArmhfMixed(MixedMixin('armhf_static'), ArmhfPlatformInfo):
name = 'armhf_mixed'
static = False
class Aarch64(ArmhfPlatformInfo):
build = 'aarch64'
arch_full = 'aarch64-linux-gnu'
toolchain_names = ['aarch64']
class Aarch64Dyn(Aarch64):
name = 'aarch64_dyn'
static = False
class Aarch64Static(Aarch64):
name = 'aarch64_static'
static = True
class Aarch64Mixed(MixedMixin('aarch64_static'), Aarch64):
name = 'aarch64_mixed'
static = False

36
kiwixbuild/platforms/base.py
View File

@ -24,6 +24,7 @@ class PlatformInfo(metaclass=_MetaPlatform):
all_running_platforms = {}
toolchain_names = []
configure_option = ""
mixed = False
libdir = None
@classmethod
@ -128,3 +129,38 @@ class MetaPlatformInfo(PlatformInfo):
platform = self.get_platform(platformName, targets)
targetDefs += platform.add_targets(targetName, targets)
return targetDefs
def MixedMixin(static_name):
class MixedMixinClass:
mixed = True
def add_targets(self, targetName, targets):
print(targetName)
if option('target') == targetName:
return super().add_targets(targetName, targets)
else:
static_platform = self.get_platform(static_name, targets)
return static_platform.add_targets(targetName, targets)
def get_fully_qualified_dep(self, dep):
if isinstance(dep, tuple):
return dep
if option('target') == dep:
return self.name, dep
return static_name, dep
def get_env(self):
env = super().get_env()
static_platform = self.get_platform(static_name)
static_buildEnv = static_platform.buildEnv
static_install_dir = static_buildEnv.install_dir
env['PATH'] = ':'.join([pj(static_install_dir, 'bin')] + [env['PATH']])
pkgconfig_path = pj(static_install_dir, static_buildEnv.libprefix, 'pkgconfig')
env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path])
env['CPPFLAGS'] = " ".join(['-I'+pj(static_install_dir, 'include'), env['CPPFLAGS']])
return env
return MixedMixinClass

30
kiwixbuild/platforms/native.py
View File

@ -1,4 +1,4 @@
from .base import PlatformInfo
from .base import PlatformInfo, MixedMixin
from kiwixbuild.utils import pj
from kiwixbuild._global import option, neutralEnv
@ -24,33 +24,7 @@ class NativeStatic(NativePlatformInfo):
static = True
compatible_hosts = ['fedora', 'debian']
class NativeMixed(NativePlatformInfo):
class NativeMixed(MixedMixin('native_static'), NativePlatformInfo):
name = 'native_mixed'
static = False
compatible_hosts = ['fedora', 'debian', 'Darwin']
def add_targets(self, targetName, targets):
print(targetName)
if option('target') == targetName:
return super().add_targets(targetName, targets)
else:
static_platform = self.get_platform('native_static', targets)
return static_platform.add_targets(targetName, targets)
def get_fully_qualified_dep(self, dep):
if isinstance(dep, tuple):
return dep
if option('target') == dep:
return 'native_mixed', dep
return 'native_static', dep
def get_env(self):
env = super().get_env()
static_platform = self.get_platform('native_static')
static_buildEnv = static_platform.buildEnv
static_install_dir = static_buildEnv.install_dir
env['PATH'] = ':'.join([pj(static_install_dir, 'bin')] + [env['PATH']])
pkgconfig_path = pj(static_install_dir, static_buildEnv.libprefix, 'pkgconfig')
env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path])
env['CPPFLAGS'] = " ".join(['-I'+pj(static_install_dir, 'include'), env['CPPFLAGS']])
return env

2
kiwixbuild/versions.py
View File

@ -39,7 +39,7 @@ release_versions = {
# This is the "version" of the whole base_deps_versions dict.
# Change this when you change base_deps_versions.
base_deps_meta_version = '82'
base_deps_meta_version = '83'
base_deps_versions = {
'zlib' : '1.2.12',