diff --git a/.github/scripts/build_projects.py b/.github/scripts/build_projects.py index fb34a87..bf325fc 100755 --- a/.github/scripts/build_projects.py +++ b/.github/scripts/build_projects.py @@ -30,11 +30,11 @@ elif PLATFORM_TARGET.startswith("native_"): TARGETS = ("libzim", "libkiwix") else: TARGETS = ("zim-tools", "kiwix-tools") -elif PLATFORM_TARGET in ("win32_static", "armhf_static", "armhf_dyn", "i586_static"): +elif PLATFORM_TARGET in ("win32_static", "armhf_static", "armhf_dyn", "aarch64_static", "aarch64_dyn", "i586_static"): TARGETS = ("kiwix-tools",) elif PLATFORM_TARGET == "flatpak": TARGETS = ("kiwix-desktop",) -elif PLATFORM_TARGET == "wasm": +elif PLATFORM_TARGET in ("wasm", "armhf_mixed", "aarch64_mixed"): TARGETS = ("libzim", ) else: TARGETS = ("libzim", "zim-tools", "libkiwix", "kiwix-tools") diff --git a/.github/scripts/build_release_nightly.py b/.github/scripts/build_release_nightly.py index 772d7e6..168359a 100755 --- a/.github/scripts/build_release_nightly.py +++ b/.github/scripts/build_release_nightly.py @@ -37,11 +37,11 @@ elif PLATFORM_TARGET.startswith("native_"): TARGETS = ("libzim", "libkiwix") else: TARGETS = ("zim-tools", "kiwix-tools") -elif PLATFORM_TARGET in ("win32_static", "armhf_static", "i586_static"): +elif PLATFORM_TARGET in ("win32_static", "armhf_static", "aarch64_static", "i586_static"): TARGETS = ("kiwix-tools",) elif PLATFORM_TARGET == "flatpak": TARGETS = ("kiwix-desktop",) -elif PLATFORM_TARGET == "wasm": +elif PLATFORM_TARGET in ("wasm", "armhf_mixed", "aarch64_mixed"): TARGETS = ("libzim", ) else: TARGETS = ("libzim", "zim-tools", "libkiwix", "kiwix-tools") diff --git a/.github/scripts/common.py b/.github/scripts/common.py index 54c35f9..22b9f0a 100644 --- a/.github/scripts/common.py +++ b/.github/scripts/common.py @@ -51,6 +51,9 @@ PLATFORM_TO_RELEASE = { "native_static": "{os}-x86_64".format(os=RELEASE_OS_NAME), "win32_static": "win-i686", "armhf_static": "{os}-armhf".format(os=RELEASE_OS_NAME), + "armhf_mixed": "{os}-armhf".format(os=RELEASE_OS_NAME), + "aarch64_static": "{os}-aarch64".format(os=RELEASE_OS_NAME), + "aarch64_mixed": "{os}-aarch64".format(os=RELEASE_OS_NAME), "i586_static": "{os}-i586".format(os=RELEASE_OS_NAME), "android_arm": "android-arm", "android_arm64": "android-arm64", @@ -278,8 +281,9 @@ def make_deps_archive(target=None, name=None, full=False): print_message("Create archive {}.", archive_name) files_to_archive = list(filter_install_dir(INSTALL_DIR)) files_to_archive += HOME.glob("BUILD_*/LOGS") - if PLATFORM_TARGET == "native_mixed": - files_to_archive += filter_install_dir(HOME / "BUILD_native_static" / "INSTALL") + if PLATFORM_TARGET.endswith("_mixed"): + static_platform = PLATFORM_TARGET.replace("_mixed", "_static") + files_to_archive += filter_install_dir(HOME / ("BUILD_" + static_platform) / "INSTALL") if PLATFORM_TARGET.startswith("android_"): files_to_archive += filter_install_dir(HOME / "BUILD_neutral" / "INSTALL") base_dir = HOME / "BUILD_{}".format(PLATFORM_TARGET) @@ -298,6 +302,11 @@ def make_deps_archive(target=None, name=None, full=False): if full: files_to_archive += ARCHIVE_DIR.glob(".*_ok") files_to_archive += BASE_DIR.glob("*/.*_ok") + # Add also static build for mixed target + if PLATFORM_TARGET.endswith("_mixed"): + static_platform = PLATFORM_TARGET.replace("_mixed", "_static") + files_to_archive += (HOME / ("BUILD_" + static_platform)).glob("*/.*_ok") + # Native dyn and static is needed for potential cross compilation that use native tools (icu) files_to_archive += (HOME / "BUILD_native_dyn").glob("*/.*_ok") files_to_archive += (HOME / "BUILD_native_static").glob("*/.*_ok") files_to_archive += HOME.glob("BUILD_android*/**/.*_ok") @@ -305,6 +314,8 @@ def make_deps_archive(target=None, name=None, full=False): files_to_archive += SOURCE_DIR.glob("zim-testing-suite-*/*") if PLATFORM_TARGET.startswith("armhf"): files_to_archive += (SOURCE_DIR / "armhf").glob("*") + if PLATFORM_TARGET.startswith("aarch64"): + files_to_archive += (SOURCE_DIR / "aarch64").glob("*") toolchains_subdirs = HOME.glob("BUILD_*/TOOLCHAINS/*/*") for subdir in toolchains_subdirs: if not subdir.match("tools"): diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5fbb656..d37b0ad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,10 @@ jobs: - wasm - armhf_static - armhf_dyn + - armhf_mixed + - aarch64_static + - aarch64_dyn + - aarch64_mixed - i586_static - i586_dyn - android_arm diff --git a/.github/workflows/releaseNigthly.yml b/.github/workflows/releaseNigthly.yml index 05af7b0..7d8a07a 100644 --- a/.github/workflows/releaseNigthly.yml +++ b/.github/workflows/releaseNigthly.yml @@ -19,6 +19,9 @@ jobs: - native_desktop - wasm - armhf_static + - armhf_mixed + - aarch64_static + - aarch64_mixed - win32_static - i586_static - android_arm @@ -44,6 +47,15 @@ jobs: - target: armhf_static image_variant: bionic lib_postfix: '/x86_64-linux-gnu' + - target: armhf_mixed + image_variant: bionic + lib_postfix: '/x86_64-linux-gnu' + - target: aarch64_static + image_variant: bionic + lib_postfix: '/x86_64-linux-gnu' + - target: aarch64_mixed + image_variant: bionic + lib_postfix: '/x86_64-linux-gnu' - target: win32_static image_variant: f35 lib_postfix: '64' diff --git a/kiwixbuild/dependencies/armhf.py b/kiwixbuild/dependencies/armhf.py index 35adca3..c0b350d 100644 --- a/kiwixbuild/dependencies/armhf.py +++ b/kiwixbuild/dependencies/armhf.py @@ -1,13 +1,36 @@ from .base import Dependency, ReleaseDownload, NoopBuilder from kiwixbuild.utils import Remotefile + +base_url = 'https://master.dl.sourceforge.net/project/raspberry-pi-cross-compilers/' + +# This is Gcc 10.3.0 and Raspberry Pi 2 and 3 only ! +armhf_base_url = base_url + 'Raspberry%20Pi%20GCC%20Cross-Compiler%20Toolchains/Stretch/GCC%2010.3.0/Raspberry%20Pi%202%2C%203/' + +# This is Gcc 10.3.0 and ALL rapsberry Pi arch64 +aarch_base_url = base_url + 'Bonus%20Raspberry%20Pi%20GCC%2064-Bit%20Toolchains/Raspberry%20Pi%20GCC%2064-Bit%20Cross-Compiler%20Toolchains/Stretch/GCC%2010.3.0/' + class armhf_toolchain(Dependency): dont_skip = True neutral = True name = 'armhf' class Source(ReleaseDownload): - archive = Remotefile('raspberrypi-tools.tar.gz', - 'e72b35436f2f23f2f7df322d6c318b9be57b21596b5ff0b8936af4ad94e04f2e') + archive = Remotefile('cross-gcc-10.3.0-pi_2-3.tar.gz', + '6aef31703fb7bfd63065dda7fb525f1f86a0509c4358c57631a51025805278b3', + armhf_base_url + 'cross-gcc-10.3.0-pi_2-3.tar.gz') + + Builder = NoopBuilder + + +class aarch64_toolchain(Dependency): + dont_skip = True + neutral = True + name = "aarch64" + + class Source(ReleaseDownload): + archive = Remotefile('cross-gcc-10.3.0-pi_64.tar.gz', + '5b3fdb7ee8c496c377ab8b11d7ffd404b4d3041f4fdcfeebcbcb734d45a5f3e9', + aarch_base_url + 'cross-gcc-10.3.0-pi_64.tar.gz') Builder = NoopBuilder diff --git a/kiwixbuild/dependencies/libkiwix.py b/kiwixbuild/dependencies/libkiwix.py index 6ccfd0c..7960350 100644 --- a/kiwixbuild/dependencies/libkiwix.py +++ b/kiwixbuild/dependencies/libkiwix.py @@ -28,7 +28,7 @@ class Libkiwix(Dependency): return '-Db_bitcode=true' if platformInfo.name == 'flatpak': return '--wrap-mode=nodownload' - if platformInfo.name == 'native_mixed' and option('target') == 'libkiwix': + if platformInfo.mixed and option('target') == 'libkiwix': return "-Dstatic-linkage=true" return '' diff --git a/kiwixbuild/dependencies/libzim.py b/kiwixbuild/dependencies/libzim.py index a7688c2..0d96d75 100644 --- a/kiwixbuild/dependencies/libzim.py +++ b/kiwixbuild/dependencies/libzim.py @@ -32,7 +32,7 @@ class Libzim(Dependency): config_options.append("-Dstatic-linkage=true") if platformInfo.build == 'iOS': config_options.append("-Db_bitcode=true") - if platformInfo.name == 'native_mixed' and option('target') == 'libzim': + if platformInfo.mixed and option('target') == 'libzim': config_options.append("-Dstatic-linkage=true") if platformInfo.name == "flatpak": config_options.append("--wrap-mode=nodownload") diff --git a/kiwixbuild/dependencies/zlib.py b/kiwixbuild/dependencies/zlib.py index 0bbe5aa..87759e0 100644 --- a/kiwixbuild/dependencies/zlib.py +++ b/kiwixbuild/dependencies/zlib.py @@ -44,3 +44,10 @@ class zlib(Dependency): binary_path=pj(self.buildEnv.install_dir, 'bin'), ) return "" + + @property + def make_target(self): + if self.buildEnv.platformInfo.static: + return "static" + else: + return "shared" diff --git a/kiwixbuild/platforms/armhf.py b/kiwixbuild/platforms/armhf.py index 0da7528..9a8a8d3 100644 --- a/kiwixbuild/platforms/armhf.py +++ b/kiwixbuild/platforms/armhf.py @@ -1,4 +1,4 @@ -from .base import PlatformInfo +from .base import PlatformInfo, MixedMixin from kiwixbuild.utils import pj from kiwixbuild._global import get_target_step @@ -29,14 +29,11 @@ class ArmhfPlatformInfo(PlatformInfo): @property def tlc_source(self): - return get_target_step('armhf', 'source') + return get_target_step(self.build, 'source') @property def root_path(self): - return pj(self.tlc_source.source_path, - 'raspberrypi-tools', - 'arm-bcm2708', - 'gcc-linaro-{}-raspbian-x64'.format(self.arch_full)) + return self.tlc_source.source_path @property def binaries(self): @@ -47,7 +44,9 @@ class ArmhfPlatformInfo(PlatformInfo): ('STRIP', 'strip'), ('WINDRES', 'windres'), ('RANLIB', 'ranlib'), - ('LD', 'ld')) + ('LD', 'ld'), + ('LDSHARED', 'g++ -shared') + ) ) binaries = {k:pj(self.root_path, 'bin', v) for k,v in binaries} @@ -72,8 +71,13 @@ class ArmhfPlatformInfo(PlatformInfo): def get_env(self): env = super().get_env() + env['LD_LIBRARY_PATH'] = ':'.join([ + pj(self.root_path, self.arch_full, 'lib64'), + pj(self.root_path, 'lib'), + env['LD_LIBRARY_PATH'] + ]) env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') - env['QEMU_LD_PREFIX'] = pj(self.root_path, "arm-linux-gnueabihf", "libc") + env['QEMU_LD_PREFIX'] = pj(self.root_path, self.arch_full, "libc") env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format( ':'.join([ pj(self.root_path, self.arch_full, "lib"), @@ -83,8 +87,8 @@ class ArmhfPlatformInfo(PlatformInfo): def set_comp_flags(self, env): super().set_comp_flags(env) - env['CFLAGS'] = " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS'] - env['CXXFLAGS'] = " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS'] + env['CFLAGS'] = " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS'] + env['CXXFLAGS'] = " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS'] def set_compiler(self, env): for k, v in self.binaries.items(): @@ -95,6 +99,7 @@ class ArmhfPlatformInfo(PlatformInfo): self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') + class ArmhfDyn(ArmhfPlatformInfo): name = 'armhf_dyn' static = False @@ -102,3 +107,26 @@ class ArmhfDyn(ArmhfPlatformInfo): class ArmhfStatic(ArmhfPlatformInfo): name = 'armhf_static' static = True + +class ArmhfMixed(MixedMixin('armhf_static'), ArmhfPlatformInfo): + name = 'armhf_mixed' + static = False + + +class Aarch64(ArmhfPlatformInfo): + build = 'aarch64' + arch_full = 'aarch64-linux-gnu' + toolchain_names = ['aarch64'] + +class Aarch64Dyn(Aarch64): + name = 'aarch64_dyn' + static = False + +class Aarch64Static(Aarch64): + name = 'aarch64_static' + static = True + + +class Aarch64Mixed(MixedMixin('aarch64_static'), Aarch64): + name = 'aarch64_mixed' + static = False diff --git a/kiwixbuild/platforms/base.py b/kiwixbuild/platforms/base.py index 2163881..0f9d6e2 100644 --- a/kiwixbuild/platforms/base.py +++ b/kiwixbuild/platforms/base.py @@ -24,6 +24,7 @@ class PlatformInfo(metaclass=_MetaPlatform): all_running_platforms = {} toolchain_names = [] configure_option = "" + mixed = False libdir = None @classmethod @@ -128,3 +129,38 @@ class MetaPlatformInfo(PlatformInfo): platform = self.get_platform(platformName, targets) targetDefs += platform.add_targets(targetName, targets) return targetDefs + + + +def MixedMixin(static_name): + class MixedMixinClass: + mixed = True + + def add_targets(self, targetName, targets): + print(targetName) + if option('target') == targetName: + return super().add_targets(targetName, targets) + else: + static_platform = self.get_platform(static_name, targets) + return static_platform.add_targets(targetName, targets) + + def get_fully_qualified_dep(self, dep): + if isinstance(dep, tuple): + return dep + if option('target') == dep: + return self.name, dep + return static_name, dep + + + def get_env(self): + env = super().get_env() + static_platform = self.get_platform(static_name) + static_buildEnv = static_platform.buildEnv + static_install_dir = static_buildEnv.install_dir + env['PATH'] = ':'.join([pj(static_install_dir, 'bin')] + [env['PATH']]) + pkgconfig_path = pj(static_install_dir, static_buildEnv.libprefix, 'pkgconfig') + env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path]) + env['CPPFLAGS'] = " ".join(['-I'+pj(static_install_dir, 'include'), env['CPPFLAGS']]) + return env + + return MixedMixinClass diff --git a/kiwixbuild/platforms/native.py b/kiwixbuild/platforms/native.py index 4ff2738..8e10401 100644 --- a/kiwixbuild/platforms/native.py +++ b/kiwixbuild/platforms/native.py @@ -1,4 +1,4 @@ -from .base import PlatformInfo +from .base import PlatformInfo, MixedMixin from kiwixbuild.utils import pj from kiwixbuild._global import option, neutralEnv @@ -24,33 +24,7 @@ class NativeStatic(NativePlatformInfo): static = True compatible_hosts = ['fedora', 'debian'] -class NativeMixed(NativePlatformInfo): +class NativeMixed(MixedMixin('native_static'), NativePlatformInfo): name = 'native_mixed' static = False compatible_hosts = ['fedora', 'debian', 'Darwin'] - - def add_targets(self, targetName, targets): - print(targetName) - if option('target') == targetName: - return super().add_targets(targetName, targets) - else: - static_platform = self.get_platform('native_static', targets) - return static_platform.add_targets(targetName, targets) - - def get_fully_qualified_dep(self, dep): - if isinstance(dep, tuple): - return dep - if option('target') == dep: - return 'native_mixed', dep - return 'native_static', dep - - def get_env(self): - env = super().get_env() - static_platform = self.get_platform('native_static') - static_buildEnv = static_platform.buildEnv - static_install_dir = static_buildEnv.install_dir - env['PATH'] = ':'.join([pj(static_install_dir, 'bin')] + [env['PATH']]) - pkgconfig_path = pj(static_install_dir, static_buildEnv.libprefix, 'pkgconfig') - env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path]) - env['CPPFLAGS'] = " ".join(['-I'+pj(static_install_dir, 'include'), env['CPPFLAGS']]) - return env diff --git a/kiwixbuild/versions.py b/kiwixbuild/versions.py index 72323b9..bcd9bfc 100644 --- a/kiwixbuild/versions.py +++ b/kiwixbuild/versions.py @@ -39,7 +39,7 @@ release_versions = { # This is the "version" of the whole base_deps_versions dict. # Change this when you change base_deps_versions. -base_deps_meta_version = '82' +base_deps_meta_version = '83' base_deps_versions = { 'zlib' : '1.2.12',