From 7807e10325f6411148b9b85c8a6fb57961e8cd32 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Fri, 6 Oct 2023 15:57:25 +0200 Subject: [PATCH 1/9] Rename files containing toolchain to `tc_*` --- kiwixbuild/dependencies/__init__.py | 10 +++++----- .../dependencies/{android_ndk.py => tc_android_ndk.py} | 0 .../dependencies/{arm_musl.py => tc_arm_musl.py} | 0 kiwixbuild/dependencies/{armhf.py => tc_armhf.py} | 0 kiwixbuild/dependencies/{emsdk.py => tc_emsdk.py} | 0 kiwixbuild/dependencies/{flatpak.py => tc_flatpak.py} | 0 6 files changed, 5 insertions(+), 5 deletions(-) rename kiwixbuild/dependencies/{android_ndk.py => tc_android_ndk.py} (100%) rename kiwixbuild/dependencies/{arm_musl.py => tc_arm_musl.py} (100%) rename kiwixbuild/dependencies/{armhf.py => tc_armhf.py} (100%) rename kiwixbuild/dependencies/{emsdk.py => tc_emsdk.py} (100%) rename kiwixbuild/dependencies/{flatpak.py => tc_flatpak.py} (100%) diff --git a/kiwixbuild/dependencies/__init__.py b/kiwixbuild/dependencies/__init__.py index 24f68c9..783f5b0 100644 --- a/kiwixbuild/dependencies/__init__.py +++ b/kiwixbuild/dependencies/__init__.py @@ -2,13 +2,13 @@ from .base import * from . import ( all_dependencies, - android_ndk, + tc_android_ndk, aria2, - armhf, - arm_musl, + tc_armhf, + tc_arm_musl, docoptcpp, - emsdk, - flatpak, + tc_emsdk, + tc_flatpak, gumbo, icu4c, ios_fat_lib, diff --git a/kiwixbuild/dependencies/android_ndk.py b/kiwixbuild/dependencies/tc_android_ndk.py similarity index 100% rename from kiwixbuild/dependencies/android_ndk.py rename to kiwixbuild/dependencies/tc_android_ndk.py diff --git a/kiwixbuild/dependencies/arm_musl.py b/kiwixbuild/dependencies/tc_arm_musl.py similarity index 100% rename from kiwixbuild/dependencies/arm_musl.py rename to kiwixbuild/dependencies/tc_arm_musl.py diff --git a/kiwixbuild/dependencies/armhf.py b/kiwixbuild/dependencies/tc_armhf.py similarity index 100% rename from kiwixbuild/dependencies/armhf.py rename to kiwixbuild/dependencies/tc_armhf.py diff --git a/kiwixbuild/dependencies/emsdk.py b/kiwixbuild/dependencies/tc_emsdk.py similarity index 100% rename from kiwixbuild/dependencies/emsdk.py rename to kiwixbuild/dependencies/tc_emsdk.py diff --git a/kiwixbuild/dependencies/flatpak.py b/kiwixbuild/dependencies/tc_flatpak.py similarity index 100% rename from kiwixbuild/dependencies/flatpak.py rename to kiwixbuild/dependencies/tc_flatpak.py From f0ad7b126a572f8674f840cb81cfec398a168acc Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Fri, 6 Oct 2023 15:58:29 +0200 Subject: [PATCH 2/9] Rename `arm_musl.py` to `musl.py`. Those files will contain toolchain for musl. Not specific to arm. --- kiwixbuild/dependencies/__init__.py | 2 +- kiwixbuild/dependencies/{tc_arm_musl.py => tc_musl.py} | 0 kiwixbuild/platforms/__init__.py | 2 +- kiwixbuild/platforms/{arm_musl.py => musl.py} | 0 4 files changed, 2 insertions(+), 2 deletions(-) rename kiwixbuild/dependencies/{tc_arm_musl.py => tc_musl.py} (100%) rename kiwixbuild/platforms/{arm_musl.py => musl.py} (100%) diff --git a/kiwixbuild/dependencies/__init__.py b/kiwixbuild/dependencies/__init__.py index 783f5b0..bdede5b 100644 --- a/kiwixbuild/dependencies/__init__.py +++ b/kiwixbuild/dependencies/__init__.py @@ -5,7 +5,7 @@ from . import ( tc_android_ndk, aria2, tc_armhf, - tc_arm_musl, + tc_musl, docoptcpp, tc_emsdk, tc_flatpak, diff --git a/kiwixbuild/dependencies/tc_arm_musl.py b/kiwixbuild/dependencies/tc_musl.py similarity index 100% rename from kiwixbuild/dependencies/tc_arm_musl.py rename to kiwixbuild/dependencies/tc_musl.py diff --git a/kiwixbuild/platforms/__init__.py b/kiwixbuild/platforms/__init__.py index 5222bd0..fac9e94 100644 --- a/kiwixbuild/platforms/__init__.py +++ b/kiwixbuild/platforms/__init__.py @@ -4,7 +4,7 @@ from .base import * from . import ( android, armhf, - arm_musl, + musl, flatpak, i586, ios, diff --git a/kiwixbuild/platforms/arm_musl.py b/kiwixbuild/platforms/musl.py similarity index 100% rename from kiwixbuild/platforms/arm_musl.py rename to kiwixbuild/platforms/musl.py From c3d3cc5a97a0342adf9b879b555087d7050b225e Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Fri, 6 Oct 2023 16:34:26 +0200 Subject: [PATCH 3/9] Add x86_64 musl toolchain and platform. --- kiwixbuild/dependencies/tc_musl.py | 13 ++++++++ kiwixbuild/platforms/musl.py | 52 ++++++++++++++++++++++++------ 2 files changed, 56 insertions(+), 9 deletions(-) diff --git a/kiwixbuild/dependencies/tc_musl.py b/kiwixbuild/dependencies/tc_musl.py index c7066ae..42bd136 100644 --- a/kiwixbuild/dependencies/tc_musl.py +++ b/kiwixbuild/dependencies/tc_musl.py @@ -12,3 +12,16 @@ class aarch64_musl_toolchain(Dependency): 'https://musl.cc/aarch64-linux-musl-cross.tgz') Builder = NoopBuilder + + +class x86_64_musl_toolchain(Dependency): + dont_skip = True + neutral = True + name = "x86-64_musl" + + class Source(ReleaseDownload): + archive = Remotefile('x86_64-linux-musl-cross.tgz', + '', + 'https://musl.cc/x86_64-linux-musl-cross.tgz') + + Builder = NoopBuilder diff --git a/kiwixbuild/platforms/musl.py b/kiwixbuild/platforms/musl.py index dc8ea10..25ab23a 100644 --- a/kiwixbuild/platforms/musl.py +++ b/kiwixbuild/platforms/musl.py @@ -4,12 +4,8 @@ from kiwixbuild.utils import pj from kiwixbuild._global import get_target_step -class Aarch64MuslPlatformInfo(PlatformInfo): - build = 'aarch64_musl' - arch_full = 'aarch64-linux-musl' - toolchain_names = ['aarch64_musl'] +class MuslPlatformInfo(PlatformInfo): compatible_hosts = ['fedora', 'debian'] - libdir = "lib/aarch64-linux-musl" def get_cross_config(self): return { @@ -21,8 +17,8 @@ class Aarch64MuslPlatformInfo(PlatformInfo): 'host_machine': { 'system': 'linux', 'lsystem': 'linux', - 'cpu_family': 'arm', - 'cpu': 'armhf', + 'cpu_family': self.cpu_family, + 'cpu': self.cpu, 'endian': 'little', 'abi': '' } @@ -57,11 +53,13 @@ class Aarch64MuslPlatformInfo(PlatformInfo): @property def exe_wrapper_def(self): try: - which('qemu-arm') + which(self.qemu) except subprocess.CalledProcessError: return "" + except AttributeError: + return "" else: - return "exe_wrapper = 'qemu-arm'" + return f"exe_wrapper = '{self.qemu}'" @property def configure_option(self): @@ -88,6 +86,10 @@ class Aarch64MuslPlatformInfo(PlatformInfo): def set_comp_flags(self, env): super().set_comp_flags(env) + env['LD_LIBRARY_PATH'] = ':'.join([ + pj(self.root_path, self.arch_full, 'lib'), + env['LD_LIBRARY_PATH'] + ]) env['CFLAGS'] = " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS'] env['CXXFLAGS'] = " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS'] @@ -101,6 +103,16 @@ class Aarch64MuslPlatformInfo(PlatformInfo): self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') +class Aarch64MuslPlatformInfo(MuslPlatformInfo): + build = 'aarch64_musl' + arch_full = 'aarch64-linux-musl' + toolchain_names = ['aarch64_musl'] + libdir = "lib/aarch64-linux-musl" + cpu_family = 'arm' + cpu = 'armhf' + qemu = 'qemu-arm' + + class Aarch64MuslDyn(Aarch64MuslPlatformInfo): name = 'aarch64_musl_dyn' static = False @@ -112,3 +124,25 @@ class Aarch64MuslStatic(Aarch64MuslPlatformInfo): class Aarch64MuslMixed(MixedMixin('aarch64_musl_static'), Aarch64MuslPlatformInfo): name = 'aarch64_musl_mixed' static = False + + +class X86_64MuslPlatformInfo(MuslPlatformInfo): + build = 'x86-64_musl' + arch_full = 'x86_64-linux-musl' + toolchain_names = ['x86-64_musl'] + libdir = "lib/x86_64-linux-musl" + cpu_family = 'x86_64' + cpu = 'x86_64' + + +class X86_64MuslDyn(X86_64MuslPlatformInfo): + name = 'x86-64_musl_dyn' + static = False + +class X86_64MuslStatic(X86_64MuslPlatformInfo): + name = 'x86-64_musl_static' + static = True + +class x86_64MuslMixed(MixedMixin('x86-64_musl_static'), X86_64MuslPlatformInfo): + name = 'x86-64_musl_mixed' + static = False From 0ae6b0766bd721a0fc3df747f0994ef7daec042e Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Fri, 6 Oct 2023 16:39:40 +0200 Subject: [PATCH 4/9] Build x86-64_musl in the CI --- .github/scripts/build_definition.py | 3 +++ .github/workflows/ci.yml | 3 +++ .github/workflows/releaseNigthly.yml | 2 ++ 3 files changed, 8 insertions(+) diff --git a/.github/scripts/build_definition.py b/.github/scripts/build_definition.py index 6f58a11..49e1666 100644 --- a/.github/scripts/build_definition.py +++ b/.github/scripts/build_definition.py @@ -53,6 +53,9 @@ BUILD_DEF = """ | | aarch64_musl_static| | | BP | BP | | linux-aarch64-musl | | | aarch64_musl_mixed | BP | | | | | linux-aarch64-musl | | | aarch64_musl_dyn | d | | B | B | | | + | | x86-64_musl_static | | | BP | BP | | linux-x86-64-musl | + | | x86-64_musl_mixed | BP | | | | | linux-x86-64-musl | + | | x86-64_musl_dyn | d | | B | B | | | | | win32_static | d | dB | dBP | dBP | | win-i686 | | | win32_dyn | d | dB | dB | dB | | | | | i586_static | | | BP | BP | | linux-i586 | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bddf776..7756a77 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,6 +27,9 @@ jobs: - aarch64_musl_static - aarch64_musl_dyn - aarch64_musl_mixed + - x86-64_musl_static + - x86-64_musl_dyn + - x86-64_musl_mixed - i586_static - i586_dyn - android_arm diff --git a/.github/workflows/releaseNigthly.yml b/.github/workflows/releaseNigthly.yml index 9f194f8..d2c4b1e 100644 --- a/.github/workflows/releaseNigthly.yml +++ b/.github/workflows/releaseNigthly.yml @@ -24,6 +24,8 @@ jobs: - aarch64_mixed - aarch64_musl_static - aarch64_musl_mixed + - x86-64_musl_static + - x86-64_musl_mixed - win32_static - i586_static - android_arm From 149c1d25fb7808102d675b56f2a76b78d88cdfe1 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 16 Oct 2023 16:13:53 +0200 Subject: [PATCH 5/9] Use musl toolchain with gcc 10. We need a "old" gcc version to be able to run on "old" systems (libstdc++.6.0.28) --- kiwixbuild/dependencies/tc_musl.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kiwixbuild/dependencies/tc_musl.py b/kiwixbuild/dependencies/tc_musl.py index 42bd136..cace7fc 100644 --- a/kiwixbuild/dependencies/tc_musl.py +++ b/kiwixbuild/dependencies/tc_musl.py @@ -8,8 +8,8 @@ class aarch64_musl_toolchain(Dependency): class Source(ReleaseDownload): archive = Remotefile('aarch64-linux-musl-cross.tgz', - 'c909817856d6ceda86aa510894fa3527eac7989f0ef6e87b5721c58737a06c38', - 'https://musl.cc/aarch64-linux-musl-cross.tgz') + '0f18a885b161815520bbb5757a4b4ab40d0898c29bebee58d0cddd6112e59cc6', + 'https://more.musl.cc/10/x86_64-linux-musl/aarch64-linux-musl-cross.tgz') Builder = NoopBuilder @@ -21,7 +21,7 @@ class x86_64_musl_toolchain(Dependency): class Source(ReleaseDownload): archive = Remotefile('x86_64-linux-musl-cross.tgz', - '', - 'https://musl.cc/x86_64-linux-musl-cross.tgz') + 'a3d55de8105739fcfb8b10eaa72cdb5d779319726bacff24149388d7608d1ed8', + 'https://more.musl.cc/10/x86_64-linux-musl/x86_64-linux-musl-cross.tgz') Builder = NoopBuilder From 750eeedc42dfba18854d2f5efdd2953bfd050e2c Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 16 Oct 2023 17:08:52 +0200 Subject: [PATCH 6/9] Update base_deps_meta_version. --- kiwixbuild/versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kiwixbuild/versions.py b/kiwixbuild/versions.py index 50e1be7..39f293f 100644 --- a/kiwixbuild/versions.py +++ b/kiwixbuild/versions.py @@ -39,7 +39,7 @@ release_versions = { # This is the "version" of the whole base_deps_versions dict. # Change this when you change base_deps_versions. -base_deps_meta_version = '90' +base_deps_meta_version = '92' base_deps_versions = { 'zlib' : '1.2.12', From b57f83ed2500c1b0d17fe2ea6349ac9295f390b6 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Fri, 20 Oct 2023 14:06:16 +0200 Subject: [PATCH 7/9] Move all toolchain installation in TOOLCHAIN directory. This is a clearer install and it simplify the base_deps archive creation. --- .github/scripts/common.py | 16 ++++----------- kiwixbuild/dependencies/base.py | 24 ++++++++++++++++++++++- kiwixbuild/dependencies/tc_android_ndk.py | 2 +- kiwixbuild/dependencies/tc_armhf.py | 11 +++++++---- kiwixbuild/dependencies/tc_emsdk.py | 8 ++++---- kiwixbuild/dependencies/tc_musl.py | 6 +++--- kiwixbuild/platforms/armhf.py | 8 ++++---- kiwixbuild/platforms/musl.py | 6 +++--- 8 files changed, 49 insertions(+), 32 deletions(-) diff --git a/.github/scripts/common.py b/.github/scripts/common.py index 83561f3..0d84142 100644 --- a/.github/scripts/common.py +++ b/.github/scripts/common.py @@ -27,6 +27,7 @@ HOME = Path(os.path.expanduser("~")) BASE_DIR = HOME / "BUILD_{}".format(PLATFORM_TARGET) SOURCE_DIR = HOME / "SOURCE" ARCHIVE_DIR = HOME / "ARCHIVE" +TOOLCHAIN_DIR = BASE_DIR / "TOOLCHAINS" INSTALL_DIR = BASE_DIR / "INSTALL" TMP_DIR = Path("/tmp") KBUILD_SOURCE_DIR = HOME / "kiwix-build" @@ -278,13 +279,9 @@ def make_deps_archive(target=None, name=None, full=False): base_dir = HOME / "BUILD_{}".format(PLATFORM_TARGET) if (base_dir / "meson_cross_file.txt").exists(): files_to_archive.append(base_dir / "meson_cross_file.txt") - # Add ndk/sdk/toolchains to allow project's CI to find them and compile - files_to_archive += HOME.glob("BUILD_*/android-ndk*") - files_to_archive += HOME.glob("BUILD_*/emsdk*") - if PLATFORM_TARGET.startswith("aarch64"): - files_to_archive += SOURCE_DIR.glob("aarch64*/*") - if PLATFORM_TARGET.startswith("armv"): - files_to_archive += SOURCE_DIR.glob("armv*/*") + # Copy any toolchain + files_to_archive += [TOOLCHAIN_DIR] + files_to_archive += HOME.glob("BUILD_neutral/TOOLCHAINS/*") if (BASE_DIR / "meson_cross_file.txt").exists(): files_to_archive.append(BASE_DIR / "meson_cross_file.txt") @@ -307,11 +304,6 @@ def make_deps_archive(target=None, name=None, full=False): files_to_archive += SOURCE_DIR.glob("*/.*_ok") files_to_archive += SOURCE_DIR.glob("zim-testing-suite-*/*") - toolchains_subdirs = HOME.glob("BUILD_*/TOOLCHAINS/*/*") - for subdir in toolchains_subdirs: - if not subdir.match("tools"): - files_to_archive.append(subdir) - archive_file = TMP_DIR / archive_name with tarfile.open(str(archive_file), "w:xz") as tar: for name in set(files_to_archive): diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index 3b5ae2f..68b8a81 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -3,7 +3,7 @@ import os import shutil import time -from kiwixbuild.utils import pj, Context, SkipCommand, WarningMessage, extract_archive, Defaultdict, StopBuild, run_command, colorize +from kiwixbuild.utils import pj, Context, SkipCommand, WarningMessage, extract_archive, Defaultdict, StopBuild, run_command, colorize, copy_tree from kiwixbuild.versions import main_project_versions, base_deps_versions from kiwixbuild._global import neutralEnv, option, get_target_step @@ -323,6 +323,28 @@ class NoopBuilder(Builder): pass +class TcCopyBuilder(Builder): + src_subdir = None + + @property + def build_path(self): + return pj(self.buildEnv.toolchain_dir, self.target.full_name()) + + def build(self): + self.command('copy', self._copy) + + def _copy(self, context): + context.try_skip(self.build_path) + if self.src_subdir: + source_path = pj(self.source_path, self.src_subdir) + else: + source_path = self.source_path + copy_tree(source_path, self.build_path) + + def make_dist(self): + pass + + class MakeBuilder(Builder): configure_option_template = "{dep_options} {static_option} {env_option} --prefix {install_dir} --libdir {libdir}" configure_option = "" diff --git a/kiwixbuild/dependencies/tc_android_ndk.py b/kiwixbuild/dependencies/tc_android_ndk.py index 88cad29..96be642 100644 --- a/kiwixbuild/dependencies/tc_android_ndk.py +++ b/kiwixbuild/dependencies/tc_android_ndk.py @@ -25,7 +25,7 @@ class android_ndk(Dependency): class Builder(Builder): @property def install_path(self): - return self.build_path + return pj(self.buildEnv.toolchain_dir, self.target.full_name()) @property def api(self): diff --git a/kiwixbuild/dependencies/tc_armhf.py b/kiwixbuild/dependencies/tc_armhf.py index 0080f6d..507aee1 100644 --- a/kiwixbuild/dependencies/tc_armhf.py +++ b/kiwixbuild/dependencies/tc_armhf.py @@ -1,4 +1,4 @@ -from .base import Dependency, ReleaseDownload, NoopBuilder +from .base import Dependency, ReleaseDownload, TcCopyBuilder from kiwixbuild.utils import Remotefile # The arm toolchains @@ -19,7 +19,9 @@ class armv6_toolchain(Dependency): '4c371c4c5b55ebd1f3d7dd26b14703632d9ba47423f901bcd9303d83ad444434', base_url + 'x-tools-armv6-rpi-linux-gnueabihf.tar.xz') - Builder = NoopBuilder + + class Builder(TcCopyBuilder): + src_subdir = "armv6-rpi-linux-gnueabihf" class armv8_toolchain(Dependency): @@ -32,7 +34,8 @@ class armv8_toolchain(Dependency): 'cc28f5c3f6a3e7d9985f98779c4e72224b4eb5a7e4dc2bcdefd90cb241fb94a5', base_url + 'x-tools-armv8-rpi3-linux-gnueabihf.tar.xz') - Builder = NoopBuilder + class Builder(TcCopyBuilder): + src_subdir = "armv8-rpi3-linux-gnueabihf" class aarch64_toolchain(Dependency): dont_skip = True @@ -44,4 +47,4 @@ class aarch64_toolchain(Dependency): '1b048bb8886ad63d21797cd9129fc37b9ea0dfaac7e3c36f888aa16fbec1d320', aarch_base_url + 'cross-gcc-6.3.0-pi_64.tar.gz') - Builder = NoopBuilder + Builder = TcCopyBuilder diff --git a/kiwixbuild/dependencies/tc_emsdk.py b/kiwixbuild/dependencies/tc_emsdk.py index bd7c283..898bbe9 100644 --- a/kiwixbuild/dependencies/tc_emsdk.py +++ b/kiwixbuild/dependencies/tc_emsdk.py @@ -23,21 +23,21 @@ class emsdk(Dependency): class Builder(Builder): @property def install_path(self): - return self.build_path + return pj(self.buildEnv.toolchain_dir, self.target.full_name()) def _copy_source(self, context): context.try_skip(self.build_path) - copy_tree(self.source_path, self.build_path) + copy_tree(self.source_path, self.install_path) def _install(self, context): context.try_skip(self.build_path) command = "./emsdk install 3.1.24" - run_command(command, self.build_path, context) + run_command(command, self.install_path, context) def _activate(self, context): context.try_skip(self.build_path) command = "./emsdk activate 3.1.24" - run_command(command, self.build_path, context) + run_command(command, self.install_path, context) def build(self): diff --git a/kiwixbuild/dependencies/tc_musl.py b/kiwixbuild/dependencies/tc_musl.py index cace7fc..4ca68f5 100644 --- a/kiwixbuild/dependencies/tc_musl.py +++ b/kiwixbuild/dependencies/tc_musl.py @@ -1,4 +1,4 @@ -from .base import Dependency, ReleaseDownload, NoopBuilder +from .base import Dependency, ReleaseDownload, TcCopyBuilder from kiwixbuild.utils import Remotefile class aarch64_musl_toolchain(Dependency): @@ -11,7 +11,7 @@ class aarch64_musl_toolchain(Dependency): '0f18a885b161815520bbb5757a4b4ab40d0898c29bebee58d0cddd6112e59cc6', 'https://more.musl.cc/10/x86_64-linux-musl/aarch64-linux-musl-cross.tgz') - Builder = NoopBuilder + Builder = TcCopyBuilder class x86_64_musl_toolchain(Dependency): @@ -24,4 +24,4 @@ class x86_64_musl_toolchain(Dependency): 'a3d55de8105739fcfb8b10eaa72cdb5d779319726bacff24149388d7608d1ed8', 'https://more.musl.cc/10/x86_64-linux-musl/x86_64-linux-musl-cross.tgz') - Builder = NoopBuilder + Builder = TcCopyBuilder diff --git a/kiwixbuild/platforms/armhf.py b/kiwixbuild/platforms/armhf.py index 2d1e0ac..d47e5e0 100644 --- a/kiwixbuild/platforms/armhf.py +++ b/kiwixbuild/platforms/armhf.py @@ -30,12 +30,12 @@ class ArmPlatformInfo(PlatformInfo): return "lib/{}".format(self.arch_full) @property - def tlc_source(self): - return get_target_step(self.build, 'source') + def toolchain(self): + return get_target_step(self.build, 'neutral') @property def root_path(self): - return pj(self.tlc_source.source_path, self.arch_full) + return self.toolchain.build_path @property def binaries(self): @@ -150,7 +150,7 @@ class Aarch64(ArmPlatformInfo): @property def root_path(self): - return self.tlc_source.source_path + return self.toolchain.build_path class Aarch64Dyn(Aarch64): name = 'aarch64_dyn' diff --git a/kiwixbuild/platforms/musl.py b/kiwixbuild/platforms/musl.py index 25ab23a..ade0c22 100644 --- a/kiwixbuild/platforms/musl.py +++ b/kiwixbuild/platforms/musl.py @@ -25,12 +25,12 @@ class MuslPlatformInfo(PlatformInfo): } @property - def tlc_source(self): - return get_target_step(self.build, 'source') + def toolchain(self): + return get_target_step(self.build, 'neutral') @property def root_path(self): - return self.tlc_source.source_path + return self.toolchain.build_path @property def binaries(self): From 9a2a9ca188381fc3b5ff8cdabf5999f6a3ef637c Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Fri, 20 Oct 2023 15:12:36 +0200 Subject: [PATCH 8/9] Do not follow symlink when we copy a directory. --- kiwixbuild/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kiwixbuild/utils.py b/kiwixbuild/utils.py index d8b1361..0c5d4ec 100644 --- a/kiwixbuild/utils.py +++ b/kiwixbuild/utils.py @@ -101,7 +101,7 @@ def copy_tree(src, dst, post_copy_function=None): os.makedirs(dstdir, exist_ok=True) for f in files: dstfile = pj(dstdir, f) - shutil.copy2(pj(root, f), dstfile) + shutil.copy2(pj(root, f), dstfile, follow_symlinks=False) if post_copy_function is not None: post_copy_function(dstfile) From 3f908f429c226d5f6ff1ef4c7c8467200bd1c15e Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 23 Oct 2023 10:25:08 +0200 Subject: [PATCH 9/9] Remove `x86-64_musl_dyn` from the CI. This target fails on the CI because of conflict between the builded libraries (lzma, libz) and native library. As they are both for the same arch but with different libc, python tries to load them and fails. There is a real bug somewhere in our toolchain but we need to build for x86_64 musl right now and this is done in `x86-64_musl_mixed` which pass. So let's remove it for now. (`x86-64_musl_dyn` fails with the docker image. It is ok with native (fedora 38) system) --- .github/scripts/build_definition.py | 1 - .github/workflows/ci.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/scripts/build_definition.py b/.github/scripts/build_definition.py index 49e1666..39716e3 100644 --- a/.github/scripts/build_definition.py +++ b/.github/scripts/build_definition.py @@ -55,7 +55,6 @@ BUILD_DEF = """ | | aarch64_musl_dyn | d | | B | B | | | | | x86-64_musl_static | | | BP | BP | | linux-x86-64-musl | | | x86-64_musl_mixed | BP | | | | | linux-x86-64-musl | - | | x86-64_musl_dyn | d | | B | B | | | | | win32_static | d | dB | dBP | dBP | | win-i686 | | | win32_dyn | d | dB | dB | dB | | | | | i586_static | | | BP | BP | | linux-i586 | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7756a77..f02aa9b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,6 @@ jobs: - aarch64_musl_dyn - aarch64_musl_mixed - x86-64_musl_static - - x86-64_musl_dyn - x86-64_musl_mixed - i586_static - i586_dyn