Merge pull request #636 from kiwix/musl

Add x86-64 musl

.github/scripts/build_definition.py

@@ -53,6 +53,8 @@ BUILD_DEF = """
     |         | aarch64_musl_static|        |          | BP        | BP          |               | linux-aarch64-musl   |
     |         | aarch64_musl_mixed | BP     |          |           |             |               | linux-aarch64-musl   |
     |         | aarch64_musl_dyn   | d      |          | B         | B           |               |                      |
+    |         | x86-64_musl_static |        |          | BP        | BP          |               | linux-x86-64-musl    |
+    |         | x86-64_musl_mixed  | BP     |          |           |             |               | linux-x86-64-musl    |
     |         | win32_static       | d      | dB       | dBP       | dBP         |               | win-i686             |
     |         | win32_dyn          | d      | dB       | dB        | dB          |               |                      |
     |         | i586_static        |        |          | BP        | BP          |               | linux-i586           |

.github/scripts/common.py

@@ -27,6 +27,7 @@ HOME = Path(os.path.expanduser("~"))
 BASE_DIR = HOME / "BUILD_{}".format(PLATFORM_TARGET)
 SOURCE_DIR = HOME / "SOURCE"
 ARCHIVE_DIR = HOME / "ARCHIVE"
+TOOLCHAIN_DIR = BASE_DIR / "TOOLCHAINS"
 INSTALL_DIR = BASE_DIR / "INSTALL"
 TMP_DIR = Path("/tmp")
 KBUILD_SOURCE_DIR = HOME / "kiwix-build"
@@ -278,13 +279,9 @@ def make_deps_archive(target=None, name=None, full=False):
         base_dir = HOME / "BUILD_{}".format(PLATFORM_TARGET)
         if (base_dir / "meson_cross_file.txt").exists():
             files_to_archive.append(base_dir / "meson_cross_file.txt")
-    # Add ndk/sdk/toolchains to allow project's CI to find them and compile
+    # Copy any toolchain
-    files_to_archive += HOME.glob("BUILD_*/android-ndk*")
+    files_to_archive += [TOOLCHAIN_DIR]
-    files_to_archive += HOME.glob("BUILD_*/emsdk*")
+    files_to_archive += HOME.glob("BUILD_neutral/TOOLCHAINS/*")
-    if PLATFORM_TARGET.startswith("aarch64"):
-        files_to_archive += SOURCE_DIR.glob("aarch64*/*")
-    if PLATFORM_TARGET.startswith("armv"):
-        files_to_archive += SOURCE_DIR.glob("armv*/*")
     if (BASE_DIR / "meson_cross_file.txt").exists():
         files_to_archive.append(BASE_DIR / "meson_cross_file.txt")
 
@@ -307,11 +304,6 @@ def make_deps_archive(target=None, name=None, full=False):
         files_to_archive += SOURCE_DIR.glob("*/.*_ok")
         files_to_archive += SOURCE_DIR.glob("zim-testing-suite-*/*")
 
-        toolchains_subdirs = HOME.glob("BUILD_*/TOOLCHAINS/*/*")
-        for subdir in toolchains_subdirs:
-            if not subdir.match("tools"):
-                files_to_archive.append(subdir)
-
     archive_file = TMP_DIR / archive_name
     with tarfile.open(str(archive_file), "w:xz") as tar:
         for name in set(files_to_archive):

.github/workflows/ci.yml

@@ -27,6 +27,8 @@ jobs:
           - aarch64_musl_static
           - aarch64_musl_dyn
           - aarch64_musl_mixed
+          - x86-64_musl_static
+          - x86-64_musl_mixed
           - i586_static
           - i586_dyn
           - android_arm

.github/workflows/releaseNigthly.yml

@@ -24,6 +24,8 @@ jobs:
           - aarch64_mixed
           - aarch64_musl_static
           - aarch64_musl_mixed
+          - x86-64_musl_static
+          - x86-64_musl_mixed
           - win32_static
           - i586_static
           - android_arm

kiwixbuild/dependencies/__init__.py

@@ -2,13 +2,13 @@
 from .base import *
 from . import (
     all_dependencies,
-    android_ndk,
+    tc_android_ndk,
     aria2,
-    armhf,
+    tc_armhf,
-    arm_musl,
+    tc_musl,
     docoptcpp,
-    emsdk,
+    tc_emsdk,
-    flatpak,
+    tc_flatpak,
     gumbo,
     icu4c,
     ios_fat_lib,

kiwixbuild/dependencies/arm_musl.py

@@ -1,14 +0,0 @@
-from .base import Dependency, ReleaseDownload, NoopBuilder
-from kiwixbuild.utils import Remotefile
-
-class aarch64_musl_toolchain(Dependency):
-    dont_skip = True
-    neutral = True
-    name = "aarch64_musl"
-
-    class Source(ReleaseDownload):
-        archive = Remotefile('aarch64-linux-musl-cross.tgz',
-                             'c909817856d6ceda86aa510894fa3527eac7989f0ef6e87b5721c58737a06c38',
-                             'https://musl.cc/aarch64-linux-musl-cross.tgz')
-
-    Builder = NoopBuilder

kiwixbuild/dependencies/base.py

@@ -3,7 +3,7 @@ import os
 import shutil
 import time
 
-from kiwixbuild.utils import pj, Context, SkipCommand, WarningMessage, extract_archive, Defaultdict, StopBuild, run_command, colorize
+from kiwixbuild.utils import pj, Context, SkipCommand, WarningMessage, extract_archive, Defaultdict, StopBuild, run_command, colorize, copy_tree
 from kiwixbuild.versions import main_project_versions, base_deps_versions
 from kiwixbuild._global import neutralEnv, option, get_target_step
 
@@ -323,6 +323,28 @@ class NoopBuilder(Builder):
         pass
 
 
+class TcCopyBuilder(Builder):
+    src_subdir = None
+
+    @property
+    def build_path(self):
+        return pj(self.buildEnv.toolchain_dir, self.target.full_name())
+
+    def build(self):
+        self.command('copy', self._copy)
+
+    def _copy(self, context):
+        context.try_skip(self.build_path)
+        if self.src_subdir:
+            source_path = pj(self.source_path, self.src_subdir)
+        else:
+            source_path = self.source_path
+        copy_tree(source_path, self.build_path)
+
+    def make_dist(self):
+        pass
+
+
 class MakeBuilder(Builder):
     configure_option_template = "{dep_options} {static_option} {env_option} --prefix {install_dir} --libdir {libdir}"
     configure_option = ""

kiwixbuild/dependencies/tc_android_ndk.py

@@ -25,7 +25,7 @@ class android_ndk(Dependency):
     class Builder(Builder):
         @property
         def install_path(self):
-            return self.build_path
+            return pj(self.buildEnv.toolchain_dir, self.target.full_name())
 
         @property
         def api(self):

kiwixbuild/dependencies/tc_armhf.py

@@ -1,4 +1,4 @@
-from .base import Dependency, ReleaseDownload, NoopBuilder
+from .base import Dependency, ReleaseDownload, TcCopyBuilder
 from kiwixbuild.utils import Remotefile
 
 # The arm toolchains
@@ -19,7 +19,9 @@ class armv6_toolchain(Dependency):
                              '4c371c4c5b55ebd1f3d7dd26b14703632d9ba47423f901bcd9303d83ad444434',
                              base_url + 'x-tools-armv6-rpi-linux-gnueabihf.tar.xz')
 
-    Builder = NoopBuilder
+
+    class Builder(TcCopyBuilder):
+        src_subdir = "armv6-rpi-linux-gnueabihf"
 
 
 class armv8_toolchain(Dependency):
@@ -32,7 +34,8 @@ class armv8_toolchain(Dependency):
                              'cc28f5c3f6a3e7d9985f98779c4e72224b4eb5a7e4dc2bcdefd90cb241fb94a5',
                              base_url + 'x-tools-armv8-rpi3-linux-gnueabihf.tar.xz')
 
-    Builder = NoopBuilder
+    class Builder(TcCopyBuilder):
+        src_subdir = "armv8-rpi3-linux-gnueabihf"
 
 class aarch64_toolchain(Dependency):
     dont_skip = True
@@ -44,4 +47,4 @@ class aarch64_toolchain(Dependency):
                              '1b048bb8886ad63d21797cd9129fc37b9ea0dfaac7e3c36f888aa16fbec1d320',
                              aarch_base_url + 'cross-gcc-6.3.0-pi_64.tar.gz')
 
-    Builder = NoopBuilder
+    Builder = TcCopyBuilder

kiwixbuild/dependencies/tc_emsdk.py

@@ -23,21 +23,21 @@ class emsdk(Dependency):
     class Builder(Builder):
         @property
         def install_path(self):
-            return self.build_path
+            return pj(self.buildEnv.toolchain_dir, self.target.full_name())
 
         def _copy_source(self, context):
             context.try_skip(self.build_path)
-            copy_tree(self.source_path, self.build_path)
+            copy_tree(self.source_path, self.install_path)
 
         def _install(self, context):
             context.try_skip(self.build_path)
             command = "./emsdk install 3.1.24"
-            run_command(command, self.build_path, context)
+            run_command(command, self.install_path, context)
 
         def _activate(self, context):
             context.try_skip(self.build_path)
             command = "./emsdk activate 3.1.24"
-            run_command(command, self.build_path, context)
+            run_command(command, self.install_path, context)
 
 
         def build(self):

kiwixbuild/dependencies/tc_musl.py

@@ -0,0 +1,27 @@
+from .base import Dependency, ReleaseDownload, TcCopyBuilder
+from kiwixbuild.utils import Remotefile
+
+class aarch64_musl_toolchain(Dependency):
+    dont_skip = True
+    neutral = True
+    name = "aarch64_musl"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('aarch64-linux-musl-cross.tgz',
+                             '0f18a885b161815520bbb5757a4b4ab40d0898c29bebee58d0cddd6112e59cc6',
+                             'https://more.musl.cc/10/x86_64-linux-musl/aarch64-linux-musl-cross.tgz')
+
+    Builder = TcCopyBuilder
+
+
+class x86_64_musl_toolchain(Dependency):
+    dont_skip = True
+    neutral = True
+    name = "x86-64_musl"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('x86_64-linux-musl-cross.tgz',
+                             'a3d55de8105739fcfb8b10eaa72cdb5d779319726bacff24149388d7608d1ed8',
+                             'https://more.musl.cc/10/x86_64-linux-musl/x86_64-linux-musl-cross.tgz')
+
+    Builder = TcCopyBuilder

kiwixbuild/platforms/__init__.py

@@ -4,7 +4,7 @@ from .base import *
 from . import (
     android,
     armhf,
-    arm_musl,
+    musl,
     flatpak,
     i586,
     ios,

kiwixbuild/platforms/armhf.py

@@ -30,12 +30,12 @@ class ArmPlatformInfo(PlatformInfo):
         return "lib/{}".format(self.arch_full)
 
     @property
-    def tlc_source(self):
+    def toolchain(self):
-        return get_target_step(self.build, 'source')
+        return get_target_step(self.build, 'neutral')
 
     @property
     def root_path(self):
-        return pj(self.tlc_source.source_path, self.arch_full)
+        return self.toolchain.build_path
 
     @property
     def binaries(self):
@@ -150,7 +150,7 @@ class Aarch64(ArmPlatformInfo):
 
     @property
     def root_path(self):
-        return self.tlc_source.source_path
+        return self.toolchain.build_path
 
 class Aarch64Dyn(Aarch64):
     name = 'aarch64_dyn'

kiwixbuild/platforms/musl.py

@@ -4,12 +4,8 @@ from kiwixbuild.utils import pj
 from kiwixbuild._global import get_target_step
 
 
-class Aarch64MuslPlatformInfo(PlatformInfo):
+class MuslPlatformInfo(PlatformInfo):
-    build = 'aarch64_musl'
-    arch_full = 'aarch64-linux-musl'
-    toolchain_names = ['aarch64_musl']
     compatible_hosts = ['fedora', 'debian']
-    libdir = "lib/aarch64-linux-musl"
 
     def get_cross_config(self):
         return {
@@ -21,20 +17,20 @@ class Aarch64MuslPlatformInfo(PlatformInfo):
             'host_machine': {
                 'system': 'linux',
                 'lsystem': 'linux',
-                'cpu_family': 'arm',
+                'cpu_family': self.cpu_family,
-                'cpu': 'armhf',
+                'cpu': self.cpu,
                 'endian': 'little',
                 'abi': ''
             }
         }
 
     @property
-    def tlc_source(self):
+    def toolchain(self):
-        return get_target_step(self.build, 'source')
+        return get_target_step(self.build, 'neutral')
 
     @property
     def root_path(self):
-        return self.tlc_source.source_path
+        return self.toolchain.build_path
 
     @property
     def binaries(self):
@@ -57,11 +53,13 @@ class Aarch64MuslPlatformInfo(PlatformInfo):
     @property
     def exe_wrapper_def(self):
         try:
-            which('qemu-arm')
+            which(self.qemu)
         except subprocess.CalledProcessError:
             return ""
+        except AttributeError:
+            return ""
         else:
-            return "exe_wrapper = 'qemu-arm'"
+            return f"exe_wrapper = '{self.qemu}'"
 
     @property
     def configure_option(self):
@@ -88,6 +86,10 @@ class Aarch64MuslPlatformInfo(PlatformInfo):
 
     def set_comp_flags(self, env):
         super().set_comp_flags(env)
+        env['LD_LIBRARY_PATH'] = ':'.join([
+            pj(self.root_path, self.arch_full, 'lib'),
+            env['LD_LIBRARY_PATH']
+        ])
         env['CFLAGS'] = " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS']
         env['CXXFLAGS'] = " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
 
@@ -101,6 +103,16 @@ class Aarch64MuslPlatformInfo(PlatformInfo):
         self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
 
 
+class Aarch64MuslPlatformInfo(MuslPlatformInfo):
+    build = 'aarch64_musl'
+    arch_full = 'aarch64-linux-musl'
+    toolchain_names = ['aarch64_musl']
+    libdir = "lib/aarch64-linux-musl"
+    cpu_family = 'arm'
+    cpu = 'armhf'
+    qemu = 'qemu-arm'
+
+
 class Aarch64MuslDyn(Aarch64MuslPlatformInfo):
     name = 'aarch64_musl_dyn'
     static = False
@@ -112,3 +124,25 @@ class Aarch64MuslStatic(Aarch64MuslPlatformInfo):
 class Aarch64MuslMixed(MixedMixin('aarch64_musl_static'), Aarch64MuslPlatformInfo):
     name = 'aarch64_musl_mixed'
     static = False
+
+
+class X86_64MuslPlatformInfo(MuslPlatformInfo):
+    build = 'x86-64_musl'
+    arch_full = 'x86_64-linux-musl'
+    toolchain_names = ['x86-64_musl']
+    libdir = "lib/x86_64-linux-musl"
+    cpu_family = 'x86_64'
+    cpu = 'x86_64'
+
+
+class X86_64MuslDyn(X86_64MuslPlatformInfo):
+    name = 'x86-64_musl_dyn'
+    static = False
+
+class X86_64MuslStatic(X86_64MuslPlatformInfo):
+    name = 'x86-64_musl_static'
+    static = True
+
+class x86_64MuslMixed(MixedMixin('x86-64_musl_static'), X86_64MuslPlatformInfo):
+    name = 'x86-64_musl_mixed'
+    static = False

kiwixbuild/utils.py

@@ -101,7 +101,7 @@ def copy_tree(src, dst, post_copy_function=None):
         os.makedirs(dstdir, exist_ok=True)
         for f in files:
             dstfile = pj(dstdir, f)
-            shutil.copy2(pj(root, f), dstfile)
+            shutil.copy2(pj(root, f), dstfile, follow_symlinks=False)
             if post_copy_function is not None:
                 post_copy_function(dstfile)
 

kiwixbuild/versions.py

@@ -39,7 +39,7 @@ release_versions = {
 
 # This is the "version" of the whole base_deps_versions dict.
 # Change this when you change base_deps_versions.
-base_deps_meta_version = '90'
+base_deps_meta_version = '92'
 
 base_deps_versions = {
   'zlib' : '1.2.12',