Merge pull request #580 from kiwix/arm_build

.github/scripts/build_projects.py

@@ -30,11 +30,11 @@ elif PLATFORM_TARGET.startswith("native_"):
             TARGETS = ("libzim", "libkiwix")
         else:
             TARGETS = ("zim-tools", "kiwix-tools")
-elif PLATFORM_TARGET in ("win32_static", "armhf_static", "armhf_dyn", "i586_static"):
+elif PLATFORM_TARGET in ("win32_static", "armhf_static", "armhf_dyn", "aarch64_static", "aarch64_dyn", "i586_static"):
     TARGETS = ("kiwix-tools",)
 elif PLATFORM_TARGET == "flatpak":
     TARGETS = ("kiwix-desktop",)
-elif PLATFORM_TARGET == "wasm":
+elif PLATFORM_TARGET in ("wasm", "armhf_mixed", "aarch64_mixed"):
     TARGETS = ("libzim", )
 else:
     TARGETS = ("libzim", "zim-tools", "libkiwix", "kiwix-tools")

.github/scripts/build_release_nightly.py

@@ -37,11 +37,11 @@ elif PLATFORM_TARGET.startswith("native_"):
             TARGETS = ("libzim", "libkiwix")
         else:
             TARGETS = ("zim-tools", "kiwix-tools")
-elif PLATFORM_TARGET in ("win32_static", "armhf_static", "i586_static"):
+elif PLATFORM_TARGET in ("win32_static", "armhf_static", "aarch64_static", "i586_static"):
     TARGETS = ("kiwix-tools",)
 elif PLATFORM_TARGET == "flatpak":
     TARGETS = ("kiwix-desktop",)
-elif PLATFORM_TARGET == "wasm":
+elif PLATFORM_TARGET in ("wasm", "armhf_mixed", "aarch64_mixed"):
     TARGETS = ("libzim", )
 else:
     TARGETS = ("libzim", "zim-tools", "libkiwix", "kiwix-tools")

.github/scripts/common.py

@@ -51,6 +51,9 @@ PLATFORM_TO_RELEASE = {
     "native_static": "{os}-x86_64".format(os=RELEASE_OS_NAME),
     "win32_static": "win-i686",
     "armhf_static": "{os}-armhf".format(os=RELEASE_OS_NAME),
+    "armhf_mixed": "{os}-armhf".format(os=RELEASE_OS_NAME),
+    "aarch64_static": "{os}-aarch64".format(os=RELEASE_OS_NAME),
+    "aarch64_mixed": "{os}-aarch64".format(os=RELEASE_OS_NAME),
     "i586_static": "{os}-i586".format(os=RELEASE_OS_NAME),
     "android_arm": "android-arm",
     "android_arm64": "android-arm64",
@@ -278,8 +281,9 @@ def make_deps_archive(target=None, name=None, full=False):
     print_message("Create archive {}.", archive_name)
     files_to_archive = list(filter_install_dir(INSTALL_DIR))
     files_to_archive += HOME.glob("BUILD_*/LOGS")
-    if PLATFORM_TARGET == "native_mixed":
+    if PLATFORM_TARGET.endswith("_mixed"):
-        files_to_archive += filter_install_dir(HOME / "BUILD_native_static" / "INSTALL")
+        static_platform = PLATFORM_TARGET.replace("_mixed", "_static")
+        files_to_archive += filter_install_dir(HOME / ("BUILD_" + static_platform) / "INSTALL")
     if PLATFORM_TARGET.startswith("android_"):
         files_to_archive += filter_install_dir(HOME / "BUILD_neutral" / "INSTALL")
         base_dir = HOME / "BUILD_{}".format(PLATFORM_TARGET)
@@ -298,6 +302,11 @@ def make_deps_archive(target=None, name=None, full=False):
     if full:
         files_to_archive += ARCHIVE_DIR.glob(".*_ok")
         files_to_archive += BASE_DIR.glob("*/.*_ok")
+        # Add also static build for mixed target
+        if PLATFORM_TARGET.endswith("_mixed"):
+            static_platform = PLATFORM_TARGET.replace("_mixed", "_static")
+            files_to_archive += (HOME / ("BUILD_" + static_platform)).glob("*/.*_ok")
+        # Native dyn and static is needed for potential cross compilation that use native tools (icu)
         files_to_archive += (HOME / "BUILD_native_dyn").glob("*/.*_ok")
         files_to_archive += (HOME / "BUILD_native_static").glob("*/.*_ok")
         files_to_archive += HOME.glob("BUILD_android*/**/.*_ok")
@@ -305,6 +314,8 @@ def make_deps_archive(target=None, name=None, full=False):
         files_to_archive += SOURCE_DIR.glob("zim-testing-suite-*/*")
         if PLATFORM_TARGET.startswith("armhf"):
             files_to_archive += (SOURCE_DIR / "armhf").glob("*")
+        if PLATFORM_TARGET.startswith("aarch64"):
+            files_to_archive += (SOURCE_DIR / "aarch64").glob("*")
         toolchains_subdirs = HOME.glob("BUILD_*/TOOLCHAINS/*/*")
         for subdir in toolchains_subdirs:
             if not subdir.match("tools"):

.github/workflows/ci.yml

@@ -18,6 +18,10 @@ jobs:
           - wasm
           - armhf_static
           - armhf_dyn
+          - armhf_mixed
+          - aarch64_static
+          - aarch64_dyn
+          - aarch64_mixed
           - i586_static
           - i586_dyn
           - android_arm

.github/workflows/releaseNigthly.yml

@@ -19,6 +19,9 @@ jobs:
           - native_desktop
           - wasm
           - armhf_static
+          - armhf_mixed
+          - aarch64_static
+          - aarch64_mixed
           - win32_static
           - i586_static
           - android_arm
@@ -44,6 +47,15 @@ jobs:
           - target: armhf_static
             image_variant: bionic
             lib_postfix: '/x86_64-linux-gnu'
+          - target: armhf_mixed
+            image_variant: bionic
+            lib_postfix: '/x86_64-linux-gnu'
+          - target: aarch64_static
+            image_variant: bionic
+            lib_postfix: '/x86_64-linux-gnu'
+          - target: aarch64_mixed
+            image_variant: bionic
+            lib_postfix: '/x86_64-linux-gnu'
           - target: win32_static
             image_variant: f35
             lib_postfix: '64'

kiwixbuild/dependencies/armhf.py

@@ -1,13 +1,36 @@
 from .base import Dependency, ReleaseDownload, NoopBuilder
 from kiwixbuild.utils import Remotefile
 
+
+base_url = 'https://master.dl.sourceforge.net/project/raspberry-pi-cross-compilers/'
+
+# This is Gcc 10.3.0 and Raspberry Pi 2 and 3 only !
+armhf_base_url = base_url + 'Raspberry%20Pi%20GCC%20Cross-Compiler%20Toolchains/Stretch/GCC%2010.3.0/Raspberry%20Pi%202%2C%203/'
+
+# This is Gcc 10.3.0 and ALL rapsberry Pi arch64
+aarch_base_url = base_url + 'Bonus%20Raspberry%20Pi%20GCC%2064-Bit%20Toolchains/Raspberry%20Pi%20GCC%2064-Bit%20Cross-Compiler%20Toolchains/Stretch/GCC%2010.3.0/'
+
 class armhf_toolchain(Dependency):
     dont_skip = True
     neutral = True
     name = 'armhf'
 
     class Source(ReleaseDownload):
-        archive = Remotefile('raspberrypi-tools.tar.gz',
+        archive = Remotefile('cross-gcc-10.3.0-pi_2-3.tar.gz',
-                             'e72b35436f2f23f2f7df322d6c318b9be57b21596b5ff0b8936af4ad94e04f2e')
+                             '6aef31703fb7bfd63065dda7fb525f1f86a0509c4358c57631a51025805278b3',
+                             armhf_base_url + 'cross-gcc-10.3.0-pi_2-3.tar.gz')
+
+    Builder = NoopBuilder
+
+
+class aarch64_toolchain(Dependency):
+    dont_skip = True
+    neutral = True
+    name = "aarch64"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('cross-gcc-10.3.0-pi_64.tar.gz',
+                             '5b3fdb7ee8c496c377ab8b11d7ffd404b4d3041f4fdcfeebcbcb734d45a5f3e9',
+                             aarch_base_url + 'cross-gcc-10.3.0-pi_64.tar.gz')
 
     Builder = NoopBuilder

kiwixbuild/dependencies/libkiwix.py

@@ -28,7 +28,7 @@ class Libkiwix(Dependency):
                 return '-Db_bitcode=true'
             if platformInfo.name == 'flatpak':
                 return '--wrap-mode=nodownload'
-            if platformInfo.name == 'native_mixed' and option('target') == 'libkiwix':
+            if platformInfo.mixed and option('target') == 'libkiwix':
                 return "-Dstatic-linkage=true"
             return ''
 

kiwixbuild/dependencies/libzim.py

@@ -32,7 +32,7 @@ class Libzim(Dependency):
                 config_options.append("-Dstatic-linkage=true")
             if platformInfo.build == 'iOS':
                 config_options.append("-Db_bitcode=true")
-            if platformInfo.name == 'native_mixed' and option('target') == 'libzim':
+            if platformInfo.mixed and option('target') == 'libzim':
                 config_options.append("-Dstatic-linkage=true")
             if platformInfo.name == "flatpak":
                 config_options.append("--wrap-mode=nodownload")

kiwixbuild/dependencies/zlib.py

@@ -44,3 +44,10 @@ class zlib(Dependency):
                     binary_path=pj(self.buildEnv.install_dir, 'bin'),
                     )
             return ""
+
+        @property
+        def make_target(self):
+            if self.buildEnv.platformInfo.static:
+                return "static"
+            else:
+                return "shared"

kiwixbuild/platforms/armhf.py

@@ -1,4 +1,4 @@
-from .base import PlatformInfo
+from .base import PlatformInfo, MixedMixin
 
 from kiwixbuild.utils import pj
 from kiwixbuild._global import get_target_step
@@ -29,14 +29,11 @@ class ArmhfPlatformInfo(PlatformInfo):
 
     @property
     def tlc_source(self):
-        return get_target_step('armhf', 'source')
+        return get_target_step(self.build, 'source')
 
     @property
     def root_path(self):
-        return pj(self.tlc_source.source_path,
+        return self.tlc_source.source_path
-                  'raspberrypi-tools',
-                  'arm-bcm2708',
-                  'gcc-linaro-{}-raspbian-x64'.format(self.arch_full))
 
     @property
     def binaries(self):
@@ -47,7 +44,9 @@ class ArmhfPlatformInfo(PlatformInfo):
                              ('STRIP', 'strip'),
                              ('WINDRES', 'windres'),
                              ('RANLIB', 'ranlib'),
-                             ('LD', 'ld'))
+                             ('LD', 'ld'),
+                             ('LDSHARED', 'g++ -shared')
+                             )
                )
         binaries = {k:pj(self.root_path, 'bin', v)
                     for k,v in binaries}
@@ -72,8 +71,13 @@ class ArmhfPlatformInfo(PlatformInfo):
 
     def get_env(self):
         env = super().get_env()
+        env['LD_LIBRARY_PATH'] = ':'.join([
+            pj(self.root_path, self.arch_full, 'lib64'),
+            pj(self.root_path, 'lib'),
+            env['LD_LIBRARY_PATH']
+        ])
         env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
-        env['QEMU_LD_PREFIX'] = pj(self.root_path, "arm-linux-gnueabihf", "libc")
+        env['QEMU_LD_PREFIX'] = pj(self.root_path, self.arch_full, "libc")
         env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format(
             ':'.join([
                 pj(self.root_path, self.arch_full, "lib"),
@@ -83,8 +87,8 @@ class ArmhfPlatformInfo(PlatformInfo):
 
     def set_comp_flags(self, env):
         super().set_comp_flags(env)
-        env['CFLAGS'] = " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS']
+        env['CFLAGS'] = " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS']
-        env['CXXFLAGS'] = " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
+        env['CXXFLAGS'] = " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
 
     def set_compiler(self, env):
         for k, v in self.binaries.items():
@@ -95,6 +99,7 @@ class ArmhfPlatformInfo(PlatformInfo):
         self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
         self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
 
+
 class ArmhfDyn(ArmhfPlatformInfo):
     name = 'armhf_dyn'
     static = False
@@ -102,3 +107,26 @@ class ArmhfDyn(ArmhfPlatformInfo):
 class ArmhfStatic(ArmhfPlatformInfo):
     name = 'armhf_static'
     static = True
+
+class ArmhfMixed(MixedMixin('armhf_static'), ArmhfPlatformInfo):
+    name = 'armhf_mixed'
+    static = False
+
+
+class Aarch64(ArmhfPlatformInfo):
+    build = 'aarch64'
+    arch_full = 'aarch64-linux-gnu'
+    toolchain_names = ['aarch64']
+
+class Aarch64Dyn(Aarch64):
+    name = 'aarch64_dyn'
+    static = False
+
+class Aarch64Static(Aarch64):
+    name = 'aarch64_static'
+    static = True
+
+
+class Aarch64Mixed(MixedMixin('aarch64_static'), Aarch64):
+    name = 'aarch64_mixed'
+    static = False

kiwixbuild/platforms/base.py

@@ -24,6 +24,7 @@ class PlatformInfo(metaclass=_MetaPlatform):
     all_running_platforms = {}
     toolchain_names = []
     configure_option = ""
+    mixed = False
     libdir = None
 
     @classmethod
@@ -128,3 +129,38 @@ class MetaPlatformInfo(PlatformInfo):
             platform = self.get_platform(platformName, targets)
             targetDefs += platform.add_targets(targetName, targets)
         return targetDefs
+
+
+
+def MixedMixin(static_name):
+    class MixedMixinClass:
+        mixed = True
+
+        def add_targets(self, targetName, targets):
+            print(targetName)
+            if option('target') == targetName:
+                return super().add_targets(targetName, targets)
+            else:
+                static_platform = self.get_platform(static_name, targets)
+                return static_platform.add_targets(targetName, targets)
+
+        def get_fully_qualified_dep(self, dep):
+            if isinstance(dep, tuple):
+                return dep
+            if option('target') == dep:
+                return self.name, dep
+            return static_name, dep
+
+
+        def get_env(self):
+            env = super().get_env()
+            static_platform = self.get_platform(static_name)
+            static_buildEnv = static_platform.buildEnv
+            static_install_dir = static_buildEnv.install_dir
+            env['PATH'] = ':'.join([pj(static_install_dir, 'bin')] + [env['PATH']])
+            pkgconfig_path = pj(static_install_dir, static_buildEnv.libprefix, 'pkgconfig')
+            env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path])
+            env['CPPFLAGS'] = " ".join(['-I'+pj(static_install_dir, 'include'), env['CPPFLAGS']])
+            return env
+
+    return MixedMixinClass

kiwixbuild/platforms/native.py

@@ -1,4 +1,4 @@
-from .base import PlatformInfo
+from .base import PlatformInfo, MixedMixin
 
 from kiwixbuild.utils import pj
 from kiwixbuild._global import option, neutralEnv
@@ -24,33 +24,7 @@ class NativeStatic(NativePlatformInfo):
     static = True
     compatible_hosts = ['fedora', 'debian']
 
-class NativeMixed(NativePlatformInfo):
+class NativeMixed(MixedMixin('native_static'), NativePlatformInfo):
     name = 'native_mixed'
     static = False
     compatible_hosts = ['fedora', 'debian', 'Darwin']
-
-    def add_targets(self, targetName, targets):
-        print(targetName)
-        if option('target') == targetName:
-            return super().add_targets(targetName, targets)
-        else:
-            static_platform = self.get_platform('native_static', targets)
-            return static_platform.add_targets(targetName, targets)
-
-    def get_fully_qualified_dep(self, dep):
-        if isinstance(dep, tuple):
-            return dep
-        if option('target') == dep:
-            return 'native_mixed', dep
-        return 'native_static', dep
-
-    def get_env(self):
-        env = super().get_env()
-        static_platform = self.get_platform('native_static')
-        static_buildEnv = static_platform.buildEnv
-        static_install_dir = static_buildEnv.install_dir
-        env['PATH'] = ':'.join([pj(static_install_dir, 'bin')] + [env['PATH']])
-        pkgconfig_path = pj(static_install_dir, static_buildEnv.libprefix, 'pkgconfig')
-        env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path])
-        env['CPPFLAGS'] = " ".join(['-I'+pj(static_install_dir, 'include'), env['CPPFLAGS']])
-        return env

kiwixbuild/versions.py

@@ -39,7 +39,7 @@ release_versions = {
 
 # This is the "version" of the whole base_deps_versions dict.
 # Change this when you change base_deps_versions.
-base_deps_meta_version = '82'
+base_deps_meta_version = '83'
 
 base_deps_versions = {
   'zlib' : '1.2.12',