Merge pull request #78 from kiwix/travis_apple

Travis build on osx

.travis.yml

@@ -1,4 +1,5 @@
 language: cpp
+
 dist: trusty
 sudo: required
 branches:
@@ -6,8 +7,11 @@ branches:
   - master
   - /\d+\.\d+\.\d+$/
 if: type != push OR tag IS present
+
 before_install:
-- eval "${MATRIX_EVAL}"
+- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then eval "${MATRIX_EVAL}"; fi
+- PATH=$PATH:${HOME}/bin
+- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then PATH=$PATH:$(brew --prefix)/opt/gettext/bin; fi
 - ${CXX} --version
 - openssl aes-256-cbc -K $encrypted_eba2f7543984_key -iv $encrypted_eba2f7543984_iv
   -in travis/travisci_builder_id_key.enc -out travis/travisci_builder_id_key -d
@@ -66,6 +70,8 @@ matrix:
       if: type!=cron AND type!=pull_request
     - env: PLATFORM="android_mips64"
       if: type!=cron AND type!=pull_request
+    - env: PLATFORM="native_dyn"
+      os: osx
 
 notifications:
   irc:

kiwixbuild/__init__.py

@@ -47,7 +47,7 @@ PACKAGE_NAME_MAPPERS = {
         'gumbo' : ['gumbo-parser-devel'],
     },
     'fedora_native_static': {
-        'COMMON': _fedora_common,
+        'COMMON': _fedora_common + ['glibc-static', 'libstdc++-static'],
         'zlib': ['zlib-devel', 'zlib-static'],
         'lzma': ['xz-devel', 'xz-static']
         # Either there is no packages, or no static or too old
@@ -124,11 +124,15 @@ PACKAGE_NAME_MAPPERS = {
     'Darwin_native_dyn': {
         'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'],
     },
-    'Darwin_iOS_static': {
+    'Darwin_iOS': {
         'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'],
     },
 }
 
+def xrun_find(name):
+    command = "xcrun -find {}".format(name)
+    output = subprocess.check_output(command, shell=True)
+    return output[:-1].decode()
 
 class TargetInfo:
     def __init__(self, build, static, toolchains, hosts=None):
@@ -222,10 +226,43 @@ class AndroidTargetInfo(TargetInfo):
         }
 
 class iOSTargetInfo(TargetInfo):
+    __arch_infos = {
+        'armv7s': ('arm-apple-darwin', 'armv7s', 'iphoneos'),
+        'arm64': ('arm-apple-darwin', 'arm64', 'iphoneos'),
+        'i386': ('', 'i386', 'iphonesimulator'),
+        'x86_64': ('', 'x86_64', 'iphonesimulator'),
+    }
+
     def __init__(self, arch):
         super().__init__('iOS', True, ['iOS_sdk'],
                          hosts=['Darwin'])
         self.arch = arch
+        self.arch_full, self.cpu, self.sdk_name = self.__arch_infos[arch]
+        self._root_path = None
+
+    @property
+    def root_path(self):
+        if self._root_path is None:
+            command = "xcodebuild -version -sdk {} | grep -E '^Path' | sed 's/Path: //'".format(self.sdk_name)
+            self._root_path = subprocess.check_output(command, shell=True)[:-1].decode()
+        return self._root_path
+
+    def __str__(self):
+        return "iOS"
+
+    def get_cross_config(self, host):
+        return {
+            'extra_libs': ['-fembed-bitcode', '-isysroot', self.root_path, '-arch', self.arch, '-miphoneos-version-min=9.0', '-stdlib=libc++'],
+            'extra_cflags': ['-fembed-bitcode', '-isysroot', self.root_path, '-arch', self.arch, '-miphoneos-version-min=9.0', '-stdlib=libc++'],
+            'host_machine': {
+                'system': 'Darwin',
+                'lsystem': 'darwin',
+                'cpu_family': self.arch,
+                'cpu': self.cpu,
+                'endian': '',
+                'abi': ''
+            },
+        }
 
 
 class BuildEnv:
@@ -252,7 +289,10 @@ class BuildEnv:
         'android_mips64': AndroidTargetInfo('mips64'),
         'android_x86': AndroidTargetInfo('x86'),
         'android_x86_64': AndroidTargetInfo('x86_64'),
+        'iOS_armv7s': iOSTargetInfo('armv7s'),
         'iOS_arm64': iOSTargetInfo('arm64'),
+        'iOS_i386': iOSTargetInfo('i386'),
+        'iOS_x86_64': iOSTargetInfo('x86_64'),
     }
 
     def __init__(self, options, targetsDict):
@@ -359,7 +399,8 @@ class BuildEnv:
         self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
 
     def setup_iOS(self):
-        pass
+        self.cmake_crossfile = self._gen_crossfile('cmake_ios_cross_file.txt')
+        self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
 
     def setup_i586(self):
         self.cmake_crossfile = self._gen_crossfile('cmake_i586_cross_file.txt')
@@ -564,6 +605,41 @@ class BuildEnv:
             pass
 
 
+class iOS_sdk(Toolchain):
+    @property
+    def root_path(self):
+        return self.buildEnv.platform_info.root_path
+
+    @property
+    def binaries(self):
+        return {
+            'CC': xrun_find('clang'),
+            'CXX': xrun_find('clang++'),
+            'AR': '/usr/bin/ar',
+            'STRIP': '/usr/bin/strip',
+            'RANLIB': '/usr/bin/ranlib',
+            'LD': '/usr/bin/ld',
+        }
+     
+    @property
+    def configure_option(self):
+        return '--host=arm-apple-darwin'
+
+    def get_bin_dir(self):
+        return [pj(self.root_path, 'bin')]
+
+    def set_env(self, env):
+        arch = self.buildEnv.platform_info.arch
+        env['CFLAGS'] = " -fembed-bitcode -isysroot {SDKROOT} -arch {arch} -miphoneos-version-min=9.0 ".format(SDKROOT=self.root_path, arch=arch) + env['CFLAGS']
+        env['CXXFLAGS'] = env['CFLAGS'] + " -stdlib=libc++ -std=c++11 "+env['CXXFLAGS']
+        env['LDFLAGS'] = " -arch {arch} -isysroot {SDKROOT} ".format(SDKROOT=self.root_path, arch=arch)
+        env['MACOSX_DEPLOYMENT_TARGET'] = "10.4"
+
+    def set_compiler(self, env):
+        env['CC'] = self.binaries['CC']
+        env['CXX'] = self.binaries['CXX']
+
+
 class Builder:
     def __init__(self, options):
         self.options = options

kiwixbuild/dependencies.py

@@ -147,7 +147,14 @@ class CTPP2(Dependency):
                   ]
 
     class Builder(CMakeBuilder):
-        configure_option = "-DMD5_SUPPORT=OFF -DICONV_SUPPORT=OFF"
+        @property
+        def configure_option(self):
+            libprefix = self.buildEnv.libprefix
+            options = "-DMD5_SUPPORT=OFF -DICONV_SUPPORT=OFF"
+            if libprefix.startswith('lib'):
+               libprefix = libprefix[3:]
+               options += " -DLIB_SUFFIX={}".format(libprefix)
+            return options
 
 
 class CTPP2C(CTPP2):
@@ -240,7 +247,8 @@ class Icu(Dependency):
         patches = ["icu4c_fix_static_lib_name_mingw.patch",
                    "icu4c_android_elf64_st_info.patch",
                    "icu4c_custom_data.patch",
-                   "icu4c_noxlocale.patch"]
+                   "icu4c_noxlocale.patch",
+                   "icu4c_rpath.patch"]
 
 
     class Builder(MakeBuilder):
@@ -248,7 +256,7 @@ class Icu(Dependency):
 
         @property
         def configure_option(self):
-            options = "--disable-samples --disable-tests --disable-extras --disable-dyload"
+            options = "--disable-samples --disable-tests --disable-extras --disable-dyload --enable-rpath"
             if self.buildEnv.platform_info.build == 'android':
                 options += " --with-data-packaging=archive"
             return options
@@ -275,7 +283,7 @@ class Icu_cross_compile(Icu):
         @property
         def configure_option(self):
             icu_native_dep = self.buildEnv.targetsDict['icu4c_native']
-            return super().configure_option + " --with-cross-build=" + icu_native_dep.builder.build_path
+            return super().configure_option + " --with-cross-build={} --disable-tools".format(icu_native_dep.builder.build_path)
 
 
 class Libzim(Dependency):

kiwixbuild/dependency_utils.py

@@ -196,6 +196,7 @@ class SvnClone(Source):
         self.buildEnv.run_command(command, self.buildEnv.source_dir, context)
 
     def _svn_update(self, context):
+        context.try_skip(self.svn_path)
         context.force_native_build = True
         self.buildEnv.run_command("svn update", self.svn_path, context)
 

kiwixbuild/patches/icu4c_rpath.patch

@@ -0,0 +1,13 @@
+diff -ur icu4c/source/config/mh-linux icu4c.rpath/source/config/mh-linux
+--- icu4c/source/config/mh-linux	2018-04-17 11:31:50.674012676 +0200
++++ icu4c.rpath/source/config/mh-linux	2018-04-17 11:28:57.776134587 +0200
+@@ -19,7 +19,7 @@
+ LIBCPPFLAGS =
+ 
+ ## Compiler switch to embed a runtime search path
+-LD_RPATH= -Wl,-zorigin,-rpath,'$$'ORIGIN 
++LD_RPATH= '-Wl,-zorigin,-rpath,\$ORIGIN' 
+ LD_RPATH_PRE = -Wl,-rpath,
+ 
+ ## These are the library specific LDFLAGS
+Les fichiers binaires icu4c/.svn/wc.db et icu4c.rpath/.svn/wc.db sont différents

kiwixbuild/utils.py

@@ -118,8 +118,8 @@ def download_remote(what, where, check_certificate=True):
                    print_progress(progress_chars[current])
                    current = (current+1)%4
                 file.write(batch)
-    except urllib.error.HTTPError:
+    except urllib.error.URLError as e:
-        print("Cannot download url {}".format(file_url))
+        print("Cannot download url {}:\n{}".format(file_url, e.reason))
         raise StopBuild()
 
     if not what.sha256:

templates/cmake_ios_cross_file.txt

@@ -0,0 +1,7 @@
+SET(CMAKE_SYSTEM_NAME {host_machine[system]})
+
+SET(CMAKE_C_COMPILER "{toolchain.binaries[CC]}")
+SET(CMAKE_CXX_COMPILER "{toolchain.binaries[CXX]}")
+
+SET(CMAKE_FIND_ROOT_PATH {toolchain.root_path})
+

travis/compile_all.py

@@ -14,6 +14,7 @@ from urllib.error import URLError
 from kiwixbuild import dependency_versions
 
 PLATFORM = environ['PLATFORM']
+TRAVIS_OS_NAME = environ['TRAVIS_OS_NAME']
 
 def home():
     return Path(os.path.expanduser('~'))
@@ -111,7 +112,8 @@ def make_archive(project, platform):
 def make_deps_archive(target, full=False):
     (BASE_DIR/'.install_packages_ok').unlink()
 
-    archive_name = "deps_{}_{}.tar.gz".format(PLATFORM, target)
+    archive_name = "deps_{}_{}_{}.tar.gz".format(
+        TRAVIS_OS_NAME, PLATFORM, target)
     files_to_archive = [BASE_DIR/'INSTALL']
     files_to_archive += BASE_DIR.glob('**/android-ndk*')
     if (BASE_DIR/'meson_cross_file.txt').exists():
@@ -159,7 +161,8 @@ make_release = re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", environ.get('TRAVIS_TAG',
 
 # The first thing we need to do is to (potentially) download already compiled base dependencies.
 BASE_DEP_VERSION = dependency_versions.base_deps_meta_version
-base_dep_archive_name = "base_deps_{}_{}.tar.gz".format(PLATFORM, BASE_DEP_VERSION)
+base_dep_archive_name = "base_deps_{}_{}_{}.tar.gz".format(
+    TRAVIS_OS_NAME, PLATFORM, BASE_DEP_VERSION)
 
 print("--- Getting archive {} ---".format(base_dep_archive_name), flush=True)
 try:
@@ -183,6 +186,9 @@ if environ['TRAVIS_EVENT_TYPE'] != 'cron' and not make_release:
     if PLATFORM.startswith('android'):
         TARGETS = ('kiwix-android',)
     elif PLATFORM.startswith('native_'):
+        if TRAVIS_OS_NAME == "osx":
+            TARGETS = ('kiwix-lib', )
+        else:
             TARGETS = ('kiwix-tools', 'zim-tools', 'zimwriterfs')
     else:
         TARGETS = ('kiwix-tools', )
@@ -201,6 +207,9 @@ if PLATFORM.startswith('android'):
     else:
         TARGETS = ('libzim', 'kiwix-lib', 'kiwix-android')
 elif PLATFORM.startswith('native_'):
+    if TRAVIS_OS_NAME == "osx":
+        TARGETS = ('libzim', 'zimwriterfs', 'zim-tools', 'kiwix-lib')
+    else:
         TARGETS = ('libzim', 'zimwriterfs', 'zim-tools', 'kiwix-lib', 'kiwix-tools')
 else:
     TARGETS = ('libzim', 'kiwix-lib', 'kiwix-tools')

travis/install_extra_deps.sh

@@ -2,13 +2,24 @@
 
 set -e
 
+if [[ "$TRAVIS_OS_NAME" == "osx" ]]
+then
+  brew update
+  brew upgrade python3
+  pip3 install pillow
+  pip3 install .
+
+  wget https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-mac.zip
+  unzip ninja-mac.zip ninja
+else
   pip3 install --user --upgrade pip wheel
   pip3 install --user pillow
-
   pip3 install --user .
 
-# ninja
   wget https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip
   unzip ninja-linux.zip ninja
+fi
+
+mkdir -p $HOME/bin
 cp ninja $HOME/bin