diff --git a/.travis.yml b/.travis.yml index 4d2edd7..c58e7e0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,5 @@ language: cpp + dist: trusty sudo: required branches: @@ -6,8 +7,11 @@ branches: - master - /\d+\.\d+\.\d+$/ if: type != push OR tag IS present + before_install: -- eval "${MATRIX_EVAL}" +- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then eval "${MATRIX_EVAL}"; fi +- PATH=$PATH:${HOME}/bin +- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then PATH=$PATH:$(brew --prefix)/opt/gettext/bin; fi - ${CXX} --version - openssl aes-256-cbc -K $encrypted_eba2f7543984_key -iv $encrypted_eba2f7543984_iv -in travis/travisci_builder_id_key.enc -out travis/travisci_builder_id_key -d @@ -66,6 +70,8 @@ matrix: if: type!=cron AND type!=pull_request - env: PLATFORM="android_mips64" if: type!=cron AND type!=pull_request + - env: PLATFORM="native_dyn" + os: osx notifications: irc: diff --git a/kiwixbuild/__init__.py b/kiwixbuild/__init__.py index 59f2ba4..a5ea5fb 100644 --- a/kiwixbuild/__init__.py +++ b/kiwixbuild/__init__.py @@ -47,7 +47,7 @@ PACKAGE_NAME_MAPPERS = { 'gumbo' : ['gumbo-parser-devel'], }, 'fedora_native_static': { - 'COMMON': _fedora_common, + 'COMMON': _fedora_common + ['glibc-static', 'libstdc++-static'], 'zlib': ['zlib-devel', 'zlib-static'], 'lzma': ['xz-devel', 'xz-static'] # Either there is no packages, or no static or too old @@ -124,11 +124,15 @@ PACKAGE_NAME_MAPPERS = { 'Darwin_native_dyn': { 'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'], }, - 'Darwin_iOS_static': { + 'Darwin_iOS': { 'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'], }, } +def xrun_find(name): + command = "xcrun -find {}".format(name) + output = subprocess.check_output(command, shell=True) + return output[:-1].decode() class TargetInfo: def __init__(self, build, static, toolchains, hosts=None): @@ -222,10 +226,43 @@ class AndroidTargetInfo(TargetInfo): } class iOSTargetInfo(TargetInfo): + __arch_infos = { + 'armv7s': ('arm-apple-darwin', 'armv7s', 'iphoneos'), + 'arm64': ('arm-apple-darwin', 'arm64', 'iphoneos'), + 'i386': ('', 'i386', 'iphonesimulator'), + 'x86_64': ('', 'x86_64', 'iphonesimulator'), + } + def __init__(self, arch): super().__init__('iOS', True, ['iOS_sdk'], hosts=['Darwin']) self.arch = arch + self.arch_full, self.cpu, self.sdk_name = self.__arch_infos[arch] + self._root_path = None + + @property + def root_path(self): + if self._root_path is None: + command = "xcodebuild -version -sdk {} | grep -E '^Path' | sed 's/Path: //'".format(self.sdk_name) + self._root_path = subprocess.check_output(command, shell=True)[:-1].decode() + return self._root_path + + def __str__(self): + return "iOS" + + def get_cross_config(self, host): + return { + 'extra_libs': ['-fembed-bitcode', '-isysroot', self.root_path, '-arch', self.arch, '-miphoneos-version-min=9.0', '-stdlib=libc++'], + 'extra_cflags': ['-fembed-bitcode', '-isysroot', self.root_path, '-arch', self.arch, '-miphoneos-version-min=9.0', '-stdlib=libc++'], + 'host_machine': { + 'system': 'Darwin', + 'lsystem': 'darwin', + 'cpu_family': self.arch, + 'cpu': self.cpu, + 'endian': '', + 'abi': '' + }, + } class BuildEnv: @@ -252,7 +289,10 @@ class BuildEnv: 'android_mips64': AndroidTargetInfo('mips64'), 'android_x86': AndroidTargetInfo('x86'), 'android_x86_64': AndroidTargetInfo('x86_64'), + 'iOS_armv7s': iOSTargetInfo('armv7s'), 'iOS_arm64': iOSTargetInfo('arm64'), + 'iOS_i386': iOSTargetInfo('i386'), + 'iOS_x86_64': iOSTargetInfo('x86_64'), } def __init__(self, options, targetsDict): @@ -359,7 +399,8 @@ class BuildEnv: self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') def setup_iOS(self): - pass + self.cmake_crossfile = self._gen_crossfile('cmake_ios_cross_file.txt') + self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') def setup_i586(self): self.cmake_crossfile = self._gen_crossfile('cmake_i586_cross_file.txt') @@ -564,6 +605,41 @@ class BuildEnv: pass +class iOS_sdk(Toolchain): + @property + def root_path(self): + return self.buildEnv.platform_info.root_path + + @property + def binaries(self): + return { + 'CC': xrun_find('clang'), + 'CXX': xrun_find('clang++'), + 'AR': '/usr/bin/ar', + 'STRIP': '/usr/bin/strip', + 'RANLIB': '/usr/bin/ranlib', + 'LD': '/usr/bin/ld', + } + + @property + def configure_option(self): + return '--host=arm-apple-darwin' + + def get_bin_dir(self): + return [pj(self.root_path, 'bin')] + + def set_env(self, env): + arch = self.buildEnv.platform_info.arch + env['CFLAGS'] = " -fembed-bitcode -isysroot {SDKROOT} -arch {arch} -miphoneos-version-min=9.0 ".format(SDKROOT=self.root_path, arch=arch) + env['CFLAGS'] + env['CXXFLAGS'] = env['CFLAGS'] + " -stdlib=libc++ -std=c++11 "+env['CXXFLAGS'] + env['LDFLAGS'] = " -arch {arch} -isysroot {SDKROOT} ".format(SDKROOT=self.root_path, arch=arch) + env['MACOSX_DEPLOYMENT_TARGET'] = "10.4" + + def set_compiler(self, env): + env['CC'] = self.binaries['CC'] + env['CXX'] = self.binaries['CXX'] + + class Builder: def __init__(self, options): self.options = options diff --git a/kiwixbuild/dependencies.py b/kiwixbuild/dependencies.py index b395414..0f109ff 100644 --- a/kiwixbuild/dependencies.py +++ b/kiwixbuild/dependencies.py @@ -147,7 +147,14 @@ class CTPP2(Dependency): ] class Builder(CMakeBuilder): - configure_option = "-DMD5_SUPPORT=OFF -DICONV_SUPPORT=OFF" + @property + def configure_option(self): + libprefix = self.buildEnv.libprefix + options = "-DMD5_SUPPORT=OFF -DICONV_SUPPORT=OFF" + if libprefix.startswith('lib'): + libprefix = libprefix[3:] + options += " -DLIB_SUFFIX={}".format(libprefix) + return options class CTPP2C(CTPP2): @@ -240,7 +247,8 @@ class Icu(Dependency): patches = ["icu4c_fix_static_lib_name_mingw.patch", "icu4c_android_elf64_st_info.patch", "icu4c_custom_data.patch", - "icu4c_noxlocale.patch"] + "icu4c_noxlocale.patch", + "icu4c_rpath.patch"] class Builder(MakeBuilder): @@ -248,7 +256,7 @@ class Icu(Dependency): @property def configure_option(self): - options = "--disable-samples --disable-tests --disable-extras --disable-dyload" + options = "--disable-samples --disable-tests --disable-extras --disable-dyload --enable-rpath" if self.buildEnv.platform_info.build == 'android': options += " --with-data-packaging=archive" return options @@ -275,7 +283,7 @@ class Icu_cross_compile(Icu): @property def configure_option(self): icu_native_dep = self.buildEnv.targetsDict['icu4c_native'] - return super().configure_option + " --with-cross-build=" + icu_native_dep.builder.build_path + return super().configure_option + " --with-cross-build={} --disable-tools".format(icu_native_dep.builder.build_path) class Libzim(Dependency): diff --git a/kiwixbuild/dependency_utils.py b/kiwixbuild/dependency_utils.py index 0af71c7..c9fe252 100644 --- a/kiwixbuild/dependency_utils.py +++ b/kiwixbuild/dependency_utils.py @@ -196,6 +196,7 @@ class SvnClone(Source): self.buildEnv.run_command(command, self.buildEnv.source_dir, context) def _svn_update(self, context): + context.try_skip(self.svn_path) context.force_native_build = True self.buildEnv.run_command("svn update", self.svn_path, context) diff --git a/kiwixbuild/patches/icu4c_rpath.patch b/kiwixbuild/patches/icu4c_rpath.patch new file mode 100644 index 0000000..13d6f15 --- /dev/null +++ b/kiwixbuild/patches/icu4c_rpath.patch @@ -0,0 +1,13 @@ +diff -ur icu4c/source/config/mh-linux icu4c.rpath/source/config/mh-linux +--- icu4c/source/config/mh-linux 2018-04-17 11:31:50.674012676 +0200 ++++ icu4c.rpath/source/config/mh-linux 2018-04-17 11:28:57.776134587 +0200 +@@ -19,7 +19,7 @@ + LIBCPPFLAGS = + + ## Compiler switch to embed a runtime search path +-LD_RPATH= -Wl,-zorigin,-rpath,'$$'ORIGIN ++LD_RPATH= '-Wl,-zorigin,-rpath,\$ORIGIN' + LD_RPATH_PRE = -Wl,-rpath, + + ## These are the library specific LDFLAGS +Les fichiers binaires icu4c/.svn/wc.db et icu4c.rpath/.svn/wc.db sont différents diff --git a/kiwixbuild/utils.py b/kiwixbuild/utils.py index 858c495..2803de6 100644 --- a/kiwixbuild/utils.py +++ b/kiwixbuild/utils.py @@ -118,8 +118,8 @@ def download_remote(what, where, check_certificate=True): print_progress(progress_chars[current]) current = (current+1)%4 file.write(batch) - except urllib.error.HTTPError: - print("Cannot download url {}".format(file_url)) + except urllib.error.URLError as e: + print("Cannot download url {}:\n{}".format(file_url, e.reason)) raise StopBuild() if not what.sha256: diff --git a/templates/cmake_ios_cross_file.txt b/templates/cmake_ios_cross_file.txt new file mode 100644 index 0000000..feaefa1 --- /dev/null +++ b/templates/cmake_ios_cross_file.txt @@ -0,0 +1,7 @@ +SET(CMAKE_SYSTEM_NAME {host_machine[system]}) + +SET(CMAKE_C_COMPILER "{toolchain.binaries[CC]}") +SET(CMAKE_CXX_COMPILER "{toolchain.binaries[CXX]}") + +SET(CMAKE_FIND_ROOT_PATH {toolchain.root_path}) + diff --git a/travis/compile_all.py b/travis/compile_all.py index 3f43196..72f7f78 100755 --- a/travis/compile_all.py +++ b/travis/compile_all.py @@ -14,6 +14,7 @@ from urllib.error import URLError from kiwixbuild import dependency_versions PLATFORM = environ['PLATFORM'] +TRAVIS_OS_NAME = environ['TRAVIS_OS_NAME'] def home(): return Path(os.path.expanduser('~')) @@ -111,7 +112,8 @@ def make_archive(project, platform): def make_deps_archive(target, full=False): (BASE_DIR/'.install_packages_ok').unlink() - archive_name = "deps_{}_{}.tar.gz".format(PLATFORM, target) + archive_name = "deps_{}_{}_{}.tar.gz".format( + TRAVIS_OS_NAME, PLATFORM, target) files_to_archive = [BASE_DIR/'INSTALL'] files_to_archive += BASE_DIR.glob('**/android-ndk*') if (BASE_DIR/'meson_cross_file.txt').exists(): @@ -159,7 +161,8 @@ make_release = re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", environ.get('TRAVIS_TAG', # The first thing we need to do is to (potentially) download already compiled base dependencies. BASE_DEP_VERSION = dependency_versions.base_deps_meta_version -base_dep_archive_name = "base_deps_{}_{}.tar.gz".format(PLATFORM, BASE_DEP_VERSION) +base_dep_archive_name = "base_deps_{}_{}_{}.tar.gz".format( + TRAVIS_OS_NAME, PLATFORM, BASE_DEP_VERSION) print("--- Getting archive {} ---".format(base_dep_archive_name), flush=True) try: @@ -183,7 +186,10 @@ if environ['TRAVIS_EVENT_TYPE'] != 'cron' and not make_release: if PLATFORM.startswith('android'): TARGETS = ('kiwix-android',) elif PLATFORM.startswith('native_'): - TARGETS = ('kiwix-tools', 'zim-tools', 'zimwriterfs') + if TRAVIS_OS_NAME == "osx": + TARGETS = ('kiwix-lib', ) + else: + TARGETS = ('kiwix-tools', 'zim-tools', 'zimwriterfs') else: TARGETS = ('kiwix-tools', ) @@ -201,7 +207,10 @@ if PLATFORM.startswith('android'): else: TARGETS = ('libzim', 'kiwix-lib', 'kiwix-android') elif PLATFORM.startswith('native_'): - TARGETS = ('libzim', 'zimwriterfs', 'zim-tools', 'kiwix-lib', 'kiwix-tools') + if TRAVIS_OS_NAME == "osx": + TARGETS = ('libzim', 'zimwriterfs', 'zim-tools', 'kiwix-lib') + else: + TARGETS = ('libzim', 'zimwriterfs', 'zim-tools', 'kiwix-lib', 'kiwix-tools') else: TARGETS = ('libzim', 'kiwix-lib', 'kiwix-tools') diff --git a/travis/install_extra_deps.sh b/travis/install_extra_deps.sh index f9e8489..2b58f2b 100755 --- a/travis/install_extra_deps.sh +++ b/travis/install_extra_deps.sh @@ -2,13 +2,24 @@ set -e -pip3 install --user --upgrade pip wheel -pip3 install --user pillow +if [[ "$TRAVIS_OS_NAME" == "osx" ]] +then + brew update + brew upgrade python3 + pip3 install pillow + pip3 install . -pip3 install --user . + wget https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-mac.zip + unzip ninja-mac.zip ninja +else + pip3 install --user --upgrade pip wheel + pip3 install --user pillow + pip3 install --user . -# ninja -wget https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip -unzip ninja-linux.zip ninja + wget https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip + unzip ninja-linux.zip ninja +fi + +mkdir -p $HOME/bin cp ninja $HOME/bin