diff --git a/.travis.yml b/.travis.yml index c42b01a..fe092af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,8 @@ language: cpp dist: trusty sudo: required before_install: +- eval "${MATRIX_EVAL}" +- ${CXX} --version - openssl aes-256-cbc -K $encrypted_eba2f7543984_key -iv $encrypted_eba2f7543984_iv -in travis/travisci_builder_id_key.enc -out travis/travisci_builder_id_key -d - chmod 600 travis/travisci_builder_id_key @@ -23,18 +25,30 @@ deploy: on: condition: ( "$TRAVIS_EVENT_TYPE" = "cron" ) env: - - PLATFORM="native_dyn" - - PLATFORM="native_static" - - PLATFORM="win32_dyn" - - PLATFORM="win32_static" - - PLATFORM="armhf_dyn" - - PLATFORM="armhf_static" - - PLATFORM="android_arm" - - PLATFORM="android_arm64" - - PLATFORM="android_mips" - - PLATFORM="android_mips64" - - PLATFORM="android_x86" - - PLATFORM="android_x86_64" + global: + - MATRIX_EVAL="CC=gcc-5 && CXX=g++-5" + matrix: + - PLATFORM="native_dyn" + - PLATFORM="native_static" + - PLATFORM="win32_dyn" + - PLATFORM="win32_static" + - PLATFORM="armhf_dyn" + - PLATFORM="armhf_static" + - PLATFORM="android_arm" + - PLATFORM="android_arm64" + - PLATFORM="android_mips" + - PLATFORM="android_mips64" + - PLATFORM="android_x86" + - PLATFORM="android_x86_64" +matrix: + allow_failures: + - env: PLATFORM="android_arm" + - env: PLATFORM="android_arm64" + - env: PLATFORM="android_mips" + - env: PLATFORM="android_mips64" + - env: PLATFORM="android_x86" + - env: PLATFORM="android_x86_64" + notifications: irc: channels: @@ -43,3 +57,8 @@ notifications: on_failure: always addons: ssh_known_hosts: download.kiwix.org + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - g++-5 diff --git a/dependencies.py b/dependencies.py index a26af48..39bd20f 100644 --- a/dependencies.py +++ b/dependencies.py @@ -196,6 +196,23 @@ class MicroHttpd(Dependency): configure_option = "--disable-https --without-libgcrypt --without-libcurl" +class Gumbo(Dependency): + name = "gumbo" + version = "0.10.1" + + class Source(ReleaseDownload): + archive = Remotefile('gumbo-0.10.1.tar.gz', + '28463053d44a5dfbc4b77bcf49c8cee119338ffa636cc17fc3378421d714efad', + 'https://github.com/google/gumbo-parser/archive/v0.10.1.tar.gz') + + def _post_prepare_script(self, context): + context.try_skip(self.extract_path) + command = "./autogen.sh" + self.buildEnv.run_command(command, self.extract_path, context) + + Builder = MakeBuilder + + class Icu(Dependency): name = "icu4c" version = "58.2" @@ -280,11 +297,11 @@ class ZimTools(Dependency): class Zimwriterfs(Dependency): name = "zimwriterfs" - extra_packages = ['file', 'gumbo'] + extra_packages = ['file'] @property def dependencies(self): - base_dependencies = ['libzim', 'zlib', 'lzma', 'xapian-core'] + base_dependencies = ['libzim', 'zlib', 'lzma', 'xapian-core', 'gumbo'] if self.buildEnv.platform_info.build != 'native': return base_dependencies + ["icu4c_cross-compile"] else: @@ -356,9 +373,9 @@ class Gradle(Dependency): version = "3.4" class Source(ReleaseDownload): - archive = Remotefile('gradle-3.4-bin.zip', - '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205', - 'https://services.gradle.org/distributions/gradle-3.4-bin.zip') + archive = Remotefile('gradle-4.1-bin.zip', + 'd55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43', + 'https://services.gradle.org/distributions/gradle-4.1-bin.zip') class Builder(BaseBuilder): def build(self): diff --git a/dependency_utils.py b/dependency_utils.py index b423597..0cd4ed3 100644 --- a/dependency_utils.py +++ b/dependency_utils.py @@ -118,6 +118,8 @@ class ReleaseDownload(Source): self.command('extract', self._extract) if hasattr(self, 'patches'): self.command('patch', self._patch) + if hasattr(self, '_post_prepare_script'): + self.command('post_prepare_script', self._post_prepare_script) class GitClone(Source): diff --git a/kiwix-build.py b/kiwix-build.py index f2d8456..a392f59 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -75,7 +75,7 @@ PACKAGE_NAME_MAPPERS = { 'COMMON': _fedora_common + ['java-1.8.0-openjdk-devel'] }, 'debian_native_dyn': { - 'COMMON': _debian_common + ['libbz2-dev'], + 'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'], 'zlib': ['zlib1g-dev'], 'uuid': ['uuid-dev'], 'ctpp2': ['libctpp2-dev'], @@ -83,7 +83,7 @@ PACKAGE_NAME_MAPPERS = { 'libmicrohttpd': ['libmicrohttpd-dev', 'ccache'] }, 'debian_native_static': { - 'COMMON': _debian_common + ['libbz2-dev'], + 'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'], 'zlib': ['zlib1g-dev'], 'uuid': ['uuid-dev'], 'ctpp2': ['libctpp2-dev'], diff --git a/travis/compile_all.sh b/travis/compile_all.sh index 15ff10b..7c1879b 100755 --- a/travis/compile_all.sh +++ b/travis/compile_all.sh @@ -8,6 +8,14 @@ SSH_KEY=${TRAVIS_BUILD_DIR}/travis/travisci_builder_id_key mkdir -p ${NIGHTLY_ARCHIVES_DIR} +function make_nightly_archive { + ARCHIVE_NAME="${1}_$(date +%Y-%m-%d).tar.gz" + ( + cd ${BASE_DIR}/INSTALL/bin + tar -czf "${NIGHTLY_ARCHIVES_DIR}/$ARCHIVE_NAME" $2 + ) +} + cd ${HOME} if [[ "$TRAVIS_EVENT_TYPE" = "cron" ]] @@ -15,6 +23,9 @@ then if [[ ${PLATFORM} = android* ]] then TARGETS="libzim kiwix-lib kiwix-android" + elif [[ ${PLATFORM} =~ native_* ]] + then + TARGETS="libzim zimwriterfs zim-tools kiwix-lib kiwix-tools" else TARGETS="libzim kiwix-lib kiwix-tools" fi @@ -57,28 +68,15 @@ EOF # We have build every thing. Now create archives for public deployement. case ${PLATFORM} in native_static) - ARCHIVE_NAME="kiwix-tools_linux64_$(date +%Y-%m-%d).tar.gz" - FILES_LIST="kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve" - ( - cd ${BASE_DIR}/INSTALL/bin - tar -czf "${NIGHTLY_ARCHIVES_DIR}/$ARCHIVE_NAME" $FILES_LIST - ) + make_nightly_archive kiwix_tools_linux64 "kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve" + make_nightly_archive zim-tools_linux64 "zimbench zimdump zimsearch zimdiff zimpatch zimsplit" + make_nightly_archive zimwriterfs_linux64 "zimwriterfs" ;; win32_static) - ARCHIVE_NAME="kiwix-tools_win32_$(date +%Y-%m-%d).tar.gz" - FILES_LIST="kiwix-install.exe kiwix-manage.exe kiwix-read.exe kiwix-search.exe kiwix-serve.exe" - ( - cd ${BASE_DIR}/INSTALL/bin - tar -czf "${NIGHTLY_ARCHIVES_DIR}/$ARCHIVE_NAME" $FILES_LIST - ) + make_nightly_archive kiwix-tools_win32 "kiwix-install.exe kiwix-manage.exe kiwix-read.exe kiwix-search.exe kiwix-serve.exe" ;; armhf_static) - ARCHIVE_NAME="kiwix-tools_armhf_$(date +%Y-%m-%d).tar.gz" - FILES_LIST="kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve" - ( - cd ${BASE_DIR}/INSTALL/bin - tar -czf "${NIGHTLY_ARCHIVES_DIR}/$ARCHIVE_NAME" $FILES_LIST - ) + make_nightly_archive kiwix-tools_armhf "kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve" ;; android_*) APK_NAME="kiwix-${PLATFORM}" @@ -91,12 +89,18 @@ else # No a cron job, we just have to build to be sure nothing is broken. if [[ ${PLATFORM} = android* ]] then - TARGET=kiwix-android + TARGETS="kiwix-android" + elif [[ ${PLATFORM} =~ native_* ]] + then + TARGETS="kiwix-tools zim-tools zimwriterfs" else - TARGET=kiwix-tools + TARGETS="kiwix-tools" fi - ${TRAVIS_BUILD_DIR}/kiwix-build.py \ - --target-platform $PLATFORM \ - --hide-progress \ - ${TARGET} + for TARGET in ${TARGETS} + do + ${TRAVIS_BUILD_DIR}/kiwix-build.py \ + --target-platform $PLATFORM \ + --hide-progress \ + ${TARGET} + done fi