From 89e60685727a41aa183fb864dfea92c99f3ba609 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 28 Nov 2017 10:02:12 +0000 Subject: [PATCH 1/7] Also compile zim-tools and zimwriterfs in the CI. Nightly will also be publish alongside with other nightly's. Fixes #89 and #90 --- travis/compile_all.sh | 52 ++++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/travis/compile_all.sh b/travis/compile_all.sh index 15ff10b..8caa843 100755 --- a/travis/compile_all.sh +++ b/travis/compile_all.sh @@ -8,6 +8,14 @@ SSH_KEY=${TRAVIS_BUILD_DIR}/travis/travisci_builder_id_key mkdir -p ${NIGHTLY_ARCHIVES_DIR} +function make_nightly_archive { + ARCHIVE_NAME="${1}_$(date +%Y-%m-%d).tar.gz" + ( + cd ${BASE_DIR}/INSTALL/bin + tar -czf "${NIGHTLY_ARCHIVES_DIR}/$ARCHIVE_NAME" $2 + ) +} + cd ${HOME} if [[ "$TRAVIS_EVENT_TYPE" = "cron" ]] @@ -16,7 +24,7 @@ then then TARGETS="libzim kiwix-lib kiwix-android" else - TARGETS="libzim kiwix-lib kiwix-tools" + TARGETS="libzim zimwriterfs zim-tools kiwix-lib kiwix-tools" fi for TARGET in ${TARGETS} @@ -57,28 +65,19 @@ EOF # We have build every thing. Now create archives for public deployement. case ${PLATFORM} in native_static) - ARCHIVE_NAME="kiwix-tools_linux64_$(date +%Y-%m-%d).tar.gz" - FILES_LIST="kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve" - ( - cd ${BASE_DIR}/INSTALL/bin - tar -czf "${NIGHTLY_ARCHIVES_DIR}/$ARCHIVE_NAME" $FILES_LIST - ) + make_nightly_archive kiwix_tools_linux64 "kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve" + make_nightly_archive zim-tools_linux64 "zimbench zimdump zimsearch zimdiff zimpatch zimsplit" + make_nightly_archive zimwriterfs_linux64 "zimwriterfs" ;; win32_static) - ARCHIVE_NAME="kiwix-tools_win32_$(date +%Y-%m-%d).tar.gz" - FILES_LIST="kiwix-install.exe kiwix-manage.exe kiwix-read.exe kiwix-search.exe kiwix-serve.exe" - ( - cd ${BASE_DIR}/INSTALL/bin - tar -czf "${NIGHTLY_ARCHIVES_DIR}/$ARCHIVE_NAME" $FILES_LIST - ) + make_nightly_archive kiwix-tools_win32 "kiwix-install.exe kiwix-manage.exe kiwix-read.exe kiwix-search.exe kiwix-serve.exe" + make_nightly_archive zim-tools_win32 "zimbench.exe zimdump.exe zimsearch.exe zimdiff.exe zimpatch.exe zimsplit.exe" + make_nightly_archive zimwriterfs_win32 "zimwriterfs.exe" ;; armhf_static) - ARCHIVE_NAME="kiwix-tools_armhf_$(date +%Y-%m-%d).tar.gz" - FILES_LIST="kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve" - ( - cd ${BASE_DIR}/INSTALL/bin - tar -czf "${NIGHTLY_ARCHIVES_DIR}/$ARCHIVE_NAME" $FILES_LIST - ) + make_nightly_archive kiwix-tools_armhf "kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve" + make_nightly_archive zim-tools_armhf "zimbench zimdump zimsearch zimdiff zimpatch zimsplit" + make_nightly_archive zimwriterfs_armhf "zimwriterfs" ;; android_*) APK_NAME="kiwix-${PLATFORM}" @@ -91,12 +90,15 @@ else # No a cron job, we just have to build to be sure nothing is broken. if [[ ${PLATFORM} = android* ]] then - TARGET=kiwix-android + TARGETS="kiwix-android" else - TARGET=kiwix-tools + TARGETS="kiwix-tools zim-tools zimwriterfs" fi - ${TRAVIS_BUILD_DIR}/kiwix-build.py \ - --target-platform $PLATFORM \ - --hide-progress \ - ${TARGET} + for TARGET in ${TARGETS} + do + ${TRAVIS_BUILD_DIR}/kiwix-build.py \ + --target-platform $PLATFORM \ + --hide-progress \ + ${TARGET} + done fi From 7fc557dd448cef93ae8da357069845607011021c Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 28 Nov 2017 11:38:16 +0000 Subject: [PATCH 2/7] Use gcc-5 to compile everything. Zimsplit use ofstream move `operator=()` or `swap()` method. This has been implemented in gcc-5 only (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=54316) So install g++-5 and use it in travis CI. --- .travis.yml | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index c42b01a..d30126c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,8 @@ language: cpp dist: trusty sudo: required before_install: +- eval "${MATRIX_EVAL}" +- ${CXX} --version - openssl aes-256-cbc -K $encrypted_eba2f7543984_key -iv $encrypted_eba2f7543984_iv -in travis/travisci_builder_id_key.enc -out travis/travisci_builder_id_key -d - chmod 600 travis/travisci_builder_id_key @@ -23,18 +25,21 @@ deploy: on: condition: ( "$TRAVIS_EVENT_TYPE" = "cron" ) env: - - PLATFORM="native_dyn" - - PLATFORM="native_static" - - PLATFORM="win32_dyn" - - PLATFORM="win32_static" - - PLATFORM="armhf_dyn" - - PLATFORM="armhf_static" - - PLATFORM="android_arm" - - PLATFORM="android_arm64" - - PLATFORM="android_mips" - - PLATFORM="android_mips64" - - PLATFORM="android_x86" - - PLATFORM="android_x86_64" + global: + - MATRIX_EVAL="CC=gcc-5 && CXX=g++-5" + matrix: + - PLATFORM="native_dyn" + - PLATFORM="native_static" + - PLATFORM="win32_dyn" + - PLATFORM="win32_static" + - PLATFORM="armhf_dyn" + - PLATFORM="armhf_static" + - PLATFORM="android_arm" + - PLATFORM="android_arm64" + - PLATFORM="android_mips" + - PLATFORM="android_mips64" + - PLATFORM="android_x86" + - PLATFORM="android_x86_64" notifications: irc: channels: @@ -43,3 +48,8 @@ notifications: on_failure: always addons: ssh_known_hosts: download.kiwix.org + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - g++-5 From 53ecc65b9d90fc82f5128f8cc58b6ec701a13790 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 28 Nov 2017 12:24:39 +0000 Subject: [PATCH 3/7] Zimwriterfs need libmagic-dev. Install the package `libmagic-dev`. There is no precompiled libmagic package from cross-compilation and we are mainly using zimwriterfs only on linux. --- kiwix-build.py | 4 ++-- travis/compile_all.sh | 12 ++++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/kiwix-build.py b/kiwix-build.py index f2d8456..a392f59 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -75,7 +75,7 @@ PACKAGE_NAME_MAPPERS = { 'COMMON': _fedora_common + ['java-1.8.0-openjdk-devel'] }, 'debian_native_dyn': { - 'COMMON': _debian_common + ['libbz2-dev'], + 'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'], 'zlib': ['zlib1g-dev'], 'uuid': ['uuid-dev'], 'ctpp2': ['libctpp2-dev'], @@ -83,7 +83,7 @@ PACKAGE_NAME_MAPPERS = { 'libmicrohttpd': ['libmicrohttpd-dev', 'ccache'] }, 'debian_native_static': { - 'COMMON': _debian_common + ['libbz2-dev'], + 'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'], 'zlib': ['zlib1g-dev'], 'uuid': ['uuid-dev'], 'ctpp2': ['libctpp2-dev'], diff --git a/travis/compile_all.sh b/travis/compile_all.sh index 8caa843..ce40dc4 100755 --- a/travis/compile_all.sh +++ b/travis/compile_all.sh @@ -23,8 +23,11 @@ then if [[ ${PLATFORM} = android* ]] then TARGETS="libzim kiwix-lib kiwix-android" - else + elif [[ ${PLATFORM} =~ native_* ]] + then TARGETS="libzim zimwriterfs zim-tools kiwix-lib kiwix-tools" + else + TARGETS="libzim zim-tools kiwix-lib kiwix-tools" fi for TARGET in ${TARGETS} @@ -72,12 +75,10 @@ EOF win32_static) make_nightly_archive kiwix-tools_win32 "kiwix-install.exe kiwix-manage.exe kiwix-read.exe kiwix-search.exe kiwix-serve.exe" make_nightly_archive zim-tools_win32 "zimbench.exe zimdump.exe zimsearch.exe zimdiff.exe zimpatch.exe zimsplit.exe" - make_nightly_archive zimwriterfs_win32 "zimwriterfs.exe" ;; armhf_static) make_nightly_archive kiwix-tools_armhf "kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve" make_nightly_archive zim-tools_armhf "zimbench zimdump zimsearch zimdiff zimpatch zimsplit" - make_nightly_archive zimwriterfs_armhf "zimwriterfs" ;; android_*) APK_NAME="kiwix-${PLATFORM}" @@ -91,8 +92,11 @@ else if [[ ${PLATFORM} = android* ]] then TARGETS="kiwix-android" - else + elif [[ ${PLATFORM} =~ native_* ]] + then TARGETS="kiwix-tools zim-tools zimwriterfs" + else + TARGETS="kiwix-tools zim-tools" fi for TARGET in ${TARGETS} do From 19730ed7c33f64d15499bb5f7c4340aa012827da Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 28 Nov 2017 12:58:35 +0000 Subject: [PATCH 4/7] Compile zim-tools only on native platform. There is no gcc-5 compiler available for cross-compiling. --- travis/compile_all.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/travis/compile_all.sh b/travis/compile_all.sh index ce40dc4..7c1879b 100755 --- a/travis/compile_all.sh +++ b/travis/compile_all.sh @@ -27,7 +27,7 @@ then then TARGETS="libzim zimwriterfs zim-tools kiwix-lib kiwix-tools" else - TARGETS="libzim zim-tools kiwix-lib kiwix-tools" + TARGETS="libzim kiwix-lib kiwix-tools" fi for TARGET in ${TARGETS} @@ -74,11 +74,9 @@ EOF ;; win32_static) make_nightly_archive kiwix-tools_win32 "kiwix-install.exe kiwix-manage.exe kiwix-read.exe kiwix-search.exe kiwix-serve.exe" - make_nightly_archive zim-tools_win32 "zimbench.exe zimdump.exe zimsearch.exe zimdiff.exe zimpatch.exe zimsplit.exe" ;; armhf_static) make_nightly_archive kiwix-tools_armhf "kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve" - make_nightly_archive zim-tools_armhf "zimbench zimdump zimsearch zimdiff zimpatch zimsplit" ;; android_*) APK_NAME="kiwix-${PLATFORM}" @@ -96,7 +94,7 @@ else then TARGETS="kiwix-tools zim-tools zimwriterfs" else - TARGETS="kiwix-tools zim-tools" + TARGETS="kiwix-tools" fi for TARGET in ${TARGETS} do From d7ffc4bd345579707f2f406cb163cadf234f0dab Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 28 Nov 2017 14:30:31 +0000 Subject: [PATCH 5/7] Add gumbo dependency. This is needed by zimwriterfs and the package doesn't exist on ubuntu trusty (used by travis CI). --- dependencies.py | 21 +++++++++++++++++++-- dependency_utils.py | 2 ++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/dependencies.py b/dependencies.py index a26af48..3588031 100644 --- a/dependencies.py +++ b/dependencies.py @@ -196,6 +196,23 @@ class MicroHttpd(Dependency): configure_option = "--disable-https --without-libgcrypt --without-libcurl" +class Gumbo(Dependency): + name = "gumbo" + version = "0.10.1" + + class Source(ReleaseDownload): + archive = Remotefile('gumbo-0.10.1.tar.gz', + '28463053d44a5dfbc4b77bcf49c8cee119338ffa636cc17fc3378421d714efad', + 'https://github.com/google/gumbo-parser/archive/v0.10.1.tar.gz') + + def _post_prepare_script(self, context): + context.try_skip(self.extract_path) + command = "./autogen.sh" + self.buildEnv.run_command(command, self.extract_path, context) + + Builder = MakeBuilder + + class Icu(Dependency): name = "icu4c" version = "58.2" @@ -280,11 +297,11 @@ class ZimTools(Dependency): class Zimwriterfs(Dependency): name = "zimwriterfs" - extra_packages = ['file', 'gumbo'] + extra_packages = ['file'] @property def dependencies(self): - base_dependencies = ['libzim', 'zlib', 'lzma', 'xapian-core'] + base_dependencies = ['libzim', 'zlib', 'lzma', 'xapian-core', 'gumbo'] if self.buildEnv.platform_info.build != 'native': return base_dependencies + ["icu4c_cross-compile"] else: diff --git a/dependency_utils.py b/dependency_utils.py index b423597..0cd4ed3 100644 --- a/dependency_utils.py +++ b/dependency_utils.py @@ -118,6 +118,8 @@ class ReleaseDownload(Source): self.command('extract', self._extract) if hasattr(self, 'patches'): self.command('patch', self._patch) + if hasattr(self, '_post_prepare_script'): + self.command('post_prepare_script', self._post_prepare_script) class GitClone(Source): From 7c7471e01c6410b90e11734faf356d2fcadae403 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 28 Nov 2017 15:18:37 +0000 Subject: [PATCH 6/7] Update version of gradle. `kiwix-android` changes the minimum version of gradle to use kiwix/kiwix-android#289 Let's adapt. --- dependencies.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dependencies.py b/dependencies.py index 3588031..39bd20f 100644 --- a/dependencies.py +++ b/dependencies.py @@ -373,9 +373,9 @@ class Gradle(Dependency): version = "3.4" class Source(ReleaseDownload): - archive = Remotefile('gradle-3.4-bin.zip', - '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205', - 'https://services.gradle.org/distributions/gradle-3.4-bin.zip') + archive = Remotefile('gradle-4.1-bin.zip', + 'd55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43', + 'https://services.gradle.org/distributions/gradle-4.1-bin.zip') class Builder(BaseBuilder): def build(self): From 603300c154d6d35052cba3b48d025f606027d140 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 28 Nov 2017 16:25:42 +0000 Subject: [PATCH 7/7] Allow android build to fails. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recent changes on android side make the build fails. While this is not fixed, do not block MR not related to kiwix-android. This should be reverted as soon as kiwix-android build is fixed. --- .travis.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.travis.yml b/.travis.yml index d30126c..fe092af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,6 +40,15 @@ env: - PLATFORM="android_mips64" - PLATFORM="android_x86" - PLATFORM="android_x86_64" +matrix: + allow_failures: + - env: PLATFORM="android_arm" + - env: PLATFORM="android_arm64" + - env: PLATFORM="android_mips" + - env: PLATFORM="android_mips64" + - env: PLATFORM="android_x86" + - env: PLATFORM="android_x86_64" + notifications: irc: channels: