Merge pull request #95 from kiwix/ci_zimwriterfr_zimtools

Also compile zim-tools and zimwriterfs in the CI.

.travis.yml

@@ -2,6 +2,8 @@ language: cpp
 dist: trusty
 sudo: required
 before_install:
+- eval "${MATRIX_EVAL}"
+- ${CXX} --version
 - openssl aes-256-cbc -K $encrypted_eba2f7543984_key -iv $encrypted_eba2f7543984_iv
   -in travis/travisci_builder_id_key.enc -out travis/travisci_builder_id_key -d
 - chmod 600 travis/travisci_builder_id_key
@@ -23,18 +25,30 @@ deploy:
   on:
     condition: ( "$TRAVIS_EVENT_TYPE" = "cron" )
 env:
- - PLATFORM="native_dyn"
- - PLATFORM="native_static"
- - PLATFORM="win32_dyn"
- - PLATFORM="win32_static"
- - PLATFORM="armhf_dyn"
- - PLATFORM="armhf_static"
- - PLATFORM="android_arm"
- - PLATFORM="android_arm64"
- - PLATFORM="android_mips"
- - PLATFORM="android_mips64"
- - PLATFORM="android_x86"
- - PLATFORM="android_x86_64"
+  global:
+    - MATRIX_EVAL="CC=gcc-5 && CXX=g++-5"
+  matrix:
+    - PLATFORM="native_dyn"
+    - PLATFORM="native_static"
+    - PLATFORM="win32_dyn"
+    - PLATFORM="win32_static"
+    - PLATFORM="armhf_dyn"
+    - PLATFORM="armhf_static"
+    - PLATFORM="android_arm"
+    - PLATFORM="android_arm64"
+    - PLATFORM="android_mips"
+    - PLATFORM="android_mips64"
+    - PLATFORM="android_x86"
+    - PLATFORM="android_x86_64"
+matrix:
+  allow_failures:
+    - env: PLATFORM="android_arm"
+    - env: PLATFORM="android_arm64"
+    - env: PLATFORM="android_mips"
+    - env: PLATFORM="android_mips64"
+    - env: PLATFORM="android_x86"
+    - env: PLATFORM="android_x86_64"
+
 notifications:
   irc:
     channels:
@@ -43,3 +57,8 @@ notifications:
     on_failure: always
 addons:
   ssh_known_hosts: download.kiwix.org
+  apt:
+    sources:
+      - ubuntu-toolchain-r-test
+    packages:
+      - g++-5

dependencies.py

@@ -196,6 +196,23 @@ class MicroHttpd(Dependency):
         configure_option = "--disable-https --without-libgcrypt --without-libcurl"
 
 
+class Gumbo(Dependency):
+    name = "gumbo"
+    version = "0.10.1"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('gumbo-0.10.1.tar.gz',
+                             '28463053d44a5dfbc4b77bcf49c8cee119338ffa636cc17fc3378421d714efad',
+                             'https://github.com/google/gumbo-parser/archive/v0.10.1.tar.gz')
+
+        def _post_prepare_script(self, context):
+            context.try_skip(self.extract_path)
+            command = "./autogen.sh"
+            self.buildEnv.run_command(command, self.extract_path, context)
+
+    Builder = MakeBuilder
+
+
 class Icu(Dependency):
     name = "icu4c"
     version = "58.2"
@@ -280,11 +297,11 @@ class ZimTools(Dependency):
 
 class Zimwriterfs(Dependency):
     name = "zimwriterfs"
-    extra_packages = ['file', 'gumbo']
+    extra_packages = ['file']
 
     @property
     def dependencies(self):
-        base_dependencies = ['libzim', 'zlib', 'lzma', 'xapian-core']
+        base_dependencies = ['libzim', 'zlib', 'lzma', 'xapian-core', 'gumbo']
         if self.buildEnv.platform_info.build != 'native':
             return base_dependencies + ["icu4c_cross-compile"]
         else:
@@ -356,9 +373,9 @@ class Gradle(Dependency):
     version = "3.4"
 
     class Source(ReleaseDownload):
-        archive = Remotefile('gradle-3.4-bin.zip',
-                             '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205',
-                             'https://services.gradle.org/distributions/gradle-3.4-bin.zip')
+        archive = Remotefile('gradle-4.1-bin.zip',
+                             'd55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43',
+                             'https://services.gradle.org/distributions/gradle-4.1-bin.zip')
 
     class Builder(BaseBuilder):
         def build(self):

dependency_utils.py

@@ -118,6 +118,8 @@ class ReleaseDownload(Source):
         self.command('extract', self._extract)
         if hasattr(self, 'patches'):
             self.command('patch', self._patch)
+        if hasattr(self, '_post_prepare_script'):
+            self.command('post_prepare_script', self._post_prepare_script)
 
 
 class GitClone(Source):

kiwix-build.py

@@ -75,7 +75,7 @@ PACKAGE_NAME_MAPPERS = {
         'COMMON': _fedora_common + ['java-1.8.0-openjdk-devel']
     },
     'debian_native_dyn': {
-        'COMMON': _debian_common + ['libbz2-dev'],
+        'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'],
         'zlib': ['zlib1g-dev'],
         'uuid': ['uuid-dev'],
         'ctpp2': ['libctpp2-dev'],
@@ -83,7 +83,7 @@ PACKAGE_NAME_MAPPERS = {
         'libmicrohttpd': ['libmicrohttpd-dev', 'ccache']
     },
     'debian_native_static': {
-        'COMMON': _debian_common + ['libbz2-dev'],
+        'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'],
         'zlib': ['zlib1g-dev'],
         'uuid': ['uuid-dev'],
         'ctpp2': ['libctpp2-dev'],

travis/compile_all.sh

@@ -8,6 +8,14 @@ SSH_KEY=${TRAVIS_BUILD_DIR}/travis/travisci_builder_id_key
 
 mkdir -p ${NIGHTLY_ARCHIVES_DIR}
 
+function make_nightly_archive {
+  ARCHIVE_NAME="${1}_$(date +%Y-%m-%d).tar.gz"
+  (
+    cd ${BASE_DIR}/INSTALL/bin
+    tar -czf "${NIGHTLY_ARCHIVES_DIR}/$ARCHIVE_NAME" $2
+  )
+}
+
 cd ${HOME}
 
 if [[ "$TRAVIS_EVENT_TYPE" = "cron" ]]
@@ -15,6 +23,9 @@ then
   if [[ ${PLATFORM} = android* ]]
   then
     TARGETS="libzim kiwix-lib kiwix-android"
+  elif [[ ${PLATFORM} =~ native_* ]]
+  then
+    TARGETS="libzim zimwriterfs zim-tools kiwix-lib kiwix-tools"
   else
     TARGETS="libzim kiwix-lib kiwix-tools"
   fi
@@ -57,28 +68,15 @@ EOF
   # We have build every thing. Now create archives for public deployement.
   case ${PLATFORM} in
     native_static)
-      ARCHIVE_NAME="kiwix-tools_linux64_$(date +%Y-%m-%d).tar.gz"
-      FILES_LIST="kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve"
-      (
-        cd ${BASE_DIR}/INSTALL/bin
-        tar -czf "${NIGHTLY_ARCHIVES_DIR}/$ARCHIVE_NAME" $FILES_LIST
-      )
+      make_nightly_archive kiwix_tools_linux64 "kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve"
+      make_nightly_archive zim-tools_linux64 "zimbench zimdump zimsearch zimdiff zimpatch zimsplit"
+      make_nightly_archive zimwriterfs_linux64 "zimwriterfs"
       ;;
     win32_static)
-      ARCHIVE_NAME="kiwix-tools_win32_$(date +%Y-%m-%d).tar.gz"
-      FILES_LIST="kiwix-install.exe kiwix-manage.exe kiwix-read.exe kiwix-search.exe kiwix-serve.exe"
-      (
-        cd ${BASE_DIR}/INSTALL/bin
-        tar -czf "${NIGHTLY_ARCHIVES_DIR}/$ARCHIVE_NAME" $FILES_LIST
-      )
+      make_nightly_archive kiwix-tools_win32 "kiwix-install.exe kiwix-manage.exe kiwix-read.exe kiwix-search.exe kiwix-serve.exe"
       ;;
     armhf_static)
-      ARCHIVE_NAME="kiwix-tools_armhf_$(date +%Y-%m-%d).tar.gz"
-      FILES_LIST="kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve"
-      (
-        cd ${BASE_DIR}/INSTALL/bin
-        tar -czf "${NIGHTLY_ARCHIVES_DIR}/$ARCHIVE_NAME" $FILES_LIST
-      )
+      make_nightly_archive kiwix-tools_armhf "kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve"
       ;;
     android_*)
       APK_NAME="kiwix-${PLATFORM}"
@@ -91,12 +89,18 @@ else
   # No a cron job, we just have to build to be sure nothing is broken.
   if [[ ${PLATFORM} = android* ]]
   then
-    TARGET=kiwix-android
+    TARGETS="kiwix-android"
+  elif [[ ${PLATFORM} =~ native_* ]]
+  then
+    TARGETS="kiwix-tools zim-tools zimwriterfs"
   else
-    TARGET=kiwix-tools
+    TARGETS="kiwix-tools"
   fi
-  ${TRAVIS_BUILD_DIR}/kiwix-build.py \
-    --target-platform $PLATFORM \
-    --hide-progress \
-    ${TARGET}
+  for TARGET in ${TARGETS}
+  do
+    ${TRAVIS_BUILD_DIR}/kiwix-build.py \
+      --target-platform $PLATFORM \
+      --hide-progress \
+      ${TARGET}
+  done
 fi