Merge pull request #25 from kiwix/prepare_deps

Prepare deps

.travis.yml

@@ -4,6 +4,7 @@ sudo: required
 before_install:
 - openssl aes-256-cbc -K $encrypted_eba2f7543984_key -iv $encrypted_eba2f7543984_iv
   -in travis/travisci_builder_id_key.enc -out travis/travisci_builder_id_key -d
+- chmod 600 travis/travisci_builder_id_key
 cache: ccache
 install: travis/install_extra_deps.sh
 script: travis/compile_all.sh
@@ -12,18 +13,18 @@ deploy:
   skip_cleanup: true
   script: travis/deploy.sh
   on:
-    condition: ( "$DEPLOY" = "true" ) && ( "$TRAVIS_EVENT_TYPE" = "cron" )
+    condition: ( "$TRAVIS_EVENT_TYPE" = "cron" )
 env:
- - BUILD_OPTION="--target-platform=native_dyn"
- - BUILD_OPTION="--target-platform=native_static" ARCHIVE_TYPE="--tar" DEPLOY=true
- - BUILD_OPTION="--target-platform=win32_dyn"
- - BUILD_OPTION="--target-platform=win32_static" ARCHIVE_TYPE="--zip" DEPLOY=true
- - BUILD_OPTION="--target-platform=android_arm kiwix-lib"
- - BUILD_OPTION="--target-platform=android_arm64 kiwix-lib"
- - BUILD_OPTION="--target-platform=android_mips kiwix-lib"
- - BUILD_OPTION="--target-platform=android_mips64 kiwix-lib"
- - BUILD_OPTION="--target-platform=android_x86 kiwix-lib"
- - BUILD_OPTION="--target-platform=android_x86_64 kiwix-lib"
+ - PLATFORM="native_dyn"
+ - PLATFORM="native_static"
+ - PLATFORM="win32_dyn"
+ - PLATFORM="win32_static"
+ - PLATFORM="android_arm"
+ - PLATFORM="android_arm64"
+ - PLATFORM="android_mips"
+ - PLATFORM="android_mips64"
+ - PLATFORM="android_x86"
+ - PLATFORM="android_x86_64"
 notifications:
   irc:
     channels:

dependencies.py

@@ -232,6 +232,7 @@ class Icu_cross_compile(Icu):
 
 class Libzim(Dependency):
     name = "libzim"
+    dependencies = ['zlib', 'lzma']
 
     class Source(GitClone):
         git_remote = "https://github.com/openzim/libzim.git"

kiwix-build.py

@@ -721,6 +721,8 @@ class Builder:
         dependencies = list(remove_duplicates(dependencies))
 
         for dep in dependencies:
+            if self.options.build_deps_only and dep == targetDef:
+                continue
             self.targets[dep] = _targets[dep]
 
     def add_targets(self, targetName, targets):
@@ -792,6 +794,9 @@ def parse_args():
                         help="Skip SSL certificate verification during download")
     parser.add_argument('--skip-source-prepare', action='store_true',
                         help="Skip the source download part")
+    parser.add_argument('--build-deps-only', action='store_true',
+                        help=("Build only the dependencies of the specified targets."))
+
     return parser.parse_args()
 
 

travis/compile_all.sh

@@ -2,4 +2,75 @@
 
 set -e
 
-./kiwix-build.py $BUILD_OPTION
+BASE_DIR="BUILD_${PLATFORM}"
+DEPS_ARCHIVES_DIR=${HOME}/DEPS_ARCHIVES
+mkdir -p ${DEPS_ARCHIVES_DIR}
+NIGHTLY_ARCHIVES_DIR=${HOME}/NIGHTLY_ARCHIVES
+mkdir -p ${NIGHTLY_ARCHIVES_DIR}
+
+cd ${HOME}
+
+if [[ "$TRAVIS_EVENT_TYPE" = "cron" ]]
+then
+  if [[ ${PLATFORM} = android* ]]
+  then
+    TARGETS="libzim kiwix-lib"
+  else
+    TARGETS="libzim kiwix-lib kiwix-tools"
+  fi
+
+  for TARGET in ${TARGETS}
+  do
+    echo $TARGET
+    ${TRAVIS_BUILD_DIR}/kiwix-build.py \
+      --target-platform $PLATFORM \
+      --build-deps-only \
+      ${TARGET}
+    rm ${BASE_DIR}/.install_packages_ok
+
+    (
+      cd ${BASE_DIR}
+      if [ -f meson_cross_file.txt ]
+      then
+        MESON_FILE=meson_cross_file.txt
+      fi
+      ANDROID_NDK_DIR=$(find . -name "android-ndk*")
+      tar -czf "${DEPS_ARCHIVES_DIR}/deps_${PLATFORM}_${TARGET}.tar.gz" INSTALL ${MESON_FILE} ${ANDROID_NDK_DIR}
+    )
+
+    ${TRAVIS_BUILD_DIR}/kiwix-build.py --target-platform $PLATFORM ${TARGET}
+    rm ${BASE_DIR}/.install_packages_ok
+  done
+
+  # We have build every thing. Now create archives for public deployement.
+  case ${PLATFORM} in
+    native_static)
+      ARCHIVE_NAME="kiwix-tools_linux64_$(date +%Y-%m-%d).tar.gz"
+      FILES_LIST="kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve"
+      (
+        cd ${BASE_DIR}/INSTALL/bin
+        tar -czf "${NIGHTLY_ARCHIVES_DIR}/$ARCHIVE_NAME" $FILES_LIST
+      )
+      ;;
+    win32_static)
+      ARCHIVE_NAME="kiwix-tools_win32_$(date +%Y-%m-%d).tar.gz"
+      FILES_LIST="kiwix-install.exe kiwix-manage.exe kiwix-read.exe kiwix-search.exe kiwix-serve.exe"
+      (
+        cd ${BASE_DIR}/INSTALL/bin
+        tar -czf "${NIGHTLY_ARCHIVES_DIR}/$ARCHIVE_NAME" $FILES_LIST
+      )
+      ;;
+  esac
+
+else
+  # No a cron job, we just have to build to be sure nothing is broken.
+  if [[ ${PLATFORM} = android* ]]
+  then
+    TARGET=kiwix-lib
+  else
+    TARGET=kiwix-tools
+  fi
+  ${TRAVIS_BUILD_DIR}/kiwix-build.py \
+    --target-platform $PLATFORM \
+    ${TARGET}
+fi

travis/deploy.sh

@@ -2,13 +2,23 @@
 
 set -e
 
+DEPS_ARCHIVES_DIR=${HOME}/DEPS_ARCHIVES
+NIGHTLY_ARCHIVES_DIR=${HOME}/NIGHTLY_ARCHIVES
+
 SSH_KEY=travis/travisci_builder_id_key
 
-chmod 600 ${SSH_KEY}
+DEPS_ARCHIVES=$(find $DEPS_ARCHIVES_DIR -type f)
+if [[ "x$DEPS_ARCHIVES" != "x" ]]
+then
+  scp -vp -i ${SSH_KEY} \
+    ${DEPS_ARCHIVES} \
+    nightlybot@download.kiwix.org:~/travis_deps/
+fi
 
-BASE_DIR="BUILD_*/INSTALL"
-./kiwix-deploy.py ${BASE_DIR} ${ARCHIVE_TYPE} \
-    --deploy \
-    --ssh_private_key=${SSH_KEY} \
-    --server=nightlybot@download.kiwix.org \
-    --base_path=/var/www/download.kiwix.org/nightly
+NIGHTLY_ARCHIVES=$(find $NIGHTLY_ARCHIVES_DIR -type f)
+if [[ "x$NIGHTLY_ARCHIVES" != "x" ]]
+then
+  scp -vrp -i ${SSH_KEY} \
+    ${NIGHTLY_ARCHIVES} \
+    nightlybot@download.kiwix.org:/var/www/download.kiwix.org/nightly/$(date +%Y-%m-%d)
+fi