Merge pull request #96 from kiwix/make_release

Make release

dependencies.py

@@ -135,6 +135,8 @@ class CTPP2(Dependency):
     version = "2.8.3"
 
     class Source(ReleaseDownload):
+        name = "ctpp2"
+        source_dir = "ctpp2-2.8.3"
         archive = Remotefile('ctpp2-2.8.3.tar.gz',
                              'a83ffd07817adb575295ef40fbf759892512e5a63059c520f9062d9ab8fb42fc')
         patches = ["ctpp2_include.patch",
@@ -276,9 +278,11 @@ class Libzim(Dependency):
     class Source(GitClone):
         git_remote = "https://github.com/openzim/libzim.git"
         git_dir = "libzim"
+        release_git_ref = "3.0.0"
 
     Builder = MesonBuilder
 
+
 class ZimTools(Dependency):
     name = "zim-tools"
     dependencies = ['libzim']
@@ -336,6 +340,7 @@ class Kiwixlib(Dependency):
     class Source(GitClone):
         git_remote = "https://github.com/kiwix/kiwix-lib.git"
         git_dir = "kiwix-lib"
+        release_git_ref = "1.0.1"
 
     class Builder(MesonBuilder):
         @property
@@ -359,6 +364,7 @@ class KiwixTools(Dependency):
     class Source(GitClone):
         git_remote = "https://github.com/kiwix/kiwix-tools.git"
         git_dir = "kiwix-tools"
+        release_git_ref = "0.3.0"
 
     class Builder(MesonBuilder):
         @property

dependency_utils.py

@@ -123,22 +123,33 @@ class ReleaseDownload(Source):
 
 
 class GitClone(Source):
-    git_ref = "master"
+    base_git_ref = "master"
+    release_git_ref = "master"
 
     @property
     def source_dir(self):
-        return self.git_dir
+        if self.buildEnv.make_release:
+            return "{}_release".format(self.git_dir)
+        else:
+            return self.git_dir
 
     @property
     def git_path(self):
-        return pj(self.buildEnv.source_dir, self.git_dir)
+        return pj(self.buildEnv.source_dir, self.source_dir)
+
+    @property
+    def git_ref(self):
+        if self.buildEnv.make_release:
+            return self.release_git_ref
+        else:
+            return self.base_git_ref
 
     def _git_clone(self, context):
         context.force_native_build = True
         if os.path.exists(self.git_path):
             raise SkipCommand()
         command = "git clone --depth=1 --branch {} {} {}".format(
-            self.git_ref, self.git_remote, self.git_dir)
+            self.git_ref, self.git_remote, self.source_dir)
         self.buildEnv.run_command(command, self.buildEnv.source_dir, context)
 
     def _git_update(self, context):
@@ -218,6 +229,12 @@ class Builder:
         if hasattr(self, '_post_build_script'):
             self.command('post_build_script', self._post_build_script)
 
+    def make_dist(self):
+        if hasattr(self, '_pre_build_script'):
+            self.command('pre_build_script', self._pre_build_script)
+        self.command('configure', self._configure)
+        self.command('make_dist', self._make_dist)
+
 
 class MakeBuilder(Builder):
     configure_option = ""
@@ -274,6 +291,11 @@ class MakeBuilder(Builder):
         )
         self.buildEnv.run_command(command, self.build_path, context)
 
+    def _make_dist(self, context):
+        context.try_skip(self.build_path)
+        command = "make dist"
+        self.buildEnv.run_command(command, self.build_path, context)
+
 
 class CMakeBuilder(MakeBuilder):
     def _configure(self, context):
@@ -357,6 +379,10 @@ class MesonBuilder(Builder):
         command = "{} -v install".format(self.buildEnv.ninja_command)
         self.buildEnv.run_command(command, self.build_path, context)
 
+    def _make_dist(self, context):
+        command = "{} -v dist".format(self.buildEnv.ninja_command)
+        self.buildEnv.run_command(command, self.build_path, context)
+
 
 class GradleBuilder(Builder):
     gradle_target = "build"

kiwix-build.py

@@ -942,9 +942,17 @@ class Builder:
 
         builders = (dep.builder for dep in self.targets.values() if (dep.builder and not dep.skip))
         for builder in builders:
+            if self.options.make_dist and builder.name == self.options.targets:
+                continue
             print("build {} :".format(builder.name))
             builder.build()
 
+        if self.options.make_dist:
+            dep = self.targets[self.options.targets]
+            builder = dep.builder
+            print("make dist {}:".format(builder.name))
+            builder.make_dist()
+
     def run(self):
         try:
             print("[INSTALL PACKAGES]")
@@ -979,7 +987,11 @@ def parse_args():
     parser.add_argument('--skip-source-prepare', action='store_true',
                         help="Skip the source download part")
     parser.add_argument('--build-deps-only', action='store_true',
-                        help=("Build only the dependencies of the specified targets."))
+                        help="Build only the dependencies of the specified targets.")
+    parser.add_argument('--make-dist', action='store_true',
+                        help="Build distrubution (dist) source archive")
+    parser.add_argument('--make-release', action='store_true',
+                        help="Build a release version")
     subgroup = parser.add_argument_group('advanced')
     subgroup.add_argument('--no-cert-check', action='store_true',
                           help="Skip SSL certificate verification during download")

travis/compile_all.sh

@@ -4,21 +4,35 @@ set -e
 
 BASE_DIR="BUILD_${PLATFORM}"
 NIGHTLY_ARCHIVES_DIR=${HOME}/NIGHTLY_ARCHIVES
+RELEASE_ARCHIVES_DIR=${HOME}/RELEASE_ARCHIVES
 SSH_KEY=${TRAVIS_BUILD_DIR}/travis/travisci_builder_id_key
 
 mkdir -p ${NIGHTLY_ARCHIVES_DIR}
+mkdir -p ${RELEASE_ARCHIVES_DIR}
 
-function make_nightly_archive {
+function make_archive {
-  ARCHIVE_NAME="${1}_$(date +%Y-%m-%d).tar.gz"
+  if [[ "$MAKE_RELEASE" == "0" ]]
+  then
+    ARCHIVE_PATH="${NIGHTLY_ARCHIVES_DIR}/${1}_$(date +%Y-%m-%d).tar.gz"
+  else
+    ARCHIVE_PATH="${RELEASE_ARCHIVES_DIR}/${1}-${TRAVIS_TAG}.tar.gz"
+  fi
   (
     cd ${BASE_DIR}/INSTALL/bin
-    tar -czf "${NIGHTLY_ARCHIVES_DIR}/$ARCHIVE_NAME" $2
+    tar -czf "${ARCHIVE_PATH}" $2
   )
 }
 
 cd ${HOME}
 
-if [[ "$TRAVIS_EVENT_TYPE" = "cron" ]]
+if [[ $TRAVIS_TAG =~ ^[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+ ]]
+then
+   MAKE_RELEASE=1
+else
+   MAKE_RELEASE=0
+fi
+
+if [[ "$TRAVIS_EVENT_TYPE" == "cron" || "$MAKE_RELEASE" == "1" ]]
 then
   if [[ ${PLATFORM} = android* ]]
   then
@@ -33,23 +47,25 @@ then
   for TARGET in ${TARGETS}
   do
     echo $TARGET
-    ${TRAVIS_BUILD_DIR}/kiwix-build.py \
+    if [[ "$TRAVIS_EVENT_TYPE" == "cron" ]]
-      --target-platform $PLATFORM \
+    then
-      --build-deps-only \
+      ${TRAVIS_BUILD_DIR}/kiwix-build.py \
-      --hide-progress \
+        --target-platform $PLATFORM \
-      ${TARGET}
+        --build-deps-only \
-    rm ${BASE_DIR}/.install_packages_ok
+        --hide-progress \
+        ${TARGET}
+      rm ${BASE_DIR}/.install_packages_ok
 
-    (
+      (
-      cd ${BASE_DIR}
+        cd ${BASE_DIR}
-      if [ -f meson_cross_file.txt ]
+        if [ -f meson_cross_file.txt ]
-      then
+        then
-        MESON_FILE=meson_cross_file.txt
+          MESON_FILE=meson_cross_file.txt
-      fi
+        fi
-      ANDROID_NDK_DIR=$(find . -name "android-ndk*")
+        ANDROID_NDK_DIR=$(find . -name "android-ndk*")
-      ARCHIVE_NAME="deps_${PLATFORM}_${TARGET}.tar.gz"
+        ARCHIVE_NAME="deps_${PLATFORM}_${TARGET}.tar.gz"
 
-      cat <<EOF > manifest.txt
+        cat <<EOF > manifest.txt
 ${ARCHIVE_NAME}
 *********************************
 
@@ -57,26 +73,49 @@ Dependencies archive for ${TARGET} on platform ${PLATFORM}
 Generated at $(date)
 EOF
 
-      tar -czf ${ARCHIVE_NAME} INSTALL manifest.txt ${MESON_FILE} ${ANDROID_NDK_DIR}
+        tar -czf ${ARCHIVE_NAME} INSTALL manifest.txt ${MESON_FILE} ${ANDROID_NDK_DIR}
-      scp -i ${SSH_KEY} ${ARCHIVE_NAME} nightlybot@download.kiwix.org:/var/www/tmp.kiwix.org/ci/
+        scp -i ${SSH_KEY} ${ARCHIVE_NAME} nightlybot@download.kiwix.org:/var/www/tmp.kiwix.org/ci/
-    )
+      )
+    fi
 
-    ${TRAVIS_BUILD_DIR}/kiwix-build.py --hide-progress --target-platform $PLATFORM ${TARGET}
+    if [[ "$MAKE_RELEASE" == "1" ]]
+    then
+        ${TRAVIS_BUILD_DIR}/kiwix-build.py \
+            --hide-progress \
+            --make-release \
+            --target-platform $PLATFORM ${TARGET}
+        if [[ "$PLATFORM" == "native_dyn" ]]
+        then
+            ${TRAVIS_BUILD_DIR}/kiwix-build.py \
+                --hide-progress \
+                --make-release \
+                --make-dist \
+                --target-platform $PLATFORM ${TARGET}
+        fi
+    else
+        ${TRAVIS_BUILD_DIR}/kiwix-build.py \
+            --hide-progress \
+            --target-platform $PLATFORM ${TARGET}
+    fi
     rm ${BASE_DIR}/.install_packages_ok
   done
 
   # We have build every thing. Now create archives for public deployement.
   case ${PLATFORM} in
+    native_dyn)
+      #[TODO] Copy archive somewhere
+      #scp ${BASE_DIR}/${TARGET}/${TARGET}-${TARGET-version}.tar.gz ${SOMEWHERE}
+      ;;
     native_static)
-      make_nightly_archive kiwix_tools_linux64 "kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve"
+      make_archive kiwix_tools_linux64 "kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve"
-      make_nightly_archive zim-tools_linux64 "zimbench zimdump zimsearch zimdiff zimpatch zimsplit"
+      make_archive zim-tools_linux64 "zimbench zimdump zimsearch zimdiff zimpatch zimsplit"
-      make_nightly_archive zimwriterfs_linux64 "zimwriterfs"
+      make_archive zimwriterfs_linux64 "zimwriterfs"
       ;;
     win32_static)
-      make_nightly_archive kiwix-tools_win32 "kiwix-install.exe kiwix-manage.exe kiwix-read.exe kiwix-search.exe kiwix-serve.exe"
+      make_archive kiwix-tools_win32 "kiwix-install.exe kiwix-manage.exe kiwix-read.exe kiwix-search.exe kiwix-serve.exe"
       ;;
     armhf_static)
-      make_nightly_archive kiwix-tools_armhf "kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve"
+      make_archive kiwix-tools_armhf "kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve"
       ;;
     android_*)
       APK_NAME="kiwix-${PLATFORM}"

travis/deploy.sh

@@ -3,6 +3,7 @@
 set -e
 
 NIGHTLY_ARCHIVES_DIR=${HOME}/NIGHTLY_ARCHIVES
+RELEASE_ARCHIVES_DIR=${HOME}/RELEASE_ARCHIVES
 SSH_KEY=travis/travisci_builder_id_key
 
 NIGHTLY_ARCHIVES=$(find $NIGHTLY_ARCHIVES_DIR -type f)
@@ -12,3 +13,11 @@ then
     ${NIGHTLY_ARCHIVES} \
     nightlybot@download.kiwix.org:/var/www/download.kiwix.org/nightly/$(date +%Y-%m-%d)
 fi
+
+RELEASE_ARCHIVES=$(find $RELEASE_ARCHIVES_DIR -type f)
+if [[ "x$RELEASE_ARCHIVES" != "x" ]]
+then
+  scp -vrp -i ${SSH_KEY} \
+    ${RELEASE_ARCHIVES} \
+    nightlybot@downoald.kiwix.org:/var/www/download.kiwix.org/releases
+fi