From e4f8b6c7a963d2708196fd6c9e7f40717a689b83 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 28 Nov 2017 16:30:20 +0000 Subject: [PATCH 1/5] Add a option to kiwix-build.py to build release version of subproject. With `--make-release` option, kiwix-build.py will build the tagged version of the subproject. --- dependencies.py | 4 ++++ dependency_utils.py | 19 +++++++++++++++---- kiwix-build.py | 4 +++- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/dependencies.py b/dependencies.py index 39bd20f..3020454 100644 --- a/dependencies.py +++ b/dependencies.py @@ -276,9 +276,11 @@ class Libzim(Dependency): class Source(GitClone): git_remote = "https://github.com/openzim/libzim.git" git_dir = "libzim" + release_git_ref = "3.0.0" Builder = MesonBuilder + class ZimTools(Dependency): name = "zim-tools" dependencies = ['libzim'] @@ -336,6 +338,7 @@ class Kiwixlib(Dependency): class Source(GitClone): git_remote = "https://github.com/kiwix/kiwix-lib.git" git_dir = "kiwix-lib" + release_git_ref = "1.0.1" class Builder(MesonBuilder): @property @@ -359,6 +362,7 @@ class KiwixTools(Dependency): class Source(GitClone): git_remote = "https://github.com/kiwix/kiwix-tools.git" git_dir = "kiwix-tools" + release_git_ref = "0.3.0" class Builder(MesonBuilder): @property diff --git a/dependency_utils.py b/dependency_utils.py index 0cd4ed3..be00774 100644 --- a/dependency_utils.py +++ b/dependency_utils.py @@ -123,22 +123,33 @@ class ReleaseDownload(Source): class GitClone(Source): - git_ref = "master" + base_git_ref = "master" + release_git_ref = "master" @property def source_dir(self): - return self.git_dir + if self.buildEnv.make_release: + return "{}_release".format(self.git_dir) + else: + return self.git_dir @property def git_path(self): - return pj(self.buildEnv.source_dir, self.git_dir) + return pj(self.buildEnv.source_dir, self.source_dir) + + @property + def git_ref(self): + if self.buildEnv.make_release: + return self.release_git_ref + else: + return self.base_git_ref def _git_clone(self, context): context.force_native_build = True if os.path.exists(self.git_path): raise SkipCommand() command = "git clone --depth=1 --branch {} {} {}".format( - self.git_ref, self.git_remote, self.git_dir) + self.git_ref, self.git_remote, self.source_dir) self.buildEnv.run_command(command, self.buildEnv.source_dir, context) def _git_update(self, context): diff --git a/kiwix-build.py b/kiwix-build.py index a392f59..8b28a88 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -979,7 +979,9 @@ def parse_args(): parser.add_argument('--skip-source-prepare', action='store_true', help="Skip the source download part") parser.add_argument('--build-deps-only', action='store_true', - help=("Build only the dependencies of the specified targets.")) + help="Build only the dependencies of the specified targets.") + parser.add_argument('--make-release', action='store_true', + help="Build a release version") subgroup = parser.add_argument_group('advanced') subgroup.add_argument('--no-cert-check', action='store_true', help="Skip SSL certificate verification during download") From bb5b85da507b459d29b14c135df77685a3c5c2ac Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 28 Nov 2017 16:43:52 +0000 Subject: [PATCH 2/5] Make travisCI publish binaries as release versions when tag is pushed. This publish the same this that nightly build except that : - Use the `--make-release` option to force kiwix-build to use tagged version of subprojects. - The archives are named using the tag name. - The archives are pushed to "releases" directory in download.kiwix.org. --- travis/compile_all.sh | 83 ++++++++++++++++++++++++++++--------------- travis/deploy.sh | 9 +++++ 2 files changed, 64 insertions(+), 28 deletions(-) diff --git a/travis/compile_all.sh b/travis/compile_all.sh index 7c1879b..c6648a5 100755 --- a/travis/compile_all.sh +++ b/travis/compile_all.sh @@ -4,21 +4,35 @@ set -e BASE_DIR="BUILD_${PLATFORM}" NIGHTLY_ARCHIVES_DIR=${HOME}/NIGHTLY_ARCHIVES +RELEASE_ARCHIVES_DIR=${HOME}/RELEASE_ARCHIVES SSH_KEY=${TRAVIS_BUILD_DIR}/travis/travisci_builder_id_key mkdir -p ${NIGHTLY_ARCHIVES_DIR} +mkdir -p ${RELEASE_ARCHIVES_DIR} -function make_nightly_archive { - ARCHIVE_NAME="${1}_$(date +%Y-%m-%d).tar.gz" +function make_archive { + if [[ "$MAKE_RELEASE" == "0" ]] + then + ARCHIVE_PATH="${NIGHTLY_ARCHIVES_DIR}/${1}_$(date +%Y-%m-%d).tar.gz" + else + ARCHIVE_PATH="${RELEASE_ARCHIVES_DIR}/${1}-${TRAVIS_TAG}.tar.gz" + fi ( cd ${BASE_DIR}/INSTALL/bin - tar -czf "${NIGHTLY_ARCHIVES_DIR}/$ARCHIVE_NAME" $2 + tar -czf "${ARCHIVE_PATH}" $2 ) } cd ${HOME} -if [[ "$TRAVIS_EVENT_TYPE" = "cron" ]] +if [[ $TRAVIS_TAG =~ ^[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+ ]] +then + MAKE_RELEASE=1 +else + MAKE_RELEASE=0 +fi + +if [[ "$TRAVIS_EVENT_TYPE" == "cron" || "$MAKE_RELEASE" == "1" ]] then if [[ ${PLATFORM} = android* ]] then @@ -33,23 +47,25 @@ then for TARGET in ${TARGETS} do echo $TARGET - ${TRAVIS_BUILD_DIR}/kiwix-build.py \ - --target-platform $PLATFORM \ - --build-deps-only \ - --hide-progress \ - ${TARGET} - rm ${BASE_DIR}/.install_packages_ok + if [[ "$TRAVIS_EVENT_TYPE" == "cron" ]] + then + ${TRAVIS_BUILD_DIR}/kiwix-build.py \ + --target-platform $PLATFORM \ + --build-deps-only \ + --hide-progress \ + ${TARGET} + rm ${BASE_DIR}/.install_packages_ok - ( - cd ${BASE_DIR} - if [ -f meson_cross_file.txt ] - then - MESON_FILE=meson_cross_file.txt - fi - ANDROID_NDK_DIR=$(find . -name "android-ndk*") - ARCHIVE_NAME="deps_${PLATFORM}_${TARGET}.tar.gz" + ( + cd ${BASE_DIR} + if [ -f meson_cross_file.txt ] + then + MESON_FILE=meson_cross_file.txt + fi + ANDROID_NDK_DIR=$(find . -name "android-ndk*") + ARCHIVE_NAME="deps_${PLATFORM}_${TARGET}.tar.gz" - cat < manifest.txt + cat < manifest.txt ${ARCHIVE_NAME} ********************************* @@ -57,26 +73,37 @@ Dependencies archive for ${TARGET} on platform ${PLATFORM} Generated at $(date) EOF - tar -czf ${ARCHIVE_NAME} INSTALL manifest.txt ${MESON_FILE} ${ANDROID_NDK_DIR} - scp -i ${SSH_KEY} ${ARCHIVE_NAME} nightlybot@download.kiwix.org:/var/www/tmp.kiwix.org/ci/ - ) + tar -czf ${ARCHIVE_NAME} INSTALL manifest.txt ${MESON_FILE} ${ANDROID_NDK_DIR} + scp -i ${SSH_KEY} ${ARCHIVE_NAME} nightlybot@download.kiwix.org:/var/www/tmp.kiwix.org/ci/ + ) + fi - ${TRAVIS_BUILD_DIR}/kiwix-build.py --hide-progress --target-platform $PLATFORM ${TARGET} + if [[ "$MAKE_RELEASE" == "1" ]] + then + ${TRAVIS_BUILD_DIR}/kiwix-build.py \ + --hide-progress \ + --make-release \ + --target-platform $PLATFORM ${TARGET} + else + ${TRAVIS_BUILD_DIR}/kiwix-build.py \ + --hide-progress \ + --target-platform $PLATFORM ${TARGET} + fi rm ${BASE_DIR}/.install_packages_ok done # We have build every thing. Now create archives for public deployement. case ${PLATFORM} in native_static) - make_nightly_archive kiwix_tools_linux64 "kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve" - make_nightly_archive zim-tools_linux64 "zimbench zimdump zimsearch zimdiff zimpatch zimsplit" - make_nightly_archive zimwriterfs_linux64 "zimwriterfs" + make_archive kiwix_tools_linux64 "kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve" + make_archive zim-tools_linux64 "zimbench zimdump zimsearch zimdiff zimpatch zimsplit" + make_archive zimwriterfs_linux64 "zimwriterfs" ;; win32_static) - make_nightly_archive kiwix-tools_win32 "kiwix-install.exe kiwix-manage.exe kiwix-read.exe kiwix-search.exe kiwix-serve.exe" + make_archive kiwix-tools_win32 "kiwix-install.exe kiwix-manage.exe kiwix-read.exe kiwix-search.exe kiwix-serve.exe" ;; armhf_static) - make_nightly_archive kiwix-tools_armhf "kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve" + make_archive kiwix-tools_armhf "kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve" ;; android_*) APK_NAME="kiwix-${PLATFORM}" diff --git a/travis/deploy.sh b/travis/deploy.sh index 929db04..5659fc7 100755 --- a/travis/deploy.sh +++ b/travis/deploy.sh @@ -3,6 +3,7 @@ set -e NIGHTLY_ARCHIVES_DIR=${HOME}/NIGHTLY_ARCHIVES +RELEASE_ARCHIVES_DIR=${HOME}/RELEASE_ARCHIVES SSH_KEY=travis/travisci_builder_id_key NIGHTLY_ARCHIVES=$(find $NIGHTLY_ARCHIVES_DIR -type f) @@ -12,3 +13,11 @@ then ${NIGHTLY_ARCHIVES} \ nightlybot@download.kiwix.org:/var/www/download.kiwix.org/nightly/$(date +%Y-%m-%d) fi + +RELEASE_ARCHIVES=$(find $RELEASE_ARCHIVES_DIR -type f) +if [[ "x$RELEASE_ARCHIVES" != "x" ]] +then + scp -vrp -i ${SSH_KEY} \ + ${RELEASE_ARCHIVES} \ + nightlybot@downoald.kiwix.org:/var/www/download.kiwix.org/releases +fi From a9ce8ee8c1be1d6c90a78cfee536acd43fac234e Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 5 Dec 2017 09:22:25 +0000 Subject: [PATCH 3/5] Add a `--make-dist` command to kiwix-build. If specified, kiwix-build will not "make/install" the target but make the dist archive (source). It will build dependencies, as the make dist will try to compile and test the target. --- dependency_utils.py | 15 +++++++++++++++ kiwix-build.py | 10 ++++++++++ 2 files changed, 25 insertions(+) diff --git a/dependency_utils.py b/dependency_utils.py index be00774..5326bd4 100644 --- a/dependency_utils.py +++ b/dependency_utils.py @@ -229,6 +229,12 @@ class Builder: if hasattr(self, '_post_build_script'): self.command('post_build_script', self._post_build_script) + def make_dist(self): + if hasattr(self, '_pre_build_script'): + self.command('pre_build_script', self._pre_build_script) + self.command('configure', self._configure) + self.command('make_dist', self._make_dist) + class MakeBuilder(Builder): configure_option = "" @@ -285,6 +291,11 @@ class MakeBuilder(Builder): ) self.buildEnv.run_command(command, self.build_path, context) + def _make_dist(self, context): + context.try_skip(self.build_path) + command = "make dist" + self.buildEnv.run_command(command, self.build_path, context) + class CMakeBuilder(MakeBuilder): def _configure(self, context): @@ -368,6 +379,10 @@ class MesonBuilder(Builder): command = "{} -v install".format(self.buildEnv.ninja_command) self.buildEnv.run_command(command, self.build_path, context) + def _make_dist(self, context): + command = "{} -v dist".format(self.buildEnv.ninja_command) + self.buildEnv.run_command(command, self.build_path, context) + class GradleBuilder(Builder): gradle_target = "build" diff --git a/kiwix-build.py b/kiwix-build.py index 8b28a88..eee3e19 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -942,9 +942,17 @@ class Builder: builders = (dep.builder for dep in self.targets.values() if (dep.builder and not dep.skip)) for builder in builders: + if self.options.make_dist and builder.name == self.options.targets: + continue print("build {} :".format(builder.name)) builder.build() + if self.options.make_dist: + dep = self.targets[self.options.targets] + builder = dep.builder + print("make dist {}:".format(builder.name)) + builder.make_dist() + def run(self): try: print("[INSTALL PACKAGES]") @@ -980,6 +988,8 @@ def parse_args(): help="Skip the source download part") parser.add_argument('--build-deps-only', action='store_true', help="Build only the dependencies of the specified targets.") + parser.add_argument('--make-dist', action='store_true', + help="Build distrubution (dist) source archive") parser.add_argument('--make-release', action='store_true', help="Build a release version") subgroup = parser.add_argument_group('advanced') From 43937caf6df9b9021b4300c0ac546ee5d11e7416 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 5 Dec 2017 09:29:01 +0000 Subject: [PATCH 4/5] Make travis create the dist archive. TODO: Copy the archive at the right place. --- travis/compile_all.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/travis/compile_all.sh b/travis/compile_all.sh index c6648a5..ef6cf3f 100755 --- a/travis/compile_all.sh +++ b/travis/compile_all.sh @@ -84,6 +84,14 @@ EOF --hide-progress \ --make-release \ --target-platform $PLATFORM ${TARGET} + if [[ "$PLATFORM" == "native_dyn" ]] + then + ${TRAVIS_BUILD_DIR}/kiwix-build.py \ + --hide-progress \ + --make-release \ + --make-dist \ + --target-platform $PLATFORM ${TARGET} + fi else ${TRAVIS_BUILD_DIR}/kiwix-build.py \ --hide-progress \ @@ -94,6 +102,10 @@ EOF # We have build every thing. Now create archives for public deployement. case ${PLATFORM} in + native_dyn) + #[TODO] Copy archive somewhere + #scp ${BASE_DIR}/${TARGET}/${TARGET}-${TARGET-version}.tar.gz ${SOMEWHERE} + ;; native_static) make_archive kiwix_tools_linux64 "kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve" make_archive zim-tools_linux64 "zimbench zimdump zimsearch zimdiff zimpatch zimsplit" From ba7028f0a8f09cfc86516a6e3d991899bc541686 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 5 Dec 2017 09:31:47 +0000 Subject: [PATCH 5/5] Force the source_dir of ctpp2 source. The `source_dir` is automatically infer from the name of the dependency. As the ctpp2 source is use in two dependencies (ctpp2 and ctpp2c), if we use the default source_dir, kiwix-build will use one name and the other dependency will not find the source directory. By forcing the `source_dir`, both dependencies will use the same directory. --- dependencies.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dependencies.py b/dependencies.py index 3020454..59d9cd9 100644 --- a/dependencies.py +++ b/dependencies.py @@ -135,6 +135,8 @@ class CTPP2(Dependency): version = "2.8.3" class Source(ReleaseDownload): + name = "ctpp2" + source_dir = "ctpp2-2.8.3" archive = Remotefile('ctpp2-2.8.3.tar.gz', 'a83ffd07817adb575295ef40fbf759892512e5a63059c520f9062d9ab8fb42fc') patches = ["ctpp2_include.patch",