diff --git a/dependencies.py b/dependencies.py index 39bd20f..59d9cd9 100644 --- a/dependencies.py +++ b/dependencies.py @@ -135,6 +135,8 @@ class CTPP2(Dependency): version = "2.8.3" class Source(ReleaseDownload): + name = "ctpp2" + source_dir = "ctpp2-2.8.3" archive = Remotefile('ctpp2-2.8.3.tar.gz', 'a83ffd07817adb575295ef40fbf759892512e5a63059c520f9062d9ab8fb42fc') patches = ["ctpp2_include.patch", @@ -276,9 +278,11 @@ class Libzim(Dependency): class Source(GitClone): git_remote = "https://github.com/openzim/libzim.git" git_dir = "libzim" + release_git_ref = "3.0.0" Builder = MesonBuilder + class ZimTools(Dependency): name = "zim-tools" dependencies = ['libzim'] @@ -336,6 +340,7 @@ class Kiwixlib(Dependency): class Source(GitClone): git_remote = "https://github.com/kiwix/kiwix-lib.git" git_dir = "kiwix-lib" + release_git_ref = "1.0.1" class Builder(MesonBuilder): @property @@ -359,6 +364,7 @@ class KiwixTools(Dependency): class Source(GitClone): git_remote = "https://github.com/kiwix/kiwix-tools.git" git_dir = "kiwix-tools" + release_git_ref = "0.3.0" class Builder(MesonBuilder): @property diff --git a/dependency_utils.py b/dependency_utils.py index 0cd4ed3..5326bd4 100644 --- a/dependency_utils.py +++ b/dependency_utils.py @@ -123,22 +123,33 @@ class ReleaseDownload(Source): class GitClone(Source): - git_ref = "master" + base_git_ref = "master" + release_git_ref = "master" @property def source_dir(self): - return self.git_dir + if self.buildEnv.make_release: + return "{}_release".format(self.git_dir) + else: + return self.git_dir @property def git_path(self): - return pj(self.buildEnv.source_dir, self.git_dir) + return pj(self.buildEnv.source_dir, self.source_dir) + + @property + def git_ref(self): + if self.buildEnv.make_release: + return self.release_git_ref + else: + return self.base_git_ref def _git_clone(self, context): context.force_native_build = True if os.path.exists(self.git_path): raise SkipCommand() command = "git clone --depth=1 --branch {} {} {}".format( - self.git_ref, self.git_remote, self.git_dir) + self.git_ref, self.git_remote, self.source_dir) self.buildEnv.run_command(command, self.buildEnv.source_dir, context) def _git_update(self, context): @@ -218,6 +229,12 @@ class Builder: if hasattr(self, '_post_build_script'): self.command('post_build_script', self._post_build_script) + def make_dist(self): + if hasattr(self, '_pre_build_script'): + self.command('pre_build_script', self._pre_build_script) + self.command('configure', self._configure) + self.command('make_dist', self._make_dist) + class MakeBuilder(Builder): configure_option = "" @@ -274,6 +291,11 @@ class MakeBuilder(Builder): ) self.buildEnv.run_command(command, self.build_path, context) + def _make_dist(self, context): + context.try_skip(self.build_path) + command = "make dist" + self.buildEnv.run_command(command, self.build_path, context) + class CMakeBuilder(MakeBuilder): def _configure(self, context): @@ -357,6 +379,10 @@ class MesonBuilder(Builder): command = "{} -v install".format(self.buildEnv.ninja_command) self.buildEnv.run_command(command, self.build_path, context) + def _make_dist(self, context): + command = "{} -v dist".format(self.buildEnv.ninja_command) + self.buildEnv.run_command(command, self.build_path, context) + class GradleBuilder(Builder): gradle_target = "build" diff --git a/kiwix-build.py b/kiwix-build.py index a392f59..eee3e19 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -942,9 +942,17 @@ class Builder: builders = (dep.builder for dep in self.targets.values() if (dep.builder and not dep.skip)) for builder in builders: + if self.options.make_dist and builder.name == self.options.targets: + continue print("build {} :".format(builder.name)) builder.build() + if self.options.make_dist: + dep = self.targets[self.options.targets] + builder = dep.builder + print("make dist {}:".format(builder.name)) + builder.make_dist() + def run(self): try: print("[INSTALL PACKAGES]") @@ -979,7 +987,11 @@ def parse_args(): parser.add_argument('--skip-source-prepare', action='store_true', help="Skip the source download part") parser.add_argument('--build-deps-only', action='store_true', - help=("Build only the dependencies of the specified targets.")) + help="Build only the dependencies of the specified targets.") + parser.add_argument('--make-dist', action='store_true', + help="Build distrubution (dist) source archive") + parser.add_argument('--make-release', action='store_true', + help="Build a release version") subgroup = parser.add_argument_group('advanced') subgroup.add_argument('--no-cert-check', action='store_true', help="Skip SSL certificate verification during download") diff --git a/travis/compile_all.sh b/travis/compile_all.sh index 7c1879b..ef6cf3f 100755 --- a/travis/compile_all.sh +++ b/travis/compile_all.sh @@ -4,21 +4,35 @@ set -e BASE_DIR="BUILD_${PLATFORM}" NIGHTLY_ARCHIVES_DIR=${HOME}/NIGHTLY_ARCHIVES +RELEASE_ARCHIVES_DIR=${HOME}/RELEASE_ARCHIVES SSH_KEY=${TRAVIS_BUILD_DIR}/travis/travisci_builder_id_key mkdir -p ${NIGHTLY_ARCHIVES_DIR} +mkdir -p ${RELEASE_ARCHIVES_DIR} -function make_nightly_archive { - ARCHIVE_NAME="${1}_$(date +%Y-%m-%d).tar.gz" +function make_archive { + if [[ "$MAKE_RELEASE" == "0" ]] + then + ARCHIVE_PATH="${NIGHTLY_ARCHIVES_DIR}/${1}_$(date +%Y-%m-%d).tar.gz" + else + ARCHIVE_PATH="${RELEASE_ARCHIVES_DIR}/${1}-${TRAVIS_TAG}.tar.gz" + fi ( cd ${BASE_DIR}/INSTALL/bin - tar -czf "${NIGHTLY_ARCHIVES_DIR}/$ARCHIVE_NAME" $2 + tar -czf "${ARCHIVE_PATH}" $2 ) } cd ${HOME} -if [[ "$TRAVIS_EVENT_TYPE" = "cron" ]] +if [[ $TRAVIS_TAG =~ ^[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+ ]] +then + MAKE_RELEASE=1 +else + MAKE_RELEASE=0 +fi + +if [[ "$TRAVIS_EVENT_TYPE" == "cron" || "$MAKE_RELEASE" == "1" ]] then if [[ ${PLATFORM} = android* ]] then @@ -33,23 +47,25 @@ then for TARGET in ${TARGETS} do echo $TARGET - ${TRAVIS_BUILD_DIR}/kiwix-build.py \ - --target-platform $PLATFORM \ - --build-deps-only \ - --hide-progress \ - ${TARGET} - rm ${BASE_DIR}/.install_packages_ok + if [[ "$TRAVIS_EVENT_TYPE" == "cron" ]] + then + ${TRAVIS_BUILD_DIR}/kiwix-build.py \ + --target-platform $PLATFORM \ + --build-deps-only \ + --hide-progress \ + ${TARGET} + rm ${BASE_DIR}/.install_packages_ok - ( - cd ${BASE_DIR} - if [ -f meson_cross_file.txt ] - then - MESON_FILE=meson_cross_file.txt - fi - ANDROID_NDK_DIR=$(find . -name "android-ndk*") - ARCHIVE_NAME="deps_${PLATFORM}_${TARGET}.tar.gz" + ( + cd ${BASE_DIR} + if [ -f meson_cross_file.txt ] + then + MESON_FILE=meson_cross_file.txt + fi + ANDROID_NDK_DIR=$(find . -name "android-ndk*") + ARCHIVE_NAME="deps_${PLATFORM}_${TARGET}.tar.gz" - cat < manifest.txt + cat < manifest.txt ${ARCHIVE_NAME} ********************************* @@ -57,26 +73,49 @@ Dependencies archive for ${TARGET} on platform ${PLATFORM} Generated at $(date) EOF - tar -czf ${ARCHIVE_NAME} INSTALL manifest.txt ${MESON_FILE} ${ANDROID_NDK_DIR} - scp -i ${SSH_KEY} ${ARCHIVE_NAME} nightlybot@download.kiwix.org:/var/www/tmp.kiwix.org/ci/ - ) + tar -czf ${ARCHIVE_NAME} INSTALL manifest.txt ${MESON_FILE} ${ANDROID_NDK_DIR} + scp -i ${SSH_KEY} ${ARCHIVE_NAME} nightlybot@download.kiwix.org:/var/www/tmp.kiwix.org/ci/ + ) + fi - ${TRAVIS_BUILD_DIR}/kiwix-build.py --hide-progress --target-platform $PLATFORM ${TARGET} + if [[ "$MAKE_RELEASE" == "1" ]] + then + ${TRAVIS_BUILD_DIR}/kiwix-build.py \ + --hide-progress \ + --make-release \ + --target-platform $PLATFORM ${TARGET} + if [[ "$PLATFORM" == "native_dyn" ]] + then + ${TRAVIS_BUILD_DIR}/kiwix-build.py \ + --hide-progress \ + --make-release \ + --make-dist \ + --target-platform $PLATFORM ${TARGET} + fi + else + ${TRAVIS_BUILD_DIR}/kiwix-build.py \ + --hide-progress \ + --target-platform $PLATFORM ${TARGET} + fi rm ${BASE_DIR}/.install_packages_ok done # We have build every thing. Now create archives for public deployement. case ${PLATFORM} in + native_dyn) + #[TODO] Copy archive somewhere + #scp ${BASE_DIR}/${TARGET}/${TARGET}-${TARGET-version}.tar.gz ${SOMEWHERE} + ;; native_static) - make_nightly_archive kiwix_tools_linux64 "kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve" - make_nightly_archive zim-tools_linux64 "zimbench zimdump zimsearch zimdiff zimpatch zimsplit" - make_nightly_archive zimwriterfs_linux64 "zimwriterfs" + make_archive kiwix_tools_linux64 "kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve" + make_archive zim-tools_linux64 "zimbench zimdump zimsearch zimdiff zimpatch zimsplit" + make_archive zimwriterfs_linux64 "zimwriterfs" ;; win32_static) - make_nightly_archive kiwix-tools_win32 "kiwix-install.exe kiwix-manage.exe kiwix-read.exe kiwix-search.exe kiwix-serve.exe" + make_archive kiwix-tools_win32 "kiwix-install.exe kiwix-manage.exe kiwix-read.exe kiwix-search.exe kiwix-serve.exe" ;; armhf_static) - make_nightly_archive kiwix-tools_armhf "kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve" + make_archive kiwix-tools_armhf "kiwix-install kiwix-manage kiwix-read kiwix-search kiwix-serve" ;; android_*) APK_NAME="kiwix-${PLATFORM}" diff --git a/travis/deploy.sh b/travis/deploy.sh index 929db04..5659fc7 100755 --- a/travis/deploy.sh +++ b/travis/deploy.sh @@ -3,6 +3,7 @@ set -e NIGHTLY_ARCHIVES_DIR=${HOME}/NIGHTLY_ARCHIVES +RELEASE_ARCHIVES_DIR=${HOME}/RELEASE_ARCHIVES SSH_KEY=travis/travisci_builder_id_key NIGHTLY_ARCHIVES=$(find $NIGHTLY_ARCHIVES_DIR -type f) @@ -12,3 +13,11 @@ then ${NIGHTLY_ARCHIVES} \ nightlybot@download.kiwix.org:/var/www/download.kiwix.org/nightly/$(date +%Y-%m-%d) fi + +RELEASE_ARCHIVES=$(find $RELEASE_ARCHIVES_DIR -type f) +if [[ "x$RELEASE_ARCHIVES" != "x" ]] +then + scp -vrp -i ${SSH_KEY} \ + ${RELEASE_ARCHIVES} \ + nightlybot@downoald.kiwix.org:/var/www/download.kiwix.org/releases +fi