diff --git a/.travis.yml b/.travis.yml index c262328..b88f619 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,12 +20,12 @@ jobs: - stage: build_docker_images script: &build_images - | - if [ ! $(curl -sflL https://hub.docker.com/v2/repositories/kiwix/kiwix-build_ci/tags/${VARIANT}-${TRAVIS_COMMIT}) ] + if [ ! $(curl -sflL https://hub.docker.com/v2/repositories/kiwix/kiwix-build_ci/tags/${VARIANT}-${DOCKER_VERSION}) ] then echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin docker build -t kiwix-build_ci -f travis/${VARIANT}_builder.dockerfile . - docker tag kiwix-build_ci kiwix/kiwix-build_ci:${VARIANT}-${TRAVIS_COMMIT} - docker push kiwix/kiwix-build_ci:${VARIANT}-${TRAVIS_COMMIT} + docker tag kiwix-build_ci kiwix/kiwix-build_ci:${VARIANT}-${DOCKER_VERSION} + docker push kiwix/kiwix-build_ci:${VARIANT}-${DOCKER_VERSION} fi env: VARIANT=xenial - stage: build_docker_images @@ -80,7 +80,8 @@ script: --mount=type=bind,src=$HOME/EXPORT,dst=/home/ci_builder/EXPORT \ --mount=type=bind,src=$HOME/.cache,dst=/home/ci_builder/.cache \ --mount=type=bind,src=$HOME/.gradle/caches,dst=/home/ci_builder/.gradle/caches \ - kiwix/kiwix-build_ci:${VARIANT}-${TRAVIS_COMMIT} + --mount=type=bind,src=$TRAVIS_BUILD_DIR,dst=/home/ci_builder/kiwix-build \ + kiwix/kiwix-build_ci:${VARIANT}-${DOCKER_VERSION} fi - if [[ $TRAVIS_OS_NAME != "linux" ]]; then python3 travis/compile_all.py; fi after_success: @@ -105,6 +106,7 @@ env: global: - NIGHTLY_DATE=$(date +%Y-%m-%d) - SSH_KEY=/tmp/private/travisci_builder_id_key + - DOCKER_VERSION=1 matrix: - PLATFORM="flatpak" DESKTOP_ONLY=1 - PLATFORM="native_dyn" diff --git a/kiwixbuild/builder.py b/kiwixbuild/builder.py index 4462fd4..b70c139 100644 --- a/kiwixbuild/builder.py +++ b/kiwixbuild/builder.py @@ -204,6 +204,7 @@ class Builder: platform.clean_intermediate_directories() else: print("SKIP") - except StopBuild: + except StopBuild as e: + print(e) sys.exit("Stopping build due to errors") diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index cdbda9a..ba6b1a1 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -446,7 +446,7 @@ class MesonBuilder(Builder): @property def build_type(self): - return 'release' if option('make_release') else 'debugoptimized' + return 'release' if option('make_release') else 'debug' @property def strip_option(self): diff --git a/kiwixbuild/platforms/win32.py b/kiwixbuild/platforms/win32.py index 10e2e20..83c57f4 100644 --- a/kiwixbuild/platforms/win32.py +++ b/kiwixbuild/platforms/win32.py @@ -6,10 +6,10 @@ from kiwixbuild._global import neutralEnv class Win32PlatformInfo(PlatformInfo): - extra_libs = ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4', '-lmsvcr100', '-liphlpapi'] build = 'win32' compatible_hosts = ['fedora', 'debian'] arch_full = 'i686-w64-mingw32' + extra_libs = ['-lwinmm', '-lshlwapi', '-lws2_32'] def get_cross_config(self): return { diff --git a/kiwixbuild/utils.py b/kiwixbuild/utils.py index 0f2844a..9cbd3f3 100644 --- a/kiwixbuild/utils.py +++ b/kiwixbuild/utils.py @@ -127,7 +127,7 @@ def download_remote(what, where): print('Sha256 for {} not set, do no verify download'.format(what.name)) elif what.sha256 != get_sha256(file_path): os.remove(file_path) - raise StopBuild() + raise StopBuild("Sha 256 doesn't correspond") class SkipCommand(Exception): @@ -135,7 +135,11 @@ class SkipCommand(Exception): class StopBuild(Exception): - pass + def __init__(self, msg=""): + self.msg = msg + + def __str__(self): + return self.msg class Remotefile(namedtuple('Remotefile', ('name', 'sha256', 'url'))): diff --git a/travis/bionic_builder.dockerfile b/travis/bionic_builder.dockerfile index cde804d..df2298c 100644 --- a/travis/bionic_builder.dockerfile +++ b/travis/bionic_builder.dockerfile @@ -29,11 +29,7 @@ USER ci_builder WORKDIR /home/ci_builder ENV PATH="/home/ci_builder/.local/bin:${PATH}" -# Install kiwix-build -COPY --chown=ci_builder:ci_builder . kiwix-build -RUN pip3 install --user -e ./kiwix-build - ENV TRAVIS_BUILD_DIR /home/ci_builder/kiwix-build ENV TRAVIS_OS_NAME linux_bionic -CMD kiwix-build/travis/compile_all.py +CMD pip3 install --user ./kiwix-build && kiwix-build/travis/compile_all.py diff --git a/travis/compile_all.py b/travis/compile_all.py index d637598..29a00a5 100755 --- a/travis/compile_all.py +++ b/travis/compile_all.py @@ -110,9 +110,11 @@ def run_kiwix_build(target, platform, print_message("Build {} (deps={}, release={}, dist={})", target, build_deps_only, make_release, make_dist) subprocess.check_call(command, cwd=str(HOME)) + print_message("Build ended") def create_desktop_image(): + print_message("creating desktop image") if make_release: postfix = main_project_versions['kiwix-desktop'] extra_postfix = release_versions.get('kiwix-desktop') @@ -131,6 +133,7 @@ def create_desktop_image(): if PLATFORM == 'flatpak': build_path = BASE_DIR/'org.kiwix.desktop.flatpak' app_name = 'org.kiwix.desktop.{}.flatpak'.format(postfix) + print_message("archive is ", build_path) else: build_path = HOME/'Kiwix-{}-x86_64.AppImage'.format(postfix) app_name = "kiwix-desktop_x86_64_{}.appimage".format(postfix) @@ -253,8 +256,16 @@ def make_flatpak_cache_archive(): os=TRAVIS_OS_NAME, platform=PLATFORM) cache = BASE_DIR/'.flatpak-builder' + print_message("make flatpak cache archive") + nb_files = 0 + def pseudo_filter(tarinfo): + nonlocal nb_files + nb_files += 1 + if (nb_files % 1000) == 0: + print('.', flush=True) + return tarinfo with tarfile.open(str(BASE_EXPORT_DIR/archive_name), 'w:xz') as tar: - tar.add(str(cache), arcname=str(cache.relative_to(BASE_DIR))) + tar.add(str(cache), arcname=str(cache.relative_to(BASE_DIR)), filter=pseudo_filter) def update_flathub_git(): @@ -264,6 +275,7 @@ def update_flathub_git(): env['GIT_AUTHOR_EMAIL'] = env['GIT_COMMITTER_EMAIL'] = "kiwixbot@kymeria.fr" def call(command, cwd=None): cwd = cwd or GIT_REPO_DIR + print_message("call ", command) subprocess.check_call(command, env=env, cwd=str(cwd)) command = ['git', 'clone', FLATPAK_HTTP_GIT_REMOTE] call(command, cwd=GIT_EXPORT_DIR) @@ -436,6 +448,7 @@ for target in TARGETS: run_kiwix_build(target, platform=PLATFORM, make_release=make_release) + print_message("target is ", target) if target == 'kiwix-desktop': create_desktop_image() if make_release and PLATFORM == 'native_dyn' and release_versions.get(target) == 0: diff --git a/travis/xenial_builder.dockerfile b/travis/xenial_builder.dockerfile index d84ac01..e826542 100644 --- a/travis/xenial_builder.dockerfile +++ b/travis/xenial_builder.dockerfile @@ -22,7 +22,7 @@ RUN apt update -q && \ # vim less grep \ && \ apt-get clean -y && \ - rm -rf /var/lib/apt/list/* /usr/share/doc/* /var/cache/debconf/* + rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/* # Create user RUN useradd --create-home ci_builder @@ -30,12 +30,8 @@ USER ci_builder WORKDIR /home/ci_builder ENV PATH="/home/ci_builder/.local/bin:${PATH}" -# Install kiwix-build -COPY --chown=ci_builder:ci_builder . kiwix-build -RUN pip3 install --user -e ./kiwix-build - ENV TRAVIS_BUILD_DIR /home/ci_builder/kiwix-build ENV GRADLE_USER_HOME /home/ci_builder ENV TRAVIS_OS_NAME linux_xenial -CMD kiwix-build/travis/compile_all.py +CMD pip3 install --user ./kiwix-build && kiwix-build/travis/compile_all.py