diff --git a/.github/ci_images/bionic_builder.dockerfile b/.github/ci_images/bionic_builder.dockerfile index c45cb78..64f8360 100644 --- a/.github/ci_images/bionic_builder.dockerfile +++ b/.github/ci_images/bionic_builder.dockerfile @@ -24,7 +24,7 @@ RUN apt update -q \ # vim less grep \ && apt-get clean -y \ && rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/* \ - && pip3 install meson==0.52.1 pytest gcovr + && pip3 install meson==0.52.1 pytest gcovr requests # Create user RUN useradd --create-home runner diff --git a/.github/ci_images/f31_builder.dockerfile b/.github/ci_images/f31_builder.dockerfile index 62ed448..38cadc4 100644 --- a/.github/ci_images/f31_builder.dockerfile +++ b/.github/ci_images/f31_builder.dockerfile @@ -12,13 +12,13 @@ RUN dnf install -y --nodocs \ mingw32-winpthreads-static mingw32-zlib-static mingw32-xz-libs-static \ mingw32-libmicrohttpd \ # python3 - python-unversioned-command \ + python3-pip python-unversioned-command \ # Other tools (to remove) # vim less grep && dnf remove -y "*-doc" \ && dnf autoremove -y \ && dnf clean all \ - && pip3 install meson==0.52.1 pytest + && pip3 install meson==0.52.1 pytest requests # Create user RUN useradd --create-home runner diff --git a/.github/ci_images/focal_builder.dockerfile b/.github/ci_images/focal_builder.dockerfile index b3b9a00..57fae3a 100644 --- a/.github/ci_images/focal_builder.dockerfile +++ b/.github/ci_images/focal_builder.dockerfile @@ -2,6 +2,7 @@ FROM ubuntu:focal ENV LANG C.UTF-8 ENV OS_NAME focal +ENV DEBIAN_FRONTEND noninteractive RUN apt update -q \ && apt install -q -y --no-install-recommends \ @@ -24,7 +25,7 @@ RUN apt update -q \ # vim less grep \ && apt-get clean -y \ && rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/* \ - && pip3 install meson==0.52.1 pytest gcovr + && pip3 install meson==0.52.1 pytest gcovr requests # Create user RUN useradd --create-home runner diff --git a/.github/ci_images/xenial_builder.dockerfile b/.github/ci_images/xenial_builder.dockerfile index 2b634a4..b060135 100644 --- a/.github/ci_images/xenial_builder.dockerfile +++ b/.github/ci_images/xenial_builder.dockerfile @@ -21,7 +21,7 @@ RUN apt update -q \ # vim less grep \ && apt-get clean -y \ && rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/* \ - && pip3 install meson==0.52.1 pytest gcovr + && pip3 install meson==0.52.1 pytest gcovr requests # Create user RUN useradd --create-home runner diff --git a/.github/scripts/build_release_nightly.py b/.github/scripts/build_release_nightly.py new file mode 100755 index 0000000..b8a694b --- /dev/null +++ b/.github/scripts/build_release_nightly.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 + +import os +import json +import shutil + +from common import ( + run_kiwix_build, + main_project_versions, + release_versions, + get_postfix, + make_archive, + create_desktop_image, + update_flathub_git, + upload_archive, + BASE_DIR, + TMP_DIR, + HOME, + OS_NAME, + PLATFORM_TARGET, + DESKTOP, +) + +from upload_to_bintray import upload_from_json + + +if os.environ.get('GITHUB_EVENT_NAME') == 'schedule': + RELEASE = False +else: + RELEASE = True + +if PLATFORM_TARGET == "android": + TARGETS = ("kiwix-lib-app",) +elif PLATFORM_TARGET.startswith("iOS"): + TARGETS = ("libzim", "kiwix-lib") +elif PLATFORM_TARGET.startswith("native_"): + if OS_NAME == "osx": + TARGETS = ("libzim", "zimwriterfs", "zim-tools", "kiwix-lib") + else: + if DESKTOP: + TARGETS = ("kiwix-desktop",) + elif PLATFORM_TARGET == "native_mixed": + TARGETS = ("libzim",) + else: + TARGETS = ("zimwriterfs", "zim-tools", "kiwix-tools") +elif PLATFORM_TARGET in ("win32_static", "armhf_static", "i586_static"): + TARGETS = ("kiwix-tools",) +elif PLATFORM_TARGET == "flatpak": + TARGETS = ("kiwix-desktop",) +else: + TARGETS = ("libzim", "zim-tools", "kiwix-lib", "kiwix-tools") + +# Filter what to build if we are doing a release. +if RELEASE: + TARGETS = tuple(filter(lambda t: release_versions.get(t) is not None, TARGETS)) + +for target in TARGETS: + run_kiwix_build(target, platform=PLATFORM_TARGET, make_release=RELEASE) + if target == "kiwix-desktop": + archive = create_desktop_image(make_release=RELEASE) + else: + archive = make_archive(target, make_release=RELEASE) + if archive: + upload_archive(archive, target, make_release=RELEASE) + +# We have few more things to do for release: +if RELEASE: + # Publish source archives + if PLATFORM_TARGET == "native_dyn" and OS_NAME != "osx": + for target in TARGETS: + if release_versions.get(target) != 0: + continue + run_kiwix_build( + target, platform=PLATFORM_TARGET, make_release=RELEASE, make_dist=True + ) + full_target_name = "{}-{}".format(target, main_project_versions[target]) + if target == "kiwix-desktop": + archive = ( + BASE_DIR / full_target_name / "{}.tar.gz".format(full_target_name) + ) + else: + archive = ( + BASE_DIR + / full_target_name + / "meson-dist" + / "{}.tar.xz".format(full_target_name) + ) + upload_archive(archive, target, make_release=RELEASE) + + # Publish flathub + if PLATFORM_TARGET == "flatpak" and "kiwix-desktop" in TARGETS: + update_flathub_git() + + if PLATFORM_TARGET == "android" and "kiwix-lib-app" in TARGETS: + postfix = get_postfix("kiwix-lib") + basename = "kiwixlib-{}".format(postfix) + + output_release_dir = ( + HOME / "BUILD_android" / "kiwix-lib-app" / "kiwixLibAndroid" / "build" + ) + shutil.copy( + str(output_release_dir / "outputs" / "aar" / "kiwixLibAndroid-release.aar"), + str(TMP_DIR / (basename + ".aar")), + ) + shutil.copy( + str(output_release_dir / "pom.xml"), str(TMP_DIR / (basename + ".pom")) + ) + + json_filename = "{}_bintray_info.json".format(basename) + data = { + "version": postfix, + "files": [basename + ext for ext in (".aar", ".pom")], + } + with open(str(TMP_DIR / json_filename), "w") as f: + json.dump(data, f) + + upload_from_json(TMP_DIR / json_filename) diff --git a/.github/scripts/common.py b/.github/scripts/common.py index ec07ebe..f3ce29f 100644 --- a/.github/scripts/common.py +++ b/.github/scripts/common.py @@ -3,11 +3,16 @@ from os import environ as _environ from pathlib import Path from datetime import date import tarfile +import zipfile import subprocess import re +import shutil -from kiwixbuild.versions import base_deps_versions - +from kiwixbuild.versions import ( + main_project_versions, + release_versions, + base_deps_versions, +) PLATFORM_TARGET = _environ["PLATFORM_TARGET"] if PLATFORM_TARGET == "native_desktop": @@ -23,6 +28,7 @@ SOURCE_DIR = HOME / "SOURCE" ARCHIVE_DIR = HOME / "ARCHIVE" INSTALL_DIR = BASE_DIR / "INSTALL" TMP_DIR = Path("/tmp") +KBUILD_SOURCE_DIR = HOME / "kiwix-build" # [TODO] KIWIX_DESKTOP_ONLY = False @@ -30,6 +36,58 @@ KIWIX_DESKTOP_ONLY = False _ref = _environ.get("GITHUB_REF", "").split("/")[-1] MAKE_RELEASE = re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", _ref) is not None +PLATFORM_TO_RELEASE = { + "native_mixed": "linux-x86_64", + "native_static": "linux-x86_64", + "win32_static": "win-i686", + "armhf_static": "linux-armhf", + "i586_static": "linux-i586", +} + +FLATPAK_HTTP_GIT_REMOTE = "https://github.com/flathub/org.kiwix.desktop.git" +FLATPAK_GIT_REMOTE = "git@github.com:flathub/org.kiwix.desktop.git" + +BIN_EXT = ".exe" if PLATFORM_TARGET.startswith("win32_") else "" + +# We have build everything. Now create archives for public deployement. +EXPORT_FILES = { + "kiwix-tools": ( + INSTALL_DIR / "bin", + [ + f + BIN_EXT + for f in ("kiwix-manage", "kiwix-read", "kiwix-search", "kiwix-serve") + ], + ), + "zim-tools": ( + INSTALL_DIR / "bin", + [ + f + BIN_EXT + for f in ( + "zimbench", + "zimcheck", + "zimdump", + "zimsearch", + "zimdiff", + "zimpatch", + "zimsplit", + ) + ], + ), + "zimwriterfs": (INSTALL_DIR / "bin", ["zimwriterfs" + BIN_EXT]), + "libzim": ( + INSTALL_DIR, + ( + "lib/x86_64-linux-gnu/libzim.so.{}".format(main_project_versions["libzim"]), + "lib/x86_64-linux-gnu/libzim.so.{}".format( + main_project_versions["libzim"][0] + ), + "include/zim/**/*.h", + ), + ), +} + +DATE = date.today().isoformat() + def print_message(message, *args, **kwargs): message = message.format(*args, **kwargs) @@ -49,10 +107,7 @@ def write_manifest(manifest_file, archive_name, target, platform): with manifest_file.open(mode="w") as f: f.write( MANIFEST_TEMPLATE.format( - archive_name=archive_name, - target=target, - platform=platform, - date=date.today().isoformat(), + archive_name=archive_name, target=target, platform=platform, date=DATE, ) ) @@ -95,12 +150,28 @@ def run_kiwix_build( make_dist, ) env = dict(_environ) - env['SKIP_BIG_MEMORY_TEST'] = '1' + env["SKIP_BIG_MEMORY_TEST"] = "1" subprocess.check_call(command, cwd=str(HOME), env=env) print_message("Build ended") def upload(file_to_upload, host, dest_path): + if not file_to_upload.exists(): + print_message("No {} to upload!", file_to_upload) + return + + command = [ + "ssh", + "-i", + _environ.get("SSH_KEY"), + "-o", + "StrictHostKeyChecking=no", + host, + "mkdir -p {}".format(dest_path), + ] + print_message("Creating dest path {}", dest_path) + subprocess.check_call(command) + command = [ "scp", "-i", @@ -114,6 +185,22 @@ def upload(file_to_upload, host, dest_path): subprocess.check_call(command) +def upload_archive(archive, project, make_release): + if project.startswith("kiwix-"): + host = "ci@download.kiwix.org" + dest_path = "/data/download/" + else: + host = "ci@download.openzim.org" + dest_path = "/data/openzim/" + + if make_release: + dest_path = dest_path + "release/" + project + else: + dest_path = dest_path + "nightly/" + DATE + + upload(archive, host, dest_path) + + def make_deps_archive(target=None, name=None, full=False): archive_name = name or "deps2_{}_{}_{}.tar.xz".format( OS_NAME, PLATFORM_TARGET, target @@ -162,3 +249,123 @@ def make_deps_archive(target=None, name=None, full=False): tar.add(str(name), arcname=str(name.relative_to(relative_path))) return archive_file + + +def get_postfix(project): + postfix = main_project_versions[project] + extra = release_versions.get(project) + if extra is not None: + postfix = "{}-{}".format(postfix, extra) + return postfix + + +def make_archive(project, make_release): + try: + platform = PLATFORM_TO_RELEASE[PLATFORM_TARGET] + except KeyError: + # We don't know how to name the release. + return None + + base_dir, export_files = EXPORT_FILES[project] + + if make_release: + postfix = get_postfix(project) + else: + postfix = DATE + + archive_name = "{}_{}-{}".format(project, platform, postfix) + + files_to_archive = [] + for export_file in export_files: + files_to_archive.extend(base_dir.glob(export_file)) + if platform == "win-i686": + open_archive = lambda a: zipfile.ZipFile( + str(a), "w", compression=zipfile.ZIP_DEFLATED + ) + archive_add = lambda a, f: a.write(str(f), arcname=str(f.relative_to(base_dir))) + archive_ext = ".zip" + else: + open_archive = lambda a: tarfile.open(str(a), "w:gz") + archive_add = lambda a, f: a.add( + str(f), arcname="{}/{}".format(archive_name, str(f.relative_to(base_dir))) + ) + archive_ext = ".tar.gz" + + archive = TMP_DIR / "{}{}".format(archive_name, archive_ext) + print_message("create archive {} with {}", archive, files_to_archive) + with open_archive(archive) as arch: + for f in files_to_archive: + archive_add(arch, f) + return archive + + +def create_desktop_image(make_release): + print_message("creating desktop image") + if make_release: + postfix = get_postfix("kiwix-desktop") + src_dir = SOURCE_DIR / "kiwix-desktop_release" + else: + postfix = DATE + src_dir = SOURCE_DIR / "kiwix-desktop" + + if PLATFORM_TARGET == "flatpak": + build_path = BASE_DIR / "org.kiwix.desktop.flatpak" + app_name = "org.kiwix.desktop.{}.flatpak".format(postfix) + print_message("archive is {}", build_path) + else: + build_path = HOME / "Kiwix-{}-x86_64.AppImage".format(postfix) + app_name = "kiwix-desktop_x86_64_{}.appimage".format(postfix) + command = [ + "kiwix-build/scripts/create_kiwix-desktop_appImage.sh", + str(INSTALL_DIR), + str(src_dir), + str(HOME / "AppDir"), + ] + env = dict(os.environ) + env["VERSION"] = postfix + print_message("Build AppImage of kiwix-desktop") + subprocess.check_call(command, cwd=str(HOME), env=env) + + print_message("Copy Build to {}".format(TMP_DIR / app_name)) + shutil.copy(str(build_path), str(TMP_DIR / app_name)) + return TMP_DIR / app_name + + +def update_flathub_git(): + git_repo_dir = TMP_DIR / "org.kiwix.desktop" + env = dict(os.environ) + env["GIT_AUTHOR_NAME"] = env["GIT_COMMITTER_NAME"] = "KiwixBot" + env["GIT_AUTHOR_EMAIL"] = env["GIT_COMMITTER_EMAIL"] = "kiwixbot@kymeria.fr" + + def call(command, cwd=None): + cwd = cwd or git_repo_dir + print_message("call {}", command) + subprocess.check_call(command, env=env, cwd=str(cwd)) + + command = ["git", "clone", FLATPAK_HTTP_GIT_REMOTE] + call(command, cwd=TMP_DIR) + shutil.copy(str(BASE_DIR / "org.kiwix.desktop.json"), str(git_repo_dir)) + patch_dir = KBUILD_SOURCE_DIR / "kiwixbuild" / "patches" + for dep in ("libaria2", "mustache", "pugixml", "xapian", "zstd"): + for f in patch_dir.glob("{}_*.patch".format(dep)): + shutil.copy(str(f), str(git_repo_dir / "patches")) + command = ["git", "add", "-A", "."] + call(command) + command = [ + "git", + "commit", + "-m", + "Update to version {}".format(main_project_versions["kiwix-desktop"]), + ] + try: + call(command) + except subprocess.CalledProcessError: + # This may fail if there is nothing to commit (a rebuild of the CI for exemple) + return + command = ["git", "config", "remote.origin.pushurl", FLATPAK_GIT_REMOTE] + call(command) + command = ["git", "push"] + env["GIT_SSH_COMMAND"] = "ssh -o StrictHostKeyChecking=no -i " + _environ.get( + "SSH_KEY" + ) + call(command) diff --git a/.github/scripts/ensure_base_deps.py b/.github/scripts/ensure_base_deps.py index 004f837..4122d4b 100755 --- a/.github/scripts/ensure_base_deps.py +++ b/.github/scripts/ensure_base_deps.py @@ -15,7 +15,6 @@ from common import ( HOME, PLATFORM_TARGET, OS_NAME, - MAKE_RELEASE, ) def download_base_archive(base_name): @@ -31,14 +30,16 @@ def download_base_archive(base_name): file.write(batch) return file_path +ARCHIVE_NAME_TEMPLATE = "base_deps2_{os}_{platform}_{version}.tar.xz" -ARCHIVE_NAME_TEMPLATE = "base_deps2_{os}_{platform}_{version}_{release}.tar.xz" -base_dep_archive_name = ARCHIVE_NAME_TEMPLATE.format( - os=OS_NAME, - platform=PLATFORM_TARGET, - version=base_deps_meta_version, - release="release" if MAKE_RELEASE else "debug", -) +if PLATFORM_TARGET == 'flatpak': + base_dep_archive_name = "base_deps2_{}_flatpak.tar.xz".format(OS_NAME) +else: + base_dep_archive_name = ARCHIVE_NAME_TEMPLATE.format( + os=OS_NAME, + platform=PLATFORM_TARGET, + version=base_deps_meta_version, + ) print_message("Getting archive {}", base_dep_archive_name) try: @@ -54,7 +55,6 @@ except URLError: os=OS_NAME, platform="android_{}".format(arch), version=base_deps_meta_version, - release="release" if MAKE_RELEASE else "debug", ) print_message("Getting archive {}", archive_name) try: @@ -64,6 +64,8 @@ except URLError: os.remove(str(local_filename)) except URLError: pass + elif PLATFORM_TARGET == "flatpak": + print_message("Cannot get archive. Move on") else: run_kiwix_build("alldependencies", platform=PLATFORM_TARGET) archive_file = make_deps_archive(name=base_dep_archive_name, full=True) diff --git a/.github/scripts/upload_to_bintray.py b/.github/scripts/upload_to_bintray.py new file mode 100755 index 0000000..643be93 --- /dev/null +++ b/.github/scripts/upload_to_bintray.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 + +import os, sys +import json +import requests + +bintray_auth = (os.environ.get('BINTRAY_USER'), os.environ.get('BINTRAY_PASS')) + +def create_version(version): + url = "https://api.bintray.com/packages/kiwix/kiwix/kiwixlib/versions" + payload = { + 'name': version, + 'desc': 'Release of kiwix-lib' + } + headers = { + 'Content-Type': 'application/json' + } + + r = requests.post(url, data=json.dumps(payload), headers=headers, auth=bintray_auth) + rcode = r.status_code + + if rcode == 409: + print("Bintray version %s already exists, skipping." % version) + return True + + rcode_family = rcode // 100 + if rcode_family in (2, 3): + print("Bintray Version created!") + return True + + print("ERROR : Bintray API response {}".format(rcode)) + return False + + +def upload(version, filepath, artefact): + url_template = "https://api.bintray.com/content/kiwix/kiwix/kiwixlib/{version}/org/kiwix/kiwixlib/kiwixlib/{version}/{artefact}" + parameters = { + 'publish': 1, + 'override': 1 + } + + # Upload the main artefact + url = url_template.format(version=version, artefact=artefact) + with open(filepath, 'rb') as f: + r = requests.put(url, data=f, auth=bintray_auth, params=parameters) + + rcode = r.status_code + rcode_family = rcode // 100 + if rcode_family not in (2, 3): + print("ERROR: Fail to upload artefact") + print(r.text) + return False + + return True + + +def upload_from_json(json_path): + basedir = os.path.dirname(str(json_path)) + with open(str(json_path)) as f: + options = json.load(f) + + if not create_version(options['version']): + raise RuntimeError("Cannot create version") + + for file_ in options['files']: + path = os.path.join(basedir, file_) + if not upload(options['version'], path, file_): + raise RuntimeError("Cannot upload file {}".format(file_)) + + +if __name__ == "__main__": + try: + info_file = sys.argv[1] + except IndexError: + print("Usage {} infofile".format(sys.argv[0])) + sys.exit(-1) + + print("Use info file {}".format(info_file)) + try: + upload_from_json(info_file) + except RuntimeError as e: + sys.exit(str(e)) + diff --git a/.github/workflows/releaseNigthly.yml b/.github/workflows/releaseNigthly.yml new file mode 100644 index 0000000..ada69cf --- /dev/null +++ b/.github/workflows/releaseNigthly.yml @@ -0,0 +1,167 @@ +name: Release&Nigthly + +on: + push: + tags: + - r_[0-9]+i + schedule: + - cron: '0 1 * * *' + +env: + DOCKER_VERSION: 28 + +jobs: + Docker: + strategy: + fail-fast: false + matrix: + variant: [xenial, bionic, f31, focal] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Build docker image + shell: bash + run: | + TAGNAME=${{matrix.variant}}-${DOCKER_VERSION} + if [ ! $(curl -sflL https://hub.docker.com/v2/repositories/kiwix/kiwix-build_ci/tags/${TAGNAME}) ] + then + echo "${{secrets.docker_password}}" | docker login -u "${{secrets.docker_username}}" --password-stdin + FULLTAGNAME=kiwix/kiwix-build_ci:${TAGNAME} + docker build -t ${FULLTAGNAME} - < ${GITHUB_WORKSPACE}/.github/ci_images/${{matrix.variant}}_builder.dockerfile + docker push ${FULLTAGNAME} + fi + + Linux: + strategy: + fail-fast: false + matrix: + target: + - native_static + - native_dyn + - native_mixed + - native_desktop + - armhf_static + - win32_static + - i586_static + - android + - flatpak + include: + - target: native_static + image_variant: xenial + lib_postfix: '/x86_64-linux-gnu' + - target: native_dyn + image_variant: xenial + lib_postfix: '/x86_64-linux-gnu' + - target: native_mixed + image_variant: xenial + lib_postfix: '/x86_64-linux-gnu' + - target: native_desktop + image_variant: bionic + lib_postfix: '/x86_64-linux-gnu' + - target: armhf_static + image_variant: xenial + lib_postfix: '/x86_64-linux-gnu' + - target: win32_static + image_variant: f31 + lib_postfix: '64' + - target: i586_static + image_variant: xenial + lib_postfix: '/x86_64-linux-gnu' + - target: android + image_variant: xenial + lib_postfix: '/x86_64-linux-gnu' + - target: flatpak + image_variant: bionic + lib_postfix: '/x86_64-linux-gnu' + env: + HOME: /home/runner + SSH_KEY: /tmp/id_rsa + runs-on: ubuntu-latest + needs: Docker + container: + image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-28" + options: "--device /dev/fuse --privileged" + steps: + - name: Checkout code + shell: bash + run: | + cd $HOME + git clone https://github.com/${REP} --depth=1 --branch ${GITHUB_REF##*heads/} + pip3 install --user --no-deps ./${REP##*/} + env: + REP: ${{github.repository}} + - name: secret + shell: bash + run: | + echo "${{secrets.ssh_key}}" > $SSH_KEY + chmod 600 $SSH_KEY + - name: Ensure base deps + shell: bash + run: | + cd $HOME + kiwix-build/.github/scripts/ensure_base_deps.py + env: + PLATFORM_TARGET: ${{matrix.target}} + - name: Build release + shell: bash + run: | + cd $HOME + kiwix-build/.github/scripts/build_release_nightly.py + env: + PLATFORM_TARGET: ${{matrix.target}} + - name: Upload failure logs + if: failure() + run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh + env: + PLATFORM_TARGET: ${{matrix.target}} + + + Macos: + strategy: + fail-fast: false + matrix: + target: + - native_dyn + runs-on: macos-latest + env: + SSH_KEY: /tmp/id_rsa + OS_NAME: osx + steps: + - name: Checkout code + uses: actions/checkout@v1 + - name: Setup python 3.6 + uses: actions/setup-python@v1 + with: + python-version: '3.6' + - name: Install packages + uses: mstksg/get-package@v1 + with: + brew: pkg-config ninja + - name: Install python modules + run: | + pip3 install meson==0.52.1 pytest requests + pip3 install --no-deps $GITHUB_WORKSPACE + - name: secret + shell: bash + run: | + echo "${{secrets.ssh_key}}" > $SSH_KEY + chmod 600 $SSH_KEY + - name: Ensure base deps + shell: bash + run: | + cd $HOME + $GITHUB_WORKSPACE/.github/scripts/ensure_base_deps.py + env: + PLATFORM_TARGET: ${{matrix.target}} + - name: Build release + shell: bash + run: | + cd $HOME + $GITHUB_WORKSPACE/.github/scripts/build_release_nightly.py + env: + PLATFORM_TARGET: ${{matrix.target}} + - name: Upload failure logs + if: failure() + run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.sh + env: + PLATFORM_TARGET: ${{matrix.target}} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index ae5522c..0000000 --- a/.travis.yml +++ /dev/null @@ -1,149 +0,0 @@ -language: cpp - -dist: xenial -sudo: required -services: -- docker -branches: - only: - - master - - /\d+\.\d+\.\d+$/ - -if: type != push OR tag IS present - -stages: -- build_docker_images -- build -- android_build - -jobs: - include: - - stage: build_docker_images - script: &build_images - - | - if [ ! $(curl -sflL https://hub.docker.com/v2/repositories/kiwix/kiwix-build_ci/tags/${VARIANT}-${DOCKER_VERSION}) ] - then - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - docker build -t kiwix-build_ci -f travis/${VARIANT}_builder.dockerfile . - docker tag kiwix-build_ci kiwix/kiwix-build_ci:${VARIANT}-${DOCKER_VERSION} - docker push kiwix/kiwix-build_ci:${VARIANT}-${DOCKER_VERSION} - fi - env: VARIANT=xenial - - stage: build_docker_images - script: *build_images - env: VARIANT=bionic - - stage: build_docker_images - script: *build_images - env: VARIANT=focal - - stage: build_docker_images - script: *build_images - env: VARIANT=f30 - - stage: android_build - env: PLATFORM="android" - if: type != pull_request - - env: PLATFORM="native_dyn" - os: osx - - env: PLATFORM="iOS_arm64" - os: osx - - - -before_install: -- PATH=$PATH:${HOME}/bin:${HOME}/.local/bin -- mkdir -p /tmp/private $HOME/EXPORT/BASE /$HOME/.cache $HOME/.gradle/caches -- chmod -R a+wX $HOME/EXPORT $HOME/.cache $HOME/.gradle -- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then PATH=$PATH:$(brew --prefix)/opt/gettext/bin; fi -- '[ -n "$encrypted_eba2f7543984_iv" ] && openssl aes-256-cbc - -K $encrypted_eba2f7543984_key -iv $encrypted_eba2f7543984_iv - -in travis/travisci_builder_id_key.enc -out $SSH_KEY -d' -- chmod 600 $SSH_KEY -before_cache: - - rm -f $HOME/.gradle/caches/modules-2/modules-2.lock - - rm -fr $HOME/.gradle/caches/*/plugin-resolution/ -cache: - ccache: true - directories: - - $HOME/.cache/pip - - $HOME/.gradle/caches/ - - $HOME/.gradle/wrapper/ - - $HOME/.android/build-cache -install: if [[ $TRAVIS_OS_NAME != "linux" ]]; then travis/install_extra_deps.sh; fi -stage: build -script: -- | - if [[ $TRAVIS_OS_NAME = "linux" ]] - then - VARIANT=xenial - if [[ $DESKTOP_ONLY = 1 ]]; then VARIANT=bionic; fi - if [[ $PLATFORM =~ ^win32_* ]]; then VARIANT=f30; fi - fi -- | - if [[ $TRAVIS_OS_NAME = "linux" ]] - then - docker run --rm \ - -e PLATFORM -e NIGHTLY_DATE -e TRAVIS_EVENT_TYPE -e DESKTOP_ONLY -e TRAVIS_TAG \ - --device /dev/fuse --cap-add ALL --privileged \ - --mount=type=bind,src=$HOME/EXPORT,dst=/home/ci_builder/EXPORT \ - --mount=type=bind,src=$HOME/.cache,dst=/home/ci_builder/.cache \ - --mount=type=bind,src=$HOME/.gradle/caches,dst=/home/ci_builder/.gradle/caches \ - --mount=type=bind,src=$TRAVIS_BUILD_DIR,dst=/home/ci_builder/kiwix-build \ - kiwix/kiwix-build_ci:${VARIANT}-${DOCKER_VERSION} - fi -- if [[ $TRAVIS_OS_NAME != "linux" ]]; then python3 travis/compile_all.py; fi -after_success: -- travis/upload_base_deps.sh -after_failure: -- travis/upload_base_deps.sh -- travis/upload_all_log.sh -deploy: - - provider: script - skip_cleanup: true - script: travis/deploy.sh - on: - tags: true - condition: $TRAVIS_BUILD_STAGE_NAME != "Build_docker_images" - - provider: script - skip_cleanup: true - script: travis/deploy.sh - on: - branch: master - condition: $TRAVIS_BUILD_STAGE_NAME != "Build_docker_images" && $TRAVIS_EVENT_TYPE = cron -env: - global: - - NIGHTLY_DATE=$(date +%Y-%m-%d) - - SSH_KEY=/tmp/private/travisci_builder_id_key - - DOCKER_VERSION=4 - matrix: - - PLATFORM="flatpak" DESKTOP_ONLY=1 - - PLATFORM="native_dyn" - - PLATFORM="native_static" - - PLATFORM="native_mixed" - - PLATFORM="armhf_dyn" - - PLATFORM="armhf_static" - - PLATFORM="win32_dyn" - - PLATFORM="win32_static" - - PLATFORM="i586_dyn" - - PLATFORM="i586_static" - - PLATFORM="native_dyn" DESKTOP_ONLY=1 - - PLATFORM="android_arm" - - PLATFORM="android_arm64" - - PLATFORM="android_x86" - - PLATFORM="android_x86_64" -addons: - ssh_known_hosts: - - download.kiwix.org - - tmp.kiwix.org - - download.openzim.org - -notifications: - irc: - channels: - - "chat.freenode.net#kiwix" - on_success: change - on_failure: always - email: - recipients: - - contact@kiwix.org - - mgautier@kymeria.fr - on_success: change - on_failure: always diff --git a/scripts/upload_kiwix_lib_android_to_bintray.py b/scripts/upload_kiwix_lib_android_to_bintray.py index cb61e43..0e7a5fe 100755 --- a/scripts/upload_kiwix_lib_android_to_bintray.py +++ b/scripts/upload_kiwix_lib_android_to_bintray.py @@ -54,6 +54,19 @@ def upload(version, filepath, artefact): return True +def upload_from_json(json_path): + basedir = os.path.dirname(json_path) + with open(str(json_path)) as f: + options = json.load(f) + + if not create_version(options['version']): + raise RuntimeError("Cannot create version") + + for file_ in options['files']: + path = os.path.join(basedir, file_) + if not upload(options['version'], path, file_): + raise RuntimeError("Cannot upload file {}".format(file_)) + if __name__ == "__main__": try: @@ -63,14 +76,8 @@ if __name__ == "__main__": sys.exit(-1) print("Use info file {}".format(info_file)) - with open(info_file) as f: - options = json.load(f) + try: + upload_from_json(info_file) + except RuntimeError as e: + sys.exit(str(e)) - if not create_version(options['version']): - sys.exit("Cannot create version") - - basedir = os.path.dirname(sys.argv[1]) - for file_ in options['files']: - path = os.path.join(basedir, file_) - if not upload(options['version'], path, file_): - sys.exit("Cannot upload file {}".format(file_)) diff --git a/travis/bionic_builder.dockerfile b/travis/bionic_builder.dockerfile deleted file mode 100644 index daecbf2..0000000 --- a/travis/bionic_builder.dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -FROM ubuntu:bionic - -ENV LANG C.UTF-8 - -RUN apt update -q && \ - apt install -q -y --no-install-recommends \ -# Base build tools - build-essential automake libtool cmake ccache pkg-config autopoint patch \ - python3-pip python3-setuptools python3-wheel git subversion wget unzip \ - ninja-build \ -# Packaged dependencies - libbz2-dev libmagic-dev uuid-dev zlib1g-dev \ - libmicrohttpd-dev aria2 \ -# Qt packages - libqt5gui5 qtbase5-dev qtwebengine5-dev libqt5svg5-dev qt5-image-formats-plugins qt5-default \ -# To create the appimage of kiwix-desktop - libfuse2 fuse patchelf \ -# Flatpak tools - elfutils flatpak flatpak-builder \ -# Cross win32 compiler - g++-mingw-w64-i686 gcc-mingw-w64-i686 gcc-mingw-w64-base mingw-w64-tools \ -# Other tools (to remove) -# vim less grep \ - && \ - apt-get clean -y && \ - rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/* - -# Create user -RUN useradd --create-home ci_builder -USER ci_builder -WORKDIR /home/ci_builder -ENV PATH="/home/ci_builder/.local/bin:${PATH}" - -ENV TRAVIS_BUILD_DIR /home/ci_builder/kiwix-build -ENV TRAVIS_OS_NAME linux_bionic - -CMD pip3 install --user ./kiwix-build && kiwix-build/travis/compile_all.py diff --git a/travis/compile_all.py b/travis/compile_all.py deleted file mode 100755 index 2414b69..0000000 --- a/travis/compile_all.py +++ /dev/null @@ -1,527 +0,0 @@ -#!/usr/bin/env python3 - -import sys, os -import shutil -from os import environ -from pathlib import Path -from datetime import date -import tarfile, zipfile -import subprocess -import re -import json -from urllib.request import urlopen -from urllib.error import URLError - -from kiwixbuild.versions import ( - main_project_versions, - release_versions, - base_deps_versions, - base_deps_meta_version) - -PLATFORM = environ['PLATFORM'] -TRAVIS_OS_NAME = environ['TRAVIS_OS_NAME'] -HOME = Path(os.path.expanduser('~')) -NIGHTLY_DATE = environ['NIGHTLY_DATE'] -KBUILD_SOURCE_DIR = Path(environ['TRAVIS_BUILD_DIR']) -KIWIX_DESKTOP_ONLY = environ.get('DESKTOP_ONLY') == '1' - -BASE_DIR = HOME/"BUILD_{}".format(PLATFORM) -EXPORT_DIR = HOME/'EXPORT' -BASE_EXPORT_DIR = EXPORT_DIR/"BASE" -GIT_EXPORT_DIR = EXPORT_DIR/"GIT" -SOURCE_DIR = HOME/"SOURCE" -ARCHIVE_DIR = HOME/"ARCHIVE" -INSTALL_DIR = BASE_DIR/"INSTALL" -NIGHTLY_KIWIX_ARCHIVES_DIR = EXPORT_DIR/'NIGHTLY_KIWIX_ARCHIVES'/NIGHTLY_DATE -RELEASE_KIWIX_ARCHIVES_DIR = EXPORT_DIR/'RELEASE_KIWIX_ARCHIVES' -NIGHTLY_ZIM_ARCHIVES_DIR = EXPORT_DIR/'NIGHTLY_ZIM_ARCHIVES'/NIGHTLY_DATE -RELEASE_ZIM_ARCHIVES_DIR = EXPORT_DIR/'RELEASE_ZIM_ARCHIVES' -DIST_KIWIX_ARCHIVES_DIR = EXPORT_DIR/'DIST_KIWIX_ARCHIVES' -DIST_ZIM_ARCHIVES_DIR = EXPORT_DIR/'DIST_ZIM_ARCHIVES' -BINTRAY_ARCHIVES_DIR = EXPORT_DIR/'BINTRAY_ARCHIVES' - -BIN_EXT = '.exe' if PLATFORM.startswith('win32_') else '' - -# We have build everything. Now create archives for public deployement. -EXPORT_FILES = { - 'kiwix-tools': - (INSTALL_DIR/'bin', [f+BIN_EXT for f in ('kiwix-manage', 'kiwix-read', 'kiwix-search', 'kiwix-serve')]), - 'zim-tools': - (INSTALL_DIR/'bin', [f+BIN_EXT for f in ('zimbench', 'zimcheck', 'zimdump', 'zimsearch', 'zimdiff', 'zimpatch', 'zimsplit')]), - 'zimwriterfs': - (INSTALL_DIR/'bin', ['zimwriterfs'+BIN_EXT]), - 'libzim': - (INSTALL_DIR, ('lib/x86_64-linux-gnu/libzim.so.{}'.format(main_project_versions['libzim']), - 'lib/x86_64-linux-gnu/libzim.so.{}'.format(main_project_versions['libzim'][0]), - 'include/zim/**/*.h')) -} - -FLATPAK_HTTP_GIT_REMOTE = 'https://github.com/flathub/org.kiwix.desktop.git' -FLATPAK_GIT_REMOTE = 'git@github.com:flathub/org.kiwix.desktop.git' - -_date = date.today().isoformat() - -def print_message(message, *args, **kwargs): - message = message.format(*args, **kwargs) - message = "{0} {1} {0}".format('-'*3, message) - print(message, flush=True) - - -def write_manifest(manifest_file, archive_name, target, platform): - with manifest_file.open(mode='w') as f: - f.write('''{archive_name} -*************************** - -Dependencies archive for {target} on platform {platform} -Generated at {date} -'''.format( - archive_name=archive_name, - target=target, - platform=platform, - date=date.today().isoformat())) - - -def run_kiwix_build(target, platform, - build_deps_only=False, - target_only=False, - make_release=False, - make_dist=False): - command = ['kiwix-build'] - command.append(target) - command.append('--hide-progress') - command.append('--fast-clone') - if platform == 'flatpak' or platform.startswith('win32_'): - command.append('--assume-packages-installed') - if target == 'kiwix-lib-app' and platform.startswith('android_'): - command.extend(['--target-platform', 'android', '--android-arch', platform[8:]]) - elif platform == 'android': - command.extend(['--target-platform', 'android']) - for arch in ('arm', 'arm64', 'x86', 'x86_64'): - command.extend(['--android-arch', arch]) - else: - command.extend(['--target-platform', platform]) - if build_deps_only: - command.append('--build-deps-only') - if target_only: - command.append('--build-nodeps') - if make_release: - command.append('--make-release') - if make_dist: - command.append('--make-dist') - print_message("Build {} (deps={}, release={}, dist={})", - target, build_deps_only, make_release, make_dist) - subprocess.check_call(command, cwd=str(HOME)) - print_message("{}", list(BASE_DIR.glob("{}*/**/*.tar.xz".format(target)))) - print_message("Build ended") - - -def create_desktop_image(): - print_message("creating desktop image") - if make_release: - postfix = main_project_versions['kiwix-desktop'] - extra_postfix = release_versions.get('kiwix-desktop') - if extra_postfix is None: - # We should not make archives for release not wanted - return - if extra_postfix: - postfix = "{}-{}".format(postfix, extra_postfix) - archive_dir = RELEASE_KIWIX_ARCHIVES_DIR/'kiwix-desktop' - src_dir = SOURCE_DIR/'kiwix-desktop_release' - else: - postfix = _date - archive_dir = NIGHTLY_KIWIX_ARCHIVES_DIR - src_dir = SOURCE_DIR/'kiwix-desktop' - - if PLATFORM == 'flatpak': - build_path = BASE_DIR/'org.kiwix.desktop.flatpak' - app_name = 'org.kiwix.desktop.{}.flatpak'.format(postfix) - print_message("archive is {}", build_path) - else: - build_path = HOME/'Kiwix-{}-x86_64.AppImage'.format(postfix) - app_name = "kiwix-desktop_x86_64_{}.appimage".format(postfix) - command = ['kiwix-build/scripts/create_kiwix-desktop_appImage.sh', - str(INSTALL_DIR), str(src_dir), str(HOME/'AppDir')] - env = dict(os.environ) - env['VERSION'] = postfix - print_message("Build AppImage of kiwix-desktop") - subprocess.check_call(command, cwd=str(HOME), env=env) - - try: - archive_dir.mkdir(parents=True) - except FileExistsError: - pass - - print_message("Copy Build to {}".format(archive_dir/app_name)) - shutil.copy(str(build_path), str(archive_dir/app_name)) - - -def make_archive(project, platform): - base_dir, export_files = EXPORT_FILES[project] - - if make_release: - postfix = main_project_versions[project] - extra_postfix = release_versions.get(project) - if extra_postfix is None: - # We should not make archives for release not wanted - return - if extra_postfix: - postfix = "{}-{}".format(postfix, extra_postfix) - if project in ('kiwix-lib', 'kiwix-tools'): - archive_dir = RELEASE_KIWIX_ARCHIVES_DIR/project - else: - archive_dir = RELEASE_ZIM_ARCHIVES_DIR/project - else: - postfix = _date - if project in ('kiwix-lib', 'kiwix-tools'): - archive_dir = NIGHTLY_KIWIX_ARCHIVES_DIR - else: - archive_dir = NIGHTLY_ZIM_ARCHIVES_DIR - - try: - archive_dir.mkdir(parents=True) - except FileExistsError: - pass - - archive_name = "{}_{}-{}".format(project, platform, postfix) - - files_to_archive = [] - for export_file in export_files: - files_to_archive.extend(base_dir.glob(export_file)) - if platform == "win-i686": - open_archive = lambda a : zipfile.ZipFile(str(a), 'w', compression=zipfile.ZIP_DEFLATED) - archive_add = lambda a, f : a.write(str(f), arcname=str(f.relative_to(base_dir))) - archive_ext = ".zip" - else: - open_archive = lambda a : tarfile.open(str(a), 'w:gz') - archive_add = lambda a, f : a.add(str(f), arcname="{}/{}".format(archive_name, str(f.relative_to(base_dir)))) - archive_ext = ".tar.gz" - - - archive = archive_dir/'{}{}'.format(archive_name, archive_ext) - print_message("create archive {} with {}", archive, files_to_archive) - with open_archive(archive) as arch: - for f in files_to_archive: - archive_add(arch, f) - - -def make_deps_archive(target=None, name=None, full=False): - archive_name = name or "deps_{}_{}_{}.tar.xz".format( - TRAVIS_OS_NAME, PLATFORM, target) - print_message("Create archive {}.", archive_name) - files_to_archive = [INSTALL_DIR] - files_to_archive += HOME.glob('BUILD_*/LOGS') - if PLATFORM == 'native_mixed': - files_to_archive += [HOME/'BUILD_native_static'/'INSTALL'] - if PLATFORM.startswith('android'): - files_to_archive.append(HOME/'BUILD_neutral'/'INSTALL') - if PLATFORM == 'android': - for arch in ('arm', 'arm64', 'x86', 'x86_64'): - base_dir = HOME/"BUILD_android_{}".format(arch) - files_to_archive.append(base_dir/'INSTALL') - if (base_dir/'meson_cross_file.txt').exists(): - files_to_archive.append(base_dir/'meson_cross_file.txt') - files_to_archive += HOME.glob('BUILD_*/android-ndk*') - files_to_archive += HOME.glob('BUILD_*/android-sdk*') - if (BASE_DIR/'meson_cross_file.txt').exists(): - files_to_archive.append(BASE_DIR/'meson_cross_file.txt') - - manifest_file = BASE_DIR/'manifest.txt' - write_manifest(manifest_file, archive_name, target, PLATFORM) - files_to_archive.append(manifest_file) - - relative_path = HOME - if full: - files_to_archive += ARCHIVE_DIR.glob(".*_ok") - files_to_archive += BASE_DIR.glob('*/.*_ok') - files_to_archive += (HOME/"BUILD_native_dyn").glob('*/.*_ok') - files_to_archive += (HOME/"BUILD_native_static").glob('*/.*_ok') - files_to_archive += HOME.glob('BUILD_android*/**/.*_ok') - files_to_archive += SOURCE_DIR.glob('*/.*_ok') - if PLATFORM.startswith('armhf'): - files_to_archive += (SOURCE_DIR/'armhf').glob('*') - toolchains_subdirs = HOME.glob('**/TOOLCHAINS/*/*') - for subdir in toolchains_subdirs: - if not subdir.match('tools'): - files_to_archive.append(subdir) - - with tarfile.open(str(BASE_EXPORT_DIR/archive_name), 'w:xz') as tar: - for name in set(files_to_archive): - print('.{}'.format(name), flush=True) - tar.add(str(name), arcname=str(name.relative_to(relative_path))) - -def make_flatpak_cache_archive(): - archive_name = "base_deps_{os}_{platform}.tar.xz".format( - os=TRAVIS_OS_NAME, - platform=PLATFORM) - cache = BASE_DIR/'.flatpak-builder' - print_message("make flatpak cache archive") - nb_files = 0 - def pseudo_filter(tarinfo): - nonlocal nb_files - nb_files += 1 - if (nb_files % 1000) == 0: - print('.', flush=True) - return tarinfo - with tarfile.open(str(BASE_EXPORT_DIR/archive_name), 'w:xz') as tar: - tar.add(str(cache), arcname=str(cache.relative_to(BASE_DIR)), filter=pseudo_filter) - - -def update_flathub_git(): - GIT_REPO_DIR = GIT_EXPORT_DIR/"org.kiwix.desktop" - env = dict(os.environ) - env['GIT_AUTHOR_NAME'] = env['GIT_COMMITTER_NAME'] = "KiwixBot" - env['GIT_AUTHOR_EMAIL'] = env['GIT_COMMITTER_EMAIL'] = "kiwixbot@kymeria.fr" - def call(command, cwd=None): - cwd = cwd or GIT_REPO_DIR - print_message("call {}", command) - subprocess.check_call(command, env=env, cwd=str(cwd)) - command = ['git', 'clone', FLATPAK_HTTP_GIT_REMOTE] - call(command, cwd=GIT_EXPORT_DIR) - shutil.copy(str(BASE_DIR/'org.kiwix.desktop.json'), str(GIT_REPO_DIR)) - patch_dir = KBUILD_SOURCE_DIR/'kiwixbuild'/'patches' - for dep in ('libaria2', 'mustache', 'pugixml', 'xapian', 'zstd'): - for f in patch_dir.glob('{}_*.patch'.format(dep)): - shutil.copy(str(f), str(GIT_REPO_DIR/'patches')) - command = ['git', 'add', '-A', '.'] - call(command) - command = ['git', 'commit', '-m', - 'Update to version {}'.format(main_project_versions['kiwix-desktop'])] - try: - call(command) - except subprocess.CalledProcessError: - # This may fail if there is nothing to commit (a rebuild of the CI for exemple) - pass - command = ['git', 'config', 'remote.origin.pushurl', FLATPAK_GIT_REMOTE] - call(command) - - - -for p in (NIGHTLY_KIWIX_ARCHIVES_DIR, - NIGHTLY_ZIM_ARCHIVES_DIR, - RELEASE_KIWIX_ARCHIVES_DIR, - RELEASE_ZIM_ARCHIVES_DIR, - DIST_KIWIX_ARCHIVES_DIR, - DIST_ZIM_ARCHIVES_DIR, - BINTRAY_ARCHIVES_DIR, - BASE_EXPORT_DIR, - GIT_EXPORT_DIR): - try: - p.mkdir(parents=True) - except FileExistsError: - pass - -make_release = re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", environ.get('TRAVIS_TAG', '')) is not None - -def download_base_archive(base_name): - url = 'http://tmp.kiwix.org/ci/{}'.format(base_name) - file_path = str(HOME/base_name) - batch_size = 1024*1024*8 - with urlopen(url) as resource, open(file_path, 'wb') as file: - while True: - batch = resource.read(batch_size) - if not batch: - break - print(".", end="", flush=True) - file.write(batch) - return file_path - -def export_dist(target): - if target in ('kiwix-lib', 'kiwix-tools', 'kiwix-desktop'): - out_dir = DIST_KIWIX_ARCHIVES_DIR - else: - out_dir = DIST_ZIM_ARCHIVES_DIR - - if target in main_project_versions: - try: - (out_dir/target).mkdir(parents=True) - except FileExistsError: - pass - - full_target_name = "{}-{}".format(target, main_project_versions[target]) - if target != 'kiwix-desktop': - in_file = BASE_DIR/full_target_name/'meson-dist'/'{}.tar.xz'.format( - full_target_name) - else: - in_file = BASE_DIR/full_target_name/'{}.tar.gz'.format(full_target_name) - if in_file.exists(): - print_message("Copying {} to {}", in_file, out_dir/target) - shutil.copy(str(in_file), str(out_dir/target)) - else: - print_message("No {} to copy.", in_file) - - -if PLATFORM != 'flatpak': - # The first thing we need to do is to (potentially) download already compiled base dependencies. - base_dep_archive_name = "base_deps_{os}_{platform}_{version}_{release}.tar.xz".format( - os=TRAVIS_OS_NAME, - platform=PLATFORM, - version=base_deps_meta_version, - release='release' if make_release else 'debug') - - print_message("Getting archive {}", base_dep_archive_name) - try: - local_filename = download_base_archive(base_dep_archive_name) - with tarfile.open(local_filename) as f: - f.extractall(str(HOME)) - except URLError: - print_message("Cannot get archive. Build dependencies") - if PLATFORM == 'android': - for arch in ('arm', 'arm64', 'x86', 'x86_64'): - archive_name = "base_deps_{os}_android_{arch}_{version}_{release}.tar.xz".format( - os=TRAVIS_OS_NAME, - arch=arch, - version=base_deps_meta_version, - release='release' if make_release else 'debug') - print_message("Getting archive {}", archive_name) - try: - local_filename = download_base_archive(archive_name) - with tarfile.open(local_filename) as f: - f.extractall(str(HOME)) - except URLError: - pass - else: - run_kiwix_build('alldependencies', platform=PLATFORM) - make_deps_archive(name=base_dep_archive_name, full=True) -else: - base_dep_archive_name = "base_deps_{os}_{platform}.tar.xz".format( - os=TRAVIS_OS_NAME, - platform=PLATFORM) - - print_message("Getting archive {}", base_dep_archive_name) - try: - local_filename = download_base_archive(base_dep_archive_name) - BASE_DIR.mkdir(parents=True) - with tarfile.open(local_filename) as f: - f.extractall(str(BASE_DIR)) - except URLError: - print_message("Cannot get archive. Move on") - - -# A basic compilation to be sure everything is working (for a PR) -if environ['TRAVIS_EVENT_TYPE'] != 'cron' and not make_release: - if PLATFORM.startswith('android'): - TARGETS = ('kiwix-lib',) - elif PLATFORM.startswith('iOS'): - TARGETS = ('kiwix-lib',) - elif PLATFORM.startswith('native_'): - if TRAVIS_OS_NAME == "osx": - TARGETS = ('kiwix-lib', 'zim-tools', 'zimwriterfs') - elif PLATFORM == 'native_dyn' and KIWIX_DESKTOP_ONLY: - TARGETS = ('kiwix-desktop', ) - elif PLATFORM == 'native_mixed': - TARGETS = ('libzim', ) - else: - TARGETS = ('kiwix-tools', 'zim-tools', 'zimwriterfs') - elif PLATFORM == 'flatpak': - TARGETS = ('kiwix-desktop', ) - else: - TARGETS = ('kiwix-tools', ) - - for target in TARGETS: - run_kiwix_build(target, - platform=PLATFORM) - - if PLATFORM == 'native_mixed': - make_archive('libzim', 'linux-x86_64') - elif PLATFORM == 'native_static': - for target in ('kiwix-tools', 'zim-tools', 'zimwriterfs'): - make_archive(target, 'linux-x86_64') - elif PLATFORM == 'win32_static': - make_archive('kiwix-tools', 'win-i686') - elif PLATFORM == 'armhf_static': - make_archive('kiwix-tools', 'linux-armhf') - elif PLATFORM == 'i586_static': - make_archive('kiwix-tools', 'linux-i586') - elif PLATFORM == 'flatpak': - make_flatpak_cache_archive() - - sys.exit(0) - - -TARGETS = tuple() -if PLATFORM == 'android': - TARGETS = ('kiwix-lib-app',) -elif PLATFORM.startswith('android_'): - TARGETS = ('libzim', 'kiwix-lib') -elif PLATFORM.startswith('iOS'): - TARGETS = ('libzim', 'kiwix-lib') -elif PLATFORM.startswith('native_'): - if TRAVIS_OS_NAME == "osx": - TARGETS = ('libzim', 'zimwriterfs', 'zim-tools', 'kiwix-lib') - else: - if PLATFORM == 'native_dyn' and KIWIX_DESKTOP_ONLY: - TARGETS = ('kiwix-desktop', ) - elif PLATFORM == 'native_mixed': - TARGETS = ('libzim', ) - else: - TARGETS = ('libzim', 'zimwriterfs', 'zim-tools', 'kiwix-lib', 'kiwix-tools') -elif PLATFORM == 'flatpak': - TARGETS = ('kiwix-desktop', ) -else: - TARGETS = ('libzim', 'zim-tools', 'kiwix-lib', 'kiwix-tools') - -for target in TARGETS: - if environ['TRAVIS_EVENT_TYPE'] == 'cron' and PLATFORM not in ('android', 'flatpak'): - run_kiwix_build(target, - platform=PLATFORM, - build_deps_only=True) - make_deps_archive(target=target) - - if target == 'kiwix-lib-app': - extra_postfix = release_versions.get('kiwix-lib') - if extra_postfix: - environ['KIWIXLIB_BUILDVERSION'] = extra_postfix - - run_kiwix_build(target, - platform=PLATFORM, - make_release=make_release) - print_message("target is {}", target) - if target == 'kiwix-desktop': - create_desktop_image() - if make_release and PLATFORM == 'native_dyn' and release_versions.get(target) == 0: - run_kiwix_build(target, - platform=PLATFORM, - make_release=True, - make_dist=True) - export_dist(target) - -# We have build everything. Now create archives for public deployement. -if PLATFORM == 'native_mixed': - make_archive('libzim', 'linux-x86_64') -elif PLATFORM == 'native_static': - for target in ('kiwix-tools', 'zim-tools', 'zimwriterfs'): - make_archive(target, 'linux-x86_64') -elif PLATFORM == 'win32_static': - make_archive('kiwix-tools', 'win-i686') -elif PLATFORM == 'armhf_static': - make_archive('kiwix-tools', 'linux-armhf') -elif PLATFORM == 'i586_static': - make_archive('kiwix-tools', 'linux-i586') -elif ( make_release - and PLATFORM == 'flatpak' - and release_versions.get('kiwix-desktop') is not None): - update_flathub_git() -elif PLATFORM == 'android' and 'kiwix-lib-app' in TARGETS: - if make_release and release_versions.get('kiwix-lib') is not None: - postfix = main_project_versions['kiwix-lib'] - extra_postfix = release_versions.get('kiwix-lib') - if extra_postfix: - postfix = "{}-{}".format(postfix, extra_postfix) - - basename = "kiwixlib-{}".format(postfix) - - output_release_dir = HOME/'BUILD_android'/'kiwix-lib-app'/'kiwixLibAndroid'/'build' - shutil.copy(str(output_release_dir/'outputs'/'aar'/'kiwixLibAndroid-release.aar'), - str(BINTRAY_ARCHIVES_DIR/(basename+'.aar'))) - shutil.copy(str(output_release_dir/'pom.xml'), - str(BINTRAY_ARCHIVES_DIR/(basename+'.pom'))) - - - json_filename = '{}_bintray_info.json'.format(basename) - data = { - 'version': postfix, - 'files': [basename+ext for ext in ('.aar', '.pom')] - } - with open(str(BINTRAY_ARCHIVES_DIR/json_filename), 'w') as f: - json.dump(data, f) - diff --git a/travis/deploy.sh b/travis/deploy.sh deleted file mode 100755 index fcb56cd..0000000 --- a/travis/deploy.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env bash - -set -e - -EXPORT_DIR=${HOME}/EXPORT -NIGHTLY_KIWIX_ARCHIVES_DIR=${EXPORT_DIR}/NIGHTLY_KIWIX_ARCHIVES/${NIGHTLY_DATE} -NIGHTLY_ZIM_ARCHIVES_DIR=${EXPORT_DIR}/NIGHTLY_ZIM_ARCHIVES/${NIGHTLY_DATE} -RELEASE_KIWIX_ARCHIVES_DIR=${EXPORT_DIR}/RELEASE_KIWIX_ARCHIVES -RELEASE_ZIM_ARCHIVES_DIR=${EXPORT_DIR}/RELEASE_ZIM_ARCHIVES -DIST_KIWIX_ARCHIVES_DIR=${EXPORT_DIR}/DIST_KIWIX_ARCHIVES -DIST_ZIM_ARCHIVES_DIR=${EXPORT_DIR}/DIST_ZIM_ARCHIVES -BINTRAY_ARCHIVES_DIR=${EXPORT_DIR}/BINTRAY_ARCHIVES - -if [[ "$TRAVIS_EVENT_TYPE" = "cron" ]] -then - echo "Publishing nightlies" - ls ${NIGHTLY_KIWIX_ARCHIVES_DIR} - scp -rp -i ${SSH_KEY} -o StrictHostKeyChecking=no \ - ${NIGHTLY_KIWIX_ARCHIVES_DIR} \ - ci@download.kiwix.org:/data/download/nightly - ls ${NIGHTLY_ZIM_ARCHIVES_DIR} - scp -rp -i ${SSH_KEY} -o StrictHostKeyChecking=no \ - ${NIGHTLY_ZIM_ARCHIVES_DIR} \ - ci@download.openzim.org:/data/openzim/nightly - -elif [[ "x$TRAVIS_TAG" != "x" ]] -then - RELEASE_ARCHIVES=$(find $RELEASE_KIWIX_ARCHIVES_DIR -type f) - if [[ "x$RELEASE_ARCHIVES" != "x" ]] - then - echo "Publishing kiwix archives $RELEASE_ARCHIVES" - for archive in $RELEASE_ARCHIVES - do - subdir=$(basename $(dirname $archive)) - scp -rp -i ${SSH_KEY} -o StrictHostKeyChecking=no \ - ${archive} \ - ci@download.kiwix.org:/data/download/release/${subdir} - done - fi - - RELEASE_ARCHIVES=$(find $RELEASE_ZIM_ARCHIVES_DIR -type f) - if [[ "x$RELEASE_ARCHIVES" != "x" ]] - then - echo "Publishing zim archives $RELEASE_ARCHIVES" - for archive in $RELEASE_ARCHIVES - do - subdir=$(basename $(dirname $archive)) - scp -rp -i ${SSH_KEY} -o StrictHostKeyChecking=no \ - ${archive} \ - ci@download.openzim.org:/data/openzim/release/${subdir} - done - fi - - DIST_KIWIX_ARCHIVES=$(find $DIST_KIWIX_ARCHIVES_DIR -type f) - if [[ "x$DIST_KIWIX_ARCHIVES" != "x" ]] - then - echo "Publishing dist kiwix archives $DIST_KIWIX_ARCHIVES" - for archive in $DIST_KIWIX_ARCHIVES - do - subdir=$(basename $(dirname $archive)) - scp -rp -i ${SSH_KEY} -o StrictHostKeyChecking=no \ - ${archive} \ - ci@download.kiwix.org:/data/download/release/${subdir} - done - fi - - DIST_ZIM_ARCHIVES=$(find $DIST_ZIM_ARCHIVES_DIR -type f) - if [[ "x$DIST_ZIM_ARCHIVES" != "x" ]] - then - echo "Publishing dist zim archives $DIST_ZIM_ARCHIVES" - for archive in $DIST_ZIM_ARCHIVES - do - subdir=$(basename $(dirname $archive)) - scp -rp -i ${SSH_KEY} -o StrictHostKeyChecking=no \ - ${archive} \ - ci@download.openzim.org:/data/openzim/release/${subdir} - done - fi - - (cd ${EXPORT_DIR}/GIT - GIT_REPOS=$(ls -l | awk '/^d/ { print $9 }') - if [[ "x$GIT_REPOS" != "x" ]] - then - echo "Pushing repositories $GIT_REPOS" - for repo in $GIT_REPOS - do - (cd $repo; - GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=no -i $SSH_KEY" git push - ) - done - fi - ) - - ls $BINTRAY_ARCHIVES_DIR - BINTRAY_ARCHIVES=$(find $BINTRAY_ARCHIVES_DIR -name "*_bintray_info.json" -type f) - if [[ "x$BINTRAY_ARCHIVES" != "x" ]] - then - echo "Publish bintray $BINTRAY_ARCHIVES" - for archive_info in $BINTRAY_ARCHIVES - do - ${TRAVIS_BUILD_DIR}/scripts/upload_kiwix_lib_android_to_bintray.py $archive_info - done - fi - -fi - diff --git a/travis/f30_builder.dockerfile b/travis/f30_builder.dockerfile deleted file mode 100644 index 99d68fe..0000000 --- a/travis/f30_builder.dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -FROM fedora:30 - -ENV LANG C.UTF-8 - -RUN dnf install -y --nodocs \ -# Base build tools - make automake libtool cmake git-core subversion pkg-config gcc-c++ \ - wget unzip ninja-build ccache which patch \ -# Cross win32 compiler - mingw32-gcc-c++ mingw32-bzip2-static mingw32-win-iconv-static \ - mingw32-winpthreads-static mingw32-zlib-static mingw32-xz-libs-static \ - mingw32-libmicrohttpd \ -# Other tools (to remove) -# vim less grep - && dnf remove -y "*-doc" \ - && dnf autoremove -y \ - && dnf clean all - -# Create user -RUN useradd --create-home ci_builder -USER ci_builder -WORKDIR /home/ci_builder -ENV PATH="/home/ci_builder/.local/bin:${PATH}" - -ENV TRAVIS_BUILD_DIR /home/ci_builder/kiwix-build -ENV TRAVIS_OS_NAME linux_f30 - -CMD pip3 install --user ./kiwix-build && kiwix-build/travis/compile_all.py diff --git a/travis/focal_builder.dockerfile b/travis/focal_builder.dockerfile deleted file mode 100644 index dba937a..0000000 --- a/travis/focal_builder.dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -FROM ubuntu:focal - -ENV LANG C.UTF-8 - -RUN apt update -q && \ - apt install -q -y --no-install-recommends \ -# Base build tools - build-essential automake libtool cmake ccache pkg-config autopoint patch \ - python3-pip python3-setuptools python3-wheel git subversion wget unzip \ - ninja-build \ -# Packaged dependencies - libbz2-dev libmagic-dev uuid-dev zlib1g-dev \ - libmicrohttpd-dev aria2 \ -# Qt packages - libqt5gui5 qtbase5-dev qtwebengine5-dev libqt5svg5-dev qt5-image-formats-plugins qt5-default \ -# To create the appimage of kiwix-desktop - libfuse2 fuse patchelf \ -# Flatpak tools - elfutils flatpak flatpak-builder \ -# Cross win32 compiler - g++-mingw-w64-i686 gcc-mingw-w64-i686 gcc-mingw-w64-base mingw-w64-tools \ -# Other tools (to remove) -# vim less grep \ - && \ - apt-get clean -y && \ - rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/* - -# Create user -RUN useradd --create-home ci_builder -USER ci_builder -WORKDIR /home/ci_builder -ENV PATH="/home/ci_builder/.local/bin:${PATH}" - -ENV TRAVIS_BUILD_DIR /home/ci_builder/kiwix-build -ENV TRAVIS_OS_NAME linux_focal - -CMD pip3 install --user ./kiwix-build && kiwix-build/travis/compile_all.py diff --git a/travis/install_extra_deps.sh b/travis/install_extra_deps.sh deleted file mode 100755 index 5777030..0000000 --- a/travis/install_extra_deps.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -set -e - -if [[ "$TRAVIS_OS_NAME" == "osx" ]] -then - pip3 install pillow - pip3 install . - - wget https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-mac.zip - unzip ninja-mac.zip ninja -else - wget https://bootstrap.pypa.io/get-pip.py - python3.5 get-pip.py --user - python3.5 -m pip install --user --upgrade pip wheel - python3.5 -m pip install --user pillow - python3.5 -m pip install --user . - python3.5 -m pip show -f kiwix-build - - wget https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip - unzip ninja-linux.zip ninja -fi - -mkdir -p $HOME/bin -cp ninja $HOME/bin - diff --git a/travis/travisci_builder_id_key.enc b/travis/travisci_builder_id_key.enc deleted file mode 100644 index 7d09269..0000000 Binary files a/travis/travisci_builder_id_key.enc and /dev/null differ diff --git a/travis/travisci_builder_id_key.pub b/travis/travisci_builder_id_key.pub deleted file mode 100644 index 12e88a2..0000000 --- a/travis/travisci_builder_id_key.pub +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCWxcsEawR4TrXmGgMXXOhuYjXtO1V/BpEQ2tuf2tTiloZ0n1cPtxeLp3ic7F91p/e+58U1QlpLkxs9Ia0REXvr2ppd3FuIRjADs6g9iqK4jgAx/KKVjy2vhtxFtDKP5xCkbqSRjDcYnSZwrllEPbsOEIgTvRBaOyyqBNbctjSN6DhB+hY1bsPNJntDL0/XLqSYFXBnxd+unRLiItRJX1C42tpTwWyr9OvpjuPCjWVo/6WqohU8xjW29xPDpmAnWCX02JD//hRamJ+zfDPxf5AGTVJhk7of4Jjj4pTunhTXRLGhA3UBPApbI//NA2nAOc8feOchu1VfO6n5JuVbnXDzUlMFVqSNJ6ihYBTEczN+T/xbdc3Uyzi3JZX7IW2lH1YpxJoGEzJ67HjNPHdKgQzqVGlKl00qUrL1aMJ+1rOm8xDvPYVjHidORPUn0x+2fvxJ0VVr4crVwmTa23ft79MS2sLbw5DbPkURVUkEH88nSwcbF9fbuCPE4RL15NS7ZNwDqRMyQIxnAe7Sf+UWz45HxTrdvgjrsk0PCYX+KCgnXEexXWDDlbXWuiJa1h2VhKb2zM3AUmtRMK0pFTi/xCrFFwDIYYIBOMxMb0nYmruv5Feh/cez1FNYq9+UKrnBAV7DAZCLibM2/u46OAgbbtZS70zaX1JvcX44yUgjpClNcQ== TravisCIBuilder diff --git a/travis/upload_all_log.sh b/travis/upload_all_log.sh deleted file mode 100755 index 057f509..0000000 --- a/travis/upload_all_log.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -set -e - -cd $HOME - -tar -czf fail_log_${PLATFORM}.tar.gz BUILD_${PLATFORM} - -scp -vrp -i ${SSH_KEY} \ - fail_log_${PLATFORM}.tar.gz \ - ci@tmp.kiwix.org:/data/tmp/ci diff --git a/travis/upload_base_deps.sh b/travis/upload_base_deps.sh deleted file mode 100755 index 6c5f232..0000000 --- a/travis/upload_base_deps.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -set -e - -BASE_EXPORT_DIR=$HOME/EXPORT/BASE -ls $BASE_EXPORT_DIR - -BASE_ARCHIVES=$(find $BASE_EXPORT_DIR -type f) -echo $BASE_ARCHIVES -if [[ "x$BASE_ARCHIVES" != "x" ]] -then - scp -vrp -i ${SSH_KEY} -o StrictHostKeyChecking=no \ - $BASE_ARCHIVES \ - ci@tmp.kiwix.org:/data/tmp/ci -fi diff --git a/travis/xenial_builder.dockerfile b/travis/xenial_builder.dockerfile deleted file mode 100644 index e0b7bcb..0000000 --- a/travis/xenial_builder.dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -FROM ubuntu:xenial - -ENV LANG C.UTF-8 - -RUN apt update -q && \ - dpkg --add-architecture i386 && \ - apt install -q -y --no-install-recommends \ -# Base build tools - build-essential automake libtool cmake ccache pkg-config autopoint patch \ - python3-pip python3-setuptools python3-wheel git subversion wget unzip \ - ninja-build \ -# Python (2) is needed to install android-ndk - python \ -# Packaged dependencies - libbz2-dev libmagic-dev uuid-dev zlib1g-dev default-jdk \ - libmicrohttpd-dev \ -# Cross compile i586 - libc6-dev-i386 lib32stdc++6 gcc-multilib g++-multilib \ -# Other tools (to remove) -# vim less grep \ - && \ - apt-get clean -y && \ - rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/* && \ - pip3 install --upgrade pip && \ - pip3 install meson==0.52.1 pytest gcovr - -# Create user -RUN useradd --create-home ci_builder -USER ci_builder -WORKDIR /home/ci_builder -ENV PATH="/home/ci_builder/.local/bin:${PATH}" - -ENV TRAVIS_BUILD_DIR /home/ci_builder/kiwix-build -ENV GRADLE_USER_HOME /home/ci_builder -ENV TRAVIS_OS_NAME linux_xenial - -CMD pip3 install --user --no-deps ./kiwix-build && kiwix-build/travis/compile_all.py