Merge pull request #443 from kiwix/release_github

.github/ci_images/bionic_builder.dockerfile

@@ -24,7 +24,7 @@ RUN apt update -q \
 #    vim less grep \
   && apt-get clean -y \
   && rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/* \
-  && pip3 install meson==0.52.1 pytest gcovr
+  && pip3 install meson==0.52.1 pytest gcovr requests
 
 # Create user
 RUN useradd --create-home runner

.github/ci_images/f31_builder.dockerfile

@@ -12,13 +12,13 @@ RUN dnf install -y --nodocs \
     mingw32-winpthreads-static mingw32-zlib-static mingw32-xz-libs-static \
     mingw32-libmicrohttpd \
 # python3
-    python-unversioned-command \
+    python3-pip python-unversioned-command \
 # Other tools (to remove)
 #    vim less grep
   && dnf remove -y "*-doc" \
   && dnf autoremove -y \
   && dnf clean all \
-  && pip3 install meson==0.52.1 pytest
+  && pip3 install meson==0.52.1 pytest requests
 
 # Create user
 RUN useradd --create-home runner

.github/ci_images/focal_builder.dockerfile

@@ -2,6 +2,7 @@ FROM ubuntu:focal
 
 ENV LANG C.UTF-8
 ENV OS_NAME focal
+ENV DEBIAN_FRONTEND noninteractive
 
 RUN apt update -q \
   && apt install -q -y --no-install-recommends \
@@ -24,7 +25,7 @@ RUN apt update -q \
 #    vim less grep \
   && apt-get clean -y \
   && rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/* \
-  && pip3 install meson==0.52.1 pytest gcovr
+  && pip3 install meson==0.52.1 pytest gcovr requests
 
 # Create user
 RUN useradd --create-home runner

.github/ci_images/xenial_builder.dockerfile

@@ -21,7 +21,7 @@ RUN apt update -q \
 #    vim less grep \
   && apt-get clean -y \
   && rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/* \
-  && pip3 install meson==0.52.1 pytest gcovr
+  && pip3 install meson==0.52.1 pytest gcovr requests
 
 # Create user
 RUN useradd --create-home runner

.github/scripts/build_release_nightly.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+
+import os
+import json
+import shutil
+
+from common import (
+    run_kiwix_build,
+    main_project_versions,
+    release_versions,
+    get_postfix,
+    make_archive,
+    create_desktop_image,
+    update_flathub_git,
+    upload_archive,
+    BASE_DIR,
+    TMP_DIR,
+    HOME,
+    OS_NAME,
+    PLATFORM_TARGET,
+    DESKTOP,
+)
+
+from upload_to_bintray import upload_from_json
+
+
+if os.environ.get('GITHUB_EVENT_NAME') == 'schedule':
+    RELEASE = False
+else:
+    RELEASE = True
+
+if PLATFORM_TARGET == "android":
+    TARGETS = ("kiwix-lib-app",)
+elif PLATFORM_TARGET.startswith("iOS"):
+    TARGETS = ("libzim", "kiwix-lib")
+elif PLATFORM_TARGET.startswith("native_"):
+    if OS_NAME == "osx":
+        TARGETS = ("libzim", "zimwriterfs", "zim-tools", "kiwix-lib")
+    else:
+        if DESKTOP:
+            TARGETS = ("kiwix-desktop",)
+        elif PLATFORM_TARGET == "native_mixed":
+            TARGETS = ("libzim",)
+        else:
+            TARGETS = ("zimwriterfs", "zim-tools", "kiwix-tools")
+elif PLATFORM_TARGET in ("win32_static", "armhf_static", "i586_static"):
+    TARGETS = ("kiwix-tools",)
+elif PLATFORM_TARGET == "flatpak":
+    TARGETS = ("kiwix-desktop",)
+else:
+    TARGETS = ("libzim", "zim-tools", "kiwix-lib", "kiwix-tools")
+
+# Filter what to build if we are doing a release.
+if RELEASE:
+    TARGETS = tuple(filter(lambda t: release_versions.get(t) is not None, TARGETS))
+
+for target in TARGETS:
+    run_kiwix_build(target, platform=PLATFORM_TARGET, make_release=RELEASE)
+    if target == "kiwix-desktop":
+        archive = create_desktop_image(make_release=RELEASE)
+    else:
+        archive = make_archive(target, make_release=RELEASE)
+    if archive:
+        upload_archive(archive, target, make_release=RELEASE)
+
+# We have few more things to do for release:
+if RELEASE:
+    # Publish source archives
+    if PLATFORM_TARGET == "native_dyn" and OS_NAME != "osx":
+        for target in TARGETS:
+            if release_versions.get(target) != 0:
+                continue
+            run_kiwix_build(
+                target, platform=PLATFORM_TARGET, make_release=RELEASE, make_dist=True
+            )
+            full_target_name = "{}-{}".format(target, main_project_versions[target])
+            if target == "kiwix-desktop":
+                archive = (
+                    BASE_DIR / full_target_name / "{}.tar.gz".format(full_target_name)
+                )
+            else:
+                archive = (
+                    BASE_DIR
+                    / full_target_name
+                    / "meson-dist"
+                    / "{}.tar.xz".format(full_target_name)
+                )
+            upload_archive(archive, target, make_release=RELEASE)
+
+    # Publish flathub
+    if PLATFORM_TARGET == "flatpak" and "kiwix-desktop" in TARGETS:
+        update_flathub_git()
+
+    if PLATFORM_TARGET == "android" and "kiwix-lib-app" in TARGETS:
+        postfix = get_postfix("kiwix-lib")
+        basename = "kiwixlib-{}".format(postfix)
+
+        output_release_dir = (
+            HOME / "BUILD_android" / "kiwix-lib-app" / "kiwixLibAndroid" / "build"
+        )
+        shutil.copy(
+            str(output_release_dir / "outputs" / "aar" / "kiwixLibAndroid-release.aar"),
+            str(TMP_DIR / (basename + ".aar")),
+        )
+        shutil.copy(
+            str(output_release_dir / "pom.xml"), str(TMP_DIR / (basename + ".pom"))
+        )
+
+        json_filename = "{}_bintray_info.json".format(basename)
+        data = {
+            "version": postfix,
+            "files": [basename + ext for ext in (".aar", ".pom")],
+        }
+        with open(str(TMP_DIR / json_filename), "w") as f:
+            json.dump(data, f)
+
+        upload_from_json(TMP_DIR / json_filename)

.github/scripts/common.py

@@ -3,11 +3,16 @@ from os import environ as _environ
 from pathlib import Path
 from datetime import date
 import tarfile
+import zipfile
 import subprocess
 import re
+import shutil
 
-from kiwixbuild.versions import base_deps_versions
+from kiwixbuild.versions import (
-
+    main_project_versions,
+    release_versions,
+    base_deps_versions,
+)
 
 PLATFORM_TARGET = _environ["PLATFORM_TARGET"]
 if PLATFORM_TARGET == "native_desktop":
@@ -23,6 +28,7 @@ SOURCE_DIR = HOME / "SOURCE"
 ARCHIVE_DIR = HOME / "ARCHIVE"
 INSTALL_DIR = BASE_DIR / "INSTALL"
 TMP_DIR = Path("/tmp")
+KBUILD_SOURCE_DIR = HOME / "kiwix-build"
 
 # [TODO]
 KIWIX_DESKTOP_ONLY = False
@@ -30,6 +36,58 @@ KIWIX_DESKTOP_ONLY = False
 _ref = _environ.get("GITHUB_REF", "").split("/")[-1]
 MAKE_RELEASE = re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", _ref) is not None
 
+PLATFORM_TO_RELEASE = {
+    "native_mixed": "linux-x86_64",
+    "native_static": "linux-x86_64",
+    "win32_static": "win-i686",
+    "armhf_static": "linux-armhf",
+    "i586_static": "linux-i586",
+}
+
+FLATPAK_HTTP_GIT_REMOTE = "https://github.com/flathub/org.kiwix.desktop.git"
+FLATPAK_GIT_REMOTE = "git@github.com:flathub/org.kiwix.desktop.git"
+
+BIN_EXT = ".exe" if PLATFORM_TARGET.startswith("win32_") else ""
+
+# We have build everything. Now create archives for public deployement.
+EXPORT_FILES = {
+    "kiwix-tools": (
+        INSTALL_DIR / "bin",
+        [
+            f + BIN_EXT
+            for f in ("kiwix-manage", "kiwix-read", "kiwix-search", "kiwix-serve")
+        ],
+    ),
+    "zim-tools": (
+        INSTALL_DIR / "bin",
+        [
+            f + BIN_EXT
+            for f in (
+                "zimbench",
+                "zimcheck",
+                "zimdump",
+                "zimsearch",
+                "zimdiff",
+                "zimpatch",
+                "zimsplit",
+            )
+        ],
+    ),
+    "zimwriterfs": (INSTALL_DIR / "bin", ["zimwriterfs" + BIN_EXT]),
+    "libzim": (
+        INSTALL_DIR,
+        (
+            "lib/x86_64-linux-gnu/libzim.so.{}".format(main_project_versions["libzim"]),
+            "lib/x86_64-linux-gnu/libzim.so.{}".format(
+                main_project_versions["libzim"][0]
+            ),
+            "include/zim/**/*.h",
+        ),
+    ),
+}
+
+DATE = date.today().isoformat()
+
 
 def print_message(message, *args, **kwargs):
     message = message.format(*args, **kwargs)
@@ -49,10 +107,7 @@ def write_manifest(manifest_file, archive_name, target, platform):
     with manifest_file.open(mode="w") as f:
         f.write(
             MANIFEST_TEMPLATE.format(
-                archive_name=archive_name,
+                archive_name=archive_name, target=target, platform=platform, date=DATE,
-                target=target,
-                platform=platform,
-                date=date.today().isoformat(),
             )
         )
 
@@ -95,12 +150,28 @@ def run_kiwix_build(
         make_dist,
     )
     env = dict(_environ)
-    env['SKIP_BIG_MEMORY_TEST'] = '1'
+    env["SKIP_BIG_MEMORY_TEST"] = "1"
     subprocess.check_call(command, cwd=str(HOME), env=env)
     print_message("Build ended")
 
 
 def upload(file_to_upload, host, dest_path):
+    if not file_to_upload.exists():
+        print_message("No {} to upload!", file_to_upload)
+        return
+
+    command = [
+        "ssh",
+        "-i",
+        _environ.get("SSH_KEY"),
+        "-o",
+        "StrictHostKeyChecking=no",
+        host,
+        "mkdir -p {}".format(dest_path),
+    ]
+    print_message("Creating dest path {}", dest_path)
+    subprocess.check_call(command)
+
     command = [
         "scp",
         "-i",
@@ -114,6 +185,22 @@ def upload(file_to_upload, host, dest_path):
     subprocess.check_call(command)
 
 
+def upload_archive(archive, project, make_release):
+    if project.startswith("kiwix-"):
+        host = "ci@download.kiwix.org"
+        dest_path = "/data/download/"
+    else:
+        host = "ci@download.openzim.org"
+        dest_path = "/data/openzim/"
+
+    if make_release:
+        dest_path = dest_path + "release/" + project
+    else:
+        dest_path = dest_path + "nightly/" + DATE
+
+    upload(archive, host, dest_path)
+
+
 def make_deps_archive(target=None, name=None, full=False):
     archive_name = name or "deps2_{}_{}_{}.tar.xz".format(
         OS_NAME, PLATFORM_TARGET, target
@@ -162,3 +249,123 @@ def make_deps_archive(target=None, name=None, full=False):
             tar.add(str(name), arcname=str(name.relative_to(relative_path)))
 
     return archive_file
+
+
+def get_postfix(project):
+    postfix = main_project_versions[project]
+    extra = release_versions.get(project)
+    if extra is not None:
+        postfix = "{}-{}".format(postfix, extra)
+    return postfix
+
+
+def make_archive(project, make_release):
+    try:
+        platform = PLATFORM_TO_RELEASE[PLATFORM_TARGET]
+    except KeyError:
+        # We don't know how to name the release.
+        return None
+
+    base_dir, export_files = EXPORT_FILES[project]
+
+    if make_release:
+        postfix = get_postfix(project)
+    else:
+        postfix = DATE
+
+    archive_name = "{}_{}-{}".format(project, platform, postfix)
+
+    files_to_archive = []
+    for export_file in export_files:
+        files_to_archive.extend(base_dir.glob(export_file))
+    if platform == "win-i686":
+        open_archive = lambda a: zipfile.ZipFile(
+            str(a), "w", compression=zipfile.ZIP_DEFLATED
+        )
+        archive_add = lambda a, f: a.write(str(f), arcname=str(f.relative_to(base_dir)))
+        archive_ext = ".zip"
+    else:
+        open_archive = lambda a: tarfile.open(str(a), "w:gz")
+        archive_add = lambda a, f: a.add(
+            str(f), arcname="{}/{}".format(archive_name, str(f.relative_to(base_dir)))
+        )
+        archive_ext = ".tar.gz"
+
+    archive = TMP_DIR / "{}{}".format(archive_name, archive_ext)
+    print_message("create archive {} with {}", archive, files_to_archive)
+    with open_archive(archive) as arch:
+        for f in files_to_archive:
+            archive_add(arch, f)
+    return archive
+
+
+def create_desktop_image(make_release):
+    print_message("creating desktop image")
+    if make_release:
+        postfix = get_postfix("kiwix-desktop")
+        src_dir = SOURCE_DIR / "kiwix-desktop_release"
+    else:
+        postfix = DATE
+        src_dir = SOURCE_DIR / "kiwix-desktop"
+
+    if PLATFORM_TARGET == "flatpak":
+        build_path = BASE_DIR / "org.kiwix.desktop.flatpak"
+        app_name = "org.kiwix.desktop.{}.flatpak".format(postfix)
+        print_message("archive is {}", build_path)
+    else:
+        build_path = HOME / "Kiwix-{}-x86_64.AppImage".format(postfix)
+        app_name = "kiwix-desktop_x86_64_{}.appimage".format(postfix)
+        command = [
+            "kiwix-build/scripts/create_kiwix-desktop_appImage.sh",
+            str(INSTALL_DIR),
+            str(src_dir),
+            str(HOME / "AppDir"),
+        ]
+        env = dict(os.environ)
+        env["VERSION"] = postfix
+        print_message("Build AppImage of kiwix-desktop")
+        subprocess.check_call(command, cwd=str(HOME), env=env)
+
+    print_message("Copy Build to {}".format(TMP_DIR / app_name))
+    shutil.copy(str(build_path), str(TMP_DIR / app_name))
+    return TMP_DIR / app_name
+
+
+def update_flathub_git():
+    git_repo_dir = TMP_DIR / "org.kiwix.desktop"
+    env = dict(os.environ)
+    env["GIT_AUTHOR_NAME"] = env["GIT_COMMITTER_NAME"] = "KiwixBot"
+    env["GIT_AUTHOR_EMAIL"] = env["GIT_COMMITTER_EMAIL"] = "kiwixbot@kymeria.fr"
+
+    def call(command, cwd=None):
+        cwd = cwd or git_repo_dir
+        print_message("call {}", command)
+        subprocess.check_call(command, env=env, cwd=str(cwd))
+
+    command = ["git", "clone", FLATPAK_HTTP_GIT_REMOTE]
+    call(command, cwd=TMP_DIR)
+    shutil.copy(str(BASE_DIR / "org.kiwix.desktop.json"), str(git_repo_dir))
+    patch_dir = KBUILD_SOURCE_DIR / "kiwixbuild" / "patches"
+    for dep in ("libaria2", "mustache", "pugixml", "xapian", "zstd"):
+        for f in patch_dir.glob("{}_*.patch".format(dep)):
+            shutil.copy(str(f), str(git_repo_dir / "patches"))
+    command = ["git", "add", "-A", "."]
+    call(command)
+    command = [
+        "git",
+        "commit",
+        "-m",
+        "Update to version {}".format(main_project_versions["kiwix-desktop"]),
+    ]
+    try:
+        call(command)
+    except subprocess.CalledProcessError:
+        # This may fail if there is nothing to commit (a rebuild of the CI for exemple)
+        return
+    command = ["git", "config", "remote.origin.pushurl", FLATPAK_GIT_REMOTE]
+    call(command)
+    command = ["git", "push"]
+    env["GIT_SSH_COMMAND"] = "ssh -o StrictHostKeyChecking=no -i " + _environ.get(
+        "SSH_KEY"
+    )
+    call(command)

.github/scripts/ensure_base_deps.py

@@ -15,7 +15,6 @@ from common import (
     HOME,
     PLATFORM_TARGET,
     OS_NAME,
-    MAKE_RELEASE,
 )
 
 def download_base_archive(base_name):
@@ -31,14 +30,16 @@ def download_base_archive(base_name):
             file.write(batch)
     return file_path
 
+ARCHIVE_NAME_TEMPLATE = "base_deps2_{os}_{platform}_{version}.tar.xz"
 
-ARCHIVE_NAME_TEMPLATE = "base_deps2_{os}_{platform}_{version}_{release}.tar.xz"
+if PLATFORM_TARGET == 'flatpak':
-base_dep_archive_name = ARCHIVE_NAME_TEMPLATE.format(
+    base_dep_archive_name = "base_deps2_{}_flatpak.tar.xz".format(OS_NAME)
+else:
+    base_dep_archive_name = ARCHIVE_NAME_TEMPLATE.format(
         os=OS_NAME,
         platform=PLATFORM_TARGET,
         version=base_deps_meta_version,
-    release="release" if MAKE_RELEASE else "debug",
+    )
-)
 
 print_message("Getting archive {}", base_dep_archive_name)
 try:
@@ -54,7 +55,6 @@ except URLError:
                 os=OS_NAME,
                 platform="android_{}".format(arch),
                 version=base_deps_meta_version,
-                release="release" if MAKE_RELEASE else "debug",
             )
             print_message("Getting archive {}", archive_name)
             try:
@@ -64,6 +64,8 @@ except URLError:
                 os.remove(str(local_filename))
             except URLError:
                 pass
+    elif PLATFORM_TARGET == "flatpak":
+        print_message("Cannot get archive. Move on")
     else:
         run_kiwix_build("alldependencies", platform=PLATFORM_TARGET)
         archive_file = make_deps_archive(name=base_dep_archive_name, full=True)

.github/scripts/upload_to_bintray.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+
+import os, sys
+import json
+import requests
+
+bintray_auth = (os.environ.get('BINTRAY_USER'), os.environ.get('BINTRAY_PASS'))
+
+def create_version(version):
+    url = "https://api.bintray.com/packages/kiwix/kiwix/kiwixlib/versions"
+    payload = {
+        'name': version,
+        'desc': 'Release of kiwix-lib'
+    }
+    headers = {
+        'Content-Type': 'application/json'
+    }
+
+    r = requests.post(url, data=json.dumps(payload), headers=headers, auth=bintray_auth)
+    rcode = r.status_code
+
+    if rcode == 409:
+        print("Bintray version %s already exists, skipping." % version)
+        return True
+
+    rcode_family = rcode // 100
+    if rcode_family in (2, 3):
+        print("Bintray Version created!")
+        return True
+
+    print("ERROR : Bintray API response {}".format(rcode))
+    return False
+
+
+def upload(version, filepath, artefact):
+    url_template = "https://api.bintray.com/content/kiwix/kiwix/kiwixlib/{version}/org/kiwix/kiwixlib/kiwixlib/{version}/{artefact}"
+    parameters = {
+        'publish': 1,
+        'override': 1
+    }
+
+    # Upload the main artefact
+    url = url_template.format(version=version, artefact=artefact)
+    with open(filepath, 'rb') as f:
+        r = requests.put(url, data=f, auth=bintray_auth, params=parameters)
+
+    rcode = r.status_code
+    rcode_family = rcode // 100
+    if rcode_family not in (2, 3):
+        print("ERROR: Fail to upload artefact")
+        print(r.text)
+        return False
+
+    return True
+
+
+def upload_from_json(json_path):
+    basedir = os.path.dirname(str(json_path))
+    with open(str(json_path)) as f:
+        options = json.load(f)
+
+    if not create_version(options['version']):
+        raise RuntimeError("Cannot create version")
+
+    for file_ in options['files']:
+        path = os.path.join(basedir, file_)
+        if not upload(options['version'], path, file_):
+            raise RuntimeError("Cannot upload file {}".format(file_))
+
+
+if __name__ == "__main__":
+    try:
+        info_file = sys.argv[1]
+    except IndexError:
+        print("Usage {} infofile".format(sys.argv[0]))
+        sys.exit(-1)
+
+    print("Use info file {}".format(info_file))
+    try:
+        upload_from_json(info_file)
+    except RuntimeError as e:
+        sys.exit(str(e))
+

.github/workflows/releaseNigthly.yml

@@ -0,0 +1,167 @@
+name: Release&Nigthly
+
+on:
+ push:
+   tags:
+     - r_[0-9]+i
+ schedule:
+   - cron: '0 1 * * *'
+
+env:
+  DOCKER_VERSION: 28
+
+jobs:
+  Docker:
+    strategy:
+      fail-fast: false
+      matrix:
+        variant: [xenial, bionic, f31, focal]
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build docker image
+      shell: bash
+      run: |
+        TAGNAME=${{matrix.variant}}-${DOCKER_VERSION}
+        if [ ! $(curl -sflL https://hub.docker.com/v2/repositories/kiwix/kiwix-build_ci/tags/${TAGNAME}) ]
+        then
+          echo "${{secrets.docker_password}}" | docker login -u "${{secrets.docker_username}}" --password-stdin
+          FULLTAGNAME=kiwix/kiwix-build_ci:${TAGNAME}
+          docker build -t ${FULLTAGNAME} - < ${GITHUB_WORKSPACE}/.github/ci_images/${{matrix.variant}}_builder.dockerfile
+          docker push ${FULLTAGNAME}
+        fi
+
+  Linux:
+    strategy:
+      fail-fast: false
+      matrix:
+        target:
+          - native_static
+          - native_dyn
+          - native_mixed
+          - native_desktop
+          - armhf_static
+          - win32_static
+          - i586_static
+          - android
+          - flatpak
+        include:
+          - target: native_static
+            image_variant: xenial
+            lib_postfix: '/x86_64-linux-gnu'
+          - target: native_dyn
+            image_variant: xenial
+            lib_postfix: '/x86_64-linux-gnu'
+          - target: native_mixed
+            image_variant: xenial
+            lib_postfix: '/x86_64-linux-gnu'
+          - target: native_desktop
+            image_variant: bionic
+            lib_postfix: '/x86_64-linux-gnu'
+          - target: armhf_static
+            image_variant: xenial
+            lib_postfix: '/x86_64-linux-gnu'
+          - target: win32_static
+            image_variant: f31
+            lib_postfix: '64'
+          - target: i586_static
+            image_variant: xenial
+            lib_postfix: '/x86_64-linux-gnu'
+          - target: android
+            image_variant: xenial
+            lib_postfix: '/x86_64-linux-gnu'
+          - target: flatpak
+            image_variant: bionic
+            lib_postfix: '/x86_64-linux-gnu'
+    env:
+      HOME: /home/runner
+      SSH_KEY: /tmp/id_rsa
+    runs-on: ubuntu-latest
+    needs: Docker
+    container:
+      image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-28"
+      options: "--device /dev/fuse --privileged"
+    steps:
+    - name: Checkout code
+      shell: bash
+      run: |
+        cd $HOME
+        git clone https://github.com/${REP} --depth=1 --branch ${GITHUB_REF##*heads/}
+        pip3 install --user --no-deps ./${REP##*/}
+      env:
+        REP: ${{github.repository}}
+    - name: secret
+      shell: bash
+      run: |
+        echo "${{secrets.ssh_key}}" > $SSH_KEY
+        chmod 600 $SSH_KEY
+    - name: Ensure base deps
+      shell: bash
+      run: |
+        cd $HOME
+        kiwix-build/.github/scripts/ensure_base_deps.py
+      env:
+        PLATFORM_TARGET: ${{matrix.target}}
+    - name: Build release
+      shell: bash
+      run: |
+        cd $HOME
+        kiwix-build/.github/scripts/build_release_nightly.py
+      env:
+        PLATFORM_TARGET: ${{matrix.target}}
+    - name: Upload failure logs
+      if: failure()
+      run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh
+      env:
+        PLATFORM_TARGET: ${{matrix.target}}
+
+
+  Macos:
+    strategy:
+      fail-fast: false
+      matrix:
+        target:
+          - native_dyn
+    runs-on: macos-latest
+    env:
+      SSH_KEY: /tmp/id_rsa
+      OS_NAME: osx
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v1
+    - name: Setup python 3.6
+      uses: actions/setup-python@v1
+      with:
+        python-version: '3.6'
+    - name: Install packages
+      uses: mstksg/get-package@v1
+      with:
+        brew: pkg-config ninja
+    - name: Install python modules
+      run: |
+        pip3 install meson==0.52.1 pytest requests
+        pip3 install --no-deps $GITHUB_WORKSPACE
+    - name: secret
+      shell: bash
+      run: |
+        echo "${{secrets.ssh_key}}" > $SSH_KEY
+        chmod 600 $SSH_KEY
+    - name: Ensure base deps
+      shell: bash
+      run: |
+        cd $HOME
+        $GITHUB_WORKSPACE/.github/scripts/ensure_base_deps.py
+      env:
+        PLATFORM_TARGET: ${{matrix.target}}
+    - name: Build release
+      shell: bash
+      run: |
+        cd $HOME
+        $GITHUB_WORKSPACE/.github/scripts/build_release_nightly.py
+      env:
+        PLATFORM_TARGET: ${{matrix.target}}
+    - name: Upload failure logs
+      if: failure()
+      run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.sh
+      env:
+        PLATFORM_TARGET: ${{matrix.target}}

.travis.yml

@@ -1,149 +0,0 @@
-language: cpp
-
-dist: xenial
-sudo: required
-services:
-- docker
-branches:
-  only:
-  - master
-  - /\d+\.\d+\.\d+$/
-
-if: type != push OR tag IS present
-
-stages:
-- build_docker_images
-- build
-- android_build
-
-jobs:
-  include:
-    - stage: build_docker_images
-      script: &build_images
-        - |
-          if [ ! $(curl -sflL https://hub.docker.com/v2/repositories/kiwix/kiwix-build_ci/tags/${VARIANT}-${DOCKER_VERSION}) ]
-          then
-            echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
-            docker build -t kiwix-build_ci -f travis/${VARIANT}_builder.dockerfile .
-            docker tag kiwix-build_ci kiwix/kiwix-build_ci:${VARIANT}-${DOCKER_VERSION}
-            docker push kiwix/kiwix-build_ci:${VARIANT}-${DOCKER_VERSION}
-          fi
-      env: VARIANT=xenial
-    - stage: build_docker_images
-      script: *build_images
-      env: VARIANT=bionic
-    - stage: build_docker_images
-      script: *build_images
-      env: VARIANT=focal
-    - stage: build_docker_images
-      script: *build_images
-      env: VARIANT=f30
-    - stage: android_build
-      env: PLATFORM="android"
-      if: type != pull_request
-    - env: PLATFORM="native_dyn"
-      os: osx
-    - env: PLATFORM="iOS_arm64"
-      os: osx
-
-
-
-before_install:
-- PATH=$PATH:${HOME}/bin:${HOME}/.local/bin
-- mkdir -p /tmp/private $HOME/EXPORT/BASE /$HOME/.cache $HOME/.gradle/caches
-- chmod -R a+wX $HOME/EXPORT $HOME/.cache $HOME/.gradle
-- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then PATH=$PATH:$(brew --prefix)/opt/gettext/bin; fi
-- '[ -n "$encrypted_eba2f7543984_iv" ] && openssl aes-256-cbc
-  -K $encrypted_eba2f7543984_key -iv $encrypted_eba2f7543984_iv
-  -in travis/travisci_builder_id_key.enc -out $SSH_KEY -d'
-- chmod 600 $SSH_KEY
-before_cache:
-  - rm -f $HOME/.gradle/caches/modules-2/modules-2.lock
-  - rm -fr $HOME/.gradle/caches/*/plugin-resolution/
-cache:
-  ccache: true
-  directories:
-    - $HOME/.cache/pip
-    - $HOME/.gradle/caches/
-    - $HOME/.gradle/wrapper/
-    - $HOME/.android/build-cache
-install: if [[ $TRAVIS_OS_NAME != "linux" ]]; then travis/install_extra_deps.sh; fi
-stage: build
-script:
-- |
-  if [[ $TRAVIS_OS_NAME = "linux" ]]
-  then
-    VARIANT=xenial
-    if [[ $DESKTOP_ONLY = 1 ]]; then VARIANT=bionic; fi
-    if [[ $PLATFORM =~ ^win32_* ]]; then VARIANT=f30; fi
-  fi
-- |
-  if [[ $TRAVIS_OS_NAME = "linux" ]]
-  then
-    docker run --rm \
-      -e PLATFORM -e NIGHTLY_DATE -e TRAVIS_EVENT_TYPE -e DESKTOP_ONLY -e TRAVIS_TAG \
-      --device /dev/fuse --cap-add ALL --privileged \
-      --mount=type=bind,src=$HOME/EXPORT,dst=/home/ci_builder/EXPORT \
-      --mount=type=bind,src=$HOME/.cache,dst=/home/ci_builder/.cache \
-      --mount=type=bind,src=$HOME/.gradle/caches,dst=/home/ci_builder/.gradle/caches \
-      --mount=type=bind,src=$TRAVIS_BUILD_DIR,dst=/home/ci_builder/kiwix-build \
-      kiwix/kiwix-build_ci:${VARIANT}-${DOCKER_VERSION}
-  fi
-- if [[ $TRAVIS_OS_NAME != "linux" ]]; then python3 travis/compile_all.py; fi
-after_success:
-- travis/upload_base_deps.sh
-after_failure:
-- travis/upload_base_deps.sh
-- travis/upload_all_log.sh
-deploy:
-  - provider: script
-    skip_cleanup: true
-    script: travis/deploy.sh
-    on:
-      tags: true
-      condition: $TRAVIS_BUILD_STAGE_NAME != "Build_docker_images"
-  - provider: script
-    skip_cleanup: true
-    script: travis/deploy.sh
-    on:
-      branch: master
-      condition: $TRAVIS_BUILD_STAGE_NAME != "Build_docker_images" && $TRAVIS_EVENT_TYPE = cron
-env:
-  global:
-    - NIGHTLY_DATE=$(date +%Y-%m-%d)
-    - SSH_KEY=/tmp/private/travisci_builder_id_key
-    - DOCKER_VERSION=4
-  matrix:
-    - PLATFORM="flatpak" DESKTOP_ONLY=1
-    - PLATFORM="native_dyn"
-    - PLATFORM="native_static"
-    - PLATFORM="native_mixed"
-    - PLATFORM="armhf_dyn"
-    - PLATFORM="armhf_static"
-    - PLATFORM="win32_dyn"
-    - PLATFORM="win32_static"
-    - PLATFORM="i586_dyn"
-    - PLATFORM="i586_static"
-    - PLATFORM="native_dyn" DESKTOP_ONLY=1
-    - PLATFORM="android_arm"
-    - PLATFORM="android_arm64"
-    - PLATFORM="android_x86"
-    - PLATFORM="android_x86_64"
-addons:
-  ssh_known_hosts:
-  - download.kiwix.org
-  - tmp.kiwix.org
-  - download.openzim.org
-
-notifications:
-  irc:
-    channels:
-     - "chat.freenode.net#kiwix"
-    on_success: change
-    on_failure: always
-  email:
-    recipients:
-      - contact@kiwix.org
-      - mgautier@kymeria.fr
-    on_success: change
-    on_failure: always

scripts/upload_kiwix_lib_android_to_bintray.py

@@ -54,6 +54,19 @@ def upload(version, filepath, artefact):
     return True
 
 
+def upload_from_json(json_path):
+    basedir = os.path.dirname(json_path)
+    with open(str(json_path)) as f:
+        options = json.load(f)
+
+    if not create_version(options['version']):
+        raise RuntimeError("Cannot create version")
+
+    for file_ in options['files']:
+        path = os.path.join(basedir, file_)
+        if not upload(options['version'], path, file_):
+            raise RuntimeError("Cannot upload file {}".format(file_))
+
 
 if __name__ == "__main__":
     try:
@@ -63,14 +76,8 @@ if __name__ == "__main__":
         sys.exit(-1)
 
     print("Use info file {}".format(info_file))
-    with open(info_file) as f:
+    try:
-        options = json.load(f)
+        upload_from_json(info_file)
+    except RuntimeError as e:
+        sys.exit(str(e))
 
-    if not create_version(options['version']):
-        sys.exit("Cannot create version")
-
-    basedir = os.path.dirname(sys.argv[1])
-    for file_ in options['files']:
-        path = os.path.join(basedir, file_)
-        if not upload(options['version'], path, file_):
-            sys.exit("Cannot upload file {}".format(file_))

travis/bionic_builder.dockerfile

@@ -1,37 +0,0 @@
-FROM ubuntu:bionic
-
-ENV LANG C.UTF-8
-
-RUN apt update -q && \
-    apt install -q -y --no-install-recommends \
-# Base build tools
-    build-essential automake libtool cmake ccache pkg-config autopoint patch \
-    python3-pip python3-setuptools python3-wheel git subversion wget unzip \
-    ninja-build \
-# Packaged dependencies
-    libbz2-dev libmagic-dev uuid-dev zlib1g-dev \
-    libmicrohttpd-dev aria2 \
-# Qt packages
-    libqt5gui5 qtbase5-dev qtwebengine5-dev libqt5svg5-dev qt5-image-formats-plugins qt5-default \
-# To create the appimage of kiwix-desktop
-    libfuse2 fuse patchelf \
-# Flatpak tools
-    elfutils flatpak flatpak-builder \
-# Cross win32 compiler
-    g++-mingw-w64-i686 gcc-mingw-w64-i686 gcc-mingw-w64-base mingw-w64-tools \
-# Other tools (to remove)
-#    vim less grep \
-  && \
-  apt-get clean -y && \
-  rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/*
-
-# Create user
-RUN useradd --create-home ci_builder
-USER ci_builder
-WORKDIR /home/ci_builder
-ENV PATH="/home/ci_builder/.local/bin:${PATH}"
-
-ENV TRAVIS_BUILD_DIR /home/ci_builder/kiwix-build
-ENV TRAVIS_OS_NAME linux_bionic
-
-CMD pip3 install --user ./kiwix-build && kiwix-build/travis/compile_all.py

travis/compile_all.py

@@ -1,527 +0,0 @@
-#!/usr/bin/env python3
-
-import sys, os
-import shutil
-from os import environ
-from pathlib import Path
-from datetime import date
-import tarfile, zipfile
-import subprocess
-import re
-import json
-from urllib.request import urlopen
-from urllib.error import URLError
-
-from kiwixbuild.versions import (
-    main_project_versions,
-    release_versions,
-    base_deps_versions,
-    base_deps_meta_version)
-
-PLATFORM = environ['PLATFORM']
-TRAVIS_OS_NAME = environ['TRAVIS_OS_NAME']
-HOME = Path(os.path.expanduser('~'))
-NIGHTLY_DATE = environ['NIGHTLY_DATE']
-KBUILD_SOURCE_DIR = Path(environ['TRAVIS_BUILD_DIR'])
-KIWIX_DESKTOP_ONLY = environ.get('DESKTOP_ONLY') == '1'
-
-BASE_DIR = HOME/"BUILD_{}".format(PLATFORM)
-EXPORT_DIR = HOME/'EXPORT'
-BASE_EXPORT_DIR = EXPORT_DIR/"BASE"
-GIT_EXPORT_DIR = EXPORT_DIR/"GIT"
-SOURCE_DIR = HOME/"SOURCE"
-ARCHIVE_DIR = HOME/"ARCHIVE"
-INSTALL_DIR = BASE_DIR/"INSTALL"
-NIGHTLY_KIWIX_ARCHIVES_DIR = EXPORT_DIR/'NIGHTLY_KIWIX_ARCHIVES'/NIGHTLY_DATE
-RELEASE_KIWIX_ARCHIVES_DIR = EXPORT_DIR/'RELEASE_KIWIX_ARCHIVES'
-NIGHTLY_ZIM_ARCHIVES_DIR = EXPORT_DIR/'NIGHTLY_ZIM_ARCHIVES'/NIGHTLY_DATE
-RELEASE_ZIM_ARCHIVES_DIR = EXPORT_DIR/'RELEASE_ZIM_ARCHIVES'
-DIST_KIWIX_ARCHIVES_DIR = EXPORT_DIR/'DIST_KIWIX_ARCHIVES'
-DIST_ZIM_ARCHIVES_DIR = EXPORT_DIR/'DIST_ZIM_ARCHIVES'
-BINTRAY_ARCHIVES_DIR = EXPORT_DIR/'BINTRAY_ARCHIVES'
-
-BIN_EXT = '.exe' if PLATFORM.startswith('win32_') else ''
-
-# We have build everything. Now create archives for public deployement.
-EXPORT_FILES = {
-    'kiwix-tools':
-        (INSTALL_DIR/'bin', [f+BIN_EXT for f in ('kiwix-manage', 'kiwix-read', 'kiwix-search', 'kiwix-serve')]),
-    'zim-tools':
-        (INSTALL_DIR/'bin', [f+BIN_EXT for f in ('zimbench', 'zimcheck', 'zimdump', 'zimsearch', 'zimdiff', 'zimpatch', 'zimsplit')]),
-    'zimwriterfs':
-        (INSTALL_DIR/'bin', ['zimwriterfs'+BIN_EXT]),
-    'libzim':
-        (INSTALL_DIR, ('lib/x86_64-linux-gnu/libzim.so.{}'.format(main_project_versions['libzim']),
-                       'lib/x86_64-linux-gnu/libzim.so.{}'.format(main_project_versions['libzim'][0]),
-                       'include/zim/**/*.h'))
-}
-
-FLATPAK_HTTP_GIT_REMOTE = 'https://github.com/flathub/org.kiwix.desktop.git'
-FLATPAK_GIT_REMOTE = 'git@github.com:flathub/org.kiwix.desktop.git'
-
-_date = date.today().isoformat()
-
-def print_message(message, *args, **kwargs):
-    message = message.format(*args, **kwargs)
-    message = "{0} {1} {0}".format('-'*3, message)
-    print(message, flush=True)
-
-
-def write_manifest(manifest_file, archive_name, target, platform):
-    with manifest_file.open(mode='w') as f:
-        f.write('''{archive_name}
-***************************
-
-Dependencies archive for {target} on platform {platform}
-Generated at {date}
-'''.format(
-    archive_name=archive_name,
-    target=target,
-    platform=platform,
-    date=date.today().isoformat()))
-
-
-def run_kiwix_build(target, platform,
-                    build_deps_only=False,
-                    target_only=False,
-                    make_release=False,
-                    make_dist=False):
-    command = ['kiwix-build']
-    command.append(target)
-    command.append('--hide-progress')
-    command.append('--fast-clone')
-    if platform == 'flatpak' or platform.startswith('win32_'):
-        command.append('--assume-packages-installed')
-    if target == 'kiwix-lib-app' and platform.startswith('android_'):
-        command.extend(['--target-platform', 'android', '--android-arch', platform[8:]])
-    elif platform == 'android':
-        command.extend(['--target-platform', 'android'])
-        for arch in ('arm', 'arm64', 'x86', 'x86_64'):
-            command.extend(['--android-arch', arch])
-    else:
-        command.extend(['--target-platform', platform])
-    if build_deps_only:
-        command.append('--build-deps-only')
-    if target_only:
-        command.append('--build-nodeps')
-    if make_release:
-        command.append('--make-release')
-    if make_dist:
-        command.append('--make-dist')
-    print_message("Build {} (deps={}, release={}, dist={})",
-        target, build_deps_only, make_release, make_dist)
-    subprocess.check_call(command, cwd=str(HOME))
-    print_message("{}", list(BASE_DIR.glob("{}*/**/*.tar.xz".format(target))))
-    print_message("Build ended")
-
-
-def create_desktop_image():
-    print_message("creating desktop image")
-    if make_release:
-        postfix = main_project_versions['kiwix-desktop']
-        extra_postfix = release_versions.get('kiwix-desktop')
-        if extra_postfix is None:
-            # We should not make archives for release not wanted
-            return
-        if extra_postfix:
-            postfix = "{}-{}".format(postfix, extra_postfix)
-        archive_dir = RELEASE_KIWIX_ARCHIVES_DIR/'kiwix-desktop'
-        src_dir = SOURCE_DIR/'kiwix-desktop_release'
-    else:
-        postfix = _date
-        archive_dir = NIGHTLY_KIWIX_ARCHIVES_DIR
-        src_dir = SOURCE_DIR/'kiwix-desktop'
-
-    if PLATFORM == 'flatpak':
-        build_path = BASE_DIR/'org.kiwix.desktop.flatpak'
-        app_name = 'org.kiwix.desktop.{}.flatpak'.format(postfix)
-        print_message("archive is {}", build_path)
-    else:
-        build_path = HOME/'Kiwix-{}-x86_64.AppImage'.format(postfix)
-        app_name = "kiwix-desktop_x86_64_{}.appimage".format(postfix)
-        command = ['kiwix-build/scripts/create_kiwix-desktop_appImage.sh',
-                   str(INSTALL_DIR), str(src_dir), str(HOME/'AppDir')]
-        env = dict(os.environ)
-        env['VERSION'] = postfix
-        print_message("Build AppImage of kiwix-desktop")
-        subprocess.check_call(command, cwd=str(HOME), env=env)
-
-    try:
-        archive_dir.mkdir(parents=True)
-    except FileExistsError:
-        pass
-
-    print_message("Copy Build to {}".format(archive_dir/app_name))
-    shutil.copy(str(build_path), str(archive_dir/app_name))
-
-
-def make_archive(project, platform):
-    base_dir, export_files = EXPORT_FILES[project]
-
-    if make_release:
-        postfix = main_project_versions[project]
-        extra_postfix = release_versions.get(project)
-        if extra_postfix is None:
-            # We should not make archives for release not wanted
-            return
-        if extra_postfix:
-            postfix = "{}-{}".format(postfix, extra_postfix)
-        if project in ('kiwix-lib', 'kiwix-tools'):
-            archive_dir = RELEASE_KIWIX_ARCHIVES_DIR/project
-        else:
-            archive_dir = RELEASE_ZIM_ARCHIVES_DIR/project
-    else:
-        postfix = _date
-        if project in ('kiwix-lib', 'kiwix-tools'):
-            archive_dir = NIGHTLY_KIWIX_ARCHIVES_DIR
-        else:
-            archive_dir = NIGHTLY_ZIM_ARCHIVES_DIR
-
-    try:
-        archive_dir.mkdir(parents=True)
-    except FileExistsError:
-        pass
-
-    archive_name = "{}_{}-{}".format(project, platform, postfix)
-
-    files_to_archive = []
-    for export_file in export_files:
-        files_to_archive.extend(base_dir.glob(export_file))
-    if platform == "win-i686":
-        open_archive = lambda a : zipfile.ZipFile(str(a), 'w', compression=zipfile.ZIP_DEFLATED)
-        archive_add = lambda a, f : a.write(str(f), arcname=str(f.relative_to(base_dir)))
-        archive_ext = ".zip"
-    else:
-        open_archive = lambda a : tarfile.open(str(a), 'w:gz')
-        archive_add = lambda a, f : a.add(str(f), arcname="{}/{}".format(archive_name, str(f.relative_to(base_dir))))
-        archive_ext = ".tar.gz"
-
-
-    archive = archive_dir/'{}{}'.format(archive_name, archive_ext)
-    print_message("create archive {} with {}", archive, files_to_archive)
-    with open_archive(archive) as arch:
-        for f in files_to_archive:
-            archive_add(arch, f)
-
-
-def make_deps_archive(target=None, name=None, full=False):
-    archive_name = name or "deps_{}_{}_{}.tar.xz".format(
-        TRAVIS_OS_NAME, PLATFORM, target)
-    print_message("Create archive {}.", archive_name)
-    files_to_archive = [INSTALL_DIR]
-    files_to_archive += HOME.glob('BUILD_*/LOGS')
-    if PLATFORM == 'native_mixed':
-        files_to_archive += [HOME/'BUILD_native_static'/'INSTALL']
-    if PLATFORM.startswith('android'):
-        files_to_archive.append(HOME/'BUILD_neutral'/'INSTALL')
-    if PLATFORM == 'android':
-        for arch in ('arm', 'arm64', 'x86', 'x86_64'):
-            base_dir = HOME/"BUILD_android_{}".format(arch)
-            files_to_archive.append(base_dir/'INSTALL')
-            if (base_dir/'meson_cross_file.txt').exists():
-                files_to_archive.append(base_dir/'meson_cross_file.txt')
-    files_to_archive += HOME.glob('BUILD_*/android-ndk*')
-    files_to_archive += HOME.glob('BUILD_*/android-sdk*')
-    if (BASE_DIR/'meson_cross_file.txt').exists():
-        files_to_archive.append(BASE_DIR/'meson_cross_file.txt')
-
-    manifest_file = BASE_DIR/'manifest.txt'
-    write_manifest(manifest_file, archive_name, target, PLATFORM)
-    files_to_archive.append(manifest_file)
-
-    relative_path = HOME
-    if full:
-        files_to_archive += ARCHIVE_DIR.glob(".*_ok")
-        files_to_archive += BASE_DIR.glob('*/.*_ok')
-        files_to_archive += (HOME/"BUILD_native_dyn").glob('*/.*_ok')
-        files_to_archive += (HOME/"BUILD_native_static").glob('*/.*_ok')
-        files_to_archive += HOME.glob('BUILD_android*/**/.*_ok')
-        files_to_archive += SOURCE_DIR.glob('*/.*_ok')
-        if PLATFORM.startswith('armhf'):
-            files_to_archive += (SOURCE_DIR/'armhf').glob('*')
-        toolchains_subdirs = HOME.glob('**/TOOLCHAINS/*/*')
-        for subdir in toolchains_subdirs:
-            if not subdir.match('tools'):
-                files_to_archive.append(subdir)
-
-    with tarfile.open(str(BASE_EXPORT_DIR/archive_name), 'w:xz') as tar:
-        for name in set(files_to_archive):
-            print('.{}'.format(name), flush=True)
-            tar.add(str(name), arcname=str(name.relative_to(relative_path)))
-
-def make_flatpak_cache_archive():
-    archive_name = "base_deps_{os}_{platform}.tar.xz".format(
-        os=TRAVIS_OS_NAME,
-        platform=PLATFORM)
-    cache = BASE_DIR/'.flatpak-builder'
-    print_message("make flatpak cache archive")
-    nb_files = 0
-    def pseudo_filter(tarinfo):
-        nonlocal nb_files
-        nb_files += 1
-        if (nb_files % 1000) == 0:
-            print('.', flush=True)
-        return tarinfo
-    with tarfile.open(str(BASE_EXPORT_DIR/archive_name), 'w:xz') as tar:
-        tar.add(str(cache), arcname=str(cache.relative_to(BASE_DIR)), filter=pseudo_filter)
-
-
-def update_flathub_git():
-    GIT_REPO_DIR = GIT_EXPORT_DIR/"org.kiwix.desktop"
-    env = dict(os.environ)
-    env['GIT_AUTHOR_NAME'] = env['GIT_COMMITTER_NAME'] = "KiwixBot"
-    env['GIT_AUTHOR_EMAIL'] = env['GIT_COMMITTER_EMAIL'] = "kiwixbot@kymeria.fr"
-    def call(command, cwd=None):
-        cwd = cwd or GIT_REPO_DIR
-        print_message("call {}", command)
-        subprocess.check_call(command, env=env, cwd=str(cwd))
-    command = ['git', 'clone', FLATPAK_HTTP_GIT_REMOTE]
-    call(command, cwd=GIT_EXPORT_DIR)
-    shutil.copy(str(BASE_DIR/'org.kiwix.desktop.json'), str(GIT_REPO_DIR))
-    patch_dir = KBUILD_SOURCE_DIR/'kiwixbuild'/'patches'
-    for dep in ('libaria2', 'mustache', 'pugixml', 'xapian', 'zstd'):
-        for f in patch_dir.glob('{}_*.patch'.format(dep)):
-            shutil.copy(str(f), str(GIT_REPO_DIR/'patches'))
-    command = ['git', 'add', '-A', '.']
-    call(command)
-    command = ['git', 'commit', '-m',
-               'Update to version {}'.format(main_project_versions['kiwix-desktop'])]
-    try:
-        call(command)
-    except subprocess.CalledProcessError:
-        # This may fail if there is nothing to commit (a rebuild of the CI for exemple)
-        pass
-    command = ['git', 'config', 'remote.origin.pushurl', FLATPAK_GIT_REMOTE]
-    call(command)
-
-
-
-for p in (NIGHTLY_KIWIX_ARCHIVES_DIR,
-          NIGHTLY_ZIM_ARCHIVES_DIR,
-          RELEASE_KIWIX_ARCHIVES_DIR,
-          RELEASE_ZIM_ARCHIVES_DIR,
-          DIST_KIWIX_ARCHIVES_DIR,
-          DIST_ZIM_ARCHIVES_DIR,
-          BINTRAY_ARCHIVES_DIR,
-          BASE_EXPORT_DIR,
-          GIT_EXPORT_DIR):
-    try:
-        p.mkdir(parents=True)
-    except FileExistsError:
-        pass
-
-make_release = re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", environ.get('TRAVIS_TAG', '')) is not None
-
-def download_base_archive(base_name):
-    url = 'http://tmp.kiwix.org/ci/{}'.format(base_name)
-    file_path = str(HOME/base_name)
-    batch_size = 1024*1024*8
-    with urlopen(url) as resource, open(file_path, 'wb') as file:
-        while True:
-            batch = resource.read(batch_size)
-            if not batch:
-                break
-            print(".", end="", flush=True)
-            file.write(batch)
-    return file_path
-
-def export_dist(target):
-    if target in ('kiwix-lib', 'kiwix-tools', 'kiwix-desktop'):
-        out_dir = DIST_KIWIX_ARCHIVES_DIR
-    else:
-        out_dir = DIST_ZIM_ARCHIVES_DIR
-
-    if target in main_project_versions:
-        try:
-            (out_dir/target).mkdir(parents=True)
-        except FileExistsError:
-            pass
-
-        full_target_name = "{}-{}".format(target, main_project_versions[target])
-        if target != 'kiwix-desktop':
-            in_file = BASE_DIR/full_target_name/'meson-dist'/'{}.tar.xz'.format(
-                full_target_name)
-        else:
-            in_file = BASE_DIR/full_target_name/'{}.tar.gz'.format(full_target_name)
-        if in_file.exists():
-            print_message("Copying {} to {}", in_file, out_dir/target)
-            shutil.copy(str(in_file), str(out_dir/target))
-        else:
-            print_message("No {} to copy.", in_file)
-
-
-if PLATFORM != 'flatpak':
-    # The first thing we need to do is to (potentially) download already compiled base dependencies.
-    base_dep_archive_name = "base_deps_{os}_{platform}_{version}_{release}.tar.xz".format(
-        os=TRAVIS_OS_NAME,
-        platform=PLATFORM,
-        version=base_deps_meta_version,
-        release='release' if make_release else 'debug')
-
-    print_message("Getting archive {}", base_dep_archive_name)
-    try:
-        local_filename = download_base_archive(base_dep_archive_name)
-        with tarfile.open(local_filename) as f:
-            f.extractall(str(HOME))
-    except URLError:
-        print_message("Cannot get archive. Build dependencies")
-        if PLATFORM == 'android':
-            for arch in ('arm', 'arm64', 'x86', 'x86_64'):
-                archive_name = "base_deps_{os}_android_{arch}_{version}_{release}.tar.xz".format(
-                    os=TRAVIS_OS_NAME,
-                    arch=arch,
-                    version=base_deps_meta_version,
-                    release='release' if make_release else 'debug')
-                print_message("Getting archive {}", archive_name)
-                try:
-                    local_filename = download_base_archive(archive_name)
-                    with tarfile.open(local_filename) as f:
-                        f.extractall(str(HOME))
-                except URLError:
-                    pass
-        else:
-            run_kiwix_build('alldependencies', platform=PLATFORM)
-            make_deps_archive(name=base_dep_archive_name, full=True)
-else:
-    base_dep_archive_name = "base_deps_{os}_{platform}.tar.xz".format(
-        os=TRAVIS_OS_NAME,
-        platform=PLATFORM)
-
-    print_message("Getting archive {}", base_dep_archive_name)
-    try:
-        local_filename = download_base_archive(base_dep_archive_name)
-        BASE_DIR.mkdir(parents=True)
-        with tarfile.open(local_filename) as f:
-            f.extractall(str(BASE_DIR))
-    except URLError:
-        print_message("Cannot get archive. Move on")
-
-
-# A basic compilation to be sure everything is working (for a PR)
-if environ['TRAVIS_EVENT_TYPE'] != 'cron' and not make_release:
-    if PLATFORM.startswith('android'):
-        TARGETS = ('kiwix-lib',)
-    elif PLATFORM.startswith('iOS'):
-        TARGETS = ('kiwix-lib',)
-    elif PLATFORM.startswith('native_'):
-        if TRAVIS_OS_NAME == "osx":
-            TARGETS = ('kiwix-lib', 'zim-tools', 'zimwriterfs')
-        elif PLATFORM == 'native_dyn' and KIWIX_DESKTOP_ONLY:
-            TARGETS = ('kiwix-desktop', )
-        elif PLATFORM == 'native_mixed':
-            TARGETS = ('libzim', )
-        else:
-            TARGETS = ('kiwix-tools', 'zim-tools', 'zimwriterfs')
-    elif PLATFORM == 'flatpak':
-        TARGETS = ('kiwix-desktop', )
-    else:
-        TARGETS = ('kiwix-tools', )
-
-    for target in TARGETS:
-        run_kiwix_build(target,
-                        platform=PLATFORM)
-
-    if PLATFORM == 'native_mixed':
-        make_archive('libzim', 'linux-x86_64')
-    elif PLATFORM == 'native_static':
-        for target in ('kiwix-tools', 'zim-tools', 'zimwriterfs'):
-            make_archive(target, 'linux-x86_64')
-    elif PLATFORM == 'win32_static':
-        make_archive('kiwix-tools', 'win-i686')
-    elif PLATFORM == 'armhf_static':
-        make_archive('kiwix-tools', 'linux-armhf')
-    elif PLATFORM == 'i586_static':
-        make_archive('kiwix-tools', 'linux-i586')
-    elif PLATFORM == 'flatpak':
-        make_flatpak_cache_archive()
-
-    sys.exit(0)
-
-
-TARGETS = tuple()
-if PLATFORM == 'android':
-    TARGETS = ('kiwix-lib-app',)
-elif PLATFORM.startswith('android_'):
-    TARGETS = ('libzim', 'kiwix-lib')
-elif PLATFORM.startswith('iOS'):
-    TARGETS = ('libzim', 'kiwix-lib')
-elif PLATFORM.startswith('native_'):
-    if TRAVIS_OS_NAME == "osx":
-        TARGETS = ('libzim', 'zimwriterfs', 'zim-tools', 'kiwix-lib')
-    else:
-        if PLATFORM == 'native_dyn' and KIWIX_DESKTOP_ONLY:
-            TARGETS = ('kiwix-desktop', )
-        elif PLATFORM == 'native_mixed':
-            TARGETS = ('libzim', )
-        else:
-            TARGETS = ('libzim', 'zimwriterfs', 'zim-tools', 'kiwix-lib', 'kiwix-tools')
-elif PLATFORM == 'flatpak':
-    TARGETS = ('kiwix-desktop', )
-else:
-    TARGETS = ('libzim', 'zim-tools', 'kiwix-lib', 'kiwix-tools')
-
-for target in TARGETS:
-    if environ['TRAVIS_EVENT_TYPE'] == 'cron' and PLATFORM not in ('android', 'flatpak'):
-        run_kiwix_build(target,
-                        platform=PLATFORM,
-                        build_deps_only=True)
-        make_deps_archive(target=target)
-
-    if target == 'kiwix-lib-app':
-        extra_postfix = release_versions.get('kiwix-lib')
-        if extra_postfix:
-            environ['KIWIXLIB_BUILDVERSION'] = extra_postfix
-
-    run_kiwix_build(target,
-                    platform=PLATFORM,
-                    make_release=make_release)
-    print_message("target is {}", target)
-    if target == 'kiwix-desktop':
-        create_desktop_image()
-    if make_release and PLATFORM == 'native_dyn' and release_versions.get(target) == 0:
-        run_kiwix_build(target,
-                        platform=PLATFORM,
-                        make_release=True,
-                        make_dist=True)
-        export_dist(target)
-
-# We have build everything. Now create archives for public deployement.
-if PLATFORM == 'native_mixed':
-    make_archive('libzim', 'linux-x86_64')
-elif PLATFORM == 'native_static':
-    for target in ('kiwix-tools', 'zim-tools', 'zimwriterfs'):
-        make_archive(target, 'linux-x86_64')
-elif PLATFORM == 'win32_static':
-    make_archive('kiwix-tools', 'win-i686')
-elif PLATFORM == 'armhf_static':
-    make_archive('kiwix-tools', 'linux-armhf')
-elif PLATFORM == 'i586_static':
-    make_archive('kiwix-tools', 'linux-i586')
-elif ( make_release
-   and PLATFORM == 'flatpak'
-   and release_versions.get('kiwix-desktop') is not None):
-    update_flathub_git()
-elif PLATFORM == 'android' and 'kiwix-lib-app' in TARGETS:
-    if make_release and release_versions.get('kiwix-lib') is not None:
-        postfix = main_project_versions['kiwix-lib']
-        extra_postfix = release_versions.get('kiwix-lib')
-        if extra_postfix:
-            postfix = "{}-{}".format(postfix, extra_postfix)
-
-        basename = "kiwixlib-{}".format(postfix)
-
-        output_release_dir = HOME/'BUILD_android'/'kiwix-lib-app'/'kiwixLibAndroid'/'build'
-        shutil.copy(str(output_release_dir/'outputs'/'aar'/'kiwixLibAndroid-release.aar'),
-                    str(BINTRAY_ARCHIVES_DIR/(basename+'.aar')))
-        shutil.copy(str(output_release_dir/'pom.xml'),
-                    str(BINTRAY_ARCHIVES_DIR/(basename+'.pom')))
-
-
-        json_filename = '{}_bintray_info.json'.format(basename)
-        data = {
-            'version': postfix,
-            'files': [basename+ext for ext in ('.aar', '.pom')]
-        }
-        with open(str(BINTRAY_ARCHIVES_DIR/json_filename), 'w') as f:
-            json.dump(data, f)
-

travis/deploy.sh

@@ -1,106 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-EXPORT_DIR=${HOME}/EXPORT
-NIGHTLY_KIWIX_ARCHIVES_DIR=${EXPORT_DIR}/NIGHTLY_KIWIX_ARCHIVES/${NIGHTLY_DATE}
-NIGHTLY_ZIM_ARCHIVES_DIR=${EXPORT_DIR}/NIGHTLY_ZIM_ARCHIVES/${NIGHTLY_DATE}
-RELEASE_KIWIX_ARCHIVES_DIR=${EXPORT_DIR}/RELEASE_KIWIX_ARCHIVES
-RELEASE_ZIM_ARCHIVES_DIR=${EXPORT_DIR}/RELEASE_ZIM_ARCHIVES
-DIST_KIWIX_ARCHIVES_DIR=${EXPORT_DIR}/DIST_KIWIX_ARCHIVES
-DIST_ZIM_ARCHIVES_DIR=${EXPORT_DIR}/DIST_ZIM_ARCHIVES
-BINTRAY_ARCHIVES_DIR=${EXPORT_DIR}/BINTRAY_ARCHIVES
-
-if [[ "$TRAVIS_EVENT_TYPE" = "cron" ]]
-then
-  echo "Publishing nightlies"
-  ls ${NIGHTLY_KIWIX_ARCHIVES_DIR}
-  scp -rp -i ${SSH_KEY} -o StrictHostKeyChecking=no \
-    ${NIGHTLY_KIWIX_ARCHIVES_DIR} \
-    ci@download.kiwix.org:/data/download/nightly
-  ls ${NIGHTLY_ZIM_ARCHIVES_DIR}
-  scp -rp -i ${SSH_KEY} -o StrictHostKeyChecking=no \
-    ${NIGHTLY_ZIM_ARCHIVES_DIR} \
-    ci@download.openzim.org:/data/openzim/nightly
-
-elif [[ "x$TRAVIS_TAG" != "x" ]]
-then
-  RELEASE_ARCHIVES=$(find $RELEASE_KIWIX_ARCHIVES_DIR -type f)
-  if [[ "x$RELEASE_ARCHIVES" != "x" ]]
-  then
-    echo "Publishing kiwix archives $RELEASE_ARCHIVES"
-    for archive in $RELEASE_ARCHIVES
-    do
-      subdir=$(basename $(dirname $archive))
-      scp -rp -i ${SSH_KEY} -o StrictHostKeyChecking=no \
-        ${archive} \
-        ci@download.kiwix.org:/data/download/release/${subdir}
-    done
-  fi
-
-  RELEASE_ARCHIVES=$(find $RELEASE_ZIM_ARCHIVES_DIR -type f)
-  if [[ "x$RELEASE_ARCHIVES" != "x" ]]
-  then
-    echo "Publishing zim archives $RELEASE_ARCHIVES"
-    for archive in $RELEASE_ARCHIVES
-    do
-      subdir=$(basename $(dirname $archive))
-      scp -rp -i ${SSH_KEY} -o StrictHostKeyChecking=no \
-        ${archive} \
-        ci@download.openzim.org:/data/openzim/release/${subdir}
-    done
-  fi
-
-  DIST_KIWIX_ARCHIVES=$(find $DIST_KIWIX_ARCHIVES_DIR -type f)
-  if [[ "x$DIST_KIWIX_ARCHIVES" != "x" ]]
-  then
-    echo "Publishing dist kiwix archives $DIST_KIWIX_ARCHIVES"
-    for archive in $DIST_KIWIX_ARCHIVES
-    do
-      subdir=$(basename $(dirname $archive))
-      scp -rp -i ${SSH_KEY} -o StrictHostKeyChecking=no \
-        ${archive} \
-        ci@download.kiwix.org:/data/download/release/${subdir}
-    done
-  fi
-
-  DIST_ZIM_ARCHIVES=$(find $DIST_ZIM_ARCHIVES_DIR -type f)
-  if [[ "x$DIST_ZIM_ARCHIVES" != "x" ]]
-  then
-    echo "Publishing dist zim archives $DIST_ZIM_ARCHIVES"
-    for archive in $DIST_ZIM_ARCHIVES
-    do
-      subdir=$(basename $(dirname $archive))
-      scp -rp -i ${SSH_KEY} -o StrictHostKeyChecking=no \
-        ${archive} \
-        ci@download.openzim.org:/data/openzim/release/${subdir}
-    done
-  fi
-
-  (cd ${EXPORT_DIR}/GIT
-  GIT_REPOS=$(ls -l | awk '/^d/ { print $9 }')
-  if [[ "x$GIT_REPOS" != "x" ]]
-  then
-    echo "Pushing repositories $GIT_REPOS"
-    for repo in $GIT_REPOS
-    do
-      (cd $repo;
-      GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=no -i $SSH_KEY" git push
-      )
-    done
-  fi
-  )
-
-  ls $BINTRAY_ARCHIVES_DIR
-  BINTRAY_ARCHIVES=$(find $BINTRAY_ARCHIVES_DIR -name "*_bintray_info.json" -type f)
-  if [[ "x$BINTRAY_ARCHIVES" != "x" ]]
-  then
-    echo "Publish bintray $BINTRAY_ARCHIVES"
-    for archive_info in $BINTRAY_ARCHIVES
-    do
-      ${TRAVIS_BUILD_DIR}/scripts/upload_kiwix_lib_android_to_bintray.py $archive_info
-    done
- fi
-
-fi
-

travis/f30_builder.dockerfile

@@ -1,28 +0,0 @@
-FROM fedora:30
-
-ENV LANG C.UTF-8
-
-RUN dnf install -y --nodocs \
-# Base build tools
-    make automake libtool cmake git-core subversion pkg-config gcc-c++ \
-    wget unzip ninja-build ccache which patch \
-# Cross win32 compiler
-    mingw32-gcc-c++ mingw32-bzip2-static mingw32-win-iconv-static \
-    mingw32-winpthreads-static mingw32-zlib-static mingw32-xz-libs-static \
-    mingw32-libmicrohttpd \
-# Other tools (to remove)
-#    vim less grep
-  && dnf remove -y "*-doc" \
-  && dnf autoremove -y \
-  && dnf clean all
-
-# Create user
-RUN useradd --create-home ci_builder
-USER ci_builder
-WORKDIR /home/ci_builder
-ENV PATH="/home/ci_builder/.local/bin:${PATH}"
-
-ENV TRAVIS_BUILD_DIR /home/ci_builder/kiwix-build
-ENV TRAVIS_OS_NAME linux_f30
-
-CMD pip3 install --user ./kiwix-build && kiwix-build/travis/compile_all.py

travis/focal_builder.dockerfile

@@ -1,37 +0,0 @@
-FROM ubuntu:focal
-
-ENV LANG C.UTF-8
-
-RUN apt update -q && \
-    apt install -q -y --no-install-recommends \
-# Base build tools
-    build-essential automake libtool cmake ccache pkg-config autopoint patch \
-    python3-pip python3-setuptools python3-wheel git subversion wget unzip \
-    ninja-build \
-# Packaged dependencies
-    libbz2-dev libmagic-dev uuid-dev zlib1g-dev \
-    libmicrohttpd-dev aria2 \
-# Qt packages
-    libqt5gui5 qtbase5-dev qtwebengine5-dev libqt5svg5-dev qt5-image-formats-plugins qt5-default \
-# To create the appimage of kiwix-desktop
-    libfuse2 fuse patchelf \
-# Flatpak tools
-    elfutils flatpak flatpak-builder \
-# Cross win32 compiler
-    g++-mingw-w64-i686 gcc-mingw-w64-i686 gcc-mingw-w64-base mingw-w64-tools \
-# Other tools (to remove)
-#    vim less grep \
-  && \
-  apt-get clean -y && \
-  rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/*
-
-# Create user
-RUN useradd --create-home ci_builder
-USER ci_builder
-WORKDIR /home/ci_builder
-ENV PATH="/home/ci_builder/.local/bin:${PATH}"
-
-ENV TRAVIS_BUILD_DIR /home/ci_builder/kiwix-build
-ENV TRAVIS_OS_NAME linux_focal
-
-CMD pip3 install --user ./kiwix-build && kiwix-build/travis/compile_all.py

travis/install_extra_deps.sh

@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-if [[ "$TRAVIS_OS_NAME" == "osx" ]]
-then
-  pip3 install pillow
-  pip3 install .
-
-  wget https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-mac.zip
-  unzip ninja-mac.zip ninja
-else
-  wget https://bootstrap.pypa.io/get-pip.py
-  python3.5 get-pip.py --user
-  python3.5 -m pip install --user --upgrade pip wheel
-  python3.5 -m pip install --user pillow
-  python3.5 -m pip install --user .
-  python3.5 -m pip show -f kiwix-build
-
-  wget https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip
-  unzip ninja-linux.zip ninja
-fi
-
-mkdir -p $HOME/bin
-cp ninja $HOME/bin
-

travis/travisci_builder_id_key.pub

@@ -1 +0,0 @@
-ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCWxcsEawR4TrXmGgMXXOhuYjXtO1V/BpEQ2tuf2tTiloZ0n1cPtxeLp3ic7F91p/e+58U1QlpLkxs9Ia0REXvr2ppd3FuIRjADs6g9iqK4jgAx/KKVjy2vhtxFtDKP5xCkbqSRjDcYnSZwrllEPbsOEIgTvRBaOyyqBNbctjSN6DhB+hY1bsPNJntDL0/XLqSYFXBnxd+unRLiItRJX1C42tpTwWyr9OvpjuPCjWVo/6WqohU8xjW29xPDpmAnWCX02JD//hRamJ+zfDPxf5AGTVJhk7of4Jjj4pTunhTXRLGhA3UBPApbI//NA2nAOc8feOchu1VfO6n5JuVbnXDzUlMFVqSNJ6ihYBTEczN+T/xbdc3Uyzi3JZX7IW2lH1YpxJoGEzJ67HjNPHdKgQzqVGlKl00qUrL1aMJ+1rOm8xDvPYVjHidORPUn0x+2fvxJ0VVr4crVwmTa23ft79MS2sLbw5DbPkURVUkEH88nSwcbF9fbuCPE4RL15NS7ZNwDqRMyQIxnAe7Sf+UWz45HxTrdvgjrsk0PCYX+KCgnXEexXWDDlbXWuiJa1h2VhKb2zM3AUmtRMK0pFTi/xCrFFwDIYYIBOMxMb0nYmruv5Feh/cez1FNYq9+UKrnBAV7DAZCLibM2/u46OAgbbtZS70zaX1JvcX44yUgjpClNcQ== TravisCIBuilder

travis/upload_all_log.sh

@@ -1,11 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-cd $HOME
-
-tar -czf fail_log_${PLATFORM}.tar.gz BUILD_${PLATFORM}
-
-scp -vrp -i ${SSH_KEY} \
-  fail_log_${PLATFORM}.tar.gz \
-  ci@tmp.kiwix.org:/data/tmp/ci

travis/upload_base_deps.sh

@@ -1,15 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-BASE_EXPORT_DIR=$HOME/EXPORT/BASE
-ls $BASE_EXPORT_DIR
-
-BASE_ARCHIVES=$(find $BASE_EXPORT_DIR -type f)
-echo $BASE_ARCHIVES
-if [[ "x$BASE_ARCHIVES" != "x" ]]
-then
-  scp -vrp -i ${SSH_KEY} -o StrictHostKeyChecking=no \
-    $BASE_ARCHIVES \
-    ci@tmp.kiwix.org:/data/tmp/ci
-fi

travis/xenial_builder.dockerfile

@@ -1,37 +0,0 @@
-FROM ubuntu:xenial
-
-ENV LANG C.UTF-8
-
-RUN apt update -q && \
-  dpkg --add-architecture i386 && \
-  apt install -q -y --no-install-recommends \
-# Base build tools
-    build-essential automake libtool cmake ccache pkg-config autopoint patch \
-    python3-pip python3-setuptools python3-wheel git subversion wget unzip \
-    ninja-build \
-# Python (2) is needed to install android-ndk
-    python \
-# Packaged dependencies
-    libbz2-dev libmagic-dev uuid-dev zlib1g-dev default-jdk \
-    libmicrohttpd-dev \
-# Cross compile i586
-    libc6-dev-i386 lib32stdc++6 gcc-multilib g++-multilib \
-# Other tools (to remove)
-#    vim less grep \
-  && \
-  apt-get clean -y && \
-  rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/* && \
-  pip3 install --upgrade pip && \
-  pip3 install meson==0.52.1 pytest gcovr
-
-# Create user
-RUN useradd --create-home ci_builder
-USER ci_builder
-WORKDIR /home/ci_builder
-ENV PATH="/home/ci_builder/.local/bin:${PATH}"
-
-ENV TRAVIS_BUILD_DIR /home/ci_builder/kiwix-build
-ENV GRADLE_USER_HOME /home/ci_builder
-ENV TRAVIS_OS_NAME linux_xenial
-
-CMD pip3 install --user --no-deps ./kiwix-build && kiwix-build/travis/compile_all.py