import os from os import environ as _environ from pathlib import Path from datetime import date import tarfile import zipfile import subprocess import re import shutil import requests from kiwixbuild.versions import ( main_project_versions, release_versions, base_deps_versions, ) PLATFORM_TARGET = _environ["PLATFORM_TARGET"] if PLATFORM_TARGET == "native_desktop": PLATFORM_TARGET = "native_dyn" DESKTOP = True else: DESKTOP = False OS_NAME = _environ["OS_NAME"] HOME = Path(os.path.expanduser("~")) BASE_DIR = HOME / "BUILD_{}".format(PLATFORM_TARGET) SOURCE_DIR = HOME / "SOURCE" ARCHIVE_DIR = HOME / "ARCHIVE" INSTALL_DIR = BASE_DIR / "INSTALL" TMP_DIR = Path("/tmp") KBUILD_SOURCE_DIR = HOME / "kiwix-build" # [TODO] KIWIX_DESKTOP_ONLY = False _ref = _environ.get("GITHUB_REF", "").split("/")[-1] MAKE_RELEASE = re.fullmatch(r"r_[0-9]+", _ref) is not None RELEASE_OS_NAME = "macos" if OS_NAME == "osx" else "linux" PLATFORM_TO_RELEASE = { "native_mixed": "{os}-x86_64".format(os=RELEASE_OS_NAME), "native_static": "{os}-x86_64".format(os=RELEASE_OS_NAME), "win32_static": "win-i686", "armhf_static": "{os}-armhf".format(os=RELEASE_OS_NAME), "i586_static": "{os}-i586".format(os=RELEASE_OS_NAME), } FLATPAK_HTTP_GIT_REMOTE = "https://github.com/flathub/org.kiwix.desktop.git" FLATPAK_GIT_REMOTE = "git@github.com:flathub/org.kiwix.desktop.git" BIN_EXT = ".exe" if PLATFORM_TARGET.startswith("win32_") else "" # We have build everything. Now create archives for public deployement. EXPORT_FILES = { "kiwix-tools": ( INSTALL_DIR / "bin", [ f + BIN_EXT for f in ("kiwix-manage", "kiwix-read", "kiwix-search", "kiwix-serve") ], ), "zim-tools": ( INSTALL_DIR / "bin", [ f + BIN_EXT for f in ( "zimbench", "zimcheck", "zimdump", "zimsearch", "zimdiff", "zimpatch", "zimsplit", "zimwriterfs", ) ], ), "libzim": ( INSTALL_DIR, ( "lib/x86_64-linux-gnu/libzim.so.{}".format(main_project_versions["libzim"]), "lib/x86_64-linux-gnu/libzim.so.{}".format( main_project_versions["libzim"][0] ), "lib/libzim.{}.dylib".format( main_project_versions["libzim"][0] ), "lib/libzim.dylib", "include/zim/**/*.h", ), ), } DATE = date.today().isoformat() def print_message(message, *args, **kwargs): message = message.format(*args, **kwargs) message = "{0} {1} {0}".format("-" * 3, message) print(message, flush=True) MANIFEST_TEMPLATE = """{archive_name} *************************** Dependencies archive for {target} on platform {platform} Generated at {date} """ def write_manifest(manifest_file, archive_name, target, platform): with manifest_file.open(mode="w") as f: f.write( MANIFEST_TEMPLATE.format( archive_name=archive_name, target=target, platform=platform, date=DATE, ) ) def run_kiwix_build( target, platform, build_deps_only=False, target_only=False, make_release=False, make_dist=False, ): command = ["kiwix-build"] command.append(target) command.append("--hide-progress") command.append("--fast-clone") if platform == "flatpak" or platform.startswith("win32_"): command.append("--assume-packages-installed") if target == "kiwix-lib-app" and platform.startswith("android_"): command.extend(["--target-platform", "android", "--android-arch", platform[8:]]) elif platform == "android": command.extend(["--target-platform", "android"]) for arch in ("arm", "arm64", "x86", "x86_64"): command.extend(["--android-arch", arch]) else: command.extend(["--target-platform", platform]) if build_deps_only: command.append("--build-deps-only") if target_only: command.append("--build-nodeps") if make_release: command.append("--make-release") if make_dist: command.append("--make-dist") print_message( "Build {} (deps={}, release={}, dist={})", target, build_deps_only, make_release, make_dist, ) env = dict(_environ) env["SKIP_BIG_MEMORY_TEST"] = "1" subprocess.check_call(command, cwd=str(HOME), env=env) print_message("Build ended") def upload(file_to_upload, host, dest_path): if not file_to_upload.exists(): print_message("No {} to upload!", file_to_upload) return command = [ "ssh", "-i", _environ.get("SSH_KEY"), "-o", "StrictHostKeyChecking=no", host, "mkdir -p {}".format(dest_path), ] print_message("Creating dest path {}", dest_path) subprocess.check_call(command) command = [ "scp", "-i", _environ.get("SSH_KEY"), "-o", "StrictHostKeyChecking=no", str(file_to_upload), "{}:{}".format(host, dest_path), ] print_message("Sending archive with command {}", command) subprocess.check_call(command) def upload_archive(archive, project, make_release): if project.startswith("kiwix-"): host = "ci@download.kiwix.org" dest_path = "/data/download/" else: host = "ci@download.openzim.org" dest_path = "/data/openzim/" if make_release: dest_path = dest_path + "release/" + project else: dest_path = dest_path + "nightly/" + DATE upload(archive, host, dest_path) def make_deps_archive(target=None, name=None, full=False): archive_name = name or "deps2_{}_{}_{}.tar.xz".format( OS_NAME, PLATFORM_TARGET, target ) print_message("Create archive {}.", archive_name) files_to_archive = [INSTALL_DIR] files_to_archive += HOME.glob("BUILD_*/LOGS") if PLATFORM_TARGET == "native_mixed": files_to_archive += [HOME / "BUILD_native_static" / "INSTALL"] if PLATFORM_TARGET.startswith("android"): files_to_archive.append(HOME / "BUILD_neutral" / "INSTALL") if PLATFORM_TARGET == "android": for arch in ("arm", "arm64", "x86", "x86_64"): base_dir = HOME / "BUILD_android_{}".format(arch) files_to_archive.append(base_dir / "INSTALL") if (base_dir / "meson_cross_file.txt").exists(): files_to_archive.append(base_dir / "meson_cross_file.txt") files_to_archive += HOME.glob("BUILD_*/android-ndk*") files_to_archive += HOME.glob("BUILD_*/android-sdk*") if (BASE_DIR / "meson_cross_file.txt").exists(): files_to_archive.append(BASE_DIR / "meson_cross_file.txt") manifest_file = BASE_DIR / "manifest.txt" write_manifest(manifest_file, archive_name, target, PLATFORM_TARGET) files_to_archive.append(manifest_file) relative_path = HOME if full: files_to_archive += ARCHIVE_DIR.glob(".*_ok") files_to_archive += BASE_DIR.glob("*/.*_ok") files_to_archive += (HOME / "BUILD_native_dyn").glob("*/.*_ok") files_to_archive += (HOME / "BUILD_native_static").glob("*/.*_ok") files_to_archive += HOME.glob("BUILD_android*/**/.*_ok") files_to_archive += SOURCE_DIR.glob("*/.*_ok") files_to_archive += SOURCE_DIR.glob("zim-testing-suite-*/*") if PLATFORM_TARGET.startswith("armhf"): files_to_archive += (SOURCE_DIR / "armhf").glob("*") toolchains_subdirs = HOME.glob("BUILD_*/TOOLCHAINS/*/*") for subdir in toolchains_subdirs: if not subdir.match("tools"): files_to_archive.append(subdir) archive_file = TMP_DIR / archive_name with tarfile.open(str(archive_file), "w:xz") as tar: for name in set(files_to_archive): print(".{}".format(name), flush=True) tar.add(str(name), arcname=str(name.relative_to(relative_path))) return archive_file def get_postfix(project): postfix = main_project_versions[project] extra = release_versions.get(project) if extra: postfix = "{}-{}".format(postfix, extra) return postfix def make_archive(project, make_release): try: platform = PLATFORM_TO_RELEASE[PLATFORM_TARGET] except KeyError: # We don't know how to name the release. return None try: base_dir, export_files = EXPORT_FILES[project] except KeyError: # No binary files to export return None if make_release: postfix = get_postfix(project) else: postfix = DATE archive_name = "{}_{}-{}".format(project, platform, postfix) files_to_archive = [] for export_file in export_files: files_to_archive.extend(base_dir.glob(export_file)) if platform == "win-i686": open_archive = lambda a: zipfile.ZipFile( str(a), "w", compression=zipfile.ZIP_DEFLATED ) archive_add = lambda a, f: a.write(str(f), arcname=str(f.relative_to(base_dir))) archive_ext = ".zip" else: open_archive = lambda a: tarfile.open(str(a), "w:gz") archive_add = lambda a, f: a.add( str(f), arcname="{}/{}".format(archive_name, str(f.relative_to(base_dir))) ) archive_ext = ".tar.gz" archive = TMP_DIR / "{}{}".format(archive_name, archive_ext) print_message("create archive {} with {}", archive, files_to_archive) with open_archive(archive) as arch: for f in files_to_archive: archive_add(arch, f) return archive def create_desktop_image(make_release): print_message("creating desktop image") if make_release: postfix = get_postfix("kiwix-desktop") src_dir = SOURCE_DIR / "kiwix-desktop_release" else: postfix = DATE src_dir = SOURCE_DIR / "kiwix-desktop" if PLATFORM_TARGET == "flatpak": build_path = BASE_DIR / "org.kiwix.desktop.flatpak" app_name = "org.kiwix.desktop.{}.flatpak".format(postfix) print_message("archive is {}", build_path) else: build_path = HOME / "Kiwix-{}-x86_64.AppImage".format(postfix) app_name = "kiwix-desktop_x86_64_{}.appimage".format(postfix) command = [ "kiwix-build/scripts/create_kiwix-desktop_appImage.sh", str(INSTALL_DIR), str(src_dir), str(HOME / "AppDir"), ] env = dict(os.environ) env["VERSION"] = postfix print_message("Build AppImage of kiwix-desktop") subprocess.check_call(command, cwd=str(HOME), env=env) print_message("Copy Build to {}".format(TMP_DIR / app_name)) shutil.copy(str(build_path), str(TMP_DIR / app_name)) return TMP_DIR / app_name def update_flathub_git(): git_repo_dir = TMP_DIR / "org.kiwix.desktop" env = dict(os.environ) env["GIT_AUTHOR_NAME"] = env["GIT_COMMITTER_NAME"] = "KiwixBot" env["GIT_AUTHOR_EMAIL"] = env["GIT_COMMITTER_EMAIL"] = "release@kiwix.org" def call(command, cwd=None): cwd = cwd or git_repo_dir print_message("call {}", command) subprocess.check_call(command, env=env, cwd=str(cwd)) command = ["git", "clone", FLATPAK_HTTP_GIT_REMOTE] call(command, cwd=TMP_DIR) shutil.copy(str(BASE_DIR / "org.kiwix.desktop.json"), str(git_repo_dir)) patch_dir = KBUILD_SOURCE_DIR / "kiwixbuild" / "patches" for dep in ("libaria2", "mustache", "pugixml", "xapian", "zstd"): for f in patch_dir.glob("{}_*.patch".format(dep)): shutil.copy(str(f), str(git_repo_dir / "patches")) command = ["git", "add", "-A", "."] call(command) command = [ "git", "commit", "-m", "Update to version {}".format(main_project_versions["kiwix-desktop"]), ] try: call(command) except subprocess.CalledProcessError: # This may fail if there is nothing to commit (a rebuild of the CI for exemple) return command = ["git", "config", "remote.origin.pushurl", FLATPAK_GIT_REMOTE] call(command) command = ["git", "push"] env["GIT_SSH_COMMAND"] = "ssh -o StrictHostKeyChecking=no -i " + _environ.get( "SSH_KEY" ) call(command) def fix_macos_rpath(project): base_dir, export_files = EXPORT_FILES[project] for file in filter(lambda f: f.endswith(".dylib"), export_files): lib = base_dir / file if lib.is_symlink(): continue command = ["install_name_tool", "-id", lib.name, str(lib)] print_message("call {}", " ".join(command)) subprocess.check_call(command, env=os.environ) def trigger_workflow(repo, workflow="docker.yml", ref="master", inputs=None): """triggers a `workflow_dispatch` event to the specified workflow on its repo repo: {user}/{repo} format workflow: workflow ID or workflow file name ref: branch or tag name inputs: dict of inputs to pass to the workflow""" print_message( "triggering workflow `{workflow}` on {repo}@{ref} " "with inputs={inputs}", workflow=workflow, repo=repo, ref=ref, inputs=inputs) url = "{base_url}/repos/{repo}/actions/workflows/{workflow}/dispatches".format( base_url=os.getenv("GITHUB_API_URL", "https://api.github.com"), repo=repo, workflow=workflow) resp = requests.post(url, headers={ "Content-Type": "application/json", "Authorization": "token {token}".format( token=os.getenv('GITHUB_PAT', '')), "Accept": "application/vnd.github.v3+json", }, json={"ref": ref, "inputs": inputs}, timeout=5) if resp.status_code != 204: raise ValueError("Unexpected HTTP {code}: {reason}".format( code=resp.status_code, reason=resp.reason)) def trigger_docker_publish(target): if target not in ("zim-tools", "kiwix-tools"): return version = get_postfix(target) repo = { "zim-tools": "openzim/zim-tools", "kiwix-tools": "kiwix/kiwix-tools"}.get(target) try: trigger_workflow(repo, workflow="docker.yml", ref="master", inputs={"version": version}) print_message("triggered docker workflow on {repo}", repo=repo) except Exception as exc: print_message("Error triggering workflow: {exc}", exc=exc) raise exc def notarize_macos_build(project): """ sign and notarize files for macOS Expects the following environment: - Imported Mac/Apple Distribution certificate (with private key) in Keychain - `SIGNING_IDENTITY` environ with Certificate name/identity - `ALTOOL_USERNAME` with Apple ID of an account with perms on the certificate - Keychain entry `ALTOOL_PASSWORD` with an app-specific password for the account - `ASC_PROVIDER` environ with Team ID """ if project != "libzim": return # currently only supports libzim use case: sign every dylib base_dir, export_files = EXPORT_FILES[project] filepaths = [base_dir.joinpath(file) for file in filter(lambda f: f.endswith(".dylib"), export_files) if not base_dir.joinpath(file).is_symlink()] if not filepaths: return for filepath in filepaths: subprocess.check_call(["/usr/bin/codesign", "--force", "--sign", os.getenv("SIGNING_IDENTITY", "no-signing-ident"), str(filepath), "--deep", "--timestamp"], env=os.environ) # create a zip of the dylibs and upload for notarization zip_name = "{}.zip".format(project) subprocess.check_call( ["/usr/bin/ditto", "-c", "-k", "--keepParent"] + [str(f) for f in filepaths] + [zip_name], env=os.environ) subprocess.check_call(["/usr/bin/xcrun", "altool", "--notarize-app", "--file", str(zip_name), "--primary-bundle-id", "org.kiwix.build.{}".format(project), "--username", os.getenv("ALTOOL_USERNAME", "missing"), "--password", "@keychain:ALTOOL_PASSWORD", "--asc-provider", os.getenv("ASC_PROVIDER")], env=os.environ) # check notarization of a file (should be in-progress atm and this != 0) subprocess.call(["/usr/sbin/spctl", "-a", "-v", "-t", "install", filepaths[-1]], env=os.environ)