import os from os import environ as _environ from pathlib import Path from datetime import date import tarfile import subprocess import re from kiwixbuild.versions import base_deps_versions PLATFORM_TARGET = _environ["PLATFORM_TARGET"] if PLATFORM_TARGET == "native_desktop": PLATFORM_TARGET = "native_dyn" DESKTOP = True else: DESKTOP = False OS_NAME = _environ["OS_NAME"] HOME = Path(os.path.expanduser("~")) BASE_DIR = HOME / "BUILD_{}".format(PLATFORM_TARGET) SOURCE_DIR = HOME / "SOURCE" ARCHIVE_DIR = HOME / "ARCHIVE" INSTALL_DIR = BASE_DIR / "INSTALL" TMP_DIR = Path("/tmp") # [TODO] KIWIX_DESKTOP_ONLY = False _ref = _environ.get("GITHUB_REF", "").split("/")[-1] MAKE_RELEASE = re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", _ref) is not None def print_message(message, *args, **kwargs): message = message.format(*args, **kwargs) message = "{0} {1} {0}".format("-" * 3, message) print(message, flush=True) MANIFEST_TEMPLATE = """{archive_name} *************************** Dependencies archive for {target} on platform {platform} Generated at {date} """ def write_manifest(manifest_file, archive_name, target, platform): with manifest_file.open(mode="w") as f: f.write( MANIFEST_TEMPLATE.format( archive_name=archive_name, target=target, platform=platform, date=date.today().isoformat(), ) ) def run_kiwix_build( target, platform, build_deps_only=False, target_only=False, make_release=False, make_dist=False, ): command = ["kiwix-build"] command.append(target) command.append("--hide-progress") if platform == "flatpak" or platform.startswith("win32_"): command.append("--assume-packages-installed") if target == "kiwix-lib-app" and platform.startswith("android_"): command.extend(["--target-platform", "android", "--android-arch", platform[8:]]) elif platform == "android": command.extend(["--target-platform", "android"]) for arch in ("arm", "arm64", "x86", "x86_64"): command.extend(["--android-arch", arch]) else: command.extend(["--target-platform", platform]) if build_deps_only: command.append("--build-deps-only") if target_only: command.append("--build-nodeps") if make_release: command.append("--make-release") if make_dist: command.append("--make-dist") print_message( "Build {} (deps={}, release={}, dist={})", target, build_deps_only, make_release, make_dist, ) env = dict(_environ) env['SKIP_BIG_MEMORY_TEST'] = '1' subprocess.check_call(command, cwd=str(HOME), env=env) print_message("Build ended") def upload(file_to_upload, host, dest_path): command = [ "scp", "-i", _environ.get("SSH_KEY"), "-o", "StrictHostKeyChecking=no", str(file_to_upload), "{}:{}".format(host, dest_path), ] print_message("Sending archive with command {}", command) subprocess.check_call(command) def make_deps_archive(target=None, name=None, full=False): archive_name = name or "deps2_{}_{}_{}.tar.xz".format( OS_NAME, PLATFORM_TARGET, target ) print_message("Create archive {}.", archive_name) files_to_archive = [INSTALL_DIR] files_to_archive += HOME.glob("BUILD_*/LOGS") if PLATFORM_TARGET == "native_mixed": files_to_archive += [HOME / "BUILD_native_static" / "INSTALL"] if PLATFORM_TARGET.startswith("android"): files_to_archive.append(HOME / "BUILD_neutral" / "INSTALL") if PLATFORM_TARGET == "android": for arch in ("arm", "arm64", "x86", "x86_64"): base_dir = HOME / "BUILD_android_{}".format(arch) files_to_archive.append(base_dir / "INSTALL") if (base_dir / "meson_cross_file.txt").exists(): files_to_archive.append(base_dir / "meson_cross_file.txt") files_to_archive += HOME.glob("BUILD_*/android-ndk*") files_to_archive += HOME.glob("BUILD_*/android-sdk*") if (BASE_DIR / "meson_cross_file.txt").exists(): files_to_archive.append(BASE_DIR / "meson_cross_file.txt") manifest_file = BASE_DIR / "manifest.txt" write_manifest(manifest_file, archive_name, target, PLATFORM_TARGET) files_to_archive.append(manifest_file) relative_path = HOME if full: files_to_archive += ARCHIVE_DIR.glob(".*_ok") files_to_archive += BASE_DIR.glob("*/.*_ok") files_to_archive += (HOME / "BUILD_native_dyn").glob("*/.*_ok") files_to_archive += (HOME / "BUILD_native_static").glob("*/.*_ok") files_to_archive += HOME.glob("BUILD_android*/**/.*_ok") files_to_archive += SOURCE_DIR.glob("*/.*_ok") files_to_archive += [ SOURCE_DIR / "pugixml-{}".format(base_deps_versions["pugixml"]) ] files_to_archive += HOME.glob( "BUILD_*/pugixml-{}".format(base_deps_versions["pugixml"]) ) if PLATFORM_TARGET.startswith("armhf"): files_to_archive += (SOURCE_DIR / "armhf").glob("*") toolchains_subdirs = HOME.glob("BUILD_*/TOOLCHAINS/*/*") for subdir in toolchains_subdirs: if not subdir.match("tools"): files_to_archive.append(subdir) archive_file = TMP_DIR / archive_name with tarfile.open(str(archive_file), "w:xz") as tar: for name in set(files_to_archive): print(".{}".format(name), flush=True) tar.add(str(name), arcname=str(name.relative_to(relative_path))) return archive_file