573 lines
19 KiB
Python
573 lines
19 KiB
Python
import os
|
|
from os import environ as _environ
|
|
from pathlib import Path
|
|
from datetime import date
|
|
import tarfile
|
|
import zipfile
|
|
import subprocess
|
|
import re
|
|
import shutil
|
|
|
|
import requests
|
|
|
|
from build_definition import get_platform_name
|
|
|
|
from kiwixbuild.versions import (
|
|
main_project_versions,
|
|
release_versions,
|
|
base_deps_versions,
|
|
)
|
|
|
|
|
|
|
|
PLATFORM_TARGET = _environ["PLATFORM_TARGET"]
|
|
OS_NAME = _environ["OS_NAME"]
|
|
HOME = Path(os.path.expanduser("~"))
|
|
|
|
BASE_DIR = HOME / "BUILD_{}".format(PLATFORM_TARGET)
|
|
SOURCE_DIR = HOME / "SOURCE"
|
|
ARCHIVE_DIR = HOME / "ARCHIVE"
|
|
TOOLCHAIN_DIR = BASE_DIR / "TOOLCHAINS"
|
|
INSTALL_DIR = BASE_DIR / "INSTALL"
|
|
TMP_DIR = Path("/tmp")
|
|
KBUILD_SOURCE_DIR = HOME / "kiwix-build"
|
|
|
|
_ref = _environ.get("GITHUB_REF", "").split("/")[-1]
|
|
MAKE_RELEASE = re.fullmatch(r"r_[0-9]+", _ref) is not None
|
|
MAKE_RELEASE = MAKE_RELEASE and (_environ.get('GITHUB_EVENT_NAME') != 'schedule')
|
|
|
|
if not MAKE_RELEASE and _ref != "main":
|
|
DEV_BRANCH = _ref
|
|
else:
|
|
DEV_BRANCH = None
|
|
|
|
FLATPAK_HTTP_GIT_REMOTE = "https://github.com/flathub/org.kiwix.desktop.git"
|
|
FLATPAK_GIT_REMOTE = "git@github.com:flathub/org.kiwix.desktop.git"
|
|
|
|
BIN_EXT = ".exe" if PLATFORM_TARGET.startswith("win32_") else ""
|
|
|
|
def major_version(version: str) -> str:
|
|
return version.split(".")[0]
|
|
|
|
# We have build everything. Now create archives for public deployement.
|
|
EXPORT_FILES = {
|
|
"kiwix-tools": (
|
|
INSTALL_DIR / "bin",
|
|
[
|
|
f + BIN_EXT
|
|
for f in ("kiwix-manage", "kiwix-search", "kiwix-serve")
|
|
],
|
|
),
|
|
"zim-tools": (
|
|
INSTALL_DIR / "bin",
|
|
[
|
|
f + BIN_EXT
|
|
for f in (
|
|
"zimbench",
|
|
"zimcheck",
|
|
"zimdump",
|
|
"zimsearch",
|
|
"zimdiff",
|
|
"zimpatch",
|
|
"zimsplit",
|
|
"zimwriterfs",
|
|
"zimrecreate"
|
|
)
|
|
],
|
|
),
|
|
"libzim": (
|
|
INSTALL_DIR,
|
|
(
|
|
# We need to package all dependencies (`*.a`) on wasm
|
|
"lib/*/libzim.a" if PLATFORM_TARGET != "wasm" else "lib/*.a",
|
|
"lib/*/libzim.so",
|
|
"lib/*/libzim.so.{version}".format(
|
|
version=main_project_versions["libzim"]
|
|
),
|
|
"lib/*/libzim.so.{version}".format(
|
|
version=major_version(main_project_versions["libzim"])
|
|
),
|
|
"lib/libzim.{}.dylib".format(
|
|
major_version(main_project_versions["libzim"])
|
|
),
|
|
"lib/libzim.dylib",
|
|
"lib/*/libzim.pc",
|
|
"include/zim/**/*.h",
|
|
"share/icu/{}/icudt{}l.dat".format(
|
|
base_deps_versions["icu4c"],
|
|
major_version(base_deps_versions["icu4c"])
|
|
)
|
|
),
|
|
),
|
|
"libkiwix": (
|
|
INSTALL_DIR,
|
|
(
|
|
"lib/*/libkiwix.so",
|
|
"lib/*/libkiwix.so.{version}".format(
|
|
version=main_project_versions["libkiwix"]
|
|
),
|
|
"lib/*/libkiwix.so.{version}".format(
|
|
version=major_version(main_project_versions["libkiwix"])
|
|
),
|
|
"lib/libkiwix.{}.dylib".format(
|
|
major_version(main_project_versions["libkiwix"])
|
|
),
|
|
"lib/libkiwix.dylib",
|
|
"lib/*/libkiwix.pc",
|
|
"include/kiwix/**/*.h",
|
|
"share/icu/{}/icudt{}l.dat".format(
|
|
base_deps_versions["icu4c"],
|
|
major_version(base_deps_versions["icu4c"])
|
|
)
|
|
),
|
|
),
|
|
}
|
|
|
|
DATE = date.today().isoformat()
|
|
|
|
def print_message(message, *args, **kwargs):
|
|
message = message.format(*args, **kwargs)
|
|
message = "{0} {1} {0}".format("-" * 3, message)
|
|
print(message, flush=True)
|
|
|
|
|
|
MANIFEST_TEMPLATE = """{archive_name}
|
|
***************************
|
|
|
|
Dependencies archive for {target} on platform {platform}
|
|
Generated at {date}
|
|
"""
|
|
|
|
|
|
def write_manifest(manifest_file, archive_name, target, platform):
|
|
with manifest_file.open(mode="w") as f:
|
|
f.write(
|
|
MANIFEST_TEMPLATE.format(
|
|
archive_name=archive_name, target=target, platform=platform, date=DATE,
|
|
)
|
|
)
|
|
|
|
|
|
def run_kiwix_build(
|
|
target,
|
|
platform,
|
|
build_deps_only=False,
|
|
target_only=False,
|
|
make_release=False,
|
|
make_dist=False,
|
|
):
|
|
command = ["kiwix-build"]
|
|
command.append(target)
|
|
command.append("--hide-progress")
|
|
command.append("--fast-clone")
|
|
command.append("--assume-packages-installed")
|
|
command.extend(["--target-platform", platform])
|
|
if build_deps_only:
|
|
command.append("--build-deps-only")
|
|
if target_only:
|
|
command.append("--build-nodeps")
|
|
if make_release:
|
|
command.append("--make-release")
|
|
if make_dist:
|
|
command.append("--make-dist")
|
|
print_message(
|
|
"Build {} (deps={}, release={}, dist={})",
|
|
target,
|
|
build_deps_only,
|
|
make_release,
|
|
make_dist,
|
|
)
|
|
env = dict(_environ)
|
|
env["SKIP_BIG_MEMORY_TEST"] = "1"
|
|
subprocess.check_call(command, cwd=str(HOME), env=env)
|
|
print_message("Build ended")
|
|
|
|
|
|
def upload(file_to_upload, host, dest_path):
|
|
if not file_to_upload.exists():
|
|
print_message("No {} to upload!", file_to_upload)
|
|
return
|
|
|
|
if ":" in host:
|
|
host, port = host.split(":", 1)
|
|
else:
|
|
port = "22"
|
|
|
|
# sending SFTP mkdir command to the sftp interactive mode and not batch (-b) mode
|
|
# as the latter would exit on any mkdir error while it is most likely
|
|
# the first parts of the destination is already present and thus can't be created
|
|
sftp_commands = "\n".join(
|
|
[
|
|
f"mkdir {part}"
|
|
for part in list(reversed(Path(dest_path).parents)) + [dest_path]
|
|
]
|
|
)
|
|
command = [
|
|
"sftp",
|
|
"-i",
|
|
_environ.get("SSH_KEY"),
|
|
"-P",
|
|
port,
|
|
"-o",
|
|
"StrictHostKeyChecking=no",
|
|
host,
|
|
]
|
|
print_message("Creating dest path {}", dest_path)
|
|
subprocess.run(command, input=sftp_commands.encode("utf-8"), check=True)
|
|
|
|
command = [
|
|
"scp",
|
|
"-c",
|
|
"aes128-ctr",
|
|
"-rp",
|
|
"-P",
|
|
port,
|
|
"-i",
|
|
_environ.get("SSH_KEY"),
|
|
"-o",
|
|
"StrictHostKeyChecking=no",
|
|
str(file_to_upload),
|
|
"{}:{}".format(host, dest_path),
|
|
]
|
|
print_message("Sending archive with command {}", command)
|
|
subprocess.check_call(command)
|
|
|
|
|
|
def upload_archive(archive, project, make_release, dev_branch=None):
|
|
if not archive.exists():
|
|
print_message("No archive {} to upload!", archive)
|
|
return
|
|
|
|
if project.startswith("kiwix-") or project in ['libkiwix']:
|
|
host = "ci@master.download.kiwix.org:30022"
|
|
dest_path = "/data/download/"
|
|
else:
|
|
host = "ci@download.openzim.org:30022"
|
|
dest_path = "/data/openzim/"
|
|
|
|
if make_release:
|
|
dest_path = dest_path + "release/" + project
|
|
else:
|
|
dest_path = dest_path + "nightly/" + DATE
|
|
|
|
if dev_branch:
|
|
dest_path = "/data/tmp/ci/dev_preview/" + dev_branch
|
|
else:
|
|
# Make the archive read only. This way, scp will preserve rights.
|
|
# If somehow we try to upload twice the same archive, scp will fails.
|
|
archive.chmod(0o444)
|
|
|
|
upload(archive, host, dest_path)
|
|
|
|
|
|
# This remove "share/doc" and "share/man" from the thing to copy in the deps archive
|
|
def filter_install_dir(path):
|
|
for dir in path.glob('*'):
|
|
if dir.name not in ['share']:
|
|
yield dir
|
|
else:
|
|
for sub_dir in dir.glob('*'):
|
|
if sub_dir.name not in ['doc', 'man']:
|
|
yield sub_dir
|
|
|
|
# Full: True if we are creating a full archive to be used as cache by kiwix-build (base_deps2_{os}_{platform}_{base_deps_version}.tar.xz)
|
|
# Full: False if we are creating a archive to be used as pre-cached dependencies for project's CI (deps2_{os}_{platform}_{target}.tar.xz)
|
|
def make_deps_archive(target=None, name=None, full=False):
|
|
archive_name = name or "deps2_{}_{}_{}.tar.xz".format(
|
|
OS_NAME, PLATFORM_TARGET, target
|
|
)
|
|
print_message("Create archive {}.", archive_name)
|
|
files_to_archive = list(filter_install_dir(INSTALL_DIR))
|
|
files_to_archive += HOME.glob("BUILD_*/LOGS")
|
|
if PLATFORM_TARGET.endswith("_mixed"):
|
|
static_platform = PLATFORM_TARGET.replace("_mixed", "_static")
|
|
files_to_archive += filter_install_dir(HOME / ("BUILD_" + static_platform) / "INSTALL")
|
|
if PLATFORM_TARGET.startswith("android_"):
|
|
files_to_archive += filter_install_dir(HOME / "BUILD_neutral" / "INSTALL")
|
|
base_dir = HOME / "BUILD_{}".format(PLATFORM_TARGET)
|
|
if (base_dir / "meson_cross_file.txt").exists():
|
|
files_to_archive.append(base_dir / "meson_cross_file.txt")
|
|
# Copy any toolchain
|
|
files_to_archive += [TOOLCHAIN_DIR]
|
|
files_to_archive += HOME.glob("BUILD_neutral/TOOLCHAINS/*")
|
|
if (BASE_DIR / "meson_cross_file.txt").exists():
|
|
files_to_archive.append(BASE_DIR / "meson_cross_file.txt")
|
|
|
|
manifest_file = BASE_DIR / "manifest.txt"
|
|
write_manifest(manifest_file, archive_name, target, PLATFORM_TARGET)
|
|
files_to_archive.append(manifest_file)
|
|
|
|
relative_path = HOME
|
|
if full:
|
|
files_to_archive += ARCHIVE_DIR.glob(".*_ok")
|
|
files_to_archive += BASE_DIR.glob("*/.*_ok")
|
|
# Add also static build for mixed target
|
|
if PLATFORM_TARGET.endswith("_mixed"):
|
|
static_platform = PLATFORM_TARGET.replace("_mixed", "_static")
|
|
files_to_archive += (HOME / ("BUILD_" + static_platform)).glob("*/.*_ok")
|
|
# Native dyn and static is needed for potential cross compilation that use native tools (icu)
|
|
files_to_archive += (HOME / "BUILD_native_dyn").glob("*/.*_ok")
|
|
files_to_archive += (HOME / "BUILD_native_static").glob("*/.*_ok")
|
|
files_to_archive += HOME.glob("BUILD_android*/**/.*_ok")
|
|
files_to_archive += SOURCE_DIR.glob("*/.*_ok")
|
|
files_to_archive += SOURCE_DIR.glob("zim-testing-suite-*/*")
|
|
|
|
archive_file = TMP_DIR / archive_name
|
|
with tarfile.open(str(archive_file), "w:xz") as tar:
|
|
for name in set(files_to_archive):
|
|
print(".{}".format(name), flush=True)
|
|
tar.add(str(name), arcname=str(name.relative_to(relative_path)))
|
|
|
|
return archive_file
|
|
|
|
|
|
def get_postfix(project):
|
|
postfix = main_project_versions[project]
|
|
extra = release_versions.get(project)
|
|
if extra:
|
|
postfix = "{}-{}".format(postfix, extra)
|
|
return postfix
|
|
|
|
|
|
def make_archive(project, make_release):
|
|
platform_name = get_platform_name()
|
|
if not platform_name:
|
|
return None
|
|
|
|
try:
|
|
base_dir, export_files = EXPORT_FILES[project]
|
|
except KeyError:
|
|
# No binary files to export
|
|
return None
|
|
|
|
if make_release:
|
|
postfix = get_postfix(project)
|
|
else:
|
|
postfix = DATE
|
|
|
|
archive_name = "{}_{}-{}".format(project, platform_name, postfix)
|
|
|
|
files_to_archive = []
|
|
for export_file in export_files:
|
|
files_to_archive.extend(base_dir.glob(export_file))
|
|
if platform_name == "win-i686":
|
|
open_archive = lambda a: zipfile.ZipFile(
|
|
str(a), "w", compression=zipfile.ZIP_DEFLATED
|
|
)
|
|
archive_add = lambda a, f: a.write(str(f), arcname=str(f.relative_to(base_dir)))
|
|
archive_ext = ".zip"
|
|
else:
|
|
open_archive = lambda a: tarfile.open(str(a), "w:gz")
|
|
archive_add = lambda a, f: a.add(
|
|
str(f), arcname="{}/{}".format(archive_name, str(f.relative_to(base_dir)))
|
|
)
|
|
archive_ext = ".tar.gz"
|
|
|
|
archive = TMP_DIR / "{}{}".format(archive_name, archive_ext)
|
|
print_message("create archive {} with {}", archive, files_to_archive)
|
|
with open_archive(archive) as arch:
|
|
for f in files_to_archive:
|
|
archive_add(arch, f)
|
|
return archive
|
|
|
|
|
|
def create_desktop_image(make_release):
|
|
print_message("creating desktop image")
|
|
if make_release:
|
|
postfix = get_postfix("kiwix-desktop")
|
|
src_dir = SOURCE_DIR / "kiwix-desktop_release"
|
|
else:
|
|
postfix = DATE
|
|
src_dir = SOURCE_DIR / "kiwix-desktop"
|
|
|
|
if PLATFORM_TARGET == "flatpak":
|
|
build_path = BASE_DIR / "org.kiwix.desktop.flatpak"
|
|
app_name = "org.kiwix.desktop.{}.flatpak".format(postfix)
|
|
print_message("archive is {}", build_path)
|
|
else:
|
|
build_path = HOME / "Kiwix-{}-x86_64.AppImage".format(postfix)
|
|
app_name = "kiwix-desktop_x86_64_{}.appimage".format(postfix)
|
|
command = [
|
|
"kiwix-build/scripts/create_kiwix-desktop_appImage.sh",
|
|
str(INSTALL_DIR),
|
|
str(src_dir),
|
|
str(HOME / "AppDir"),
|
|
]
|
|
env = dict(os.environ)
|
|
env["VERSION"] = postfix
|
|
print_message("Build AppImage of kiwix-desktop")
|
|
subprocess.check_call(command, cwd=str(HOME), env=env)
|
|
|
|
print_message("Copy Build to {}".format(TMP_DIR / app_name))
|
|
shutil.copy(str(build_path), str(TMP_DIR / app_name))
|
|
return TMP_DIR / app_name
|
|
|
|
|
|
def update_flathub_git():
|
|
git_repo_dir = TMP_DIR / "org.kiwix.desktop"
|
|
env = dict(os.environ)
|
|
env["GIT_AUTHOR_NAME"] = env["GIT_COMMITTER_NAME"] = "KiwixBot"
|
|
env["GIT_AUTHOR_EMAIL"] = env["GIT_COMMITTER_EMAIL"] = "release@kiwix.org"
|
|
|
|
def call(command, cwd=None):
|
|
cwd = cwd or git_repo_dir
|
|
print_message("call {}", command)
|
|
subprocess.check_call(command, env=env, cwd=str(cwd))
|
|
|
|
command = ["git", "clone", FLATPAK_HTTP_GIT_REMOTE]
|
|
call(command, cwd=TMP_DIR)
|
|
|
|
branch_name = "v_{}".format(get_postfix("kiwix-desktop"))
|
|
command = ["git", "checkout", "-b", branch_name]
|
|
call(command)
|
|
shutil.copy(str(BASE_DIR / "org.kiwix.desktop.json"), str(git_repo_dir))
|
|
patch_dir = KBUILD_SOURCE_DIR / "kiwixbuild" / "patches"
|
|
for dep in ["pugixml"]:
|
|
for f in patch_dir.glob("{}_*.patch".format(dep)):
|
|
shutil.copy(str(f), str(git_repo_dir / "patches"))
|
|
command = ["git", "add", "-A", "."]
|
|
call(command)
|
|
command = [
|
|
"git",
|
|
"commit",
|
|
"-m",
|
|
"Update to version {}".format(get_postfix("kiwix-desktop")),
|
|
]
|
|
try:
|
|
call(command)
|
|
except subprocess.CalledProcessError:
|
|
# This may fail if there is nothing to commit (a rebuild of the CI for exemple)
|
|
return
|
|
command = ["git", "config", "remote.origin.pushurl", FLATPAK_GIT_REMOTE]
|
|
call(command)
|
|
command = ["git", "push", "origin", branch_name]
|
|
env["GIT_SSH_COMMAND"] = "ssh -o StrictHostKeyChecking=no -i " + _environ.get(
|
|
"SSH_KEY"
|
|
)
|
|
call(command)
|
|
|
|
|
|
def fix_macos_rpath(project):
|
|
|
|
base_dir, export_files = EXPORT_FILES[project]
|
|
for file in filter(lambda f: f.endswith(".dylib"), export_files):
|
|
lib = base_dir / file
|
|
if lib.is_symlink():
|
|
continue
|
|
command = ["install_name_tool", "-id", lib.name, str(lib)]
|
|
print_message("call {}", " ".join(command))
|
|
subprocess.check_call(command, env=os.environ)
|
|
|
|
|
|
def trigger_workflow(repo, workflow="docker.yml", ref="main", inputs=None):
|
|
"""triggers a `workflow_dispatch` event to the specified workflow on its repo
|
|
|
|
repo: {user}/{repo} format
|
|
workflow: workflow ID or workflow file name
|
|
ref: branch or tag name
|
|
inputs: dict of inputs to pass to the workflow"""
|
|
print_message(
|
|
"triggering workflow `{workflow}` on {repo}@{ref} "
|
|
"with inputs={inputs}", workflow=workflow, repo=repo, ref=ref, inputs=inputs)
|
|
|
|
url = "{base_url}/repos/{repo}/actions/workflows/{workflow}/dispatches".format(
|
|
base_url=os.getenv("GITHUB_API_URL", "https://api.github.com"),
|
|
repo=repo, workflow=workflow)
|
|
|
|
resp = requests.post(url, headers={
|
|
"Content-Type": "application/json",
|
|
"Authorization": "token {token}".format(
|
|
token=os.getenv('GITHUB_PAT', '')),
|
|
"Accept": "application/vnd.github.v3+json",
|
|
}, json={"ref": ref, "inputs": inputs}, timeout=5)
|
|
if resp.status_code != 204:
|
|
raise ValueError("Unexpected HTTP {code}: {reason}".format(
|
|
code=resp.status_code, reason=resp.reason))
|
|
|
|
|
|
def trigger_docker_publish(target):
|
|
if target not in ("zim-tools", "kiwix-tools"):
|
|
return
|
|
|
|
version = get_postfix(target)
|
|
repo = {
|
|
"zim-tools": "openzim/zim-tools",
|
|
"kiwix-tools": "kiwix/kiwix-tools"}.get(target)
|
|
|
|
try:
|
|
trigger_workflow(repo, workflow="docker.yml", ref="main",
|
|
inputs={"version": version})
|
|
print_message("triggered docker workflow on {repo}", repo=repo)
|
|
except Exception as exc:
|
|
print_message("Error triggering workflow: {exc}", exc=exc)
|
|
raise exc
|
|
|
|
|
|
def notarize_macos_build(project):
|
|
""" sign and notarize files for macOS
|
|
|
|
Expects the following environment:
|
|
- `SIGNING_IDENTITY` environ with Certificate name/identity
|
|
- `KEYCHAIN` environ with path to the keychain storing credentials
|
|
- `KEYCHAIN_PROFILE` environ with name of the profile in that keychain
|
|
- `KEYCHAIN_PASSWORD` environ with password to unlock the keychain
|
|
"""
|
|
if project != "libzim":
|
|
return
|
|
|
|
# currently only supports libzim use case: sign every dylib
|
|
base_dir, export_files = EXPORT_FILES[project]
|
|
filepaths = [base_dir.joinpath(file)
|
|
for file in filter(lambda f: f.endswith(".dylib"), export_files)
|
|
if not base_dir.joinpath(file).is_symlink()]
|
|
|
|
if not filepaths:
|
|
return
|
|
|
|
for filepath in filepaths:
|
|
subprocess.check_call(["/usr/bin/codesign", "--force", "--sign",
|
|
os.getenv("SIGNING_IDENTITY", "no-signing-ident"),
|
|
"--keychain",
|
|
os.getenv("KEYCHAIN", "no-keychain-path"),
|
|
str(filepath), "--deep", "--timestamp"], env=os.environ)
|
|
|
|
# create a zip of the dylibs and upload for notarization
|
|
zip_name = "{}.zip".format(project)
|
|
subprocess.check_call(
|
|
["/usr/bin/ditto", "-c", "-k", "--keepParent"]
|
|
+ [str(f) for f in filepaths] + [zip_name],
|
|
env=os.environ)
|
|
|
|
# make sure keychain is unlocked
|
|
subprocess.check_call(
|
|
[
|
|
"/usr/bin/security",
|
|
"unlock-keychain",
|
|
"-p",
|
|
os.getenv("KEYCHAIN_PASSWORD", "no-keychain-password"),
|
|
os.getenv("KEYCHAIN", "no-keychain-path"),
|
|
],
|
|
env=os.environ,
|
|
)
|
|
|
|
subprocess.check_call(
|
|
[
|
|
"/usr/bin/xcrun",
|
|
"notarytool",
|
|
"submit",
|
|
"--keychain",
|
|
os.getenv("KEYCHAIN", "no-keychain-path"),
|
|
"--keychain-profile",
|
|
os.getenv("KEYCHAIN_PROFILE", "no-keychain-profile"),
|
|
"--wait",
|
|
str(zip_name),
|
|
],
|
|
env=os.environ,
|
|
)
|
|
|
|
# check notarization of a file (should be in-progress atm and this != 0)
|
|
subprocess.call(
|
|
["/usr/sbin/spctl", "--assess", "-vv", "--type", "install", filepaths[-1]],
|
|
env=os.environ,
|
|
)
|