[CI/CD] Use config instead of target in the CI.

This commit is contained in:
Matthieu Gautier 2024-02-05 18:00:35 +01:00
parent c0ec9c44b8
commit c25a2e63e3
9 changed files with 167 additions and 138 deletions

View File

@ -3,7 +3,7 @@ import csv, io, re
# Definition of what to build. # Definition of what to build.
# Array is read line by line. # Array is read line by line.
# Empty cells under (OS_NAME, PLATFORM_TARGET) mean "always match" (catch all, or `.*` regex) # Empty cells under (OS_NAME, COMPILE_CONFIG) mean "always match" (catch all, or `.*` regex)
# Once a cell doesn't match, skip to the next line. # Once a cell doesn't match, skip to the next line.
# Once a line matches, other lines are not read, so put more specific combination first. # Once a line matches, other lines are not read, so put more specific combination first.
# Lines composed of `-` , or `=`, or starting by `#` are ignored. # Lines composed of `-` , or `=`, or starting by `#` are ignored.
@ -15,7 +15,7 @@ import csv, io, re
# 'D' letter means we trigger the docker forkflow to build the docker image. # 'D' letter means we trigger the docker forkflow to build the docker image.
# If a cell contains several letters, all are done. # If a cell contains several letters, all are done.
BUILD_DEF = """ BUILD_DEF = """
| OS_NAME | PLATFORM_TARGET | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | platform_name | | OS_NAME | COMPILE_CONFIG | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | platform_name |
===================================================================================================================== =====================================================================================================================
# Bionic is a special case as we need to compile libzim on old arch for python # Bionic is a special case as we need to compile libzim on old arch for python
| bionic | native_mixed | BP | | | | | linux-x86_64-bionic | | bionic | native_mixed | BP | | | | | linux-x86_64-bionic |
@ -92,10 +92,10 @@ def selector_match(selector, value):
class Context(NamedTuple): class Context(NamedTuple):
OS_NAME: str OS_NAME: str
PLATFORM_TARGET: str COMPILE_CONFIG: str
def match(self, row): def match(self, row):
for key in ["OS_NAME", "PLATFORM_TARGET"]: for key in ["OS_NAME", "COMPILE_CONFIG"]:
context_value = getattr(self, key) context_value = getattr(self, key)
selector = row[key] selector = row[key]
if not selector_match(selector, context_value): if not selector_match(selector, context_value):
@ -109,10 +109,11 @@ SOURCE_PUBLISH = "S"
DEPS = "d" DEPS = "d"
DOCKER = "D" DOCKER = "D"
def select_build_targets(criteria):
from common import PLATFORM_TARGET, OS_NAME
context = Context(PLATFORM_TARGET=PLATFORM_TARGET, OS_NAME=OS_NAME) def select_build_targets(criteria):
from common import COMPILE_CONFIG, OS_NAME
context = Context(COMPILE_CONFIG=COMPILE_CONFIG, OS_NAME=OS_NAME)
reader = csv.DictReader(strip_array(BUILD_DEF), dialect=TableDialect()) reader = csv.DictReader(strip_array(BUILD_DEF), dialect=TableDialect())
for row in reader: for row in reader:
@ -133,10 +134,11 @@ def select_build_targets(criteria):
raise ValueError("No definition match with current context.") raise ValueError("No definition match with current context.")
def get_platform_name():
from common import PLATFORM_TARGET, OS_NAME
context = Context(PLATFORM_TARGET=PLATFORM_TARGET, OS_NAME=OS_NAME) def get_platform_name():
from common import COMPILE_CONFIG, OS_NAME
context = Context(COMPILE_CONFIG=COMPILE_CONFIG, OS_NAME=OS_NAME)
reader = csv.DictReader(strip_array(BUILD_DEF), dialect=TableDialect()) reader = csv.DictReader(strip_array(BUILD_DEF), dialect=TableDialect())
for row in reader: for row in reader:

View File

@ -8,16 +8,16 @@ from common import (
fix_macos_rpath, fix_macos_rpath,
upload_archive, upload_archive,
OS_NAME, OS_NAME,
PLATFORM_TARGET, COMPILE_CONFIG,
DEV_BRANCH, DEV_BRANCH,
) )
for target in select_build_targets(BUILD): for target in select_build_targets(BUILD):
run_kiwix_build(target, platform=PLATFORM_TARGET) run_kiwix_build(target, config=COMPILE_CONFIG)
if target == "kiwix-desktop": if target == "kiwix-desktop":
archive = create_desktop_image(make_release=False) archive = create_desktop_image(make_release=False)
else: else:
if PLATFORM_TARGET == "native_mixed" and OS_NAME == "macos": if COMPILE_CONFIG == "native_mixed" and OS_NAME == "macos":
fix_macos_rpath(target) fix_macos_rpath(target)
archive = make_archive(target, make_release=False) archive = make_archive(target, make_release=False)
if archive and DEV_BRANCH: if archive and DEV_BRANCH:

View File

@ -13,7 +13,7 @@ from common import (
fix_macos_rpath, fix_macos_rpath,
BASE_DIR, BASE_DIR,
OS_NAME, OS_NAME,
PLATFORM_TARGET, COMPILE_CONFIG,
MAKE_RELEASE, MAKE_RELEASE,
notarize_macos_build, notarize_macos_build,
) )
@ -21,23 +21,23 @@ from common import (
from build_definition import select_build_targets, BUILD, PUBLISH, SOURCE_PUBLISH from build_definition import select_build_targets, BUILD, PUBLISH, SOURCE_PUBLISH
# Filter what to build if we are doing a release. # Filter what to build if we are doing a release.
if MAKE_RELEASE: if MAKE_RELEASE:
TARGETS = select_build_targets(PUBLISH) TARGETS = select_build_targets(PUBLISH)
def release_filter(project): def release_filter(project):
return release_versions.get(project) is not None return release_versions.get(project) is not None
TARGETS = tuple(filter(release_filter, TARGETS)) TARGETS = tuple(filter(release_filter, TARGETS))
else: else:
TARGETS = select_build_targets(BUILD) TARGETS = select_build_targets(BUILD)
for target in TARGETS: for target in TARGETS:
run_kiwix_build(target, platform=PLATFORM_TARGET, make_release=MAKE_RELEASE) run_kiwix_build(target, config=COMPILE_CONFIG, make_release=MAKE_RELEASE)
if target == "kiwix-desktop": if target == "kiwix-desktop":
archive = create_desktop_image(make_release=MAKE_RELEASE) archive = create_desktop_image(make_release=MAKE_RELEASE)
else: else:
if OS_NAME == "macos" and PLATFORM_TARGET.endswith("_mixed"): if OS_NAME == "macos" and COMPILE_CONFIG.endswith("_mixed"):
fix_macos_rpath(target) fix_macos_rpath(target)
notarize_macos_build(target) notarize_macos_build(target)
archive = make_archive(target, make_release=MAKE_RELEASE) archive = make_archive(target, make_release=MAKE_RELEASE)
@ -57,13 +57,11 @@ if MAKE_RELEASE:
if target not in source_published_targets: if target not in source_published_targets:
continue continue
run_kiwix_build( run_kiwix_build(
target, platform=PLATFORM_TARGET, make_release=MAKE_RELEASE, make_dist=True target, config=COMPILE_CONFIG, make_release=MAKE_RELEASE, make_dist=True
) )
full_target_name = "{}-{}".format(target, main_project_versions[target]) full_target_name = "{}-{}".format(target, main_project_versions[target])
if target == "kiwix-desktop": if target == "kiwix-desktop":
archive = ( archive = BASE_DIR / full_target_name / "{}.tar.gz".format(full_target_name)
BASE_DIR / full_target_name / "{}.tar.gz".format(full_target_name)
)
else: else:
archive = ( archive = (
BASE_DIR BASE_DIR
@ -74,5 +72,5 @@ if MAKE_RELEASE:
upload_archive(archive, target, make_release=MAKE_RELEASE) upload_archive(archive, target, make_release=MAKE_RELEASE)
# Publish flathub # Publish flathub
if PLATFORM_TARGET == "flatpak" and "kiwix-desktop" in TARGETS: if COMPILE_CONFIG == "flatpak" and "kiwix-desktop" in TARGETS:
update_flathub_git() update_flathub_git()

View File

@ -20,12 +20,11 @@ from kiwixbuild.versions import (
) )
COMPILE_CONFIG = _environ["COMPILE_CONFIG"]
PLATFORM_TARGET = _environ["PLATFORM_TARGET"]
OS_NAME = _environ["OS_NAME"] OS_NAME = _environ["OS_NAME"]
HOME = Path(os.path.expanduser("~")) HOME = Path(os.path.expanduser("~"))
BASE_DIR = HOME / "BUILD_{}".format(PLATFORM_TARGET) BASE_DIR = HOME / "BUILD_{}".format(COMPILE_CONFIG)
SOURCE_DIR = HOME / "SOURCE" SOURCE_DIR = HOME / "SOURCE"
ARCHIVE_DIR = HOME / "ARCHIVE" ARCHIVE_DIR = HOME / "ARCHIVE"
TOOLCHAIN_DIR = BASE_DIR / "TOOLCHAINS" TOOLCHAIN_DIR = BASE_DIR / "TOOLCHAINS"
@ -35,7 +34,7 @@ KBUILD_SOURCE_DIR = HOME / "kiwix-build"
_ref = _environ.get("GITHUB_REF", "").split("/")[-1] _ref = _environ.get("GITHUB_REF", "").split("/")[-1]
MAKE_RELEASE = re.fullmatch(r"r_[0-9]+", _ref) is not None MAKE_RELEASE = re.fullmatch(r"r_[0-9]+", _ref) is not None
MAKE_RELEASE = MAKE_RELEASE and (_environ.get('GITHUB_EVENT_NAME') != 'schedule') MAKE_RELEASE = MAKE_RELEASE and (_environ.get("GITHUB_EVENT_NAME") != "schedule")
if not MAKE_RELEASE and _ref != "main": if not MAKE_RELEASE and _ref != "main":
DEV_BRANCH = _ref DEV_BRANCH = _ref
@ -45,19 +44,18 @@ else:
FLATPAK_HTTP_GIT_REMOTE = "https://github.com/flathub/org.kiwix.desktop.git" FLATPAK_HTTP_GIT_REMOTE = "https://github.com/flathub/org.kiwix.desktop.git"
FLATPAK_GIT_REMOTE = "git@github.com:flathub/org.kiwix.desktop.git" FLATPAK_GIT_REMOTE = "git@github.com:flathub/org.kiwix.desktop.git"
BIN_EXT = ".exe" if PLATFORM_TARGET.startswith("win32_") else "" BIN_EXT = ".exe" if COMPILE_CONFIG.startswith("win32_") else ""
def major_version(version: str) -> str: def major_version(version: str) -> str:
return version.split(".")[0] return version.split(".")[0]
# We have build everything. Now create archives for public deployement. # We have build everything. Now create archives for public deployement.
EXPORT_FILES = { EXPORT_FILES = {
"kiwix-tools": ( "kiwix-tools": (
INSTALL_DIR / "bin", INSTALL_DIR / "bin",
[ [f + BIN_EXT for f in ("kiwix-manage", "kiwix-search", "kiwix-serve")],
f + BIN_EXT
for f in ("kiwix-manage", "kiwix-search", "kiwix-serve")
],
), ),
"zim-tools": ( "zim-tools": (
INSTALL_DIR / "bin", INSTALL_DIR / "bin",
@ -72,7 +70,7 @@ EXPORT_FILES = {
"zimpatch", "zimpatch",
"zimsplit", "zimsplit",
"zimwriterfs", "zimwriterfs",
"zimrecreate" "zimrecreate",
) )
], ],
), ),
@ -80,11 +78,9 @@ EXPORT_FILES = {
INSTALL_DIR, INSTALL_DIR,
( (
# We need to package all dependencies (`*.a`) on wasm # We need to package all dependencies (`*.a`) on wasm
"lib/*/libzim.a" if PLATFORM_TARGET != "wasm" else "lib/*.a", "lib/*/libzim.a" if COMPILE_CONFIG != "wasm" else "lib/*.a",
"lib/*/libzim.so", "lib/*/libzim.so",
"lib/*/libzim.so.{version}".format( "lib/*/libzim.so.{version}".format(version=main_project_versions["libzim"]),
version=main_project_versions["libzim"]
),
"lib/*/libzim.so.{version}".format( "lib/*/libzim.so.{version}".format(
version=major_version(main_project_versions["libzim"]) version=major_version(main_project_versions["libzim"])
), ),
@ -95,9 +91,8 @@ EXPORT_FILES = {
"lib/*/libzim.pc", "lib/*/libzim.pc",
"include/zim/**/*.h", "include/zim/**/*.h",
"share/icu/{}/icudt{}l.dat".format( "share/icu/{}/icudt{}l.dat".format(
base_deps_versions["icu4c"], base_deps_versions["icu4c"], major_version(base_deps_versions["icu4c"])
major_version(base_deps_versions["icu4c"]) ),
)
), ),
), ),
"libkiwix": ( "libkiwix": (
@ -118,15 +113,15 @@ EXPORT_FILES = {
"lib/*/libkiwix.pc", "lib/*/libkiwix.pc",
"include/kiwix/**/*.h", "include/kiwix/**/*.h",
"share/icu/{}/icudt{}l.dat".format( "share/icu/{}/icudt{}l.dat".format(
base_deps_versions["icu4c"], base_deps_versions["icu4c"], major_version(base_deps_versions["icu4c"])
major_version(base_deps_versions["icu4c"]) ),
)
), ),
), ),
} }
DATE = date.today().isoformat() DATE = date.today().isoformat()
def print_message(message, *args, **kwargs): def print_message(message, *args, **kwargs):
message = message.format(*args, **kwargs) message = message.format(*args, **kwargs)
message = "{0} {1} {0}".format("-" * 3, message) message = "{0} {1} {0}".format("-" * 3, message)
@ -136,23 +131,26 @@ def print_message(message, *args, **kwargs):
MANIFEST_TEMPLATE = """{archive_name} MANIFEST_TEMPLATE = """{archive_name}
*************************** ***************************
Dependencies archive for {target} on platform {platform} Dependencies archive for {target} using config {config}
Generated at {date} Generated at {date}
""" """
def write_manifest(manifest_file, archive_name, target, platform): def write_manifest(manifest_file, archive_name, target, config):
with manifest_file.open(mode="w") as f: with manifest_file.open(mode="w") as f:
f.write( f.write(
MANIFEST_TEMPLATE.format( MANIFEST_TEMPLATE.format(
archive_name=archive_name, target=target, platform=platform, date=DATE, archive_name=archive_name,
target=target,
config=config,
date=DATE,
) )
) )
def run_kiwix_build( def run_kiwix_build(
target, target,
platform, config,
build_deps_only=False, build_deps_only=False,
target_only=False, target_only=False,
make_release=False, make_release=False,
@ -163,7 +161,7 @@ def run_kiwix_build(
command.append("--hide-progress") command.append("--hide-progress")
command.append("--fast-clone") command.append("--fast-clone")
command.append("--assume-packages-installed") command.append("--assume-packages-installed")
command.extend(["--target-platform", platform]) command.extend(["--config", config])
if build_deps_only: if build_deps_only:
command.append("--build-deps-only") command.append("--build-deps-only")
if target_only: if target_only:
@ -240,7 +238,7 @@ def upload_archive(archive, project, make_release, dev_branch=None):
print_message("No archive {} to upload!", archive) print_message("No archive {} to upload!", archive)
return return
if project.startswith("kiwix-") or project in ['libkiwix']: if project.startswith("kiwix-") or project in ["libkiwix"]:
host = "ci@master.download.kiwix.org:30022" host = "ci@master.download.kiwix.org:30022"
dest_path = "/data/download/" dest_path = "/data/download/"
else: else:
@ -264,36 +262,39 @@ def upload_archive(archive, project, make_release, dev_branch=None):
# This remove "share/doc" and "share/man" from the thing to copy in the deps archive # This remove "share/doc" and "share/man" from the thing to copy in the deps archive
def filter_install_dir(path): def filter_install_dir(path):
for dir in path.glob('*'): for dir in path.glob("*"):
if dir.name not in ['share']: if dir.name not in ["share"]:
yield dir yield dir
else: else:
for sub_dir in dir.glob('*'): for sub_dir in dir.glob("*"):
if sub_dir.name not in ['doc', 'man']: if sub_dir.name not in ["doc", "man"]:
yield sub_dir yield sub_dir
# Full: True if we are creating a full archive to be used as cache by kiwix-build (base_deps2_{os}_{platform}_{base_deps_version}.tar.xz)
# Full: False if we are creating a archive to be used as pre-cached dependencies for project's CI (deps2_{os}_{platform}_{target}.tar.xz) # Full: True if we are creating a full archive to be used as cache by kiwix-build (base_deps2_{os}_{config}_{base_deps_version}.tar.xz)
# Full: False if we are creating a archive to be used as pre-cached dependencies for project's CI (deps2_{os}_{config}_{target}.tar.xz)
def make_deps_archive(target=None, name=None, full=False): def make_deps_archive(target=None, name=None, full=False):
archive_name = name or "deps2_{}_{}_{}.tar.xz".format( archive_name = name or "deps2_{}_{}_{}.tar.xz".format(
OS_NAME, PLATFORM_TARGET, target OS_NAME, COMPILE_CONFIG, target
) )
print_message("Create archive {}.", archive_name) print_message("Create archive {}.", archive_name)
files_to_archive = list(filter_install_dir(INSTALL_DIR)) files_to_archive = list(filter_install_dir(INSTALL_DIR))
files_to_archive += HOME.glob("BUILD_*/LOGS") files_to_archive += HOME.glob("BUILD_*/LOGS")
if PLATFORM_TARGET == "apple_all_static": if COMPILE_CONFIG == "apple_all_static":
for subplatform in AppleXCFramework.subPlatformNames: for subconfig in AppleXCFramework.subConfigNames:
base_dir = HOME / "BUILD_{}".format(subplatform) base_dir = HOME / "BUILD_{}".format(subconfig)
files_to_archive += filter_install_dir(base_dir / "INSTALL") files_to_archive += filter_install_dir(base_dir / "INSTALL")
if (base_dir / "meson_cross_file.txt").exists(): if (base_dir / "meson_cross_file.txt").exists():
files_to_archive.append(base_dir / "meson_cross_file.txt") files_to_archive.append(base_dir / "meson_cross_file.txt")
if PLATFORM_TARGET.endswith("_mixed"): if COMPILE_CONFIG.endswith("_mixed"):
static_platform = PLATFORM_TARGET.replace("_mixed", "_static") static_config = COMPILE_CONFIG.replace("_mixed", "_static")
files_to_archive += filter_install_dir(HOME / ("BUILD_" + static_platform) / "INSTALL") files_to_archive += filter_install_dir(
if PLATFORM_TARGET.startswith("android_"): HOME / ("BUILD_" + static_config) / "INSTALL"
)
if COMPILE_CONFIG.startswith("android_"):
files_to_archive += filter_install_dir(HOME / "BUILD_neutral" / "INSTALL") files_to_archive += filter_install_dir(HOME / "BUILD_neutral" / "INSTALL")
base_dir = HOME / "BUILD_{}".format(PLATFORM_TARGET) base_dir = HOME / "BUILD_{}".format(COMPILE_CONFIG)
if (base_dir / "meson_cross_file.txt").exists(): if (base_dir / "meson_cross_file.txt").exists():
files_to_archive.append(base_dir / "meson_cross_file.txt") files_to_archive.append(base_dir / "meson_cross_file.txt")
# Copy any toolchain # Copy any toolchain
@ -303,7 +304,7 @@ def make_deps_archive(target=None, name=None, full=False):
files_to_archive.append(BASE_DIR / "meson_cross_file.txt") files_to_archive.append(BASE_DIR / "meson_cross_file.txt")
manifest_file = BASE_DIR / "manifest.txt" manifest_file = BASE_DIR / "manifest.txt"
write_manifest(manifest_file, archive_name, target, PLATFORM_TARGET) write_manifest(manifest_file, archive_name, target, COMPILE_CONFIG)
files_to_archive.append(manifest_file) files_to_archive.append(manifest_file)
relative_path = HOME relative_path = HOME
@ -311,9 +312,9 @@ def make_deps_archive(target=None, name=None, full=False):
files_to_archive += ARCHIVE_DIR.glob(".*_ok") files_to_archive += ARCHIVE_DIR.glob(".*_ok")
files_to_archive += BASE_DIR.glob("*/.*_ok") files_to_archive += BASE_DIR.glob("*/.*_ok")
# Add also static build for mixed target # Add also static build for mixed target
if PLATFORM_TARGET.endswith("_mixed"): if COMPILE_CONFIG.endswith("_mixed"):
static_platform = PLATFORM_TARGET.replace("_mixed", "_static") static_config = COMPILE_CONFIG.replace("_mixed", "_static")
files_to_archive += (HOME / ("BUILD_" + static_platform)).glob("*/.*_ok") files_to_archive += (HOME / ("BUILD_" + static_config)).glob("*/.*_ok")
# Native dyn and static is needed for potential cross compilation that use native tools (icu) # Native dyn and static is needed for potential cross compilation that use native tools (icu)
files_to_archive += (HOME / "BUILD_native_dyn").glob("*/.*_ok") files_to_archive += (HOME / "BUILD_native_dyn").glob("*/.*_ok")
files_to_archive += (HOME / "BUILD_native_static").glob("*/.*_ok") files_to_archive += (HOME / "BUILD_native_static").glob("*/.*_ok")
@ -391,7 +392,7 @@ def create_desktop_image(make_release):
postfix = DATE postfix = DATE
src_dir = SOURCE_DIR / "kiwix-desktop" src_dir = SOURCE_DIR / "kiwix-desktop"
if PLATFORM_TARGET == "flatpak": if COMPILE_CONFIG == "flatpak":
build_path = BASE_DIR / "org.kiwix.desktop.flatpak" build_path = BASE_DIR / "org.kiwix.desktop.flatpak"
app_name = "org.kiwix.desktop.{}.flatpak".format(postfix) app_name = "org.kiwix.desktop.{}.flatpak".format(postfix)
print_message("archive is {}", build_path) print_message("archive is {}", build_path)
@ -459,7 +460,6 @@ def update_flathub_git():
def fix_macos_rpath(project): def fix_macos_rpath(project):
base_dir, export_files = EXPORT_FILES[project] base_dir, export_files = EXPORT_FILES[project]
for file in filter(lambda f: f.endswith(".dylib"), export_files): for file in filter(lambda f: f.endswith(".dylib"), export_files):
lib = base_dir / file lib = base_dir / file
@ -478,22 +478,35 @@ def trigger_workflow(repo, workflow="docker.yml", ref="main", inputs=None):
ref: branch or tag name ref: branch or tag name
inputs: dict of inputs to pass to the workflow""" inputs: dict of inputs to pass to the workflow"""
print_message( print_message(
"triggering workflow `{workflow}` on {repo}@{ref} " "triggering workflow `{workflow}` on {repo}@{ref} " "with inputs={inputs}",
"with inputs={inputs}", workflow=workflow, repo=repo, ref=ref, inputs=inputs) workflow=workflow,
repo=repo,
ref=ref,
inputs=inputs,
)
url = "{base_url}/repos/{repo}/actions/workflows/{workflow}/dispatches".format( url = "{base_url}/repos/{repo}/actions/workflows/{workflow}/dispatches".format(
base_url=os.getenv("GITHUB_API_URL", "https://api.github.com"), base_url=os.getenv("GITHUB_API_URL", "https://api.github.com"),
repo=repo, workflow=workflow) repo=repo,
workflow=workflow,
)
resp = requests.post(url, headers={ resp = requests.post(
url,
headers={
"Content-Type": "application/json", "Content-Type": "application/json",
"Authorization": "token {token}".format( "Authorization": "token {token}".format(token=os.getenv("GITHUB_PAT", "")),
token=os.getenv('GITHUB_PAT', '')),
"Accept": "application/vnd.github.v3+json", "Accept": "application/vnd.github.v3+json",
}, json={"ref": ref, "inputs": inputs}, timeout=5) },
json={"ref": ref, "inputs": inputs},
timeout=5,
)
if resp.status_code != 204: if resp.status_code != 204:
raise ValueError("Unexpected HTTP {code}: {reason}".format( raise ValueError(
code=resp.status_code, reason=resp.reason)) "Unexpected HTTP {code}: {reason}".format(
code=resp.status_code, reason=resp.reason
)
)
def trigger_docker_publish(target): def trigger_docker_publish(target):
@ -501,13 +514,14 @@ def trigger_docker_publish(target):
return return
version = get_postfix(target) version = get_postfix(target)
repo = { repo = {"zim-tools": "openzim/zim-tools", "kiwix-tools": "kiwix/kiwix-tools"}.get(
"zim-tools": "openzim/zim-tools", target
"kiwix-tools": "kiwix/kiwix-tools"}.get(target) )
try: try:
trigger_workflow(repo, workflow="docker.yml", ref="main", trigger_workflow(
inputs={"version": version}) repo, workflow="docker.yml", ref="main", inputs={"version": version}
)
print_message("triggered docker workflow on {repo}", repo=repo) print_message("triggered docker workflow on {repo}", repo=repo)
except Exception as exc: except Exception as exc:
print_message("Error triggering workflow: {exc}", exc=exc) print_message("Error triggering workflow: {exc}", exc=exc)
@ -515,39 +529,52 @@ def trigger_docker_publish(target):
def notarize_macos_build(project): def notarize_macos_build(project):
""" sign and notarize files for macOS """sign and notarize files for macOS
Expects the following environment: Expects the following environment:
- `SIGNING_IDENTITY` environ with Certificate name/identity - `SIGNING_IDENTITY` environ with Certificate name/identity
- `KEYCHAIN` environ with path to the keychain storing credentials - `KEYCHAIN` environ with path to the keychain storing credentials
- `KEYCHAIN_PROFILE` environ with name of the profile in that keychain - `KEYCHAIN_PROFILE` environ with name of the profile in that keychain
- `KEYCHAIN_PASSWORD` environ with password to unlock the keychain - `KEYCHAIN_PASSWORD` environ with password to unlock the keychain
""" """
if project != "libzim": if project != "libzim":
return return
# currently only supports libzim use case: sign every dylib # currently only supports libzim use case: sign every dylib
base_dir, export_files = EXPORT_FILES[project] base_dir, export_files = EXPORT_FILES[project]
filepaths = [base_dir.joinpath(file) filepaths = [
for file in filter(lambda f: f.endswith(".dylib"), export_files) base_dir.joinpath(file)
if not base_dir.joinpath(file).is_symlink()] for file in filter(lambda f: f.endswith(".dylib"), export_files)
if not base_dir.joinpath(file).is_symlink()
]
if not filepaths: if not filepaths:
return return
for filepath in filepaths: for filepath in filepaths:
subprocess.check_call(["/usr/bin/codesign", "--force", "--sign", subprocess.check_call(
os.getenv("SIGNING_IDENTITY", "no-signing-ident"), [
"--keychain", "/usr/bin/codesign",
os.getenv("KEYCHAIN", "no-keychain-path"), "--force",
str(filepath), "--deep", "--timestamp"], env=os.environ) "--sign",
os.getenv("SIGNING_IDENTITY", "no-signing-ident"),
"--keychain",
os.getenv("KEYCHAIN", "no-keychain-path"),
str(filepath),
"--deep",
"--timestamp",
],
env=os.environ,
)
# create a zip of the dylibs and upload for notarization # create a zip of the dylibs and upload for notarization
zip_name = "{}.zip".format(project) zip_name = "{}.zip".format(project)
subprocess.check_call( subprocess.check_call(
["/usr/bin/ditto", "-c", "-k", "--keepParent"] ["/usr/bin/ditto", "-c", "-k", "--keepParent"]
+ [str(f) for f in filepaths] + [zip_name], + [str(f) for f in filepaths]
env=os.environ) + [zip_name],
env=os.environ,
)
# make sure keychain is unlocked # make sure keychain is unlocked
subprocess.check_call( subprocess.check_call(

View File

@ -6,13 +6,13 @@ from common import (
run_kiwix_build, run_kiwix_build,
make_deps_archive, make_deps_archive,
upload, upload,
PLATFORM_TARGET, COMPILE_CONFIG,
DEV_BRANCH, DEV_BRANCH,
) )
from build_definition import select_build_targets, DEPS from build_definition import select_build_targets, DEPS
for target in select_build_targets(DEPS): for target in select_build_targets(DEPS):
run_kiwix_build(target, platform=PLATFORM_TARGET, build_deps_only=True) run_kiwix_build(target, config=COMPILE_CONFIG, build_deps_only=True)
archive_file = make_deps_archive(target=target) archive_file = make_deps_archive(target=target)
if DEV_BRANCH: if DEV_BRANCH:
destination = "/data/tmp/ci/dev_preview/" + DEV_BRANCH destination = "/data/tmp/ci/dev_preview/" + DEV_BRANCH

View File

@ -13,10 +13,11 @@ from common import (
upload, upload,
make_deps_archive, make_deps_archive,
HOME, HOME,
PLATFORM_TARGET, COMPILE_CONFIG,
OS_NAME, OS_NAME,
) )
def download_base_archive(base_name): def download_base_archive(base_name):
url = "http://tmp.kiwix.org/ci/{}".format(base_name) url = "http://tmp.kiwix.org/ci/{}".format(base_name)
file_path = str(HOME / base_name) file_path = str(HOME / base_name)
@ -30,14 +31,15 @@ def download_base_archive(base_name):
file.write(batch) file.write(batch)
return file_path return file_path
ARCHIVE_NAME_TEMPLATE = "base_deps2_{os}_{platform}_{version}.tar.xz"
if PLATFORM_TARGET == 'flatpak': ARCHIVE_NAME_TEMPLATE = "base_deps2_{os}_{config}_{version}.tar.xz"
if COMPILE_CONFIG == "flatpak":
base_dep_archive_name = "base_deps2_flatpak.tar.xz" base_dep_archive_name = "base_deps2_flatpak.tar.xz"
else: else:
base_dep_archive_name = ARCHIVE_NAME_TEMPLATE.format( base_dep_archive_name = ARCHIVE_NAME_TEMPLATE.format(
os=OS_NAME, os=OS_NAME,
platform=PLATFORM_TARGET, config=COMPILE_CONFIG,
version=base_deps_meta_version, version=base_deps_meta_version,
) )
@ -48,11 +50,11 @@ try:
f.extractall(str(HOME)) f.extractall(str(HOME))
os.remove(str(local_filename)) os.remove(str(local_filename))
except URLError: except URLError:
if PLATFORM_TARGET == "flatpak": if COMPILE_CONFIG == "flatpak":
print_message("Cannot get archive. Move on") print_message("Cannot get archive. Move on")
else: else:
print_message("Cannot get archive. Build dependencies") print_message("Cannot get archive. Build dependencies")
run_kiwix_build("alldependencies", platform=PLATFORM_TARGET) run_kiwix_build("alldependencies", config=COMPILE_CONFIG)
archive_file = make_deps_archive(name=base_dep_archive_name, full=True) archive_file = make_deps_archive(name=base_dep_archive_name, full=True)
upload(archive_file, "ci@tmp.kiwix.org:30022", "/data/tmp/ci") upload(archive_file, "ci@tmp.kiwix.org:30022", "/data/tmp/ci")
os.remove(str(archive_file)) os.remove(str(archive_file))

View File

@ -4,7 +4,7 @@ set -e
cd $HOME cd $HOME
ARCHIVE_NAME=fail_log_${OS_NAME}_${PLATFORM_TARGET}.tar.gz ARCHIVE_NAME=fail_log_${OS_NAME}_${COMPILE_CONFIG}.tar.gz
tar -czf ${ARCHIVE_NAME} $HOME/BUILD_* $HOME/SOURCE $HOME/LOGS $HOME/TOOLCHAINS tar -czf ${ARCHIVE_NAME} $HOME/BUILD_* $HOME/SOURCE $HOME/LOGS $HOME/TOOLCHAINS
echo "Uploading archive $ARCHIVE_NAME" echo "Uploading archive $ARCHIVE_NAME"

View File

@ -10,7 +10,7 @@ jobs:
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
target: config:
- native_static - native_static
- native_dyn - native_dyn
- native_mixed - native_mixed
@ -37,13 +37,13 @@ jobs:
- android_x86_64 - android_x86_64
image_variant: ['focal'] image_variant: ['focal']
include: include:
- target: native_mixed - config: native_mixed
image_variant: bionic image_variant: bionic
- target: aarch64_mixed - config: aarch64_mixed
image_variant: bionic image_variant: bionic
- target: win32_static - config: win32_static
image_variant: f35 image_variant: f35
- target: win32_dyn - config: win32_dyn
image_variant: f35 image_variant: f35
env: env:
HOME: /home/runner HOME: /home/runner
@ -74,26 +74,26 @@ jobs:
cd $HOME cd $HOME
kiwix-build/.github/scripts/ensure_base_deps.py kiwix-build/.github/scripts/ensure_base_deps.py
env: env:
PLATFORM_TARGET: ${{matrix.target}} COMPILE_CONFIG: ${{matrix.config}}
- name: Compile all deps - name: Compile all deps
shell: bash shell: bash
run: | run: |
cd $HOME cd $HOME
kiwix-build/.github/scripts/compile_all_deps.py kiwix-build/.github/scripts/compile_all_deps.py
env: env:
PLATFORM_TARGET: ${{matrix.target}} COMPILE_CONFIG: ${{matrix.config}}
- name: Build projects - name: Build projects
shell: bash shell: bash
run: | run: |
cd $HOME cd $HOME
kiwix-build/.github/scripts/build_projects.py kiwix-build/.github/scripts/build_projects.py
env: env:
PLATFORM_TARGET: ${{matrix.target}} COMPILE_CONFIG: ${{matrix.config}}
- name: Upload failure logs - name: Upload failure logs
if: failure() if: failure()
run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh
env: env:
PLATFORM_TARGET: ${{matrix.target}} COMPILE_CONFIG: ${{matrix.config}}
Flatpak: Flatpak:
strategy: strategy:
@ -101,7 +101,7 @@ jobs:
env: env:
HOME: /home/runner HOME: /home/runner
SSH_KEY: /tmp/id_rsa SSH_KEY: /tmp/id_rsa
PLATFORM_TARGET: flatpak COMPILE_CONFIG: flatpak
OS_NAME: focal OS_NAME: focal
runs-on: ubuntu-22.04 runs-on: ubuntu-22.04
steps: steps:
@ -142,7 +142,7 @@ jobs:
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
target: config:
- native_dyn - native_dyn
- native_static - native_static
- native_mixed - native_mixed
@ -185,23 +185,23 @@ jobs:
cd $HOME cd $HOME
$GITHUB_WORKSPACE/.github/scripts/ensure_base_deps.py $GITHUB_WORKSPACE/.github/scripts/ensure_base_deps.py
env: env:
PLATFORM_TARGET: ${{matrix.target}} COMPILE_CONFIG: ${{matrix.config}}
- name: Compile all deps - name: Compile all deps
shell: bash shell: bash
run: | run: |
cd $HOME cd $HOME
$GITHUB_WORKSPACE/.github/scripts/compile_all_deps.py $GITHUB_WORKSPACE/.github/scripts/compile_all_deps.py
env: env:
PLATFORM_TARGET: ${{matrix.target}} COMPILE_CONFIG: ${{matrix.config}}
- name: Build projects - name: Build projects
shell: bash shell: bash
run: | run: |
cd $HOME cd $HOME
$GITHUB_WORKSPACE/.github/scripts/build_projects.py $GITHUB_WORKSPACE/.github/scripts/build_projects.py
env: env:
PLATFORM_TARGET: ${{matrix.target}} COMPILE_CONFIG: ${{matrix.config}}
- name: Upload failure logs - name: Upload failure logs
if: failure() if: failure()
run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.sh run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.sh
env: env:
PLATFORM_TARGET: ${{matrix.target}} COMPILE_CONFIG: ${{matrix.config}}

View File

@ -12,7 +12,7 @@ jobs:
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
target: config:
- native_static - native_static
- native_mixed - native_mixed
- native_dyn - native_dyn
@ -34,11 +34,11 @@ jobs:
- android_x86_64 - android_x86_64
image_variant: ['focal'] image_variant: ['focal']
include: include:
- target: native_mixed - config: native_mixed
image_variant: bionic image_variant: bionic
- target: aarch64_mixed - config: aarch64_mixed
image_variant: bionic image_variant: bionic
- target: win32_static - config: win32_static
image_variant: f35 image_variant: f35
env: env:
HOME: /home/runner HOME: /home/runner
@ -69,21 +69,21 @@ jobs:
cd $HOME cd $HOME
kiwix-build/.github/scripts/ensure_base_deps.py kiwix-build/.github/scripts/ensure_base_deps.py
env: env:
PLATFORM_TARGET: ${{matrix.target}} COMPILE_CONFIG: ${{matrix.config}}
- name: Build release - name: Build release
shell: bash shell: bash
run: | run: |
cd $HOME cd $HOME
kiwix-build/.github/scripts/build_release_nightly.py kiwix-build/.github/scripts/build_release_nightly.py
env: env:
PLATFORM_TARGET: ${{matrix.target}} COMPILE_CONFIG: ${{matrix.config}}
BINTRAY_USER: kiwix BINTRAY_USER: kiwix
BINTRAY_PASS: ${{secrets.bintray_pass}} BINTRAY_PASS: ${{secrets.bintray_pass}}
- name: Upload failure logs - name: Upload failure logs
if: failure() if: failure()
run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh
env: env:
PLATFORM_TARGET: ${{matrix.target}} COMPILE_CONFIG: ${{matrix.config}}
Flatpak: Flatpak:
strategy: strategy:
@ -91,7 +91,7 @@ jobs:
env: env:
HOME: /home/runner HOME: /home/runner
SSH_KEY: /tmp/id_rsa SSH_KEY: /tmp/id_rsa
PLATFORM_TARGET: flatpak COMPILE_CONFIG: flatpak
OS_NAME: focal OS_NAME: focal
runs-on: ubuntu-22.04 runs-on: ubuntu-22.04
steps: steps:
@ -132,7 +132,7 @@ jobs:
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
target: config:
- native_dyn - native_dyn
- native_static - native_static
- native_mixed - native_mixed
@ -195,26 +195,26 @@ jobs:
cd $HOME cd $HOME
$GITHUB_WORKSPACE/.github/scripts/ensure_base_deps.py $GITHUB_WORKSPACE/.github/scripts/ensure_base_deps.py
env: env:
PLATFORM_TARGET: ${{matrix.target}} COMPILE_CONFIG: ${{matrix.config}}
- name: Build release - name: Build release
shell: bash shell: bash
run: | run: |
cd $HOME cd $HOME
$GITHUB_WORKSPACE/.github/scripts/build_release_nightly.py $GITHUB_WORKSPACE/.github/scripts/build_release_nightly.py
env: env:
PLATFORM_TARGET: ${{matrix.target}} COMPILE_CONFIG: ${{matrix.config}}
- name: Upload failure logs - name: Upload failure logs
if: failure() if: failure()
run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.sh run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.sh
env: env:
PLATFORM_TARGET: ${{matrix.target}} COMPILE_CONFIG: ${{matrix.config}}
Trigger_Docker: Trigger_Docker:
needs: [Linux] needs: [Linux]
runs-on: ubuntu-22.04 runs-on: ubuntu-22.04
env: env:
PLATFORM_TARGET: native_static COMPILE_CONFIG: native_static
OS_NAME: linux OS_NAME: linux
steps: steps:
- name: Checkout code - name: Checkout code