diff --git a/.github/scripts/build_definition.py b/.github/scripts/build_definition.py new file mode 100644 index 0000000..b902abe --- /dev/null +++ b/.github/scripts/build_definition.py @@ -0,0 +1,120 @@ +from typing import NamedTuple +import csv, io, re + +# Definition of what to build. +# Array is read line by line. +# Empty cells under (OS_NAME, PLATFORM_TARGET) mean "always match" (catch all, or `.*` regex) +# Once a cell doesn't match, skip to the next line. +# Once a line matches, other lines are not read, so put more specific combination first. +# Lines composed of `-` , or `=`, or starting by `#` are ignored. +# 'B' letter means that the project is build in the CI +# 'd' letter means that the project's dependencies are build and published to be used by the project's CI. +# If a cell contains both, both are done. +BUILD_DEF = """ + | OS_NAME | PLATFORM_TARGET | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | + ============================================================================================== +# Bionic is a special case as we need to compile libzim on old arch for python + | bionic | | B | | | | | + ---------------------------------------------------------------------------------------------- +# Osx builds, build binaries on native_dyn and native_static. On anyother things, build only the libraries + | macos | native_dyn | d | d | dB | B | | + | macos | native_static | | | B | B | | + | macos | native_mixed | B | B | | | | + | macos | iOS_arm64 | dB | B | | | | + | macos | iOS_x86_64 | dB | B | | | | + | macos | iOS_Mac_ABI | B | B | | | | + | macos | macOS_arm64_static | B | B | | | | + | macos | macOS_arm64_mixed | B | B | | | | + | macos | macOS_x86_64 | B | B | | | | + ---------------------------------------------------------------------------------------------- + | | flatpak | | | | | B | + | | native_static | d | d | dB | dB | | + | | native_dyn | d | d | dB | dB | B | + | | native_mixed | B | B | | | | +# libzim CI is building alpine_dyn but not us + | | android_arm | dB | dB | | | | + | | android_arm64 | dB | dB | | | | + | | android_x86 | B | B | | | | + | | android_x86_64 | B | B | | | | + | | armv6_static | | | B | B | | + | | armv6_dyn | | | B | B | | + | | armv6_mixed | B | | | | | + | | armv8_static | | | B | B | | + | | armv8_dyn | | | B | B | | + | | armv8_mixed | B | | | | | + | | aarch64_static | | | B | B | | + | | aarch64_dyn | d | | B | B | | + | | aarch64_mixed | B | | | | | + | | win32_static | d | dB | dB | dB | | + | | win32_dyn | d | dB | dB | dB | | + | | i586_static | | | B | B | | + | | i586_dyn | | | B | B | | + | | wasm | dB | | | | | +""" + + +class TableDialect(csv.Dialect): + delimiter = "|" + quoting = csv.QUOTE_NONE + lineterminator = "\n" + + +def strip_array(array_str): + """Return a iterable of lines, skiping "decorative lines" and with all values in the line's cells stripped""" + for line in array_str.splitlines(): + line = line.strip() + line_set = set(line) + if ( + not line + or line.startswith("#") + or (len(line_set) == 1 and line_set.pop() in "-=") + ): + continue + yield "|".join(c.strip() for c in line.split("|")) + + +def selector_match(selector, value): + if not selector: + return True + return re.fullmatch(selector, value) is not None + + +class Context(NamedTuple): + OS_NAME: str + PLATFORM_TARGET: str + + def match(self, row): + for key in ["OS_NAME", "PLATFORM_TARGET"]: + context_value = getattr(self, key) + selector = row[key] + if not selector_match(selector, context_value): + return False + return True + + +BUILD = "B" +DEPS = "d" + +def select_build_targets(criteria): + from common import PLATFORM_TARGET, OS_NAME + + context = Context(PLATFORM_TARGET=PLATFORM_TARGET, OS_NAME=OS_NAME) + + reader = csv.DictReader(strip_array(BUILD_DEF), dialect=TableDialect()) + for row in reader: + if context.match(row): + build_order = [ + k + for k in ( + "libzim", + "libkiwix", + "zim-tools", + "kiwix-tools", + "kiwix-desktop", + ) + if criteria in row[k] + ] + print(build_order) + return build_order + + raise "No definition match with current context." diff --git a/.github/scripts/build_projects.py b/.github/scripts/build_projects.py index 7b837c7..b267b64 100755 --- a/.github/scripts/build_projects.py +++ b/.github/scripts/build_projects.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 - +from build_definition import select_build_targets, BUILD from common import ( run_kiwix_build, make_archive, @@ -12,55 +12,12 @@ from common import ( DEV_BRANCH, ) - -def select_build_target(): - from common import ( - PLATFORM_TARGET, - DESKTOP, - OS_NAME - ) - if OS_NAME == "bionic" and PLATFORM_TARGET.endswith("_mixed"): - return ("libzim", ) - elif (PLATFORM_TARGET.startswith("android_") - or PLATFORM_TARGET.startswith("iOS")): - return ("libzim", "libkiwix") - elif PLATFORM_TARGET.startswith("macOS"): - if PLATFORM_TARGET.endswith("_mixed"): - return ("libzim", "libkiwix") - elif PLATFORM_TARGET.endswith("_dyn"): - return ("zim-tools", "kiwix-tools") - else: - return [] - elif PLATFORM_TARGET.startswith("native_"): - if OS_NAME == "osx": - if PLATFORM_TARGET.endswith("_mixed"): - return ("libzim", "libkiwix") - else: - return ("zim-tools", "kiwix-tools") - else: - if DESKTOP: - return ("kiwix-desktop",) - elif PLATFORM_TARGET == "native_mixed": - return ("libzim", "libkiwix") - else: - return ("zim-tools", "kiwix-tools") - elif PLATFORM_TARGET in ("win32_static", "armv6_static", "armv6_dyn", "armv8_static", "armv8_dyn", "aarch64_static", "aarch64_dyn", "i586_static"): - return ("zim-tools", "kiwix-tools") - elif PLATFORM_TARGET == "flatpak": - return ("kiwix-desktop",) - elif PLATFORM_TARGET in ("wasm", "armv6_mixed", "armv8_mixed", "aarch64_mixed"): - return ("libzim", ) - else: - return ("libzim", "zim-tools", "libkiwix", "kiwix-tools") - -TARGETS = select_build_target() - -for target in TARGETS: +for target in select_build_targets(BUILD): run_kiwix_build(target, platform=PLATFORM_TARGET) if target == "kiwix-desktop": archive = create_desktop_image(make_release=False) else: - if PLATFORM_TARGET == "native_mixed" and OS_NAME == "osx": + if PLATFORM_TARGET == "native_mixed" and OS_NAME == "macos": fix_macos_rpath(target) archive = make_archive(target, make_release=False) if archive and DEV_BRANCH: diff --git a/.github/scripts/build_release_nightly.py b/.github/scripts/build_release_nightly.py index e7e176e..f320544 100755 --- a/.github/scripts/build_release_nightly.py +++ b/.github/scripts/build_release_nightly.py @@ -13,17 +13,15 @@ from common import ( fix_macos_rpath, trigger_docker_publish, BASE_DIR, - TMP_DIR, - HOME, OS_NAME, PLATFORM_TARGET, MAKE_RELEASE, notarize_macos_build, ) -from build_projects import select_build_target +from build_definition import select_build_targets, BUILD -TARGETS = select_build_target() +TARGETS = select_build_target(BUILD) # Filter what to build if we are doing a release. if MAKE_RELEASE: @@ -36,7 +34,7 @@ for target in TARGETS: if target == "kiwix-desktop": archive = create_desktop_image(make_release=MAKE_RELEASE) else: - if OS_NAME == "osx" and PLATFORM_TARGET.endswith("_mixed"): + if OS_NAME == "macos" and PLATFORM_TARGET.endswith("_mixed"): fix_macos_rpath(target) notarize_macos_build(target) archive = make_archive(target, make_release=MAKE_RELEASE) @@ -48,7 +46,7 @@ for target in TARGETS: # We have few more things to do for release: if MAKE_RELEASE: # Publish source archives - if PLATFORM_TARGET in ("native_dyn", "native_mixed") and OS_NAME != "osx": + if PLATFORM_TARGET in ("native_dyn", "native_mixed") and OS_NAME != "macos": for target in TARGETS: if release_versions.get(target) != 0: continue diff --git a/.github/scripts/common.py b/.github/scripts/common.py index 600ac2e..f483291 100644 --- a/.github/scripts/common.py +++ b/.github/scripts/common.py @@ -17,11 +17,6 @@ from kiwixbuild.versions import ( ) PLATFORM_TARGET = _environ["PLATFORM_TARGET"] -if PLATFORM_TARGET == "native_desktop": - PLATFORM_TARGET = "native_dyn" - DESKTOP = True -else: - DESKTOP = False OS_NAME = _environ["OS_NAME"] HOME = Path(os.path.expanduser("~")) @@ -32,9 +27,6 @@ INSTALL_DIR = BASE_DIR / "INSTALL" TMP_DIR = Path("/tmp") KBUILD_SOURCE_DIR = HOME / "kiwix-build" -# [TODO] -KIWIX_DESKTOP_ONLY = False - _ref = _environ.get("GITHUB_REF", "").split("/")[-1] MAKE_RELEASE = re.fullmatch(r"r_[0-9]+", _ref) is not None MAKE_RELEASE = MAKE_RELEASE and (_environ.get('GITHUB_EVENT_NAME') != 'schedule') @@ -44,7 +36,7 @@ if not MAKE_RELEASE and _ref != "main": else: DEV_BRANCH = None -RELEASE_OS_NAME = "macos" if OS_NAME == "osx" else "linux" +RELEASE_OS_NAME = "macos" if OS_NAME == "macos" else "linux" EXTRA_NAME = "-bionic" if OS_NAME == "bionic" else "" PLATFORM_TO_RELEASE = { diff --git a/.github/scripts/compile_all_deps.py b/.github/scripts/compile_all_deps.py index c690af1..87a0761 100755 --- a/.github/scripts/compile_all_deps.py +++ b/.github/scripts/compile_all_deps.py @@ -8,28 +8,10 @@ from common import ( upload, OS_NAME, PLATFORM_TARGET, - DESKTOP, - KIWIX_DESKTOP_ONLY, ) +from build_definition import select_build_targets, DEPS -if PLATFORM_TARGET.startswith("android_") or PLATFORM_TARGET.startswith("iOS"): - TARGETS = ("libzim", "libkiwix") -elif PLATFORM_TARGET == "wasm": - TARGETS = ("libzim", ) -elif PLATFORM_TARGET.startswith("native_"): - if OS_NAME == "osx": - TARGETS = ("libzim", "zim-tools", "libkiwix") - else: - if DESKTOP: - TARGETS = ("kiwix-desktop",) - elif PLATFORM_TARGET == "native_mixed": - TARGETS = ("libzim",) - else: - TARGETS = ("libzim", "zim-tools", "libkiwix", "kiwix-tools") -else: - TARGETS = ("libzim", "zim-tools", "libkiwix", "kiwix-tools") - -for target in TARGETS: +for target in select_build_targets(DEPS): run_kiwix_build(target, platform=PLATFORM_TARGET, build_deps_only=True) archive_file = make_deps_archive(target=target) upload(archive_file, "ci@tmp.kiwix.org:30022", "/data/tmp/ci") diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 084b599..d35f98a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,6 @@ jobs: - native_static - native_dyn - native_mixed - - native_desktop - wasm - armv6_static - armv6_dyn @@ -151,7 +150,7 @@ jobs: runs-on: macos-11 env: SSH_KEY: /tmp/id_rsa - OS_NAME: osx + OS_NAME: macos steps: - name: Checkout code uses: actions/checkout@v3 diff --git a/.github/workflows/releaseNigthly.yml b/.github/workflows/releaseNigthly.yml index db07360..3791f36 100644 --- a/.github/workflows/releaseNigthly.yml +++ b/.github/workflows/releaseNigthly.yml @@ -16,7 +16,6 @@ jobs: - native_static - native_dyn - native_mixed - - native_desktop - wasm - armv6_static - armv6_mixed @@ -39,8 +38,6 @@ jobs: image_variant: focal - target: native_mixed image_variant: bionic - - target: native_desktop - image_variant: focal - target: wasm image_variant: focal - target: armv6_static @@ -171,7 +168,7 @@ jobs: runs-on: macos-11 env: SSH_KEY: /tmp/id_rsa - OS_NAME: osx + OS_NAME: macos CERTIFICATE: /tmp/wmch-devid.p12 SIGNING_IDENTITY: ${{ secrets.APPLE_SIGNING_IDENTITY }} KEYCHAIN: /Users/runner/build.keychain-db