diff --git a/.github/scripts/build_definition.py b/.github/scripts/build_definition.py index 7a9675f..50b16e9 100644 --- a/.github/scripts/build_definition.py +++ b/.github/scripts/build_definition.py @@ -9,50 +9,55 @@ import csv, io, re # Lines composed of `-` , or `=`, or starting by `#` are ignored. # 'B' letter means that the project is build in the CI # 'd' letter means that the project's dependencies are build and published to be used by the project's CI. -# If a cell contains both, both are done. +# 'P' letter means that (build) project must be publish when we do a release. +# (This is used to avoid two publication of the same archive) +# 'S' letter means that source code must be publish (almost by definition, S should be put only with a P) +# 'D' letter means we trigger the docker forkflow to build the docker image. +# If a cell contains several letters, all are done. BUILD_DEF = """ - | OS_NAME | PLATFORM_TARGET | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | - ============================================================================================== + | OS_NAME | PLATFORM_TARGET | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | platform_name | + ===================================================================================================================== # Bionic is a special case as we need to compile libzim on old arch for python - | bionic | | B | | | | | - ---------------------------------------------------------------------------------------------- + | bionic | native_mixed | BP | | | | | linux-x86_64-bionic | + | bionic | aarch64_mixed | BP | | | | | linux-aarch64-bionic | + -------------------------------------------------------------------------------------------------------------------- # Osx builds, build binaries on native_dyn and native_static. On anyother things, build only the libraries - | macos | native_dyn | d | d | dB | B | | - | macos | native_static | | | B | B | | - | macos | native_mixed | B | B | | | | - | macos | iOS_arm64 | dB | B | | | | - | macos | iOS_x86_64 | dB | B | | | | - | macos | iOS_Mac_ABI | B | B | | | | - | macos | macOS_arm64_static | | | | | | - | macos | macOS_arm64_mixed | B | B | | | | - | macos | macOS_x86_64 | B | B | | | | + | macos | native_dyn | d | d | dB | B | | | + | macos | native_static | | | BP | BP | | macos-x86_64 | + | macos | native_mixed | BP | BP | | | | macos-x86_64 | + | macos | iOS_arm64 | dB | B | | | | | + | macos | iOS_x86_64 | dB | B | | | | | + | macos | iOS_Mac_ABI | B | B | | | | | + | macos | macOS_arm64_static | | | | | | | + | macos | macOS_arm64_mixed | BP | BP | | | | macos-arm64 | + | macos | macOS_x86_64 | B | B | | | | | ---------------------------------------------------------------------------------------------- - | | flatpak | | | | | B | - | | native_static | d | d | dB | dB | | - | | native_dyn | d | d | dB | dB | B | - | | native_mixed | B | B | | | | -# libzim CI is building alpine_dyn but not us - | | android_arm | dB | dB | | | | - | | android_arm64 | dB | dB | | | | - | | android_x86 | B | B | | | | - | | android_x86_64 | B | B | | | | - | | armv6_static | | | B | B | | - | | armv6_dyn | | | B | B | | - | | armv6_mixed | B | | | | | - | | armv8_static | | | B | B | | - | | armv8_dyn | | | B | B | | - | | armv8_mixed | B | | | | | - | | aarch64_static | | | B | B | | - | | aarch64_dyn | d | | B | B | | - | | aarch64_mixed | B | | | | | - | | aarch64_musl_static| | | B | B | | - | | aarch64_musl_dyn | d | | B | B | | - | | aarch64_musl_mixed | B | | | | | - | | win32_static | d | dB | dB | dB | | - | | win32_dyn | d | dB | dB | dB | | - | | i586_static | | | B | B | | - | | i586_dyn | | | B | B | | - | | wasm | dB | | | | | + | | flatpak | | | | | BP | | + | | native_static | d | d | dBPSD | dBPSD | | linux-x86_64 | + | | native_mixed | BPS | BPS | | | | linux-x86_64 | + | | native_dyn | d | d | dB | dB | BPS | | + # libzim CI is building alpine_dyn but not us + | | android_arm | dBP | dBP | | | | android-arm | + | | android_arm64 | dBP | dBP | | | | android-arm64 | + | | android_x86 | BP | BP | | | | android-x86 | + | | android_x86_64 | BP | BP | | | | android-x86_64 | + | | armv6_static | | | BP | BP | | linux-armv6 | + | | armv6_mixed | BP | | | | | linux-armv6 | + | | armv6_dyn | | | B | B | | | + | | armv8_static | | | BP | BP | | linux-armv8 | + | | armv8_mixed | BP | | | | | linux-armv8 | + | | armv8_dyn | | | B | B | | | + | | aarch64_static | | | BP | BP | | linux-aarch64 | + | | aarch64_mixed | BP | | | | | linux-aarch64 | + | | aarch64_dyn | d | | B | B | | | + | | aarch64_musl_static| | | BP | BP | | linux-aarch64-musl | + | | aarch64_musl_mixed | BP | | | | | linux-aarch64-musl | + | | aarch64_musl_dyn | d | | B | B | | | + | | win32_static | d | dB | dBP | dBP | | win-i686 | + | | win32_dyn | d | dB | dB | dB | | | + | | i586_static | | | BP | BP | | linux-i586 | + | | i586_dyn | | | B | B | | | + | | wasm | dBP | | | | | wasm-emscripten | """ @@ -96,7 +101,10 @@ class Context(NamedTuple): BUILD = "B" +PUBLISH = "P" +SOURCE_PUBLISH = "S" DEPS = "d" +DOCKER = "D" def select_build_targets(criteria): from common import PLATFORM_TARGET, OS_NAME @@ -121,3 +129,16 @@ def select_build_targets(criteria): return build_order raise "No definition match with current context." + +def get_platform_name(): + from common import PLATFORM_TARGET, OS_NAME + + context = Context(PLATFORM_TARGET=PLATFORM_TARGET, OS_NAME=OS_NAME) + + reader = csv.DictReader(strip_array(BUILD_DEF), dialect=TableDialect()) + for row in reader: + if context.match(row): + name = row["platform_name"] + return name or None + + raise "No definition match with current context." diff --git a/.github/scripts/build_release_nightly.py b/.github/scripts/build_release_nightly.py index c8503d9..e97cda6 100755 --- a/.github/scripts/build_release_nightly.py +++ b/.github/scripts/build_release_nightly.py @@ -11,7 +11,6 @@ from common import ( update_flathub_git, upload_archive, fix_macos_rpath, - trigger_docker_publish, BASE_DIR, OS_NAME, PLATFORM_TARGET, @@ -19,15 +18,19 @@ from common import ( notarize_macos_build, ) -from build_definition import select_build_targets, BUILD +from build_definition import select_build_targets, BUILD, PUBLISH, SOURCE_PUBLISH + + -TARGETS = select_build_targets(BUILD) # Filter what to build if we are doing a release. if MAKE_RELEASE: + TARGETS = select_build_targets(PUBLISH) def release_filter(project): return release_versions.get(project) is not None TARGETS = tuple(filter(release_filter, TARGETS)) +else: + TARGETS = select_build_targets(BUILD) for target in TARGETS: run_kiwix_build(target, platform=PLATFORM_TARGET, make_release=MAKE_RELEASE) @@ -40,32 +43,35 @@ for target in TARGETS: archive = make_archive(target, make_release=MAKE_RELEASE) if archive: upload_archive(archive, target, make_release=MAKE_RELEASE) - if MAKE_RELEASE and target in ("zim-tools", "kiwix-tools"): - trigger_docker_publish(target) # We have few more things to do for release: if MAKE_RELEASE: # Publish source archives - if PLATFORM_TARGET in ("native_dyn", "native_mixed") and OS_NAME != "macos": - for target in TARGETS: - if release_versions.get(target) != 0: - continue - run_kiwix_build( - target, platform=PLATFORM_TARGET, make_release=MAKE_RELEASE, make_dist=True + source_published_targets = select_build_targets(SOURCE_PUBLISH) + for target in TARGETS: + # Looping on TARGETS instead of source_published_targets ensures we are + # publishing sources only for target we are building. + # (source_published_targets must be a subset of TARGETS) + if release_versions.get(target) != 0: + continue + if target not in source_published_targets: + continue + run_kiwix_build( + target, platform=PLATFORM_TARGET, make_release=MAKE_RELEASE, make_dist=True + ) + full_target_name = "{}-{}".format(target, main_project_versions[target]) + if target == "kiwix-desktop": + archive = ( + BASE_DIR / full_target_name / "{}.tar.gz".format(full_target_name) ) - full_target_name = "{}-{}".format(target, main_project_versions[target]) - if target == "kiwix-desktop": - archive = ( - BASE_DIR / full_target_name / "{}.tar.gz".format(full_target_name) - ) - else: - archive = ( - BASE_DIR - / full_target_name - / "meson-dist" - / "{}.tar.xz".format(full_target_name) - ) - upload_archive(archive, target, make_release=MAKE_RELEASE) + else: + archive = ( + BASE_DIR + / full_target_name + / "meson-dist" + / "{}.tar.xz".format(full_target_name) + ) + upload_archive(archive, target, make_release=MAKE_RELEASE) # Publish flathub if PLATFORM_TARGET == "flatpak" and "kiwix-desktop" in TARGETS: diff --git a/.github/scripts/common.py b/.github/scripts/common.py index a9192df..83561f3 100644 --- a/.github/scripts/common.py +++ b/.github/scripts/common.py @@ -10,12 +10,16 @@ import shutil import requests +from build_definition import get_platform_name + from kiwixbuild.versions import ( main_project_versions, release_versions, base_deps_versions, ) + + PLATFORM_TARGET = _environ["PLATFORM_TARGET"] OS_NAME = _environ["OS_NAME"] HOME = Path(os.path.expanduser("~")) @@ -36,31 +40,6 @@ if not MAKE_RELEASE and _ref != "main": else: DEV_BRANCH = None -RELEASE_OS_NAME = "macos" if OS_NAME == "macos" else "linux" -EXTRA_NAME = "-bionic" if OS_NAME == "bionic" else "" - -PLATFORM_TO_RELEASE = { - "native_mixed": "{os}-x86_64{extra}".format(os=RELEASE_OS_NAME, extra=EXTRA_NAME), - "native_static": "{os}-x86_64".format(os=RELEASE_OS_NAME), - "win32_static": "win-i686", - "armv6_static": "{os}-armv6".format(os=RELEASE_OS_NAME), - "armv6_mixed": "{os}-armv6".format(os=RELEASE_OS_NAME), - "armv8_static": "{os}-armv8".format(os=RELEASE_OS_NAME), - "armv8_mixed": "{os}-armv8".format(os=RELEASE_OS_NAME), - "aarch64_static": "{os}-aarch64".format(os=RELEASE_OS_NAME), - "aarch64_mixed": "{os}-aarch64{extra}".format(os=RELEASE_OS_NAME, extra=EXTRA_NAME), - "aarch64_musl_static": "{os}-aarch64-musl".format(os=RELEASE_OS_NAME), - "aarch64_musl_mixed": "{os}-aarch64-musl".format(os=RELEASE_OS_NAME), - "i586_static": "{os}-i586".format(os=RELEASE_OS_NAME), - "macOS_arm64_static": "{os}-arm64".format(os=RELEASE_OS_NAME), - "macOS_arm64_mixed": "{os}-arm64".format(os=RELEASE_OS_NAME), - "android_arm": "android-arm", - "android_arm64": "android-arm64", - "android_x86": "android-x86", - "android_x86_64": "android-x86_64", - "wasm": "wasm-emscripten", -} - FLATPAK_HTTP_GIT_REMOTE = "https://github.com/flathub/org.kiwix.desktop.git" FLATPAK_GIT_REMOTE = "git@github.com:flathub/org.kiwix.desktop.git" @@ -351,10 +330,8 @@ def get_postfix(project): def make_archive(project, make_release): - try: - platform = PLATFORM_TO_RELEASE[PLATFORM_TARGET] - except KeyError: - # We don't know how to name the release. + platform_name = get_platform_name() + if not platform_name: return None try: @@ -368,12 +345,12 @@ def make_archive(project, make_release): else: postfix = DATE - archive_name = "{}_{}-{}".format(project, platform, postfix) + archive_name = "{}_{}-{}".format(project, platform_name, postfix) files_to_archive = [] for export_file in export_files: files_to_archive.extend(base_dir.glob(export_file)) - if platform == "win-i686": + if platform_name == "win-i686": open_archive = lambda a: zipfile.ZipFile( str(a), "w", compression=zipfile.ZIP_DEFLATED ) diff --git a/.github/scripts/trigger_docker_workflow.py b/.github/scripts/trigger_docker_workflow.py new file mode 100755 index 0000000..27280de --- /dev/null +++ b/.github/scripts/trigger_docker_workflow.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 + +from common import ( + trigger_docker_publish, + MAKE_RELEASE, +) + +from build_definition import select_build_targets, DOCKER + +# Filter what to build if we are doing a release. +if MAKE_RELEASE: + docker_trigger = select_build_targets(DOCKER) +else: + docker_trigger = [] + +for target in docker_trigger: + trigger_docker_publish(target) diff --git a/.github/workflows/releaseNigthly.yml b/.github/workflows/releaseNigthly.yml index c95d8da..d00cd3a 100644 --- a/.github/workflows/releaseNigthly.yml +++ b/.github/workflows/releaseNigthly.yml @@ -110,7 +110,6 @@ jobs: PLATFORM_TARGET: ${{matrix.target}} BINTRAY_USER: kiwix BINTRAY_PASS: ${{secrets.bintray_pass}} - GITHUB_PAT: ${{secrets.GHCR_TOKEN}} - name: Upload failure logs if: failure() run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh @@ -236,3 +235,21 @@ jobs: run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.sh env: PLATFORM_TARGET: ${{matrix.target}} + + + Trigger_Docker: + needs: [Linux] + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Install python modules + shell: bash + run: | + pip3 install --user --no-deps $GITHUB_WORKSPACE + - name: Trigger docker workflow + shell: bash + run: | + cd $HOME + $GITHUB_WORKSPACE/.github/scripts/trigger_docker_workflow.py + env: + GITHUB_PAT: ${{secrets.DOCKER_TRIGGER_GITHUB_PAT}}