smallsolar
/
kiwix-build
mirror of https://github.com/kiwix/kiwix-build.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #630 from kiwix/better_ci_def

This commit is contained in:
Matthieu Gautier 2023-07-27 16:11:51 +02:00 committed by GitHub
parent 1876b5f542 45b459a03a
commit e9c039dc9b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 134 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

99
.github/scripts/build_definition.py vendored
View File

@ -9,50 +9,55 @@ import csv, io, re
# Lines composed of `-` , or `=`, or starting by `#` are ignored. # Lines composed of `-` , or `=`, or starting by `#` are ignored.
# 'B' letter means that the project is build in the CI # 'B' letter means that the project is build in the CI
# 'd' letter means that the project's dependencies are build and published to be used by the project's CI. # 'd' letter means that the project's dependencies are build and published to be used by the project's CI.
# If a cell contains both, both are done. # 'P' letter means that (build) project must be publish when we do a release.
# (This is used to avoid two publication of the same archive)
# 'S' letter means that source code must be publish (almost by definition, S should be put only with a P)
# 'D' letter means we trigger the docker forkflow to build the docker image.
# If a cell contains several letters, all are done.
BUILD_DEF = """ BUILD_DEF = """
| OS_NAME | PLATFORM_TARGET | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | | OS_NAME | PLATFORM_TARGET | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | platform_name |
============================================================================================== =====================================================================================================================
# Bionic is a special case as we need to compile libzim on old arch for python # Bionic is a special case as we need to compile libzim on old arch for python
| bionic | | B | | | | | | bionic | native_mixed | BP | | | | | linux-x86_64-bionic |
---------------------------------------------------------------------------------------------- | bionic | aarch64_mixed | BP | | | | | linux-aarch64-bionic |
--------------------------------------------------------------------------------------------------------------------
# Osx builds, build binaries on native_dyn and native_static. On anyother things, build only the libraries # Osx builds, build binaries on native_dyn and native_static. On anyother things, build only the libraries
| macos | native_dyn | d | d | dB | B | | | macos | native_dyn | d | d | dB | B | | |
| macos | native_static | | | B | B | | | macos | native_static | | | BP | BP | | macos-x86_64 |
| macos | native_mixed | B | B | | | | | macos | native_mixed | BP | BP | | | | macos-x86_64 |
| macos | iOS_arm64 | dB | B | | | | | macos | iOS_arm64 | dB | B | | | | |
| macos | iOS_x86_64 | dB | B | | | | | macos | iOS_x86_64 | dB | B | | | | |
| macos | iOS_Mac_ABI | B | B | | | | | macos | iOS_Mac_ABI | B | B | | | | |
| macos | macOS_arm64_static | | | | | | | macos | macOS_arm64_static | | | | | | |
| macos | macOS_arm64_mixed | B | B | | | | | macos | macOS_arm64_mixed | BP | BP | | | | macos-arm64 |
| macos | macOS_x86_64 | B | B | | | | | macos | macOS_x86_64 | B | B | | | | |
---------------------------------------------------------------------------------------------- ----------------------------------------------------------------------------------------------
| | flatpak | | | | | B | | | flatpak | | | | | BP | |
| | native_static | d | d | dB | dB | | | | native_static | d | d | dBPSD | dBPSD | | linux-x86_64 |
| | native_dyn | d | d | dB | dB | B | | | native_mixed | BPS | BPS | | | | linux-x86_64 |
| | native_mixed | B | B | | | | | | native_dyn | d | d | dB | dB | BPS | |
# libzim CI is building alpine_dyn but not us # libzim CI is building alpine_dyn but not us
| | android_arm | dB | dB | | | | | | android_arm | dBP | dBP | | | | android-arm |
| | android_arm64 | dB | dB | | | | | | android_arm64 | dBP | dBP | | | | android-arm64 |
| | android_x86 | B | B | | | | | | android_x86 | BP | BP | | | | android-x86 |
| | android_x86_64 | B | B | | | | | | android_x86_64 | BP | BP | | | | android-x86_64 |
| | armv6_static | | | B | B | | | | armv6_static | | | BP | BP | | linux-armv6 |
| | armv6_dyn | | | B | B | | | | armv6_mixed | BP | | | | | linux-armv6 |
| | armv6_mixed | B | | | | | | | armv6_dyn | | | B | B | | |
| | armv8_static | | | B | B | | | | armv8_static | | | BP | BP | | linux-armv8 |
| | armv8_dyn | | | B | B | | | | armv8_mixed | BP | | | | | linux-armv8 |
| | armv8_mixed | B | | | | | | | armv8_dyn | | | B | B | | |
| | aarch64_static | | | B | B | | | | aarch64_static | | | BP | BP | | linux-aarch64 |
| | aarch64_dyn | d | | B | B | | | | aarch64_mixed | BP | | | | | linux-aarch64 |
| | aarch64_mixed | B | | | | | | | aarch64_dyn | d | | B | B | | |
| | aarch64_musl_static| | | B | B | | | | aarch64_musl_static| | | BP | BP | | linux-aarch64-musl |
| | aarch64_musl_dyn | d | | B | B | | | | aarch64_musl_mixed | BP | | | | | linux-aarch64-musl |
| | aarch64_musl_mixed | B | | | | | | | aarch64_musl_dyn | d | | B | B | | |
| | win32_static | d | dB | dB | dB | | | | win32_static | d | dB | dBP | dBP | | win-i686 |
| | win32_dyn | d | dB | dB | dB | | | | win32_dyn | d | dB | dB | dB | | |
| | i586_static | | | B | B | | | | i586_static | | | BP | BP | | linux-i586 |
| | i586_dyn | | | B | B | | | | i586_dyn | | | B | B | | |
| | wasm | dB | | | | | | | wasm | dBP | | | | | wasm-emscripten |
""" """
@ -96,7 +101,10 @@ class Context(NamedTuple):
BUILD = "B" BUILD = "B"
PUBLISH = "P"
SOURCE_PUBLISH = "S"
DEPS = "d" DEPS = "d"
DOCKER = "D"
def select_build_targets(criteria): def select_build_targets(criteria):
from common import PLATFORM_TARGET, OS_NAME from common import PLATFORM_TARGET, OS_NAME
@ -121,3 +129,16 @@ def select_build_targets(criteria):
return build_order return build_order
raise "No definition match with current context." raise "No definition match with current context."
def get_platform_name():
from common import PLATFORM_TARGET, OS_NAME
context = Context(PLATFORM_TARGET=PLATFORM_TARGET, OS_NAME=OS_NAME)
reader = csv.DictReader(strip_array(BUILD_DEF), dialect=TableDialect())
for row in reader:
if context.match(row):
name = row["platform_name"]
return name or None
raise "No definition match with current context."

18
.github/scripts/build_release_nightly.py vendored
View File

@ -11,7 +11,6 @@ from common import (
update_flathub_git, update_flathub_git,
upload_archive, upload_archive,
fix_macos_rpath, fix_macos_rpath,
trigger_docker_publish,
BASE_DIR, BASE_DIR,
OS_NAME, OS_NAME,
PLATFORM_TARGET, PLATFORM_TARGET,
@ -19,15 +18,19 @@ from common import (
notarize_macos_build, notarize_macos_build,
) )
from build_definition import select_build_targets, BUILD from build_definition import select_build_targets, BUILD, PUBLISH, SOURCE_PUBLISH
TARGETS = select_build_targets(BUILD)
# Filter what to build if we are doing a release. # Filter what to build if we are doing a release.
if MAKE_RELEASE: if MAKE_RELEASE:
TARGETS = select_build_targets(PUBLISH)
def release_filter(project): def release_filter(project):
return release_versions.get(project) is not None return release_versions.get(project) is not None
TARGETS = tuple(filter(release_filter, TARGETS)) TARGETS = tuple(filter(release_filter, TARGETS))
else:
TARGETS = select_build_targets(BUILD)
for target in TARGETS: for target in TARGETS:
run_kiwix_build(target, platform=PLATFORM_TARGET, make_release=MAKE_RELEASE) run_kiwix_build(target, platform=PLATFORM_TARGET, make_release=MAKE_RELEASE)
@ -40,16 +43,19 @@ for target in TARGETS:
archive = make_archive(target, make_release=MAKE_RELEASE) archive = make_archive(target, make_release=MAKE_RELEASE)
if archive: if archive:
upload_archive(archive, target, make_release=MAKE_RELEASE) upload_archive(archive, target, make_release=MAKE_RELEASE)
if MAKE_RELEASE and target in ("zim-tools", "kiwix-tools"):
trigger_docker_publish(target)
# We have few more things to do for release: # We have few more things to do for release:
if MAKE_RELEASE: if MAKE_RELEASE:
# Publish source archives # Publish source archives
if PLATFORM_TARGET in ("native_dyn", "native_mixed") and OS_NAME != "macos": source_published_targets = select_build_targets(SOURCE_PUBLISH)
for target in TARGETS: for target in TARGETS:
# Looping on TARGETS instead of source_published_targets ensures we are
# publishing sources only for target we are building.
# (source_published_targets must be a subset of TARGETS)
if release_versions.get(target) != 0: if release_versions.get(target) != 0:
continue continue
if target not in source_published_targets:
continue
run_kiwix_build( run_kiwix_build(
target, platform=PLATFORM_TARGET, make_release=MAKE_RELEASE, make_dist=True target, platform=PLATFORM_TARGET, make_release=MAKE_RELEASE, make_dist=True
) )

39
.github/scripts/common.py vendored
View File

@ -10,12 +10,16 @@ import shutil
import requests import requests
from build_definition import get_platform_name
from kiwixbuild.versions import ( from kiwixbuild.versions import (
main_project_versions, main_project_versions,
release_versions, release_versions,
base_deps_versions, base_deps_versions,
) )
PLATFORM_TARGET = _environ["PLATFORM_TARGET"] PLATFORM_TARGET = _environ["PLATFORM_TARGET"]
OS_NAME = _environ["OS_NAME"] OS_NAME = _environ["OS_NAME"]
HOME = Path(os.path.expanduser("~")) HOME = Path(os.path.expanduser("~"))
@ -36,31 +40,6 @@ if not MAKE_RELEASE and _ref != "main":
else: else:
DEV_BRANCH = None DEV_BRANCH = None
RELEASE_OS_NAME = "macos" if OS_NAME == "macos" else "linux"
EXTRA_NAME = "-bionic" if OS_NAME == "bionic" else ""
PLATFORM_TO_RELEASE = {
"native_mixed": "{os}-x86_64{extra}".format(os=RELEASE_OS_NAME, extra=EXTRA_NAME),
"native_static": "{os}-x86_64".format(os=RELEASE_OS_NAME),
"win32_static": "win-i686",
"armv6_static": "{os}-armv6".format(os=RELEASE_OS_NAME),
"armv6_mixed": "{os}-armv6".format(os=RELEASE_OS_NAME),
"armv8_static": "{os}-armv8".format(os=RELEASE_OS_NAME),
"armv8_mixed": "{os}-armv8".format(os=RELEASE_OS_NAME),
"aarch64_static": "{os}-aarch64".format(os=RELEASE_OS_NAME),
"aarch64_mixed": "{os}-aarch64{extra}".format(os=RELEASE_OS_NAME, extra=EXTRA_NAME),
"aarch64_musl_static": "{os}-aarch64-musl".format(os=RELEASE_OS_NAME),
"aarch64_musl_mixed": "{os}-aarch64-musl".format(os=RELEASE_OS_NAME),
"i586_static": "{os}-i586".format(os=RELEASE_OS_NAME),
"macOS_arm64_static": "{os}-arm64".format(os=RELEASE_OS_NAME),
"macOS_arm64_mixed": "{os}-arm64".format(os=RELEASE_OS_NAME),
"android_arm": "android-arm",
"android_arm64": "android-arm64",
"android_x86": "android-x86",
"android_x86_64": "android-x86_64",
"wasm": "wasm-emscripten",
}
FLATPAK_HTTP_GIT_REMOTE = "https://github.com/flathub/org.kiwix.desktop.git" FLATPAK_HTTP_GIT_REMOTE = "https://github.com/flathub/org.kiwix.desktop.git"
FLATPAK_GIT_REMOTE = "git@github.com:flathub/org.kiwix.desktop.git" FLATPAK_GIT_REMOTE = "git@github.com:flathub/org.kiwix.desktop.git"
@ -351,10 +330,8 @@ def get_postfix(project):
def make_archive(project, make_release): def make_archive(project, make_release):
try: platform_name = get_platform_name()
platform = PLATFORM_TO_RELEASE[PLATFORM_TARGET] if not platform_name:
except KeyError:
# We don't know how to name the release.
return None return None
try: try:
@ -368,12 +345,12 @@ def make_archive(project, make_release):
else: else:
postfix = DATE postfix = DATE
archive_name = "{}_{}-{}".format(project, platform, postfix) archive_name = "{}_{}-{}".format(project, platform_name, postfix)
files_to_archive = [] files_to_archive = []
for export_file in export_files: for export_file in export_files:
files_to_archive.extend(base_dir.glob(export_file)) files_to_archive.extend(base_dir.glob(export_file))
if platform == "win-i686": if platform_name == "win-i686":
open_archive = lambda a: zipfile.ZipFile( open_archive = lambda a: zipfile.ZipFile(
str(a), "w", compression=zipfile.ZIP_DEFLATED str(a), "w", compression=zipfile.ZIP_DEFLATED
) )

17
.github/scripts/trigger_docker_workflow.py vendored Executable file
View File

@ -0,0 +1,17 @@
#!/usr/bin/env python3
from common import (
trigger_docker_publish,
MAKE_RELEASE,
)
from build_definition import select_build_targets, DOCKER
# Filter what to build if we are doing a release.
if MAKE_RELEASE:
docker_trigger = select_build_targets(DOCKER)
else:
docker_trigger = []
for target in docker_trigger:
trigger_docker_publish(target)

19
.github/workflows/releaseNigthly.yml vendored
View File

@ -110,7 +110,6 @@ jobs:
PLATFORM_TARGET: ${{matrix.target}} PLATFORM_TARGET: ${{matrix.target}}
BINTRAY_USER: kiwix BINTRAY_USER: kiwix
BINTRAY_PASS: ${{secrets.bintray_pass}} BINTRAY_PASS: ${{secrets.bintray_pass}}
GITHUB_PAT: ${{secrets.GHCR_TOKEN}}
- name: Upload failure logs - name: Upload failure logs
if: failure() if: failure()
run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh
@ -236,3 +235,21 @@ jobs:
run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.sh run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.sh
env: env:
PLATFORM_TARGET: ${{matrix.target}} PLATFORM_TARGET: ${{matrix.target}}
Trigger_Docker:
needs: [Linux]
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Install python modules
shell: bash
run: |
pip3 install --user --no-deps $GITHUB_WORKSPACE
- name: Trigger docker workflow
shell: bash
run: |
cd $HOME
$GITHUB_WORKSPACE/.github/scripts/trigger_docker_workflow.py
env:
GITHUB_PAT: ${{secrets.DOCKER_TRIGGER_GITHUB_PAT}}