smallsolar
/
kiwix-build
mirror of https://github.com/kiwix/kiwix-build.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #612 from kiwix/better_build_def2

This commit is contained in:
Matthieu Gautier 2023-06-07 14:03:33 +02:00 committed by GitHub
parent 7f7156ece5 6fe37ba0de
commit 07d443d96b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 132 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
.github/scripts/build_definition.py vendored Normal file
View File

@ -0,0 +1,120 @@
from typing import NamedTuple
import csv, io, re
# Definition of what to build.
# Array is read line by line.
# Empty cells under (OS_NAME, PLATFORM_TARGET) mean "always match" (catch all, or `.*` regex)
# Once a cell doesn't match, skip to the next line.
# Once a line matches, other lines are not read, so put more specific combination first.
# Lines composed of `-` , or `=`, or starting by `#` are ignored.
# 'B' letter means that the project is build in the CI
# 'd' letter means that the project's dependencies are build and published to be used by the project's CI.
# If a cell contains both, both are done.
BUILD_DEF = """
| OS_NAME | PLATFORM_TARGET | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop |
==============================================================================================
# Bionic is a special case as we need to compile libzim on old arch for python
| bionic | | B | | | | |
----------------------------------------------------------------------------------------------
# Osx builds, build binaries on native_dyn and native_static. On anyother things, build only the libraries
| macos | native_dyn | d | d | dB | B | |
| macos | native_static | | | B | B | |
| macos | native_mixed | B | B | | | |
| macos | iOS_arm64 | dB | B | | | |
| macos | iOS_x86_64 | dB | B | | | |
| macos | iOS_Mac_ABI | B | B | | | |
| macos | macOS_arm64_static | B | B | | | |
| macos | macOS_arm64_mixed | B | B | | | |
| macos | macOS_x86_64 | B | B | | | |
----------------------------------------------------------------------------------------------
| | flatpak | | | | | B |
| | native_static | d | d | dB | dB | |
| | native_dyn | d | d | dB | dB | B |
| | native_mixed | B | B | | | |
# libzim CI is building alpine_dyn but not us
| | android_arm | dB | dB | | | |
| | android_arm64 | dB | dB | | | |
| | android_x86 | B | B | | | |
| | android_x86_64 | B | B | | | |
| | armv6_static | | | B | B | |
| | armv6_dyn | | | B | B | |
| | armv6_mixed | B | | | | |
| | armv8_static | | | B | B | |
| | armv8_dyn | | | B | B | |
| | armv8_mixed | B | | | | |
| | aarch64_static | | | B | B | |
| | aarch64_dyn | d | | B | B | |
| | aarch64_mixed | B | | | | |
| | win32_static | d | dB | dB | dB | |
| | win32_dyn | d | dB | dB | dB | |
| | i586_static | | | B | B | |
| | i586_dyn | | | B | B | |
| | wasm | dB | | | | |
"""
class TableDialect(csv.Dialect):
delimiter = "|"
quoting = csv.QUOTE_NONE
lineterminator = "\n"
def strip_array(array_str):
"""Return a iterable of lines, skiping "decorative lines" and with all values in the line's cells stripped"""
for line in array_str.splitlines():
line = line.strip()
line_set = set(line)
if (
not line
or line.startswith("#")
or (len(line_set) == 1 and line_set.pop() in "-=")
):
continue
yield "|".join(c.strip() for c in line.split("|"))
def selector_match(selector, value):
if not selector:
return True
return re.fullmatch(selector, value) is not None
class Context(NamedTuple):
OS_NAME: str
PLATFORM_TARGET: str
def match(self, row):
for key in ["OS_NAME", "PLATFORM_TARGET"]:
context_value = getattr(self, key)
selector = row[key]
if not selector_match(selector, context_value):
return False
return True
BUILD = "B"
DEPS = "d"
def select_build_targets(criteria):
from common import PLATFORM_TARGET, OS_NAME
context = Context(PLATFORM_TARGET=PLATFORM_TARGET, OS_NAME=OS_NAME)
reader = csv.DictReader(strip_array(BUILD_DEF), dialect=TableDialect())
for row in reader:
if context.match(row):
build_order = [
k
for k in (
"libzim",
"libkiwix",
"zim-tools",
"kiwix-tools",
"kiwix-desktop",
)
if criteria in row[k]
]
print(build_order)
return build_order
raise "No definition match with current context."

49
.github/scripts/build_projects.py vendored
View File

@ -1,6 +1,6 @@
#!/usr/bin/env python3
from build_definition import select_build_targets, BUILD
from common import (
run_kiwix_build,
make_archive,
@ -12,55 +12,12 @@ from common import (
DEV_BRANCH,
)
def select_build_target():
from common import (
PLATFORM_TARGET,
DESKTOP,
OS_NAME
)
if OS_NAME == "bionic" and PLATFORM_TARGET.endswith("_mixed"):
return ("libzim", )
elif (PLATFORM_TARGET.startswith("android_")
or PLATFORM_TARGET.startswith("iOS")):
return ("libzim", "libkiwix")
elif PLATFORM_TARGET.startswith("macOS"):
if PLATFORM_TARGET.endswith("_mixed"):
return ("libzim", "libkiwix")
elif PLATFORM_TARGET.endswith("_dyn"):
return ("zim-tools", "kiwix-tools")
else:
return []
elif PLATFORM_TARGET.startswith("native_"):
if OS_NAME == "osx":
if PLATFORM_TARGET.endswith("_mixed"):
return ("libzim", "libkiwix")
else:
return ("zim-tools", "kiwix-tools")
else:
if DESKTOP:
return ("kiwix-desktop",)
elif PLATFORM_TARGET == "native_mixed":
return ("libzim", "libkiwix")
else:
return ("zim-tools", "kiwix-tools")
elif PLATFORM_TARGET in ("win32_static", "armv6_static", "armv6_dyn", "armv8_static", "armv8_dyn", "aarch64_static", "aarch64_dyn", "i586_static"):
return ("zim-tools", "kiwix-tools")
elif PLATFORM_TARGET == "flatpak":
return ("kiwix-desktop",)
elif PLATFORM_TARGET in ("wasm", "armv6_mixed", "armv8_mixed", "aarch64_mixed"):
return ("libzim", )
else:
return ("libzim", "zim-tools", "libkiwix", "kiwix-tools")
TARGETS = select_build_target()
for target in TARGETS:
for target in select_build_targets(BUILD):
run_kiwix_build(target, platform=PLATFORM_TARGET)
if target == "kiwix-desktop":
archive = create_desktop_image(make_release=False)
else:
if PLATFORM_TARGET == "native_mixed" and OS_NAME == "osx":
if PLATFORM_TARGET == "native_mixed" and OS_NAME == "macos":
fix_macos_rpath(target)
archive = make_archive(target, make_release=False)
if archive and DEV_BRANCH:

10
.github/scripts/build_release_nightly.py vendored
View File

@ -13,17 +13,15 @@ from common import (
fix_macos_rpath,
trigger_docker_publish,
BASE_DIR,
TMP_DIR,
HOME,
OS_NAME,
PLATFORM_TARGET,
MAKE_RELEASE,
notarize_macos_build,
)
from build_projects import select_build_target
from build_definition import select_build_targets, BUILD
TARGETS = select_build_target()
TARGETS = select_build_target(BUILD)
# Filter what to build if we are doing a release.
if MAKE_RELEASE:
@ -36,7 +34,7 @@ for target in TARGETS:
if target == "kiwix-desktop":
archive = create_desktop_image(make_release=MAKE_RELEASE)
else:
if OS_NAME == "osx" and PLATFORM_TARGET.endswith("_mixed"):
if OS_NAME == "macos" and PLATFORM_TARGET.endswith("_mixed"):
fix_macos_rpath(target)
notarize_macos_build(target)
archive = make_archive(target, make_release=MAKE_RELEASE)
@ -48,7 +46,7 @@ for target in TARGETS:
# We have few more things to do for release:
if MAKE_RELEASE:
# Publish source archives
if PLATFORM_TARGET in ("native_dyn", "native_mixed") and OS_NAME != "osx":
if PLATFORM_TARGET in ("native_dyn", "native_mixed") and OS_NAME != "macos":
for target in TARGETS:
if release_versions.get(target) != 0:
continue

10
.github/scripts/common.py vendored
View File

@ -17,11 +17,6 @@ from kiwixbuild.versions import (
)
PLATFORM_TARGET = _environ["PLATFORM_TARGET"]
if PLATFORM_TARGET == "native_desktop":
PLATFORM_TARGET = "native_dyn"
DESKTOP = True
else:
DESKTOP = False
OS_NAME = _environ["OS_NAME"]
HOME = Path(os.path.expanduser("~"))
@ -32,9 +27,6 @@ INSTALL_DIR = BASE_DIR / "INSTALL"
TMP_DIR = Path("/tmp")
KBUILD_SOURCE_DIR = HOME / "kiwix-build"
# [TODO]
KIWIX_DESKTOP_ONLY = False
_ref = _environ.get("GITHUB_REF", "").split("/")[-1]
MAKE_RELEASE = re.fullmatch(r"r_[0-9]+", _ref) is not None
MAKE_RELEASE = MAKE_RELEASE and (_environ.get('GITHUB_EVENT_NAME') != 'schedule')
@ -44,7 +36,7 @@ if not MAKE_RELEASE and _ref != "main":
else:
DEV_BRANCH = None
RELEASE_OS_NAME = "macos" if OS_NAME == "osx" else "linux"
RELEASE_OS_NAME = "macos" if OS_NAME == "macos" else "linux"
EXTRA_NAME = "-bionic" if OS_NAME == "bionic" else ""
PLATFORM_TO_RELEASE = {

22
.github/scripts/compile_all_deps.py vendored
View File

@ -8,28 +8,10 @@ from common import (
upload,
OS_NAME,
PLATFORM_TARGET,
DESKTOP,
KIWIX_DESKTOP_ONLY,
)
from build_definition import select_build_targets, DEPS
if PLATFORM_TARGET.startswith("android_") or PLATFORM_TARGET.startswith("iOS"):
TARGETS = ("libzim", "libkiwix")
elif PLATFORM_TARGET == "wasm":
TARGETS = ("libzim", )
elif PLATFORM_TARGET.startswith("native_"):
if OS_NAME == "osx":
TARGETS = ("libzim", "zim-tools", "libkiwix")
else:
if DESKTOP:
TARGETS = ("kiwix-desktop",)
elif PLATFORM_TARGET == "native_mixed":
TARGETS = ("libzim",)
else:
TARGETS = ("libzim", "zim-tools", "libkiwix", "kiwix-tools")
else:
TARGETS = ("libzim", "zim-tools", "libkiwix", "kiwix-tools")
for target in TARGETS:
for target in select_build_targets(DEPS):
run_kiwix_build(target, platform=PLATFORM_TARGET, build_deps_only=True)
archive_file = make_deps_archive(target=target)
upload(archive_file, "ci@tmp.kiwix.org:30022", "/data/tmp/ci")

3
.github/workflows/ci.yml vendored
View File

@ -14,7 +14,6 @@ jobs:
- native_static
- native_dyn
- native_mixed
- native_desktop
- wasm
- armv6_static
- armv6_dyn
@ -151,7 +150,7 @@ jobs:
runs-on: macos-11
env:
SSH_KEY: /tmp/id_rsa
OS_NAME: osx
OS_NAME: macos
steps:
- name: Checkout code
uses: actions/checkout@v3

5
.github/workflows/releaseNigthly.yml vendored
View File

@ -16,7 +16,6 @@ jobs:
- native_static
- native_dyn
- native_mixed
- native_desktop
- wasm
- armv6_static
- armv6_mixed
@ -39,8 +38,6 @@ jobs:
image_variant: focal
- target: native_mixed
image_variant: bionic
- target: native_desktop
image_variant: focal
- target: wasm
image_variant: focal
- target: armv6_static
@ -171,7 +168,7 @@ jobs:
runs-on: macos-11
env:
SSH_KEY: /tmp/id_rsa
OS_NAME: osx
OS_NAME: macos
CERTIFICATE: /tmp/wmch-devid.p12
SIGNING_IDENTITY: ${{ secrets.APPLE_SIGNING_IDENTITY }}
KEYCHAIN: /Users/runner/build.keychain-db