From fd46c52473005bbcd5aff08d20089dfcc0944ff5 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 11 May 2023 16:12:53 +0200 Subject: [PATCH 1/8] Move the definition of what to build in a table. This provide a better visual definition (and potential better explicit) of the previous code base on `if/else`. --- .github/scripts/build_definition.py | 105 +++++++++++++++++++++++ .github/scripts/build_projects.py | 47 +--------- .github/scripts/build_release_nightly.py | 2 +- 3 files changed, 108 insertions(+), 46 deletions(-) create mode 100644 .github/scripts/build_definition.py diff --git a/.github/scripts/build_definition.py b/.github/scripts/build_definition.py new file mode 100644 index 0000000..4cfe1ac --- /dev/null +++ b/.github/scripts/build_definition.py @@ -0,0 +1,105 @@ +from typing import NamedTuple +import csv, io, re + +# Definition of what to build. +# Array is read line by line. +# Empty cells under (OS_NAME, DESKTOP, PLATFORM_TARGET) mean "always match" (catch all, or `.*` regex) +# Once a cell doesn't match, skip to the next line. +# Once a line matches, other lines are not read, so put more specific combination first. +# Lines composed of `-` , or `=`, or starting by `#` are ignored. +BUILD_DEF = """ + | OS_NAME | DESKTOP | PLATFORM_TARGET | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | + ======================================================================================================= +# Bionic is a special case as we need to compile libzim on old arch for python + | bionic | | | x | | | | | + ------------------------------------------------------------------------------------------------------- +# Osx builds, build binaries on native_dyn and native_static. On any other things, build only the libraries + | osx | | native_dyn | | | x | x | | + | osx | | native_static | | | x | x | | + | osx | | | x | x | | | | + ------------------------------------------------------------------------------------------------------- +# Build kiwix-desktop only on specific targets + | | eval'True | | | | | | x | + | | | flatpak | | | | | x | + ------------------------------------------------------------------------------------------------------- +# Library builds, on embedded archs or on all *_mixed targets + | | | android_.* | x | x | | | | + | | | native_mixed | x | x | | | | + | | | .*_mixed | x | | | | | + | | | wasm | x | | | | | +# Build binaries on *_static targets or on all others "non mixed" targets (where we have already build libs) + | | | native_.* | | | x | x | | + | | | .*_static | | | x | x | | + | | | armv[68]_.* | | | x | x | | + | | | aarch64_.* | | | x | x | | +# Else, let's build everything. + | | | | x | x | x | x | | +""" + + +class TableDialect(csv.Dialect): + delimiter = "|" + quoting = csv.QUOTE_NONE + lineterminator = "\n" + + +def strip_array(array_str): + """Return a iterable of lines, skiping "decorative lines" and with all values in the line's cells stripped""" + for line in array_str.splitlines(): + line = line.strip() + line_set = set(line) + if ( + not line + or line.startswith("#") + or (len(line_set) == 1 and line_set.pop() in "-=") + ): + continue + yield "|".join(c.strip() for c in line.split("|")) + + +def selector_match(selector, value): + if not selector: + return True + if selector.startswith("eval'"): + selector = eval(selector[5:]) + return selector == value + return re.fullmatch(selector, value) is not None + + +class Context(NamedTuple): + OS_NAME: str + DESKTOP: bool + PLATFORM_TARGET: str + + def match(self, row): + for key in ["OS_NAME", "DESKTOP", "PLATFORM_TARGET"]: + context_value = getattr(self, key) + selector = row[key] + if not selector_match(selector, context_value): + return False + return True + + +def select_build_targets(): + from common import PLATFORM_TARGET, DESKTOP, OS_NAME + + context = Context(PLATFORM_TARGET=PLATFORM_TARGET, DESKTOP=DESKTOP, OS_NAME=OS_NAME) + + reader = csv.DictReader(strip_array(BUILD_DEF), dialect=TableDialect()) + for row in reader: + if context.match(row): + build_order = [ + k + for k in ( + "libzim", + "libkiwix", + "zim-tools", + "kiwix-tools", + "kiwix-desktop", + ) + if row[k] == "x" + ] + print(build_order) + return build_order + + raise "No definition match with current context." diff --git a/.github/scripts/build_projects.py b/.github/scripts/build_projects.py index 7b837c7..9795b03 100755 --- a/.github/scripts/build_projects.py +++ b/.github/scripts/build_projects.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 - +from build_definition import select_build_targets from common import ( run_kiwix_build, make_archive, @@ -12,50 +12,7 @@ from common import ( DEV_BRANCH, ) - -def select_build_target(): - from common import ( - PLATFORM_TARGET, - DESKTOP, - OS_NAME - ) - if OS_NAME == "bionic" and PLATFORM_TARGET.endswith("_mixed"): - return ("libzim", ) - elif (PLATFORM_TARGET.startswith("android_") - or PLATFORM_TARGET.startswith("iOS")): - return ("libzim", "libkiwix") - elif PLATFORM_TARGET.startswith("macOS"): - if PLATFORM_TARGET.endswith("_mixed"): - return ("libzim", "libkiwix") - elif PLATFORM_TARGET.endswith("_dyn"): - return ("zim-tools", "kiwix-tools") - else: - return [] - elif PLATFORM_TARGET.startswith("native_"): - if OS_NAME == "osx": - if PLATFORM_TARGET.endswith("_mixed"): - return ("libzim", "libkiwix") - else: - return ("zim-tools", "kiwix-tools") - else: - if DESKTOP: - return ("kiwix-desktop",) - elif PLATFORM_TARGET == "native_mixed": - return ("libzim", "libkiwix") - else: - return ("zim-tools", "kiwix-tools") - elif PLATFORM_TARGET in ("win32_static", "armv6_static", "armv6_dyn", "armv8_static", "armv8_dyn", "aarch64_static", "aarch64_dyn", "i586_static"): - return ("zim-tools", "kiwix-tools") - elif PLATFORM_TARGET == "flatpak": - return ("kiwix-desktop",) - elif PLATFORM_TARGET in ("wasm", "armv6_mixed", "armv8_mixed", "aarch64_mixed"): - return ("libzim", ) - else: - return ("libzim", "zim-tools", "libkiwix", "kiwix-tools") - -TARGETS = select_build_target() - -for target in TARGETS: +for target in select_build_targets(): run_kiwix_build(target, platform=PLATFORM_TARGET) if target == "kiwix-desktop": archive = create_desktop_image(make_release=False) diff --git a/.github/scripts/build_release_nightly.py b/.github/scripts/build_release_nightly.py index e7e176e..f39ce95 100755 --- a/.github/scripts/build_release_nightly.py +++ b/.github/scripts/build_release_nightly.py @@ -21,7 +21,7 @@ from common import ( notarize_macos_build, ) -from build_projects import select_build_target +from build_definition import select_build_targets TARGETS = select_build_target() From fc81551555ac102001c0d14a15dff9427b460073 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Fri, 12 May 2023 17:00:53 +0200 Subject: [PATCH 2/8] Remove unused import. --- .github/scripts/build_release_nightly.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/scripts/build_release_nightly.py b/.github/scripts/build_release_nightly.py index f39ce95..ee6310e 100755 --- a/.github/scripts/build_release_nightly.py +++ b/.github/scripts/build_release_nightly.py @@ -13,8 +13,6 @@ from common import ( fix_macos_rpath, trigger_docker_publish, BASE_DIR, - TMP_DIR, - HOME, OS_NAME, PLATFORM_TARGET, MAKE_RELEASE, From e59e3698b247a4e7abf2a7ca491fc8ac0f4cc26c Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 11 May 2023 16:17:21 +0200 Subject: [PATCH 3/8] Open build definition system to more complex build definition. --- .github/scripts/build_definition.py | 38 +++++++++++++----------- .github/scripts/build_projects.py | 4 +-- .github/scripts/build_release_nightly.py | 4 +-- 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/.github/scripts/build_definition.py b/.github/scripts/build_definition.py index 4cfe1ac..34107db 100644 --- a/.github/scripts/build_definition.py +++ b/.github/scripts/build_definition.py @@ -11,29 +11,29 @@ BUILD_DEF = """ | OS_NAME | DESKTOP | PLATFORM_TARGET | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | ======================================================================================================= # Bionic is a special case as we need to compile libzim on old arch for python - | bionic | | | x | | | | | + | bionic | | | b | | | | | ------------------------------------------------------------------------------------------------------- # Osx builds, build binaries on native_dyn and native_static. On any other things, build only the libraries - | osx | | native_dyn | | | x | x | | - | osx | | native_static | | | x | x | | - | osx | | | x | x | | | | + | osx | | native_dyn | | | b | b | | + | osx | | native_static | | | b | b | | + | osx | | | b | b | | | | ------------------------------------------------------------------------------------------------------- # Build kiwix-desktop only on specific targets - | | eval'True | | | | | | x | - | | | flatpak | | | | | x | + | | eval'True | | | | | | b | + | | | flatpak | | | | | b | ------------------------------------------------------------------------------------------------------- # Library builds, on embedded archs or on all *_mixed targets - | | | android_.* | x | x | | | | - | | | native_mixed | x | x | | | | - | | | .*_mixed | x | | | | | - | | | wasm | x | | | | | + | | | android_.* | b | b | | | | + | | | native_mixed | b | b | | | | + | | | .*_mixed | b | | | | | + | | | wasm | b | | | | | # Build binaries on *_static targets or on all others "non mixed" targets (where we have already build libs) - | | | native_.* | | | x | x | | - | | | .*_static | | | x | x | | - | | | armv[68]_.* | | | x | x | | - | | | aarch64_.* | | | x | x | | + | | | native_.* | | | b | b | | + | | | .*_static | | | b | b | | + | | | armv[68]_.* | | | b | b | | + | | | aarch64_.* | | | b | b | | # Else, let's build everything. - | | | | x | x | x | x | | + | | | | b | b | b | b | | """ @@ -80,7 +80,11 @@ class Context(NamedTuple): return True -def select_build_targets(): +BUILD = "b" +DEPS = "d" + + +def select_build_targets(criteria): from common import PLATFORM_TARGET, DESKTOP, OS_NAME context = Context(PLATFORM_TARGET=PLATFORM_TARGET, DESKTOP=DESKTOP, OS_NAME=OS_NAME) @@ -97,7 +101,7 @@ def select_build_targets(): "kiwix-tools", "kiwix-desktop", ) - if row[k] == "x" + if criteria in row[k] ] print(build_order) return build_order diff --git a/.github/scripts/build_projects.py b/.github/scripts/build_projects.py index 9795b03..5e5b659 100755 --- a/.github/scripts/build_projects.py +++ b/.github/scripts/build_projects.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -from build_definition import select_build_targets +from build_definition import select_build_targets, BUILD from common import ( run_kiwix_build, make_archive, @@ -12,7 +12,7 @@ from common import ( DEV_BRANCH, ) -for target in select_build_targets(): +for target in select_build_targets(BUILD): run_kiwix_build(target, platform=PLATFORM_TARGET) if target == "kiwix-desktop": archive = create_desktop_image(make_release=False) diff --git a/.github/scripts/build_release_nightly.py b/.github/scripts/build_release_nightly.py index ee6310e..a35c2ab 100755 --- a/.github/scripts/build_release_nightly.py +++ b/.github/scripts/build_release_nightly.py @@ -19,9 +19,9 @@ from common import ( notarize_macos_build, ) -from build_definition import select_build_targets +from build_definition import select_build_targets, BUILD -TARGETS = select_build_target() +TARGETS = select_build_target(BUILD) # Filter what to build if we are doing a release. if MAKE_RELEASE: From 3defd4eba7357747b8930deffc1b58e631c470f7 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 11 May 2023 16:38:09 +0200 Subject: [PATCH 4/8] Be explicit about all our combinations. --- .github/scripts/build_definition.py | 47 +++++++++++++++++++---------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/.github/scripts/build_definition.py b/.github/scripts/build_definition.py index 34107db..a4fbeee 100644 --- a/.github/scripts/build_definition.py +++ b/.github/scripts/build_definition.py @@ -8,32 +8,47 @@ import csv, io, re # Once a line matches, other lines are not read, so put more specific combination first. # Lines composed of `-` , or `=`, or starting by `#` are ignored. BUILD_DEF = """ - | OS_NAME | DESKTOP | PLATFORM_TARGET | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | + | OS_NAME | DESKTOP | PLATFORM_TARGET | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | ======================================================================================================= # Bionic is a special case as we need to compile libzim on old arch for python - | bionic | | | b | | | | | + | bionic | | | b | | | | | ------------------------------------------------------------------------------------------------------- -# Osx builds, build binaries on native_dyn and native_static. On any other things, build only the libraries +# Osx builds, build binaries on native_dyn and native_static. On anyother things, build only the libraries | osx | | native_dyn | | | b | b | | | osx | | native_static | | | b | b | | - | osx | | | b | b | | | | + | osx | | native_mixed | b | b | | | | + | osx | | iOS_arm64 | b | b | | | | + | osx | | iOS_x86_64 | b | b | | | | + | osx | | iOS_Mac_ABI | b | b | | | | + | osx | | macOS_arm64_static | b | b | | | | + | osx | | macOS_arm64_mixed | b | b | | | | + | osx | | macOS_x86_64 | b | b | | | | ------------------------------------------------------------------------------------------------------- # Build kiwix-desktop only on specific targets - | | eval'True | | | | | | b | - | | | flatpak | | | | | b | + | | eval'True | | | | | | b | + | | | flatpak | | | | | b | ------------------------------------------------------------------------------------------------------- -# Library builds, on embedded archs or on all *_mixed targets - | | | android_.* | b | b | | | | + | | | native_static | | | b | b | | + | | | native_dyn | | | b | b | | | | | native_mixed | b | b | | | | - | | | .*_mixed | b | | | | | + | | | android_arm | b | b | | | | + | | | android_arm64 | b | b | | | | + | | | android_x86 | b | b | | | | + | | | android_x86_64 | b | b | | | | + | | | armv6_static | | | b | b | | + | | | armv6_dyn | | | b | b | | + | | | armv6_mixed | b | | | | | + | | | armv8_static | | | b | b | | + | | | armv8_dyn | | | b | b | | + | | | armv8_mixed | b | | | | | + | | | aarch64_static | | | b | b | | + | | | aarch64_dyn | | | b | b | | + | | | aarch64_mixed | b | | | | | + | | | win32_static | | | b | b | | + | | | win32_dyn | | | b | b | | + | | | i586_static | | | b | b | | + | | | i586_dyn | | | b | b | | | | | wasm | b | | | | | -# Build binaries on *_static targets or on all others "non mixed" targets (where we have already build libs) - | | | native_.* | | | b | b | | - | | | .*_static | | | b | b | | - | | | armv[68]_.* | | | b | b | | - | | | aarch64_.* | | | b | b | | -# Else, let's build everything. - | | | | b | b | b | b | | """ From 7db434ee6dd53b53f6a3ef463b26367d30b2d054 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 11 May 2023 16:44:35 +0200 Subject: [PATCH 5/8] Use the common build definition to specify which deps build. The build definition update follow what we build in the project's CI, not what was defined by the replaced python code. --- .github/scripts/build_definition.py | 72 +++++++++++++++-------------- .github/scripts/compile_all_deps.py | 20 +------- 2 files changed, 40 insertions(+), 52 deletions(-) diff --git a/.github/scripts/build_definition.py b/.github/scripts/build_definition.py index a4fbeee..3c01d7f 100644 --- a/.github/scripts/build_definition.py +++ b/.github/scripts/build_definition.py @@ -7,48 +7,52 @@ import csv, io, re # Once a cell doesn't match, skip to the next line. # Once a line matches, other lines are not read, so put more specific combination first. # Lines composed of `-` , or `=`, or starting by `#` are ignored. +# 'B' letter means that the project is build in the CI +# 'd' letter means that the project's dependencies are build and published to be used by the project's CI. +# If a cell contains both, both are done. BUILD_DEF = """ | OS_NAME | DESKTOP | PLATFORM_TARGET | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | ======================================================================================================= # Bionic is a special case as we need to compile libzim on old arch for python - | bionic | | | b | | | | | + | bionic | | | B | | | | | ------------------------------------------------------------------------------------------------------- # Osx builds, build binaries on native_dyn and native_static. On anyother things, build only the libraries - | osx | | native_dyn | | | b | b | | - | osx | | native_static | | | b | b | | - | osx | | native_mixed | b | b | | | | - | osx | | iOS_arm64 | b | b | | | | - | osx | | iOS_x86_64 | b | b | | | | - | osx | | iOS_Mac_ABI | b | b | | | | - | osx | | macOS_arm64_static | b | b | | | | - | osx | | macOS_arm64_mixed | b | b | | | | - | osx | | macOS_x86_64 | b | b | | | | + | osx | | native_dyn | d | d | dB | B | | + | osx | | native_static | | | B | B | | + | osx | | native_mixed | B | B | | | | + | osx | | iOS_arm64 | dB | B | | | | + | osx | | iOS_x86_64 | dB | B | | | | + | osx | | iOS_Mac_ABI | B | B | | | | + | osx | | macOS_arm64_static | B | B | | | | + | osx | | macOS_arm64_mixed | B | B | | | | + | osx | | macOS_x86_64 | B | B | | | | ------------------------------------------------------------------------------------------------------- # Build kiwix-desktop only on specific targets - | | eval'True | | | | | | b | - | | | flatpak | | | | | b | + | | eval'True | | | | | | B | + | | | flatpak | | | | | B | ------------------------------------------------------------------------------------------------------- - | | | native_static | | | b | b | | - | | | native_dyn | | | b | b | | - | | | native_mixed | b | b | | | | - | | | android_arm | b | b | | | | - | | | android_arm64 | b | b | | | | - | | | android_x86 | b | b | | | | - | | | android_x86_64 | b | b | | | | - | | | armv6_static | | | b | b | | - | | | armv6_dyn | | | b | b | | - | | | armv6_mixed | b | | | | | - | | | armv8_static | | | b | b | | - | | | armv8_dyn | | | b | b | | - | | | armv8_mixed | b | | | | | - | | | aarch64_static | | | b | b | | - | | | aarch64_dyn | | | b | b | | - | | | aarch64_mixed | b | | | | | - | | | win32_static | | | b | b | | - | | | win32_dyn | | | b | b | | - | | | i586_static | | | b | b | | - | | | i586_dyn | | | b | b | | - | | | wasm | b | | | | | + | | | native_static | d | d | dB | dB | | + | | | native_dyn | d | d | dB | dB | | + | | | native_mixed | B | B | | | | +# libzim CI is building alpine_dyn but not us + | | | android_arm | dB | dB | | | | + | | | android_arm64 | dB | dB | | | | + | | | android_x86 | B | B | | | | + | | | android_x86_64 | B | B | | | | + | | | armv6_static | | | B | B | | + | | | armv6_dyn | | | B | B | | + | | | armv6_mixed | B | | | | | + | | | armv8_static | | | B | B | | + | | | armv8_dyn | | | B | B | | + | | | armv8_mixed | B | | | | | + | | | aarch64_static | | | B | B | | + | | | aarch64_dyn | d | | B | B | | + | | | aarch64_mixed | B | | | | | + | | | win32_static | d | dB | dB | dB | | + | | | win32_dyn | d | dB | dB | dB | | + | | | i586_static | | | B | B | | + | | | i586_dyn | | | B | B | | + | | | wasm | dB | | | | | """ @@ -95,7 +99,7 @@ class Context(NamedTuple): return True -BUILD = "b" +BUILD = "B" DEPS = "d" diff --git a/.github/scripts/compile_all_deps.py b/.github/scripts/compile_all_deps.py index c690af1..ac70bbe 100755 --- a/.github/scripts/compile_all_deps.py +++ b/.github/scripts/compile_all_deps.py @@ -11,25 +11,9 @@ from common import ( DESKTOP, KIWIX_DESKTOP_ONLY, ) +from build_definition import select_build_targets, DEPS -if PLATFORM_TARGET.startswith("android_") or PLATFORM_TARGET.startswith("iOS"): - TARGETS = ("libzim", "libkiwix") -elif PLATFORM_TARGET == "wasm": - TARGETS = ("libzim", ) -elif PLATFORM_TARGET.startswith("native_"): - if OS_NAME == "osx": - TARGETS = ("libzim", "zim-tools", "libkiwix") - else: - if DESKTOP: - TARGETS = ("kiwix-desktop",) - elif PLATFORM_TARGET == "native_mixed": - TARGETS = ("libzim",) - else: - TARGETS = ("libzim", "zim-tools", "libkiwix", "kiwix-tools") -else: - TARGETS = ("libzim", "zim-tools", "libkiwix", "kiwix-tools") - -for target in TARGETS: +for target in select_build_targets(DEPS): run_kiwix_build(target, platform=PLATFORM_TARGET, build_deps_only=True) archive_file = make_deps_archive(target=target) upload(archive_file, "ci@tmp.kiwix.org:30022", "/data/tmp/ci") From 8237adf9504adad31949ecf9ebfd5f0aaf221b5e Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Fri, 12 May 2023 10:30:42 +0200 Subject: [PATCH 6/8] Do not use a specific target `native_desktop`. `native_desktop` is not a real target_platform. It is workaround to specify "build only kiwix-desktop on native_dyn". It was useful, when we need to build kiwix-desktop on a different build env than the other project (because of packaging constraints). Now we can build kiwix-desktop and all other projects on the same build env, we can remove it and build kiwix-desktop with other `native_dyn`. --- .github/scripts/build_definition.py | 2 +- .github/workflows/ci.yml | 1 - .github/workflows/releaseNigthly.yml | 3 --- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/scripts/build_definition.py b/.github/scripts/build_definition.py index 3c01d7f..9ab92cd 100644 --- a/.github/scripts/build_definition.py +++ b/.github/scripts/build_definition.py @@ -32,7 +32,7 @@ BUILD_DEF = """ | | | flatpak | | | | | B | ------------------------------------------------------------------------------------------------------- | | | native_static | d | d | dB | dB | | - | | | native_dyn | d | d | dB | dB | | + | | | native_dyn | d | d | dB | dB | B | | | | native_mixed | B | B | | | | # libzim CI is building alpine_dyn but not us | | | android_arm | dB | dB | | | | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 084b599..43b2edf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,6 @@ jobs: - native_static - native_dyn - native_mixed - - native_desktop - wasm - armv6_static - armv6_dyn diff --git a/.github/workflows/releaseNigthly.yml b/.github/workflows/releaseNigthly.yml index db07360..46398ff 100644 --- a/.github/workflows/releaseNigthly.yml +++ b/.github/workflows/releaseNigthly.yml @@ -16,7 +16,6 @@ jobs: - native_static - native_dyn - native_mixed - - native_desktop - wasm - armv6_static - armv6_mixed @@ -39,8 +38,6 @@ jobs: image_variant: focal - target: native_mixed image_variant: bionic - - target: native_desktop - image_variant: focal - target: wasm image_variant: focal - target: armv6_static From 7b6ed275ed5ec41596cd70495ea1ce7a514b8386 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Fri, 12 May 2023 10:36:49 +0200 Subject: [PATCH 7/8] Remove code for kiwix-desktop workaround. As we don't use the workaround now, we can remove the code associated to it. It simplify our build definition. --- .github/scripts/build_definition.py | 88 +++++++++++++---------------- .github/scripts/common.py | 8 --- .github/scripts/compile_all_deps.py | 2 - 3 files changed, 40 insertions(+), 58 deletions(-) diff --git a/.github/scripts/build_definition.py b/.github/scripts/build_definition.py index 9ab92cd..82901f0 100644 --- a/.github/scripts/build_definition.py +++ b/.github/scripts/build_definition.py @@ -3,7 +3,7 @@ import csv, io, re # Definition of what to build. # Array is read line by line. -# Empty cells under (OS_NAME, DESKTOP, PLATFORM_TARGET) mean "always match" (catch all, or `.*` regex) +# Empty cells under (OS_NAME, PLATFORM_TARGET) mean "always match" (catch all, or `.*` regex) # Once a cell doesn't match, skip to the next line. # Once a line matches, other lines are not read, so put more specific combination first. # Lines composed of `-` , or `=`, or starting by `#` are ignored. @@ -11,48 +11,45 @@ import csv, io, re # 'd' letter means that the project's dependencies are build and published to be used by the project's CI. # If a cell contains both, both are done. BUILD_DEF = """ - | OS_NAME | DESKTOP | PLATFORM_TARGET | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | - ======================================================================================================= + | OS_NAME | PLATFORM_TARGET | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | + ============================================================================================== # Bionic is a special case as we need to compile libzim on old arch for python - | bionic | | | B | | | | | - ------------------------------------------------------------------------------------------------------- + | bionic | | B | | | | | + ---------------------------------------------------------------------------------------------- # Osx builds, build binaries on native_dyn and native_static. On anyother things, build only the libraries - | osx | | native_dyn | d | d | dB | B | | - | osx | | native_static | | | B | B | | - | osx | | native_mixed | B | B | | | | - | osx | | iOS_arm64 | dB | B | | | | - | osx | | iOS_x86_64 | dB | B | | | | - | osx | | iOS_Mac_ABI | B | B | | | | - | osx | | macOS_arm64_static | B | B | | | | - | osx | | macOS_arm64_mixed | B | B | | | | - | osx | | macOS_x86_64 | B | B | | | | - ------------------------------------------------------------------------------------------------------- -# Build kiwix-desktop only on specific targets - | | eval'True | | | | | | B | - | | | flatpak | | | | | B | - ------------------------------------------------------------------------------------------------------- - | | | native_static | d | d | dB | dB | | - | | | native_dyn | d | d | dB | dB | B | - | | | native_mixed | B | B | | | | + | osx | native_dyn | d | d | dB | B | | + | osx | native_static | | | B | B | | + | osx | native_mixed | B | B | | | | + | osx | iOS_arm64 | dB | B | | | | + | osx | iOS_x86_64 | dB | B | | | | + | osx | iOS_Mac_ABI | B | B | | | | + | osx | macOS_arm64_static | B | B | | | | + | osx | macOS_arm64_mixed | B | B | | | | + | osx | macOS_x86_64 | B | B | | | | + ---------------------------------------------------------------------------------------------- + | | flatpak | | | | | B | + | | native_static | d | d | dB | dB | | + | | native_dyn | d | d | dB | dB | B | + | | native_mixed | B | B | | | | # libzim CI is building alpine_dyn but not us - | | | android_arm | dB | dB | | | | - | | | android_arm64 | dB | dB | | | | - | | | android_x86 | B | B | | | | - | | | android_x86_64 | B | B | | | | - | | | armv6_static | | | B | B | | - | | | armv6_dyn | | | B | B | | - | | | armv6_mixed | B | | | | | - | | | armv8_static | | | B | B | | - | | | armv8_dyn | | | B | B | | - | | | armv8_mixed | B | | | | | - | | | aarch64_static | | | B | B | | - | | | aarch64_dyn | d | | B | B | | - | | | aarch64_mixed | B | | | | | - | | | win32_static | d | dB | dB | dB | | - | | | win32_dyn | d | dB | dB | dB | | - | | | i586_static | | | B | B | | - | | | i586_dyn | | | B | B | | - | | | wasm | dB | | | | | + | | android_arm | dB | dB | | | | + | | android_arm64 | dB | dB | | | | + | | android_x86 | B | B | | | | + | | android_x86_64 | B | B | | | | + | | armv6_static | | | B | B | | + | | armv6_dyn | | | B | B | | + | | armv6_mixed | B | | | | | + | | armv8_static | | | B | B | | + | | armv8_dyn | | | B | B | | + | | armv8_mixed | B | | | | | + | | aarch64_static | | | B | B | | + | | aarch64_dyn | d | | B | B | | + | | aarch64_mixed | B | | | | | + | | win32_static | d | dB | dB | dB | | + | | win32_dyn | d | dB | dB | dB | | + | | i586_static | | | B | B | | + | | i586_dyn | | | B | B | | + | | wasm | dB | | | | | """ @@ -79,19 +76,15 @@ def strip_array(array_str): def selector_match(selector, value): if not selector: return True - if selector.startswith("eval'"): - selector = eval(selector[5:]) - return selector == value return re.fullmatch(selector, value) is not None class Context(NamedTuple): OS_NAME: str - DESKTOP: bool PLATFORM_TARGET: str def match(self, row): - for key in ["OS_NAME", "DESKTOP", "PLATFORM_TARGET"]: + for key in ["OS_NAME", "PLATFORM_TARGET"]: context_value = getattr(self, key) selector = row[key] if not selector_match(selector, context_value): @@ -102,11 +95,10 @@ class Context(NamedTuple): BUILD = "B" DEPS = "d" - def select_build_targets(criteria): - from common import PLATFORM_TARGET, DESKTOP, OS_NAME + from common import PLATFORM_TARGET, OS_NAME - context = Context(PLATFORM_TARGET=PLATFORM_TARGET, DESKTOP=DESKTOP, OS_NAME=OS_NAME) + context = Context(PLATFORM_TARGET=PLATFORM_TARGET, OS_NAME=OS_NAME) reader = csv.DictReader(strip_array(BUILD_DEF), dialect=TableDialect()) for row in reader: diff --git a/.github/scripts/common.py b/.github/scripts/common.py index 600ac2e..22eb754 100644 --- a/.github/scripts/common.py +++ b/.github/scripts/common.py @@ -17,11 +17,6 @@ from kiwixbuild.versions import ( ) PLATFORM_TARGET = _environ["PLATFORM_TARGET"] -if PLATFORM_TARGET == "native_desktop": - PLATFORM_TARGET = "native_dyn" - DESKTOP = True -else: - DESKTOP = False OS_NAME = _environ["OS_NAME"] HOME = Path(os.path.expanduser("~")) @@ -32,9 +27,6 @@ INSTALL_DIR = BASE_DIR / "INSTALL" TMP_DIR = Path("/tmp") KBUILD_SOURCE_DIR = HOME / "kiwix-build" -# [TODO] -KIWIX_DESKTOP_ONLY = False - _ref = _environ.get("GITHUB_REF", "").split("/")[-1] MAKE_RELEASE = re.fullmatch(r"r_[0-9]+", _ref) is not None MAKE_RELEASE = MAKE_RELEASE and (_environ.get('GITHUB_EVENT_NAME') != 'schedule') diff --git a/.github/scripts/compile_all_deps.py b/.github/scripts/compile_all_deps.py index ac70bbe..87a0761 100755 --- a/.github/scripts/compile_all_deps.py +++ b/.github/scripts/compile_all_deps.py @@ -8,8 +8,6 @@ from common import ( upload, OS_NAME, PLATFORM_TARGET, - DESKTOP, - KIWIX_DESKTOP_ONLY, ) from build_definition import select_build_targets, DEPS From 6fe37ba0de5613644703799da620ec721bbfa55f Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Fri, 12 May 2023 17:15:39 +0200 Subject: [PATCH 8/8] Rename osx to macos. --- .github/scripts/build_definition.py | 18 +++++++++--------- .github/scripts/build_projects.py | 2 +- .github/scripts/build_release_nightly.py | 4 ++-- .github/scripts/common.py | 2 +- .github/workflows/ci.yml | 2 +- .github/workflows/releaseNigthly.yml | 2 +- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/scripts/build_definition.py b/.github/scripts/build_definition.py index 82901f0..b902abe 100644 --- a/.github/scripts/build_definition.py +++ b/.github/scripts/build_definition.py @@ -17,15 +17,15 @@ BUILD_DEF = """ | bionic | | B | | | | | ---------------------------------------------------------------------------------------------- # Osx builds, build binaries on native_dyn and native_static. On anyother things, build only the libraries - | osx | native_dyn | d | d | dB | B | | - | osx | native_static | | | B | B | | - | osx | native_mixed | B | B | | | | - | osx | iOS_arm64 | dB | B | | | | - | osx | iOS_x86_64 | dB | B | | | | - | osx | iOS_Mac_ABI | B | B | | | | - | osx | macOS_arm64_static | B | B | | | | - | osx | macOS_arm64_mixed | B | B | | | | - | osx | macOS_x86_64 | B | B | | | | + | macos | native_dyn | d | d | dB | B | | + | macos | native_static | | | B | B | | + | macos | native_mixed | B | B | | | | + | macos | iOS_arm64 | dB | B | | | | + | macos | iOS_x86_64 | dB | B | | | | + | macos | iOS_Mac_ABI | B | B | | | | + | macos | macOS_arm64_static | B | B | | | | + | macos | macOS_arm64_mixed | B | B | | | | + | macos | macOS_x86_64 | B | B | | | | ---------------------------------------------------------------------------------------------- | | flatpak | | | | | B | | | native_static | d | d | dB | dB | | diff --git a/.github/scripts/build_projects.py b/.github/scripts/build_projects.py index 5e5b659..b267b64 100755 --- a/.github/scripts/build_projects.py +++ b/.github/scripts/build_projects.py @@ -17,7 +17,7 @@ for target in select_build_targets(BUILD): if target == "kiwix-desktop": archive = create_desktop_image(make_release=False) else: - if PLATFORM_TARGET == "native_mixed" and OS_NAME == "osx": + if PLATFORM_TARGET == "native_mixed" and OS_NAME == "macos": fix_macos_rpath(target) archive = make_archive(target, make_release=False) if archive and DEV_BRANCH: diff --git a/.github/scripts/build_release_nightly.py b/.github/scripts/build_release_nightly.py index a35c2ab..f320544 100755 --- a/.github/scripts/build_release_nightly.py +++ b/.github/scripts/build_release_nightly.py @@ -34,7 +34,7 @@ for target in TARGETS: if target == "kiwix-desktop": archive = create_desktop_image(make_release=MAKE_RELEASE) else: - if OS_NAME == "osx" and PLATFORM_TARGET.endswith("_mixed"): + if OS_NAME == "macos" and PLATFORM_TARGET.endswith("_mixed"): fix_macos_rpath(target) notarize_macos_build(target) archive = make_archive(target, make_release=MAKE_RELEASE) @@ -46,7 +46,7 @@ for target in TARGETS: # We have few more things to do for release: if MAKE_RELEASE: # Publish source archives - if PLATFORM_TARGET in ("native_dyn", "native_mixed") and OS_NAME != "osx": + if PLATFORM_TARGET in ("native_dyn", "native_mixed") and OS_NAME != "macos": for target in TARGETS: if release_versions.get(target) != 0: continue diff --git a/.github/scripts/common.py b/.github/scripts/common.py index 22eb754..f483291 100644 --- a/.github/scripts/common.py +++ b/.github/scripts/common.py @@ -36,7 +36,7 @@ if not MAKE_RELEASE and _ref != "main": else: DEV_BRANCH = None -RELEASE_OS_NAME = "macos" if OS_NAME == "osx" else "linux" +RELEASE_OS_NAME = "macos" if OS_NAME == "macos" else "linux" EXTRA_NAME = "-bionic" if OS_NAME == "bionic" else "" PLATFORM_TO_RELEASE = { diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43b2edf..d35f98a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -150,7 +150,7 @@ jobs: runs-on: macos-11 env: SSH_KEY: /tmp/id_rsa - OS_NAME: osx + OS_NAME: macos steps: - name: Checkout code uses: actions/checkout@v3 diff --git a/.github/workflows/releaseNigthly.yml b/.github/workflows/releaseNigthly.yml index 46398ff..3791f36 100644 --- a/.github/workflows/releaseNigthly.yml +++ b/.github/workflows/releaseNigthly.yml @@ -168,7 +168,7 @@ jobs: runs-on: macos-11 env: SSH_KEY: /tmp/id_rsa - OS_NAME: osx + OS_NAME: macos CERTIFICATE: /tmp/wmch-devid.p12 SIGNING_IDENTITY: ${{ secrets.APPLE_SIGNING_IDENTITY }} KEYCHAIN: /Users/runner/build.keychain-db