Merge pull request #612 from kiwix/better_build_def2

.github/scripts/build_definition.py

@@ -0,0 +1,120 @@
+from typing import NamedTuple
+import csv, io, re
+
+# Definition of what to build.
+# Array is read line by line.
+# Empty cells under (OS_NAME, PLATFORM_TARGET) mean "always match" (catch all, or `.*` regex)
+# Once a cell doesn't match, skip to the next line.
+# Once a line matches, other lines are not read, so put more specific combination first.
+# Lines composed of `-` , or `=`, or starting by `#` are ignored.
+# 'B' letter means that the project is build in the CI
+# 'd' letter means that the project's dependencies are build and published to be used by the project's CI.
+# If a cell contains both, both are done.
+BUILD_DEF = """
+    | OS_NAME | PLATFORM_TARGET    | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop |
+    ==============================================================================================
+# Bionic is a special case as we need to compile libzim on old arch for python
+    | bionic  |                    | B      |          |           |             |               |
+    ----------------------------------------------------------------------------------------------
+# Osx builds, build binaries on native_dyn and native_static. On anyother things, build only the libraries
+    | macos   | native_dyn         | d      | d        | dB        | B           |               |
+    | macos   | native_static      |        |          | B         | B           |               |
+    | macos   | native_mixed       | B      | B        |           |             |               |
+    | macos   | iOS_arm64          | dB     | B        |           |             |               |
+    | macos   | iOS_x86_64         | dB     | B        |           |             |               |
+    | macos   | iOS_Mac_ABI        | B      | B        |           |             |               |
+    | macos   | macOS_arm64_static | B      | B        |           |             |               |
+    | macos   | macOS_arm64_mixed  | B      | B        |           |             |               |
+    | macos   | macOS_x86_64       | B      | B        |           |             |               |
+    ----------------------------------------------------------------------------------------------
+    |         | flatpak            |        |          |           |             | B             |
+    |         | native_static      | d      | d        | dB        | dB          |               |
+    |         | native_dyn         | d      | d        | dB        | dB          | B             |
+    |         | native_mixed       | B      | B        |           |             |               |
+# libzim CI is building alpine_dyn but not us
+    |         | android_arm        | dB     | dB       |           |             |               |
+    |         | android_arm64      | dB     | dB       |           |             |               |
+    |         | android_x86        | B      | B        |           |             |               |
+    |         | android_x86_64     | B      | B        |           |             |               |
+    |         | armv6_static       |        |          | B         | B           |               |
+    |         | armv6_dyn          |        |          | B         | B           |               |
+    |         | armv6_mixed        | B      |          |           |             |               |
+    |         | armv8_static       |        |          | B         | B           |               |
+    |         | armv8_dyn          |        |          | B         | B           |               |
+    |         | armv8_mixed        | B      |          |           |             |               |
+    |         | aarch64_static     |        |          | B         | B           |               |
+    |         | aarch64_dyn        | d      |          | B         | B           |               |
+    |         | aarch64_mixed      | B      |          |           |             |               |
+    |         | win32_static       | d      | dB       | dB        | dB          |               |
+    |         | win32_dyn          | d      | dB       | dB        | dB          |               |
+    |         | i586_static        |        |          | B         | B           |               |
+    |         | i586_dyn           |        |          | B         | B           |               |
+    |         | wasm               | dB     |          |           |             |               |
+"""
+
+
+class TableDialect(csv.Dialect):
+    delimiter = "|"
+    quoting = csv.QUOTE_NONE
+    lineterminator = "\n"
+
+
+def strip_array(array_str):
+    """Return a iterable of lines, skiping "decorative lines" and with all values in the line's cells stripped"""
+    for line in array_str.splitlines():
+        line = line.strip()
+        line_set = set(line)
+        if (
+            not line
+            or line.startswith("#")
+            or (len(line_set) == 1 and line_set.pop() in "-=")
+        ):
+            continue
+        yield "|".join(c.strip() for c in line.split("|"))
+
+
+def selector_match(selector, value):
+    if not selector:
+        return True
+    return re.fullmatch(selector, value) is not None
+
+
+class Context(NamedTuple):
+    OS_NAME: str
+    PLATFORM_TARGET: str
+
+    def match(self, row):
+        for key in ["OS_NAME", "PLATFORM_TARGET"]:
+            context_value = getattr(self, key)
+            selector = row[key]
+            if not selector_match(selector, context_value):
+                return False
+        return True
+
+
+BUILD = "B"
+DEPS = "d"
+
+def select_build_targets(criteria):
+    from common import PLATFORM_TARGET, OS_NAME
+
+    context = Context(PLATFORM_TARGET=PLATFORM_TARGET, OS_NAME=OS_NAME)
+
+    reader = csv.DictReader(strip_array(BUILD_DEF), dialect=TableDialect())
+    for row in reader:
+        if context.match(row):
+            build_order = [
+                k
+                for k in (
+                    "libzim",
+                    "libkiwix",
+                    "zim-tools",
+                    "kiwix-tools",
+                    "kiwix-desktop",
+                )
+                if criteria in row[k]
+            ]
+            print(build_order)
+            return build_order
+
+    raise "No definition match with current context."

.github/scripts/build_projects.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-
+from build_definition import select_build_targets, BUILD
 from common import (
     run_kiwix_build,
     make_archive,
@@ -12,55 +12,12 @@ from common import (
     DEV_BRANCH,
 )
 
-
+for target in select_build_targets(BUILD):
-def select_build_target():
-    from common import (
-        PLATFORM_TARGET,
-        DESKTOP,
-        OS_NAME
-    )
-    if OS_NAME == "bionic" and PLATFORM_TARGET.endswith("_mixed"):
-        return ("libzim", )
-    elif (PLATFORM_TARGET.startswith("android_")
-     or PLATFORM_TARGET.startswith("iOS")):
-        return ("libzim", "libkiwix")
-    elif PLATFORM_TARGET.startswith("macOS"):
-        if PLATFORM_TARGET.endswith("_mixed"):
-            return ("libzim", "libkiwix")
-        elif PLATFORM_TARGET.endswith("_dyn"):
-            return ("zim-tools", "kiwix-tools")
-        else:
-            return []
-    elif PLATFORM_TARGET.startswith("native_"):
-        if OS_NAME == "osx":
-            if PLATFORM_TARGET.endswith("_mixed"):
-                return ("libzim", "libkiwix")
-            else:
-                return ("zim-tools", "kiwix-tools")
-        else:
-            if DESKTOP:
-                return ("kiwix-desktop",)
-            elif PLATFORM_TARGET == "native_mixed":
-                return ("libzim", "libkiwix")
-            else:
-                return ("zim-tools", "kiwix-tools")
-    elif PLATFORM_TARGET in ("win32_static", "armv6_static", "armv6_dyn", "armv8_static", "armv8_dyn", "aarch64_static", "aarch64_dyn", "i586_static"):
-        return ("zim-tools", "kiwix-tools")
-    elif PLATFORM_TARGET == "flatpak":
-        return ("kiwix-desktop",)
-    elif PLATFORM_TARGET in ("wasm", "armv6_mixed", "armv8_mixed", "aarch64_mixed"):
-        return ("libzim", )
-    else:
-        return ("libzim", "zim-tools", "libkiwix", "kiwix-tools")
-
-TARGETS = select_build_target()
-
-for target in TARGETS:
     run_kiwix_build(target, platform=PLATFORM_TARGET)
     if target == "kiwix-desktop":
         archive = create_desktop_image(make_release=False)
     else:
-        if PLATFORM_TARGET == "native_mixed" and OS_NAME == "osx":
+        if PLATFORM_TARGET == "native_mixed" and OS_NAME == "macos":
             fix_macos_rpath(target)
         archive = make_archive(target, make_release=False)
     if archive and DEV_BRANCH:

.github/scripts/build_release_nightly.py

@@ -13,17 +13,15 @@ from common import (
     fix_macos_rpath,
     trigger_docker_publish,
     BASE_DIR,
-    TMP_DIR,
-    HOME,
     OS_NAME,
     PLATFORM_TARGET,
     MAKE_RELEASE,
     notarize_macos_build,
 )
 
-from build_projects import select_build_target
+from build_definition import select_build_targets, BUILD
 
-TARGETS = select_build_target()
+TARGETS = select_build_target(BUILD)
 
 # Filter what to build if we are doing a release.
 if MAKE_RELEASE:
@@ -36,7 +34,7 @@ for target in TARGETS:
     if target == "kiwix-desktop":
         archive = create_desktop_image(make_release=MAKE_RELEASE)
     else:
-        if OS_NAME == "osx" and PLATFORM_TARGET.endswith("_mixed"):
+        if OS_NAME == "macos" and PLATFORM_TARGET.endswith("_mixed"):
             fix_macos_rpath(target)
             notarize_macos_build(target)
         archive = make_archive(target, make_release=MAKE_RELEASE)
@@ -48,7 +46,7 @@ for target in TARGETS:
 # We have few more things to do for release:
 if MAKE_RELEASE:
     # Publish source archives
-    if PLATFORM_TARGET in ("native_dyn", "native_mixed") and OS_NAME != "osx":
+    if PLATFORM_TARGET in ("native_dyn", "native_mixed") and OS_NAME != "macos":
         for target in TARGETS:
             if release_versions.get(target) != 0:
                 continue

.github/scripts/common.py

@@ -17,11 +17,6 @@ from kiwixbuild.versions import (
 )
 
 PLATFORM_TARGET = _environ["PLATFORM_TARGET"]
-if PLATFORM_TARGET == "native_desktop":
-    PLATFORM_TARGET = "native_dyn"
-    DESKTOP = True
-else:
-    DESKTOP = False
 OS_NAME = _environ["OS_NAME"]
 HOME = Path(os.path.expanduser("~"))
 
@@ -32,9 +27,6 @@ INSTALL_DIR = BASE_DIR / "INSTALL"
 TMP_DIR = Path("/tmp")
 KBUILD_SOURCE_DIR = HOME / "kiwix-build"
 
-# [TODO]
-KIWIX_DESKTOP_ONLY = False
-
 _ref = _environ.get("GITHUB_REF", "").split("/")[-1]
 MAKE_RELEASE = re.fullmatch(r"r_[0-9]+", _ref) is not None
 MAKE_RELEASE = MAKE_RELEASE and (_environ.get('GITHUB_EVENT_NAME') != 'schedule')
@@ -44,7 +36,7 @@ if not MAKE_RELEASE and _ref != "main":
 else:
     DEV_BRANCH = None
 
-RELEASE_OS_NAME = "macos" if OS_NAME == "osx" else "linux"
+RELEASE_OS_NAME = "macos" if OS_NAME == "macos" else "linux"
 EXTRA_NAME = "-bionic" if OS_NAME == "bionic" else ""
 
 PLATFORM_TO_RELEASE = {

.github/scripts/compile_all_deps.py

@@ -8,28 +8,10 @@ from common import (
     upload,
     OS_NAME,
     PLATFORM_TARGET,
-    DESKTOP,
-    KIWIX_DESKTOP_ONLY,
 )
+from build_definition import select_build_targets, DEPS
 
-if PLATFORM_TARGET.startswith("android_") or PLATFORM_TARGET.startswith("iOS"):
+for target in select_build_targets(DEPS):
-    TARGETS = ("libzim", "libkiwix")
-elif PLATFORM_TARGET == "wasm":
-    TARGETS = ("libzim", )
-elif PLATFORM_TARGET.startswith("native_"):
-    if OS_NAME == "osx":
-        TARGETS = ("libzim", "zim-tools", "libkiwix")
-    else:
-        if DESKTOP:
-            TARGETS = ("kiwix-desktop",)
-        elif PLATFORM_TARGET == "native_mixed":
-            TARGETS = ("libzim",)
-        else:
-            TARGETS = ("libzim", "zim-tools", "libkiwix", "kiwix-tools")
-else:
-    TARGETS = ("libzim", "zim-tools", "libkiwix", "kiwix-tools")
-
-for target in TARGETS:
     run_kiwix_build(target, platform=PLATFORM_TARGET, build_deps_only=True)
     archive_file = make_deps_archive(target=target)
     upload(archive_file, "ci@tmp.kiwix.org:30022", "/data/tmp/ci")

.github/workflows/ci.yml

@@ -14,7 +14,6 @@ jobs:
           - native_static
           - native_dyn
           - native_mixed
-          - native_desktop
           - wasm
           - armv6_static
           - armv6_dyn
@@ -151,7 +150,7 @@ jobs:
     runs-on: macos-11
     env:
       SSH_KEY: /tmp/id_rsa
-      OS_NAME: osx
+      OS_NAME: macos
     steps:
     - name: Checkout code
       uses: actions/checkout@v3

.github/workflows/releaseNigthly.yml

@@ -16,7 +16,6 @@ jobs:
           - native_static
           - native_dyn
           - native_mixed
-          - native_desktop
           - wasm
           - armv6_static
           - armv6_mixed
@@ -39,8 +38,6 @@ jobs:
             image_variant: focal
           - target: native_mixed
             image_variant: bionic
-          - target: native_desktop
-            image_variant: focal
           - target: wasm
             image_variant: focal
           - target: armv6_static
@@ -171,7 +168,7 @@ jobs:
     runs-on: macos-11
     env:
       SSH_KEY: /tmp/id_rsa
-      OS_NAME: osx
+      OS_NAME: macos
       CERTIFICATE: /tmp/wmch-devid.p12
       SIGNING_IDENTITY: ${{ secrets.APPLE_SIGNING_IDENTITY }}
       KEYCHAIN: /Users/runner/build.keychain-db