smallsolar
/
kiwix-build
mirror of https://github.com/kiwix/kiwix-build.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #685 from kiwix/build_dir

This commit is contained in:
Matthieu Gautier 2024-04-09 11:33:30 +02:00 committed by GitHub
parent 6501464f76 a93b3c8896
commit d6285adaf9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 152 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
.github/scripts/build_definition.py vendored
View File

@ -15,52 +15,52 @@ import csv, io, re
# 'D' letter means we trigger the docker forkflow to build the docker image.
# If a cell contains several letters, all are done.
BUILD_DEF = """
| OS_NAME | COMPILE_CONFIG | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | platform_name |
=====================================================================================================================
| OS_NAME | COMPILE_CONFIG | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | platform_name | dependency_name |
==============================================================================================================================================
# Bionic is a special case as we need to compile libzim on old arch for python
| bionic | native_mixed | BP | | | | | linux-x86_64-bionic |
| bionic | aarch64_mixed | BP | | | | | linux-aarch64-bionic |
--------------------------------------------------------------------------------------------------------------------
| bionic | native_mixed | BP | | | | | linux-x86_64-bionic | |
| bionic | aarch64_mixed | BP | | | | | linux-aarch64-bionic | |
----------------------------------------------------------------------------------------------------------------------------------------------
# Osx builds, build binaries on native_dyn and native_static. On anyother things, build only the libraries
| macos | native_dyn | d | d | dB | B | | |
| macos | native_static | | | BP | BP | | macos-x86_64 |
| macos | native_mixed | BP | BP | | | | macos-x86_64 |
| macos | iOS_arm64 | dB | dB | | | | |
| macos | iOSSimulator_x86_64| dB | dB | | | | |
| macos | iOSSimulator_arm64 | dB | dB | | | | |
| macos | macOS_arm64_static | | | BP | BP | | macos-arm64 |
| macos | macOS_arm64_mixed | BP | BP | | | | macos-arm64 |
| macos | macOS_x86_64 | B | B | | | | |
| macos | apple_all_static | | BP | | | | xcframework |
----------------------------------------------------------------------------------------------
| | flatpak | | | | | BP | |
| | native_static | d | d | dBPSD | dBPSD | | linux-x86_64 |
| | native_mixed | BPS | BPS | | | | linux-x86_64 |
| | native_dyn | d | d | dB | dB | dBPS | |
| macos | native_dyn | d | d | dB | B | | | macos-x86_64-dyn |
| macos | native_static | | | BP | BP | | macos-x86_64 | |
| macos | native_mixed | BP | BP | | | | macos-x86_64 | |
| macos | iOS_arm64 | dB | dB | | | | | ios-arm64-dyn |
| macos | iOSSimulator_x86_64| dB | dB | | | | | ios-x86_64-dyn |
| macos | iOSSimulator_arm64 | B | B | | | | | |
| macos | macOS_arm64_static | | | BP | BP | | macos-arm64 | |
| macos | macOS_arm64_mixed | BP | BP | | | | macos-arm64 | |
| macos | macOS_x86_64 | B | B | | | | | |
| macos | apple_all_static | | BP | | | | xcframework | |
----------------------------------------------------------------------------------------------------------------------------------------------
| | flatpak | | | | | BP | | |
| | native_static | d | d | dBPSD | dBPSD | | linux-x86_64 | linux-x86_64-static |
| | native_mixed | BPS | BPS | | | | linux-x86_64 | |
| | native_dyn | d | d | dB | dB | dBPS | | linux-x86_64-dyn |
# libzim CI is building alpine_dyn but not us
| | android_arm | dBP | dBP | | | | android-arm |
| | android_arm64 | dBP | dBP | | | | android-arm64 |
| | android_x86 | BP | BP | | | | android-x86 |
| | android_x86_64 | BP | BP | | | | android-x86_64 |
| | armv6_static | | | BP | BP | | linux-armv6 |
| | armv6_mixed | BP | | | | | linux-armv6 |
| | armv6_dyn | | | B | B | | |
| | armv8_static | | | BP | BP | | linux-armv8 |
| | armv8_mixed | BP | | | | | linux-armv8 |
| | armv8_dyn | | | B | B | | |
| | aarch64_static | | | BP | BP | | linux-aarch64 |
| | aarch64_mixed | BP | | | | | linux-aarch64 |
| | aarch64_dyn | d | | B | B | | |
| | aarch64_musl_static| | | BP | BP | | linux-aarch64-musl |
| | aarch64_musl_mixed | BP | | | | | linux-aarch64-musl |
| | aarch64_musl_dyn | d | | B | B | | |
| | x86-64_musl_static | | | BP | BP | | linux-x86_64-musl |
| | x86-64_musl_mixed | BP | | | | | linux-x86_64-musl |
| | win32_static | d | dB | dBP | dBP | | win-i686 |
| | win32_dyn | d | dB | dB | dB | | |
| | i586_static | | | BP | BP | | linux-i586 |
| | i586_dyn | | | B | B | | |
| | wasm | dBP | | | | | wasm-emscripten |
| | android_arm | dBP | dBP | | | | android-arm | android-arm |
| | android_arm64 | dBP | dBP | | | | android-arm64 | android-arm64 |
| | android_x86 | BP | BP | | | | android-x86 | |
| | android_x86_64 | BP | BP | | | | android-x86_64 | |
| | armv6_static | | | BP | BP | | linux-armv6 | |
| | armv6_mixed | BP | | | | | linux-armv6 | |
| | armv6_dyn | | | B | B | | | |
| | armv8_static | | | BP | BP | | linux-armv8 | |
| | armv8_mixed | BP | | | | | linux-armv8 | |
| | armv8_dyn | | | B | B | | | |
| | aarch64_static | | | BP | BP | | linux-aarch64 | |
| | aarch64_mixed | BP | | | | | linux-aarch64 | |
| | aarch64_dyn | d | | B | B | | | linux-aarch64-dyn |
| | aarch64_musl_static| | | BP | BP | | linux-aarch64-musl | |
| | aarch64_musl_mixed | BP | | | | | linux-aarch64-musl | |
| | aarch64_musl_dyn | d | | B | B | | | linux-aarch64-musl-dyn |
| | x86-64_musl_static | | | BP | BP | | linux-x86_64-musl | |
| | x86-64_musl_mixed | BP | | | | | linux-x86_64-musl | |
| | win32_static | d | dB | dBP | dBP | | win-i686 |win32-static |
| | win32_dyn | d | dB | dB | dB | | |win32-dyn |
| | i586_static | | | BP | BP | | linux-i586 | |
| | i586_dyn | | | B | B | | | |
| | wasm | dBP | | | | | wasm-emscripten | wasm |
"""
@ -135,7 +135,7 @@ def select_build_targets(criteria):
raise ValueError("No definition match with current context.")
def get_platform_name():
def get_column_value(column_name):
from common import COMPILE_CONFIG, OS_NAME
context = Context(COMPILE_CONFIG=COMPILE_CONFIG, OS_NAME=OS_NAME)
@ -143,7 +143,15 @@ def get_platform_name():
reader = csv.DictReader(strip_array(BUILD_DEF), dialect=TableDialect())
for row in reader:
if context.match(row):
name = row["platform_name"]
name = row[column_name]
return name or None
raise ValueError("No definition match with current context.")
def get_platform_name():
return get_column_value("platform_name")
def get_dependency_archive_name():
return get_column_value("dependency_name")

45
.github/scripts/common.py vendored
View File

@ -10,7 +10,7 @@ import shutil
import requests
from build_definition import get_platform_name
from build_definition import get_platform_name, get_dependency_archive_name
from kiwixbuild.dependencies.apple_xcframework import AppleXCFramework
from kiwixbuild.versions import (
@ -20,11 +20,23 @@ from kiwixbuild.versions import (
)
def get_build_dir(config) -> Path:
command = ["kiwix-build"]
command.extend(["--config", config])
command.append("--get-build-dir")
command.append("--use-target-arch-name")
return Path(
subprocess.run(command, cwd=str(HOME), check=True, stdout=subprocess.PIPE)
.stdout[:-1]
.decode("utf8")
)
COMPILE_CONFIG = _environ["COMPILE_CONFIG"]
OS_NAME = _environ["OS_NAME"]
HOME = Path(os.path.expanduser("~"))
BASE_DIR = HOME / "BUILD_{}".format(COMPILE_CONFIG)
BASE_DIR = get_build_dir(COMPILE_CONFIG)
SOURCE_DIR = HOME / "SOURCE"
ARCHIVE_DIR = HOME / "ARCHIVE"
TOOLCHAIN_DIR = BASE_DIR / "TOOLCHAINS"
@ -161,6 +173,7 @@ def run_kiwix_build(
command.append("--hide-progress")
command.append("--fast-clone")
command.append("--assume-packages-installed")
command.append("--use-target-arch-name")
command.extend(["--config", config])
if build_deps_only:
command.append("--build-deps-only")
@ -271,30 +284,28 @@ def filter_install_dir(path):
yield sub_dir
# Full: True if we are creating a full archive to be used as cache by kiwix-build (base_deps2_{os}_{config}_{base_deps_version}.tar.xz)
# Full: False if we are creating a archive to be used as pre-cached dependencies for project's CI (deps2_{os}_{config}_{target}.tar.xz)
# Full: True if we are creating a full archive to be used as cache by kiwix-build (base_deps_{os}_{config}_{base_deps_version}.tar.xz)
# Full: False if we are creating a archive to be used as pre-cached dependencies for project's CI (deps_{config}_{target}.tar.xz)
def make_deps_archive(target=None, name=None, full=False):
archive_name = name or "deps2_{}_{}_{}.tar.xz".format(
OS_NAME, COMPILE_CONFIG, target
archive_name = name or "deps_{}_{}.tar.xz".format(
get_dependency_archive_name(), target
)
print_message("Create archive {}.", archive_name)
files_to_archive = list(filter_install_dir(INSTALL_DIR))
files_to_archive += HOME.glob("BUILD_*/LOGS")
if COMPILE_CONFIG == "apple_all_static":
for subconfig in AppleXCFramework.subConfigNames:
base_dir = HOME / "BUILD_{}".format(subconfig)
base_dir = get_build_dir(subconfig)
files_to_archive += filter_install_dir(base_dir / "INSTALL")
if (base_dir / "meson_cross_file.txt").exists():
files_to_archive.append(base_dir / "meson_cross_file.txt")
if COMPILE_CONFIG.endswith("_mixed"):
static_config = COMPILE_CONFIG.replace("_mixed", "_static")
files_to_archive += filter_install_dir(
HOME / ("BUILD_" + static_config) / "INSTALL"
)
files_to_archive += filter_install_dir(get_build_dir(static_config) / "INSTALL")
if COMPILE_CONFIG.startswith("android_"):
files_to_archive += filter_install_dir(HOME / "BUILD_neutral" / "INSTALL")
base_dir = HOME / "BUILD_{}".format(COMPILE_CONFIG)
base_dir = get_build_dir(COMPILE_CONFIG)
if (base_dir / "meson_cross_file.txt").exists():
files_to_archive.append(base_dir / "meson_cross_file.txt")
# Copy any toolchain
@ -314,13 +325,13 @@ def make_deps_archive(target=None, name=None, full=False):
# Add also static build for mixed target
if COMPILE_CONFIG.endswith("_mixed"):
static_config = COMPILE_CONFIG.replace("_mixed", "_static")
files_to_archive += (HOME / ("BUILD_" + static_config)).glob("*/.*_ok")
files_to_archive += get_build_dir(static_config).glob("*/.*_ok")
# Native dyn and static is needed for potential cross compilation that use native tools (icu)
files_to_archive += (HOME / "BUILD_native_dyn").glob("*/.*_ok")
files_to_archive += (HOME / "BUILD_native_static").glob("*/.*_ok")
files_to_archive += HOME.glob("BUILD_android*/**/.*_ok")
files_to_archive += HOME.glob("BUILD_macOS*/**/.*_ok")
files_to_archive += HOME.glob("BUILD_iOS*/**/.*_ok")
files_to_archive += get_build_dir("native_dyn").glob("*/.*_ok")
files_to_archive += get_build_dir("native_static").glob("*/.*_ok")
files_to_archive += HOME.glob("BUILD_*android*/**/.*_ok")
files_to_archive += HOME.glob("BUILD_*apple-macos*/**/.*_ok")
files_to_archive += HOME.glob("BUILD_*apple-ios*/**/.*_ok")
files_to_archive += SOURCE_DIR.glob("*/.*_ok")
files_to_archive += SOURCE_DIR.glob("zim-testing-suite-*/*")

4
.github/scripts/ensure_base_deps.py vendored
View File

@ -32,10 +32,10 @@ def download_base_archive(base_name):
return file_path
ARCHIVE_NAME_TEMPLATE = "base_deps2_{os}_{config}_{version}.tar.xz"
ARCHIVE_NAME_TEMPLATE = "base_deps_{os}_{config}_{version}.tar.xz"
if COMPILE_CONFIG == "flatpak":
base_dep_archive_name = "base_deps2_flatpak.tar.xz"
base_dep_archive_name = "base_deps_flatpak.tar.xz"
else:
base_dep_archive_name = ARCHIVE_NAME_TEMPLATE.format(
os=OS_NAME,

1
actions/dl_deps_archive/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
node_modules

5
actions/dl_deps_archive/dist/index.js vendored
View File

@ -30484,7 +30484,6 @@ function addLocalPath(inputPath) {
async function run() {
try {
const base_url = core.getInput("base_url");
const os_name = getInput("os_name", process.env["OS_NAME"]);
const target = core.getInput("target_platform");
const project = getInput(
"project",
@ -30501,11 +30500,11 @@ async function run() {
let archivePath;
try {
const archive_url = `${base_url}/dev_preview/${branch}/deps2_${os_name}_${target}_${project}.tar.xz`;
const archive_url = `${base_url}/dev_preview/${branch}/deps_${target}_${project}.tar.xz`;
process.stdout.write("Downloading " + archive_url + "\n");
archivePath = await tc.downloadTool(archive_url);
} catch (error) {
const archive_url = `${base_url}/deps2_${os_name}_${target}_${project}.tar.xz`;
const archive_url = `${base_url}/deps_${target}_${project}.tar.xz`;
process.stdout.write("Downloading " + archive_url + "\n");
archivePath = await tc.downloadTool(archive_url);
}

5
actions/dl_deps_archive/index.js
View File

@ -18,7 +18,6 @@ function addLocalPath(inputPath) {
async function run() {
try {
const base_url = core.getInput("base_url");
const os_name = getInput("os_name", process.env["OS_NAME"]);
const target = core.getInput("target_platform");
const project = getInput(
"project",
@ -35,11 +34,11 @@ async function run() {
let archivePath;
try {
const archive_url = `${base_url}/dev_preview/${branch}/deps2_${os_name}_${target}_${project}.tar.xz`;
const archive_url = `${base_url}/dev_preview/${branch}/deps_${target}_${project}.tar.xz`;
process.stdout.write("Downloading " + archive_url + "\n");
archivePath = await tc.downloadTool(archive_url);
} catch (error) {
const archive_url = `${base_url}/deps2_${os_name}_${target}_${project}.tar.xz`;
const archive_url = `${base_url}/deps_${target}_${project}.tar.xz`;
process.stdout.write("Downloading " + archive_url + "\n");
archivePath = await tc.downloadTool(archive_url);
}

18
kiwixbuild/__init__.py
View File

@ -125,6 +125,21 @@ def parse_args():
"to develop with the cloned sources."
),
)
subgroup.add_argument(
"--use-target-arch-name",
action="store_true",
help=(
"Name the build directory using the arch name instead of the config name.\n"
"Different configs may create binary for the same arch so this option is "
"not recommended when working with several config on the same computer.\n"
"However, when generating dependencies for other it is better to have a "
"directory named using the target instead of the used config.\n"
"Intended to be used in CI only."
),
)
subgroup.add_argument(
"--get-build-dir", action="store_true", help="Print the output directory."
)
options = parser.parse_args()
if not options.android_arch:
@ -145,4 +160,7 @@ def main():
builder = FlatpakBuilder()
else:
builder = Builder()
if options.get_build_dir:
print(ConfigInfo.get_config(options.config).buildEnv.build_dir)
else:
builder.run()

10
kiwixbuild/buildenv.py
View File

@ -36,7 +36,8 @@ class NeutralEnv:
if _platform == "Windows":
print(
"ERROR: kiwix-build is not intented to run on Windows platform.\n"
"It should probably not work, but well, you still can have a try."
"It should probably not work, but well, you still can have a try.",
file=sys.stderr,
)
cont = input("Do you want to continue ? [y/N]")
if cont.lower() != "y":
@ -70,15 +71,18 @@ class NeutralEnv:
if required:
sys.exit("ERROR: {} command not found".format(name))
else:
print("WARNING: {} command not found".format(name))
print("WARNING: {} command not found".format(name), file=sys.stderr)
return ["{}_NOT_FOUND".format(name.upper())]
class BuildEnv:
def __init__(self, configInfo):
build_dir = "BUILD_{}".format(configInfo.name)
self.configInfo = configInfo
self.base_build_dir = pj(option("working_dir"), option("build_dir"))
build_dir = (
configInfo.arch_name if option("use_target_arch_name") else configInfo.name
)
build_dir = f"BUILD_{build_dir}"
self.build_dir = pj(self.base_build_dir, build_dir)
self.install_dir = pj(self.build_dir, "INSTALL")
self.toolchain_dir = pj(self.build_dir, "TOOLCHAINS")

4
kiwixbuild/configs/android.py
View File

@ -147,6 +147,10 @@ class Android(MetaConfigInfo):
name = "android"
compatible_hosts = ["fedora", "debian"]
@property
def arch_name(self):
return "multi-linux-android"
@property
def subConfigNames(self):
return ["android_{}".format(arch) for arch in option("android_arch")]

5
kiwixbuild/configs/base.py
View File

@ -27,6 +27,10 @@ class ConfigInfo(metaclass=_MetaConfig):
mixed = False
libdir = None
@property
def arch_name(self):
return self.arch_full
@classmethod
def get_config(cls, name, targets=None):
if name not in cls.all_running_configs:
@ -131,7 +135,6 @@ def MixedMixin(static_name):
static = False
def add_targets(self, targetName, targets):
print(targetName)
if option("target") == targetName:
return super().add_targets(targetName, targets)
else:

1
kiwixbuild/configs/flatpak.py
View File

@ -4,6 +4,7 @@ from kiwixbuild._global import option, neutralEnv
class FlatpakConfigInfo(ConfigInfo):
name = "flatpak"
arch_name = "flatpak"
build = "flatpak"
static = ""
toolchain_names = ["org.kde", "io.qt.qtwebengine"]

12
kiwixbuild/configs/ios.py
View File

@ -24,6 +24,10 @@ class AppleConfigInfo(ConfigInfo):
super().__init__(*args, **kwargs)
self._root_path = None
@property
def arch_name(self):
return self.target
@property
def root_path(self):
if self._root_path is None:
@ -209,6 +213,10 @@ class IOS(MetaConfigInfo):
name = "iOS_multi"
compatible_hosts = ["Darwin"]
@property
def arch_name(self):
return self.name
@property
def subConfigNames(self):
return ["iOS_{}".format(arch) for arch in option("ios_arch")]
@ -225,6 +233,10 @@ class AppleStaticAll(MetaConfigInfo):
name = "apple_all_static"
compatible_hosts = ["Darwin"]
@property
def arch_name(self):
return self.name
@property
def subConfigNames(self):
return AppleXCFramework.subConfigNames

11
kiwixbuild/configs/native.py
View File

@ -3,6 +3,9 @@ from .base import ConfigInfo, MixedMixin
from kiwixbuild.utils import pj
from kiwixbuild._global import option, neutralEnv
from kiwixbuild.configs.ios import MIN_MACOS_VERSION
import sysconfig
import platform
import sys
class NativeConfigInfo(ConfigInfo):
@ -16,6 +19,12 @@ class NativeConfigInfo(ConfigInfo):
env["CFLAGS"] += f"-mmacosx-version-min={MIN_MACOS_VERSION}"
return env
@property
def arch_name(self):
if sys.platform == "darwin":
return f"{platform.machine()}-apple-darwin"
return sysconfig.get_platform()
class NativeDyn(NativeConfigInfo):
name = "native_dyn"
@ -26,7 +35,7 @@ class NativeDyn(NativeConfigInfo):
class NativeStatic(NativeConfigInfo):
name = "native_static"
static = True
compatible_hosts = ["fedora", "debian"]
compatible_hosts = ["fedora", "debian", "Darwin"]
class NativeMixed(MixedMixin("native_static"), NativeConfigInfo):

1
kiwixbuild/configs/neutral.py
View File

@ -3,6 +3,7 @@ from .base import ConfigInfo
class NeutralConfigInfo(ConfigInfo):
name = "neutral"
arch_name = "neutral"
static = ""
compatible_hosts = ["fedora", "debian", "Darwin"]

8
kiwixbuild/utils.py
View File

@ -13,7 +13,9 @@ from collections import namedtuple, defaultdict
from kiwixbuild._global import neutralEnv, option
pj = os.path.join
def pj(*args):
return os.path.normpath(os.path.join(*args))
COLORS = {
@ -77,7 +79,7 @@ def remove_duplicates(iterable, key_function=None):
def get_sha256(path):
progress_chars = "/-\|"
progress_chars = "/-\\|"
current = 0
batch_size = 1024 * 8
sha256 = hashlib.sha256()
@ -137,7 +139,7 @@ def download_remote(what, where):
context = None
batch_size = 1024 * 8
extra_args = {"context": context} if sys.version_info >= (3, 4, 3) else {}
progress_chars = "/-\|"
progress_chars = "/-\\|"
try:
with urllib.request.urlopen(what.url, **extra_args) as resource, open(
file_path, "wb"

2
kiwixbuild/versions.py
View File

@ -39,7 +39,7 @@ release_versions = {
# This is the "version" of the whole base_deps_versions dict.
# Change this when you change base_deps_versions.
base_deps_meta_version = "98"
base_deps_meta_version = "99"
base_deps_versions = {
"zlib": "1.2.12",