smallsolar
/
kiwix-build
mirror of https://github.com/kiwix/kiwix-build.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #679 from kiwix/target_config 

Rename option `--target-platform` to `--config`
This commit is contained in:
Matthieu Gautier 2024-02-08 17:58:52 +01:00 committed by GitHub
parent 939f323709 d4acd108ca
commit 4766ae5b56
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
74 changed files with 2837 additions and 2445 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
.github/scripts/build_definition.py vendored
View File

@ -3,7 +3,7 @@ import csv, io, re
# Definition of what to build. # Definition of what to build.
# Array is read line by line. # Array is read line by line.
# Empty cells under (OS_NAME, PLATFORM_TARGET) mean "always match" (catch all, or `.*` regex) # Empty cells under (OS_NAME, COMPILE_CONFIG) mean "always match" (catch all, or `.*` regex)
# Once a cell doesn't match, skip to the next line. # Once a cell doesn't match, skip to the next line.
# Once a line matches, other lines are not read, so put more specific combination first. # Once a line matches, other lines are not read, so put more specific combination first.
# Lines composed of `-` , or `=`, or starting by `#` are ignored. # Lines composed of `-` , or `=`, or starting by `#` are ignored.
@ -15,7 +15,7 @@ import csv, io, re
# 'D' letter means we trigger the docker forkflow to build the docker image. # 'D' letter means we trigger the docker forkflow to build the docker image.
# If a cell contains several letters, all are done. # If a cell contains several letters, all are done.
BUILD_DEF = """ BUILD_DEF = """
| OS_NAME | PLATFORM_TARGET | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | platform_name | | OS_NAME | COMPILE_CONFIG | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | platform_name |
===================================================================================================================== =====================================================================================================================
# Bionic is a special case as we need to compile libzim on old arch for python # Bionic is a special case as we need to compile libzim on old arch for python
| bionic | native_mixed | BP | | | | | linux-x86_64-bionic | | bionic | native_mixed | BP | | | | | linux-x86_64-bionic |
@ -92,10 +92,10 @@ def selector_match(selector, value):
class Context(NamedTuple): class Context(NamedTuple):
OS_NAME: str OS_NAME: str
PLATFORM_TARGET: str COMPILE_CONFIG: str
def match(self, row): def match(self, row):
for key in ["OS_NAME", "PLATFORM_TARGET"]: for key in ["OS_NAME", "COMPILE_CONFIG"]:
context_value = getattr(self, key) context_value = getattr(self, key)
selector = row[key] selector = row[key]
if not selector_match(selector, context_value): if not selector_match(selector, context_value):
@ -109,10 +109,11 @@ SOURCE_PUBLISH = "S"
DEPS = "d" DEPS = "d"
DOCKER = "D" DOCKER = "D"
def select_build_targets(criteria):
from common import PLATFORM_TARGET, OS_NAME
context = Context(PLATFORM_TARGET=PLATFORM_TARGET, OS_NAME=OS_NAME) def select_build_targets(criteria):
from common import COMPILE_CONFIG, OS_NAME
context = Context(COMPILE_CONFIG=COMPILE_CONFIG, OS_NAME=OS_NAME)
reader = csv.DictReader(strip_array(BUILD_DEF), dialect=TableDialect()) reader = csv.DictReader(strip_array(BUILD_DEF), dialect=TableDialect())
for row in reader: for row in reader:
@ -133,10 +134,11 @@ def select_build_targets(criteria):
raise ValueError("No definition match with current context.") raise ValueError("No definition match with current context.")
def get_platform_name():
from common import PLATFORM_TARGET, OS_NAME
context = Context(PLATFORM_TARGET=PLATFORM_TARGET, OS_NAME=OS_NAME) def get_platform_name():
from common import COMPILE_CONFIG, OS_NAME
context = Context(COMPILE_CONFIG=COMPILE_CONFIG, OS_NAME=OS_NAME)
reader = csv.DictReader(strip_array(BUILD_DEF), dialect=TableDialect()) reader = csv.DictReader(strip_array(BUILD_DEF), dialect=TableDialect())
for row in reader: for row in reader:

6
.github/scripts/build_projects.py vendored
View File

@ -8,16 +8,16 @@ from common import (
fix_macos_rpath, fix_macos_rpath,
upload_archive, upload_archive,
OS_NAME, OS_NAME,
PLATFORM_TARGET, COMPILE_CONFIG,
DEV_BRANCH, DEV_BRANCH,
) )
for target in select_build_targets(BUILD): for target in select_build_targets(BUILD):
run_kiwix_build(target, platform=PLATFORM_TARGET) run_kiwix_build(target, config=COMPILE_CONFIG)
if target == "kiwix-desktop": if target == "kiwix-desktop":
archive = create_desktop_image(make_release=False) archive = create_desktop_image(make_release=False)
else: else:
if PLATFORM_TARGET == "native_mixed" and OS_NAME == "macos": if COMPILE_CONFIG == "native_mixed" and OS_NAME == "macos":
fix_macos_rpath(target) fix_macos_rpath(target)
archive = make_archive(target, make_release=False) archive = make_archive(target, make_release=False)
if archive and DEV_BRANCH: if archive and DEV_BRANCH:

18
.github/scripts/build_release_nightly.py vendored
View File

@ -13,7 +13,7 @@ from common import (
fix_macos_rpath, fix_macos_rpath,
BASE_DIR, BASE_DIR,
OS_NAME, OS_NAME,
PLATFORM_TARGET, COMPILE_CONFIG,
MAKE_RELEASE, MAKE_RELEASE,
notarize_macos_build, notarize_macos_build,
) )
@ -21,23 +21,23 @@ from common import (
from build_definition import select_build_targets, BUILD, PUBLISH, SOURCE_PUBLISH from build_definition import select_build_targets, BUILD, PUBLISH, SOURCE_PUBLISH
# Filter what to build if we are doing a release. # Filter what to build if we are doing a release.
if MAKE_RELEASE: if MAKE_RELEASE:
TARGETS = select_build_targets(PUBLISH) TARGETS = select_build_targets(PUBLISH)
def release_filter(project): def release_filter(project):
return release_versions.get(project) is not None return release_versions.get(project) is not None
TARGETS = tuple(filter(release_filter, TARGETS)) TARGETS = tuple(filter(release_filter, TARGETS))
else: else:
TARGETS = select_build_targets(BUILD) TARGETS = select_build_targets(BUILD)
for target in TARGETS: for target in TARGETS:
run_kiwix_build(target, platform=PLATFORM_TARGET, make_release=MAKE_RELEASE) run_kiwix_build(target, config=COMPILE_CONFIG, make_release=MAKE_RELEASE)
if target == "kiwix-desktop": if target == "kiwix-desktop":
archive = create_desktop_image(make_release=MAKE_RELEASE) archive = create_desktop_image(make_release=MAKE_RELEASE)
else: else:
if OS_NAME == "macos" and PLATFORM_TARGET.endswith("_mixed"): if OS_NAME == "macos" and COMPILE_CONFIG.endswith("_mixed"):
fix_macos_rpath(target) fix_macos_rpath(target)
notarize_macos_build(target) notarize_macos_build(target)
archive = make_archive(target, make_release=MAKE_RELEASE) archive = make_archive(target, make_release=MAKE_RELEASE)
@ -57,13 +57,11 @@ if MAKE_RELEASE:
if target not in source_published_targets: if target not in source_published_targets:
continue continue
run_kiwix_build( run_kiwix_build(
target, platform=PLATFORM_TARGET, make_release=MAKE_RELEASE, make_dist=True target, config=COMPILE_CONFIG, make_release=MAKE_RELEASE, make_dist=True
) )
full_target_name = "{}-{}".format(target, main_project_versions[target]) full_target_name = "{}-{}".format(target, main_project_versions[target])
if target == "kiwix-desktop": if target == "kiwix-desktop":
archive = ( archive = BASE_DIR / full_target_name / "{}.tar.gz".format(full_target_name)
BASE_DIR / full_target_name / "{}.tar.gz".format(full_target_name)
)
else: else:
archive = ( archive = (
BASE_DIR BASE_DIR
@ -74,5 +72,5 @@ if MAKE_RELEASE:
upload_archive(archive, target, make_release=MAKE_RELEASE) upload_archive(archive, target, make_release=MAKE_RELEASE)
# Publish flathub # Publish flathub
if PLATFORM_TARGET == "flatpak" and "kiwix-desktop" in TARGETS: if COMPILE_CONFIG == "flatpak" and "kiwix-desktop" in TARGETS:
update_flathub_git() update_flathub_git()

161
.github/scripts/common.py vendored
View File

@ -20,12 +20,11 @@ from kiwixbuild.versions import (
) )
COMPILE_CONFIG = _environ["COMPILE_CONFIG"]
PLATFORM_TARGET = _environ["PLATFORM_TARGET"]
OS_NAME = _environ["OS_NAME"] OS_NAME = _environ["OS_NAME"]
HOME = Path(os.path.expanduser("~")) HOME = Path(os.path.expanduser("~"))
BASE_DIR = HOME / "BUILD_{}".format(PLATFORM_TARGET) BASE_DIR = HOME / "BUILD_{}".format(COMPILE_CONFIG)
SOURCE_DIR = HOME / "SOURCE" SOURCE_DIR = HOME / "SOURCE"
ARCHIVE_DIR = HOME / "ARCHIVE" ARCHIVE_DIR = HOME / "ARCHIVE"
TOOLCHAIN_DIR = BASE_DIR / "TOOLCHAINS" TOOLCHAIN_DIR = BASE_DIR / "TOOLCHAINS"
@ -35,7 +34,7 @@ KBUILD_SOURCE_DIR = HOME / "kiwix-build"
_ref = _environ.get("GITHUB_REF", "").split("/")[-1] _ref = _environ.get("GITHUB_REF", "").split("/")[-1]
MAKE_RELEASE = re.fullmatch(r"r_[0-9]+", _ref) is not None MAKE_RELEASE = re.fullmatch(r"r_[0-9]+", _ref) is not None
MAKE_RELEASE = MAKE_RELEASE and (_environ.get('GITHUB_EVENT_NAME') != 'schedule') MAKE_RELEASE = MAKE_RELEASE and (_environ.get("GITHUB_EVENT_NAME") != "schedule")
if not MAKE_RELEASE and _ref != "main": if not MAKE_RELEASE and _ref != "main":
DEV_BRANCH = _ref DEV_BRANCH = _ref
@ -45,19 +44,18 @@ else:
FLATPAK_HTTP_GIT_REMOTE = "https://github.com/flathub/org.kiwix.desktop.git" FLATPAK_HTTP_GIT_REMOTE = "https://github.com/flathub/org.kiwix.desktop.git"
FLATPAK_GIT_REMOTE = "git@github.com:flathub/org.kiwix.desktop.git" FLATPAK_GIT_REMOTE = "git@github.com:flathub/org.kiwix.desktop.git"
BIN_EXT = ".exe" if PLATFORM_TARGET.startswith("win32_") else "" BIN_EXT = ".exe" if COMPILE_CONFIG.startswith("win32_") else ""
def major_version(version: str) -> str: def major_version(version: str) -> str:
return version.split(".")[0] return version.split(".")[0]
# We have build everything. Now create archives for public deployement. # We have build everything. Now create archives for public deployement.
EXPORT_FILES = { EXPORT_FILES = {
"kiwix-tools": ( "kiwix-tools": (
INSTALL_DIR / "bin", INSTALL_DIR / "bin",
[ [f + BIN_EXT for f in ("kiwix-manage", "kiwix-search", "kiwix-serve")],
f + BIN_EXT
for f in ("kiwix-manage", "kiwix-search", "kiwix-serve")
],
), ),
"zim-tools": ( "zim-tools": (
INSTALL_DIR / "bin", INSTALL_DIR / "bin",
@ -72,7 +70,7 @@ EXPORT_FILES = {
"zimpatch", "zimpatch",
"zimsplit", "zimsplit",
"zimwriterfs", "zimwriterfs",
"zimrecreate" "zimrecreate",
) )
], ],
), ),
@ -80,11 +78,9 @@ EXPORT_FILES = {
INSTALL_DIR, INSTALL_DIR,
( (
# We need to package all dependencies (`*.a`) on wasm # We need to package all dependencies (`*.a`) on wasm
"lib/*/libzim.a" if PLATFORM_TARGET != "wasm" else "lib/*.a", "lib/*/libzim.a" if COMPILE_CONFIG != "wasm" else "lib/*.a",
"lib/*/libzim.so", "lib/*/libzim.so",
"lib/*/libzim.so.{version}".format( "lib/*/libzim.so.{version}".format(version=main_project_versions["libzim"]),
version=main_project_versions["libzim"]
),
"lib/*/libzim.so.{version}".format( "lib/*/libzim.so.{version}".format(
version=major_version(main_project_versions["libzim"]) version=major_version(main_project_versions["libzim"])
), ),
@ -95,9 +91,8 @@ EXPORT_FILES = {
"lib/*/libzim.pc", "lib/*/libzim.pc",
"include/zim/**/*.h", "include/zim/**/*.h",
"share/icu/{}/icudt{}l.dat".format( "share/icu/{}/icudt{}l.dat".format(
base_deps_versions["icu4c"], base_deps_versions["icu4c"], major_version(base_deps_versions["icu4c"])
major_version(base_deps_versions["icu4c"]) ),
)
), ),
), ),
"libkiwix": ( "libkiwix": (
@ -118,15 +113,15 @@ EXPORT_FILES = {
"lib/*/libkiwix.pc", "lib/*/libkiwix.pc",
"include/kiwix/**/*.h", "include/kiwix/**/*.h",
"share/icu/{}/icudt{}l.dat".format( "share/icu/{}/icudt{}l.dat".format(
base_deps_versions["icu4c"], base_deps_versions["icu4c"], major_version(base_deps_versions["icu4c"])
major_version(base_deps_versions["icu4c"]) ),
)
), ),
), ),
} }
DATE = date.today().isoformat() DATE = date.today().isoformat()
def print_message(message, *args, **kwargs): def print_message(message, *args, **kwargs):
message = message.format(*args, **kwargs) message = message.format(*args, **kwargs)
message = "{0} {1} {0}".format("-" * 3, message) message = "{0} {1} {0}".format("-" * 3, message)
@ -136,23 +131,26 @@ def print_message(message, *args, **kwargs):
MANIFEST_TEMPLATE = """{archive_name} MANIFEST_TEMPLATE = """{archive_name}
*************************** ***************************
Dependencies archive for {target} on platform {platform} Dependencies archive for {target} using config {config}
Generated at {date} Generated at {date}
""" """
def write_manifest(manifest_file, archive_name, target, platform): def write_manifest(manifest_file, archive_name, target, config):
with manifest_file.open(mode="w") as f: with manifest_file.open(mode="w") as f:
f.write( f.write(
MANIFEST_TEMPLATE.format( MANIFEST_TEMPLATE.format(
archive_name=archive_name, target=target, platform=platform, date=DATE, archive_name=archive_name,
target=target,
config=config,
date=DATE,
) )
) )
def run_kiwix_build( def run_kiwix_build(
target, target,
platform, config,
build_deps_only=False, build_deps_only=False,
target_only=False, target_only=False,
make_release=False, make_release=False,
@ -163,7 +161,7 @@ def run_kiwix_build(
command.append("--hide-progress") command.append("--hide-progress")
command.append("--fast-clone") command.append("--fast-clone")
command.append("--assume-packages-installed") command.append("--assume-packages-installed")
command.extend(["--target-platform", platform]) command.extend(["--config", config])
if build_deps_only: if build_deps_only:
command.append("--build-deps-only") command.append("--build-deps-only")
if target_only: if target_only:
@ -240,7 +238,7 @@ def upload_archive(archive, project, make_release, dev_branch=None):
print_message("No archive {} to upload!", archive) print_message("No archive {} to upload!", archive)
return return
if project.startswith("kiwix-") or project in ['libkiwix']: if project.startswith("kiwix-") or project in ["libkiwix"]:
host = "ci@master.download.kiwix.org:30022" host = "ci@master.download.kiwix.org:30022"
dest_path = "/data/download/" dest_path = "/data/download/"
else: else:
@ -264,36 +262,39 @@ def upload_archive(archive, project, make_release, dev_branch=None):
# This remove "share/doc" and "share/man" from the thing to copy in the deps archive # This remove "share/doc" and "share/man" from the thing to copy in the deps archive
def filter_install_dir(path): def filter_install_dir(path):
for dir in path.glob('*'): for dir in path.glob("*"):
if dir.name not in ['share']: if dir.name not in ["share"]:
yield dir yield dir
else: else:
for sub_dir in dir.glob('*'): for sub_dir in dir.glob("*"):
if sub_dir.name not in ['doc', 'man']: if sub_dir.name not in ["doc", "man"]:
yield sub_dir yield sub_dir
# Full: True if we are creating a full archive to be used as cache by kiwix-build (base_deps2_{os}_{platform}_{base_deps_version}.tar.xz)
# Full: False if we are creating a archive to be used as pre-cached dependencies for project's CI (deps2_{os}_{platform}_{target}.tar.xz) # Full: True if we are creating a full archive to be used as cache by kiwix-build (base_deps2_{os}_{config}_{base_deps_version}.tar.xz)
# Full: False if we are creating a archive to be used as pre-cached dependencies for project's CI (deps2_{os}_{config}_{target}.tar.xz)
def make_deps_archive(target=None, name=None, full=False): def make_deps_archive(target=None, name=None, full=False):
archive_name = name or "deps2_{}_{}_{}.tar.xz".format( archive_name = name or "deps2_{}_{}_{}.tar.xz".format(
OS_NAME, PLATFORM_TARGET, target OS_NAME, COMPILE_CONFIG, target
) )
print_message("Create archive {}.", archive_name) print_message("Create archive {}.", archive_name)
files_to_archive = list(filter_install_dir(INSTALL_DIR)) files_to_archive = list(filter_install_dir(INSTALL_DIR))
files_to_archive += HOME.glob("BUILD_*/LOGS") files_to_archive += HOME.glob("BUILD_*/LOGS")
if PLATFORM_TARGET == "apple_all_static": if COMPILE_CONFIG == "apple_all_static":
for subplatform in AppleXCFramework.subPlatformNames: for subconfig in AppleXCFramework.subConfigNames:
base_dir = HOME / "BUILD_{}".format(subplatform) base_dir = HOME / "BUILD_{}".format(subconfig)
files_to_archive += filter_install_dir(base_dir / "INSTALL") files_to_archive += filter_install_dir(base_dir / "INSTALL")
if (base_dir / "meson_cross_file.txt").exists(): if (base_dir / "meson_cross_file.txt").exists():
files_to_archive.append(base_dir / "meson_cross_file.txt") files_to_archive.append(base_dir / "meson_cross_file.txt")
if PLATFORM_TARGET.endswith("_mixed"): if COMPILE_CONFIG.endswith("_mixed"):
static_platform = PLATFORM_TARGET.replace("_mixed", "_static") static_config = COMPILE_CONFIG.replace("_mixed", "_static")
files_to_archive += filter_install_dir(HOME / ("BUILD_" + static_platform) / "INSTALL") files_to_archive += filter_install_dir(
if PLATFORM_TARGET.startswith("android_"): HOME / ("BUILD_" + static_config) / "INSTALL"
)
if COMPILE_CONFIG.startswith("android_"):
files_to_archive += filter_install_dir(HOME / "BUILD_neutral" / "INSTALL") files_to_archive += filter_install_dir(HOME / "BUILD_neutral" / "INSTALL")
base_dir = HOME / "BUILD_{}".format(PLATFORM_TARGET) base_dir = HOME / "BUILD_{}".format(COMPILE_CONFIG)
if (base_dir / "meson_cross_file.txt").exists(): if (base_dir / "meson_cross_file.txt").exists():
files_to_archive.append(base_dir / "meson_cross_file.txt") files_to_archive.append(base_dir / "meson_cross_file.txt")
# Copy any toolchain # Copy any toolchain
@ -303,7 +304,7 @@ def make_deps_archive(target=None, name=None, full=False):
files_to_archive.append(BASE_DIR / "meson_cross_file.txt") files_to_archive.append(BASE_DIR / "meson_cross_file.txt")
manifest_file = BASE_DIR / "manifest.txt" manifest_file = BASE_DIR / "manifest.txt"
write_manifest(manifest_file, archive_name, target, PLATFORM_TARGET) write_manifest(manifest_file, archive_name, target, COMPILE_CONFIG)
files_to_archive.append(manifest_file) files_to_archive.append(manifest_file)
relative_path = HOME relative_path = HOME
@ -311,9 +312,9 @@ def make_deps_archive(target=None, name=None, full=False):
files_to_archive += ARCHIVE_DIR.glob(".*_ok") files_to_archive += ARCHIVE_DIR.glob(".*_ok")
files_to_archive += BASE_DIR.glob("*/.*_ok") files_to_archive += BASE_DIR.glob("*/.*_ok")
# Add also static build for mixed target # Add also static build for mixed target
if PLATFORM_TARGET.endswith("_mixed"): if COMPILE_CONFIG.endswith("_mixed"):
static_platform = PLATFORM_TARGET.replace("_mixed", "_static") static_config = COMPILE_CONFIG.replace("_mixed", "_static")
files_to_archive += (HOME / ("BUILD_" + static_platform)).glob("*/.*_ok") files_to_archive += (HOME / ("BUILD_" + static_config)).glob("*/.*_ok")
# Native dyn and static is needed for potential cross compilation that use native tools (icu) # Native dyn and static is needed for potential cross compilation that use native tools (icu)
files_to_archive += (HOME / "BUILD_native_dyn").glob("*/.*_ok") files_to_archive += (HOME / "BUILD_native_dyn").glob("*/.*_ok")
files_to_archive += (HOME / "BUILD_native_static").glob("*/.*_ok") files_to_archive += (HOME / "BUILD_native_static").glob("*/.*_ok")
@ -391,7 +392,7 @@ def create_desktop_image(make_release):
postfix = DATE postfix = DATE
src_dir = SOURCE_DIR / "kiwix-desktop" src_dir = SOURCE_DIR / "kiwix-desktop"
if PLATFORM_TARGET == "flatpak": if COMPILE_CONFIG == "flatpak":
build_path = BASE_DIR / "org.kiwix.desktop.flatpak" build_path = BASE_DIR / "org.kiwix.desktop.flatpak"
app_name = "org.kiwix.desktop.{}.flatpak".format(postfix) app_name = "org.kiwix.desktop.{}.flatpak".format(postfix)
print_message("archive is {}", build_path) print_message("archive is {}", build_path)
@ -459,7 +460,6 @@ def update_flathub_git():
def fix_macos_rpath(project): def fix_macos_rpath(project):
base_dir, export_files = EXPORT_FILES[project] base_dir, export_files = EXPORT_FILES[project]
for file in filter(lambda f: f.endswith(".dylib"), export_files): for file in filter(lambda f: f.endswith(".dylib"), export_files):
lib = base_dir / file lib = base_dir / file
@ -478,22 +478,35 @@ def trigger_workflow(repo, workflow="docker.yml", ref="main", inputs=None):
ref: branch or tag name ref: branch or tag name
inputs: dict of inputs to pass to the workflow""" inputs: dict of inputs to pass to the workflow"""
print_message( print_message(
"triggering workflow `{workflow}` on {repo}@{ref} " "triggering workflow `{workflow}` on {repo}@{ref} " "with inputs={inputs}",
"with inputs={inputs}", workflow=workflow, repo=repo, ref=ref, inputs=inputs) workflow=workflow,
repo=repo,
ref=ref,
inputs=inputs,
)
url = "{base_url}/repos/{repo}/actions/workflows/{workflow}/dispatches".format( url = "{base_url}/repos/{repo}/actions/workflows/{workflow}/dispatches".format(
base_url=os.getenv("GITHUB_API_URL", "https://api.github.com"), base_url=os.getenv("GITHUB_API_URL", "https://api.github.com"),
repo=repo, workflow=workflow) repo=repo,
workflow=workflow,
)
resp = requests.post(url, headers={ resp = requests.post(
url,
headers={
"Content-Type": "application/json", "Content-Type": "application/json",
"Authorization": "token {token}".format( "Authorization": "token {token}".format(token=os.getenv("GITHUB_PAT", "")),
token=os.getenv('GITHUB_PAT', '')),
"Accept": "application/vnd.github.v3+json", "Accept": "application/vnd.github.v3+json",
}, json={"ref": ref, "inputs": inputs}, timeout=5) },
json={"ref": ref, "inputs": inputs},
timeout=5,
)
if resp.status_code != 204: if resp.status_code != 204:
raise ValueError("Unexpected HTTP {code}: {reason}".format( raise ValueError(
code=resp.status_code, reason=resp.reason)) "Unexpected HTTP {code}: {reason}".format(
code=resp.status_code, reason=resp.reason
)
)
def trigger_docker_publish(target): def trigger_docker_publish(target):
@ -501,13 +514,14 @@ def trigger_docker_publish(target):
return return
version = get_postfix(target) version = get_postfix(target)
repo = { repo = {"zim-tools": "openzim/zim-tools", "kiwix-tools": "kiwix/kiwix-tools"}.get(
"zim-tools": "openzim/zim-tools", target
"kiwix-tools": "kiwix/kiwix-tools"}.get(target) )
try: try:
trigger_workflow(repo, workflow="docker.yml", ref="main", trigger_workflow(
inputs={"version": version}) repo, workflow="docker.yml", ref="main", inputs={"version": version}
)
print_message("triggered docker workflow on {repo}", repo=repo) print_message("triggered docker workflow on {repo}", repo=repo)
except Exception as exc: except Exception as exc:
print_message("Error triggering workflow: {exc}", exc=exc) print_message("Error triggering workflow: {exc}", exc=exc)
@ -528,26 +542,39 @@ def notarize_macos_build(project):
# currently only supports libzim use case: sign every dylib # currently only supports libzim use case: sign every dylib
base_dir, export_files = EXPORT_FILES[project] base_dir, export_files = EXPORT_FILES[project]
filepaths = [base_dir.joinpath(file) filepaths = [
base_dir.joinpath(file)
for file in filter(lambda f: f.endswith(".dylib"), export_files) for file in filter(lambda f: f.endswith(".dylib"), export_files)
if not base_dir.joinpath(file).is_symlink()] if not base_dir.joinpath(file).is_symlink()
]
if not filepaths: if not filepaths:
return return
for filepath in filepaths: for filepath in filepaths:
subprocess.check_call(["/usr/bin/codesign", "--force", "--sign", subprocess.check_call(
[
"/usr/bin/codesign",
"--force",
"--sign",
os.getenv("SIGNING_IDENTITY", "no-signing-ident"), os.getenv("SIGNING_IDENTITY", "no-signing-ident"),
"--keychain", "--keychain",
os.getenv("KEYCHAIN", "no-keychain-path"), os.getenv("KEYCHAIN", "no-keychain-path"),
str(filepath), "--deep", "--timestamp"], env=os.environ) str(filepath),
"--deep",
"--timestamp",
],
env=os.environ,
)
# create a zip of the dylibs and upload for notarization # create a zip of the dylibs and upload for notarization
zip_name = "{}.zip".format(project) zip_name = "{}.zip".format(project)
subprocess.check_call( subprocess.check_call(
["/usr/bin/ditto", "-c", "-k", "--keepParent"] ["/usr/bin/ditto", "-c", "-k", "--keepParent"]
+ [str(f) for f in filepaths] + [zip_name], + [str(f) for f in filepaths]
env=os.environ) + [zip_name],
env=os.environ,
)
# make sure keychain is unlocked # make sure keychain is unlocked
subprocess.check_call( subprocess.check_call(

4
.github/scripts/compile_all_deps.py vendored
View File

@ -6,13 +6,13 @@ from common import (
run_kiwix_build, run_kiwix_build,
make_deps_archive, make_deps_archive,
upload, upload,
PLATFORM_TARGET, COMPILE_CONFIG,
DEV_BRANCH, DEV_BRANCH,
) )
from build_definition import select_build_targets, DEPS from build_definition import select_build_targets, DEPS
for target in select_build_targets(DEPS): for target in select_build_targets(DEPS):
run_kiwix_build(target, platform=PLATFORM_TARGET, build_deps_only=True) run_kiwix_build(target, config=COMPILE_CONFIG, build_deps_only=True)
archive_file = make_deps_archive(target=target) archive_file = make_deps_archive(target=target)
if DEV_BRANCH: if DEV_BRANCH:
destination = "/data/tmp/ci/dev_preview/" + DEV_BRANCH destination = "/data/tmp/ci/dev_preview/" + DEV_BRANCH

14
.github/scripts/ensure_base_deps.py vendored
View File

@ -13,10 +13,11 @@ from common import (
upload, upload,
make_deps_archive, make_deps_archive,
HOME, HOME,
PLATFORM_TARGET, COMPILE_CONFIG,
OS_NAME, OS_NAME,
) )
def download_base_archive(base_name): def download_base_archive(base_name):
url = "http://tmp.kiwix.org/ci/{}".format(base_name) url = "http://tmp.kiwix.org/ci/{}".format(base_name)
file_path = str(HOME / base_name) file_path = str(HOME / base_name)
@ -30,14 +31,15 @@ def download_base_archive(base_name):
file.write(batch) file.write(batch)
return file_path return file_path
ARCHIVE_NAME_TEMPLATE = "base_deps2_{os}_{platform}_{version}.tar.xz"
if PLATFORM_TARGET == 'flatpak': ARCHIVE_NAME_TEMPLATE = "base_deps2_{os}_{config}_{version}.tar.xz"
if COMPILE_CONFIG == "flatpak":
base_dep_archive_name = "base_deps2_flatpak.tar.xz" base_dep_archive_name = "base_deps2_flatpak.tar.xz"
else: else:
base_dep_archive_name = ARCHIVE_NAME_TEMPLATE.format( base_dep_archive_name = ARCHIVE_NAME_TEMPLATE.format(
os=OS_NAME, os=OS_NAME,
platform=PLATFORM_TARGET, config=COMPILE_CONFIG,
version=base_deps_meta_version, version=base_deps_meta_version,
) )
@ -48,11 +50,11 @@ try:
f.extractall(str(HOME)) f.extractall(str(HOME))
os.remove(str(local_filename)) os.remove(str(local_filename))
except URLError: except URLError:
if PLATFORM_TARGET == "flatpak": if COMPILE_CONFIG == "flatpak":
print_message("Cannot get archive. Move on") print_message("Cannot get archive. Move on")
else: else:
print_message("Cannot get archive. Build dependencies") print_message("Cannot get archive. Build dependencies")
run_kiwix_build("alldependencies", platform=PLATFORM_TARGET) run_kiwix_build("alldependencies", config=COMPILE_CONFIG)
archive_file = make_deps_archive(name=base_dep_archive_name, full=True) archive_file = make_deps_archive(name=base_dep_archive_name, full=True)
upload(archive_file, "ci@tmp.kiwix.org:30022", "/data/tmp/ci") upload(archive_file, "ci@tmp.kiwix.org:30022", "/data/tmp/ci")
os.remove(str(archive_file)) os.remove(str(archive_file))

2
.github/scripts/upload_failure_logs.sh vendored
View File

@ -4,7 +4,7 @@ set -e
cd $HOME cd $HOME
ARCHIVE_NAME=fail_log_${OS_NAME}_${PLATFORM_TARGET}.tar.gz ARCHIVE_NAME=fail_log_${OS_NAME}_${COMPILE_CONFIG}.tar.gz
tar -czf ${ARCHIVE_NAME} $HOME/BUILD_* $HOME/SOURCE $HOME/LOGS $HOME/TOOLCHAINS tar -czf ${ARCHIVE_NAME} $HOME/BUILD_* $HOME/SOURCE $HOME/LOGS $HOME/TOOLCHAINS
echo "Uploading archive $ARCHIVE_NAME" echo "Uploading archive $ARCHIVE_NAME"

30
.github/workflows/ci.yml vendored
View File

@ -10,7 +10,7 @@ jobs:
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
target: config:
- native_static - native_static
- native_dyn - native_dyn
- native_mixed - native_mixed
@ -37,13 +37,13 @@ jobs:
- android_x86_64 - android_x86_64
image_variant: ['focal'] image_variant: ['focal']
include: include:
- target: native_mixed - config: native_mixed
image_variant: bionic image_variant: bionic
- target: aarch64_mixed - config: aarch64_mixed
image_variant: bionic image_variant: bionic
- target: win32_static - config: win32_static
image_variant: f35 image_variant: f35
- target: win32_dyn - config: win32_dyn
image_variant: f35 image_variant: f35
env: env:
HOME: /home/runner HOME: /home/runner
@ -74,26 +74,26 @@ jobs:
cd $HOME cd $HOME
kiwix-build/.github/scripts/ensure_base_deps.py kiwix-build/.github/scripts/ensure_base_deps.py
env: env:
PLATFORM_TARGET: ${{matrix.target}} COMPILE_CONFIG: ${{matrix.config}}
- name: Compile all deps - name: Compile all deps
shell: bash shell: bash
run: | run: |
cd $HOME cd $HOME
kiwix-build/.github/scripts/compile_all_deps.py kiwix-build/.github/scripts/compile_all_deps.py
env: env:
PLATFORM_TARGET: ${{matrix.target}} COMPILE_CONFIG: ${{matrix.config}}
- name: Build projects - name: Build projects
shell: bash shell: bash
run: | run: |
cd $HOME cd $HOME
kiwix-build/.github/scripts/build_projects.py kiwix-build/.github/scripts/build_projects.py
env: env:
PLATFORM_TARGET: ${{matrix.target}} COMPILE_CONFIG: ${{matrix.config}}
- name: Upload failure logs - name: Upload failure logs
if: failure() if: failure()
run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh
env: env:
PLATFORM_TARGET: ${{matrix.target}} COMPILE_CONFIG: ${{matrix.config}}
Flatpak: Flatpak:
strategy: strategy:
@ -101,7 +101,7 @@ jobs:
env: env:
HOME: /home/runner HOME: /home/runner
SSH_KEY: /tmp/id_rsa SSH_KEY: /tmp/id_rsa
PLATFORM_TARGET: flatpak COMPILE_CONFIG: flatpak
OS_NAME: focal OS_NAME: focal
runs-on: ubuntu-22.04 runs-on: ubuntu-22.04
steps: steps:
@ -142,7 +142,7 @@ jobs:
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
target: config:
- native_dyn - native_dyn
- native_static - native_static
- native_mixed - native_mixed
@ -185,23 +185,23 @@ jobs:
cd $HOME cd $HOME
$GITHUB_WORKSPACE/.github/scripts/ensure_base_deps.py $GITHUB_WORKSPACE/.github/scripts/ensure_base_deps.py
env: env:
PLATFORM_TARGET: ${{matrix.target}} COMPILE_CONFIG: ${{matrix.config}}
- name: Compile all deps - name: Compile all deps
shell: bash shell: bash
run: | run: |
cd $HOME cd $HOME
$GITHUB_WORKSPACE/.github/scripts/compile_all_deps.py $GITHUB_WORKSPACE/.github/scripts/compile_all_deps.py
env: env:
PLATFORM_TARGET: ${{matrix.target}} COMPILE_CONFIG: ${{matrix.config}}
- name: Build projects - name: Build projects
shell: bash shell: bash
run: | run: |
cd $HOME cd $HOME
$GITHUB_WORKSPACE/.github/scripts/build_projects.py $GITHUB_WORKSPACE/.github/scripts/build_projects.py
env: env:
PLATFORM_TARGET: ${{matrix.target}} COMPILE_CONFIG: ${{matrix.config}}
- name: Upload failure logs - name: Upload failure logs
if: failure() if: failure()
run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.sh run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.sh
env: env:
PLATFORM_TARGET: ${{matrix.target}} COMPILE_CONFIG: ${{matrix.config}}

26
.github/workflows/releaseNigthly.yml vendored
View File

@ -12,7 +12,7 @@ jobs:
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
target: config:
- native_static - native_static
- native_mixed - native_mixed
- native_dyn - native_dyn
@ -34,11 +34,11 @@ jobs:
- android_x86_64 - android_x86_64
image_variant: ['focal'] image_variant: ['focal']
include: include:
- target: native_mixed - config: native_mixed
image_variant: bionic image_variant: bionic
- target: aarch64_mixed - config: aarch64_mixed
image_variant: bionic image_variant: bionic
- target: win32_static - config: win32_static
image_variant: f35 image_variant: f35
env: env:
HOME: /home/runner HOME: /home/runner
@ -69,21 +69,21 @@ jobs:
cd $HOME cd $HOME
kiwix-build/.github/scripts/ensure_base_deps.py kiwix-build/.github/scripts/ensure_base_deps.py
env: env:
PLATFORM_TARGET: ${{matrix.target}} COMPILE_CONFIG: ${{matrix.config}}
- name: Build release - name: Build release
shell: bash shell: bash
run: | run: |
cd $HOME cd $HOME
kiwix-build/.github/scripts/build_release_nightly.py kiwix-build/.github/scripts/build_release_nightly.py
env: env:
PLATFORM_TARGET: ${{matrix.target}} COMPILE_CONFIG: ${{matrix.config}}
BINTRAY_USER: kiwix BINTRAY_USER: kiwix
BINTRAY_PASS: ${{secrets.bintray_pass}} BINTRAY_PASS: ${{secrets.bintray_pass}}
- name: Upload failure logs - name: Upload failure logs
if: failure() if: failure()
run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh
env: env:
PLATFORM_TARGET: ${{matrix.target}} COMPILE_CONFIG: ${{matrix.config}}
Flatpak: Flatpak:
strategy: strategy:
@ -91,7 +91,7 @@ jobs:
env: env:
HOME: /home/runner HOME: /home/runner
SSH_KEY: /tmp/id_rsa SSH_KEY: /tmp/id_rsa
PLATFORM_TARGET: flatpak COMPILE_CONFIG: flatpak
OS_NAME: focal OS_NAME: focal
runs-on: ubuntu-22.04 runs-on: ubuntu-22.04
steps: steps:
@ -132,7 +132,7 @@ jobs:
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
target: config:
- native_dyn - native_dyn
- native_static - native_static
- native_mixed - native_mixed
@ -195,26 +195,26 @@ jobs:
cd $HOME cd $HOME
$GITHUB_WORKSPACE/.github/scripts/ensure_base_deps.py $GITHUB_WORKSPACE/.github/scripts/ensure_base_deps.py
env: env:
PLATFORM_TARGET: ${{matrix.target}} COMPILE_CONFIG: ${{matrix.config}}
- name: Build release - name: Build release
shell: bash shell: bash
run: | run: |
cd $HOME cd $HOME
$GITHUB_WORKSPACE/.github/scripts/build_release_nightly.py $GITHUB_WORKSPACE/.github/scripts/build_release_nightly.py
env: env:
PLATFORM_TARGET: ${{matrix.target}} COMPILE_CONFIG: ${{matrix.config}}
- name: Upload failure logs - name: Upload failure logs
if: failure() if: failure()
run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.sh run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.sh
env: env:
PLATFORM_TARGET: ${{matrix.target}} COMPILE_CONFIG: ${{matrix.config}}
Trigger_Docker: Trigger_Docker:
needs: [Linux] needs: [Linux]
runs-on: ubuntu-22.04 runs-on: ubuntu-22.04
env: env:
PLATFORM_TARGET: native_static COMPILE_CONFIG: native_static
OS_NAME: linux OS_NAME: linux
steps: steps:
- name: Checkout code - name: Checkout code

50
README.md
View File

@ -78,12 +78,12 @@ invalid choice: 'not-existing-target' (choose from 'alldependencies', 'android-n
... ...
``` ```
#### Target platform #### Config
If no target platform is specified, the default will be `native_dyn`. If no config is specified, the default will be `native_dyn`.
You can select another target platform using the option You can select another config using the option
`--target-platform`. For now, there is ten different supported `--config`. For now, there is ten different supported
platforms: platforms:
- native_dyn - native_dyn
@ -98,9 +98,12 @@ platforms:
- android_x86_64 - android_x86_64
- flatpak - flatpak
So, if you want to compile `kiwix-tools` for win32 using static linkage: All `native_*` config means using the native compiler without any cross-compilation option.
Other may simply use cross-compilation or may download a specific toolchain to use.
If you want to compile `kiwix-tools` for win32 using static linkage:
```bash ```bash
kiwix-build --target-platform win32_dyn kiwix-build --config win32_dyn
``` ```
Android Android
@ -108,32 +111,23 @@ Android
`kiwix-android` (https://github.com/kiwix/kiwix-android) depends of `kiwix-android` (https://github.com/kiwix/kiwix-android) depends of
the `libkiwix` project. the `libkiwix` project.
It uses a special `.aar` file that represent (and embed) the libkiwix for
all supported android arch. This is a kind of fat archive we have for MacOs.
The `.aar` file is build using the `libkiwix-app` project.
`libkiwix-app` itself is architecture independent (it is just a packaging of
other archives) but it use `libkiwix` who is architecture dependent.
When building `libkiwix`, you should directly use the When building `libkiwix`, you should directly use the
target-platform `android_<arch>`: target-platform `android_<arch>`:
```bash ```bash
kiwix-build libkiwix --target-platform android_arm kiwix-build libkiwix --config android_arm
``` ```
But, `libkiwix-app` is mainly multi arch. You may directly use the special config `android` which will build different android architectures
To compile `libkiwix-app`, you must use the `android` platform:
```bash ```bash
kiwix-build --target-platform android libkiwix-app kiwix-build --config android libkiwix
kiwix-build libkiwix-app # because `android` platform is the default for `libkiwix-app`
``` ```
By default, when using platform `android`, `libkiwix` will be build for By default, it will build for all android architecture,
all architectures. This can be changed by using the option `--android-arch`: you can limit this with option `--android-arch`:
```bash ```bash
kiwix-build libkiwix-app # aar with all architectures kiwix-build libkiwix --config android --android-arch arm # aar with arm architecture
kiwix-build libkiwix-app --android-arch arm # aar with arm architecture kiwix-build libkiwix --config android --android-arch arm --android-arch arm64 # aan with arm and arm64 architectures
kiwix-build libkiwix-app --android-arch arm --android-arch arm64 # aan with arm and arm64 architectures
``` ```
To build `kiwix-android` itself, you should see the documentation of `kiwix-android`. To build `kiwix-android` itself, you should see the documentation of `kiwix-android`.
@ -148,13 +142,13 @@ To do so, you should directly use the target-platfrom `ios_multi`.
As for `android`, `kiwix-build` will build the library several times As for `android`, `kiwix-build` will build the library several times
(once for each platform) and then create the fat library. (once for each platform) and then create the fat library.
```bash ```bash
kiwix-build --target-platform iOS_multi libkiwix kiwix-build --config iOS_multi libkiwix
``` ```
You can specify the supported architectures with the option `--ios-arch`: You can specify the supported architectures with the option `--ios-arch`:
```bash ```bash
kiwix-build --target-platform iOS_multi libkiwix # all architetures kiwix-build --config iOS_multi libkiwix # all architetures
kiwix-build --target-platform iOS_multi --ios-arch arm --ios-arch arm64 # arm and arm64 arch only kiwix-build --config iOS_multi --ios-arch arm --ios-arch arm64 # arm and arm64 arch only
``` ```
Outputs Outputs
@ -163,9 +157,9 @@ Outputs
Kiwix-build.py will create several directories: Kiwix-build.py will create several directories:
- `ARCHIVES`: All the downloaded archives go there. - `ARCHIVES`: All the downloaded archives go there.
- `SOURCES`: All the sources (extracted from archives and patched) go there. - `SOURCES`: All the sources (extracted from archives and patched) go there.
- `BUILD_<target_platform>`: All the build files go there. - `BUILD_<config>`: All the build files go there.
- `BUILD_<target_platform>/INSTALL`: The installed files go there. - `BUILD_<config>/INSTALL`: The installed files go there.
- `BUILD_<target_platform>/LOGS`: The logs files of the build. - `BUILD_<config>/LOGS`: The logs files of the build.
If you want to install all those directories elsewhere, you can pass the If you want to install all those directories elsewhere, you can pass the
`--working-dir` option to `kiwix-build`: `--working-dir` option to `kiwix-build`:

168
kiwixbuild/__init__.py
View File

@ -4,81 +4,145 @@ import os, sys
import argparse import argparse
from .dependencies import Dependency from .dependencies import Dependency
from .platforms import PlatformInfo from .configs import ConfigInfo
from .builder import Builder from .builder import Builder
from .flatpak_builder import FlatpakBuilder from .flatpak_builder import FlatpakBuilder
from . import _global from . import _global
def parse_args(): def parse_args():
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('target', default='kiwix-tools', nargs='?', metavar='TARGET', parser.add_argument(
choices=Dependency.all_deps.keys()) "target",
parser.add_argument('--working-dir', default=".", default="kiwix-tools",
help=("Directory where kiwix-build puts all its files " nargs="?",
metavar="TARGET",
choices=Dependency.all_deps.keys(),
)
parser.add_argument(
"--working-dir",
default=".",
help=(
"Directory where kiwix-build puts all its files "
"(source, archive and build)\n" "(source, archive and build)\n"
"working-dir can be absolute path or a relative (to cwd) one.")) "working-dir can be absolute path or a relative (to cwd) one."
parser.add_argument('--build-dir', default=".", ),
help=("Directory where kiwix-build puts all build files.\n" )
"build-dir can be absolute path or a relative (to working-dir) one.")) parser.add_argument(
parser.add_argument('--libprefix', default=None) "--build-dir",
parser.add_argument('--target-platform', choices=PlatformInfo.all_platforms) default=".",
parser.add_argument('--verbose', '-v', action="store_true", help=(
help=("Print all logs on stdout instead of in specific" "Directory where kiwix-build puts all build files.\n"
" log files per commands")) "build-dir can be absolute path or a relative (to working-dir) one."
parser.add_argument('--hide-progress', action='store_false', dest='show_progress', ),
help="Hide intermediate progress information.") )
parser.add_argument('--skip-source-prepare', action='store_true', parser.add_argument("--libprefix", default=None)
help="Skip the source download part") parser.add_argument(
parser.add_argument('--build-deps-only', action='store_true', "--config", choices=ConfigInfo.all_configs, default="native_dyn"
help="Build only the dependencies of the specified target.") )
parser.add_argument('--build-nodeps', action='store_true', parser.add_argument(
help="Build only the target, not its dependencies.") "--verbose",
parser.add_argument('--make-dist', action='store_true', "-v",
help="Build distrubution (dist) source archive") action="store_true",
parser.add_argument('--make-release', action='store_true', help=(
help="Build a release version") "Print all logs on stdout instead of in specific" " log files per commands"
subgroup = parser.add_argument_group('advanced') ),
subgroup.add_argument('--no-cert-check', action='store_true', )
help="Skip SSL certificate verification during download") parser.add_argument(
subgroup.add_argument('--clean-at-end', action='store_true', "--hide-progress",
help="Clean all intermediate files after the (successfull) build") action="store_false",
subgroup.add_argument('--dont-install-packages', action='store_true', dest="show_progress",
help="Do not try to install packages before compiling") help="Hide intermediate progress information.",
subgroup.add_argument('--assume-packages-installed', action='store_true', )
help="Assume the package to install to be aleady installed") parser.add_argument(
subgroup.add_argument('--android-arch', action='append', "--skip-source-prepare",
help=("Specify the architecture to build for android application/libraries.\n" action="store_true",
help="Skip the source download part",
)
parser.add_argument(
"--build-deps-only",
action="store_true",
help="Build only the dependencies of the specified target.",
)
parser.add_argument(
"--build-nodeps",
action="store_true",
help="Build only the target, not its dependencies.",
)
parser.add_argument(
"--make-dist",
action="store_true",
help="Build distrubution (dist) source archive",
)
parser.add_argument(
"--make-release", action="store_true", help="Build a release version"
)
subgroup = parser.add_argument_group("advanced")
subgroup.add_argument(
"--no-cert-check",
action="store_true",
help="Skip SSL certificate verification during download",
)
subgroup.add_argument(
"--clean-at-end",
action="store_true",
help="Clean all intermediate files after the (successfull) build",
)
subgroup.add_argument(
"--dont-install-packages",
action="store_true",
help="Do not try to install packages before compiling",
)
subgroup.add_argument(
"--assume-packages-installed",
action="store_true",
help="Assume the package to install to be aleady installed",
)
subgroup.add_argument(
"--android-arch",
action="append",
help=(
"Specify the architecture to build for android application/libraries.\n"
"Can be specified several times to build for several architectures.\n" "Can be specified several times to build for several architectures.\n"
"If not specified, all architectures will be build.")) "If not specified, all architectures will be build."
subgroup.add_argument('--ios-arch', action='append', ),
help=("Specify the architecture to build for ios application/libraries.\n" )
subgroup.add_argument(
"--ios-arch",
action="append",
help=(
"Specify the architecture to build for ios application/libraries.\n"
"Can be specified several times to build for several architectures.\n" "Can be specified several times to build for several architectures.\n"
"If not specified, all architectures will be build.")) "If not specified, all architectures will be build."
subgroup.add_argument('--fast-clone', action='store_true', ),
help=("Do not clone the whole repository.\n" )
subgroup.add_argument(
"--fast-clone",
action="store_true",
help=(
"Do not clone the whole repository.\n"
"This is useful for one shot build but it is not recommended if you want " "This is useful for one shot build but it is not recommended if you want "
"to develop with the cloned sources.")) "to develop with the cloned sources."
),
)
options = parser.parse_args() options = parser.parse_args()
if not options.android_arch: if not options.android_arch:
options.android_arch = ['arm', 'arm64', 'x86', 'x86_64'] options.android_arch = ["arm", "arm64", "x86", "x86_64"]
if not options.ios_arch: if not options.ios_arch:
options.ios_arch = ['arm64', 'x86_64'] options.ios_arch = ["arm64", "x86_64"]
if not options.target_platform:
options.target_platform = 'native_dyn'
return options return options
def main(): def main():
options = parse_args() options = parse_args()
options.working_dir = os.path.abspath(options.working_dir) options.working_dir = os.path.abspath(options.working_dir)
_global.set_options(options) _global.set_options(options)
neutralEnv = buildenv.PlatformNeutralEnv() neutralEnv = buildenv.NeutralEnv()
_global.set_neutralEnv(neutralEnv) _global.set_neutralEnv(neutralEnv)
if options.target_platform == 'flatpak': if options.config == "flatpak":
builder = FlatpakBuilder() builder = FlatpakBuilder()
else: else:
builder = Builder() builder = Builder()
builder.run() builder.run()

26
kiwixbuild/_global.py
View File

@ -5,23 +5,29 @@ _neutralEnv = None
_options = None _options = None
_target_steps = _OrderedDict() _target_steps = _OrderedDict()
def set_neutralEnv(env): def set_neutralEnv(env):
global _neutralEnv global _neutralEnv
_neutralEnv = env _neutralEnv = env
def neutralEnv(what): def neutralEnv(what):
return getattr(_neutralEnv, what) return getattr(_neutralEnv, what)
def set_options(options): def set_options(options):
global _options global _options
_options = options _options = options
def option(what): def option(what):
return getattr(_options, what) return getattr(_options, what)
def add_target_step(key, what): def add_target_step(key, what):
_target_steps[key] = what _target_steps[key] = what
def get_target_step(key, default_context=None): def get_target_step(key, default_context=None):
if isinstance(key, tuple): if isinstance(key, tuple):
context, target = key context, target = key
@ -29,24 +35,6 @@ def get_target_step(key, default_context=None):
context, target = default_context, key context, target = default_context, key
return _target_steps[(context, target)] return _target_steps[(context, target)]
def target_steps(): def target_steps():
return _target_steps return _target_steps
def backend():
global _backend
if _backend is not None:
return _backend
_platform = platform.system()
if _platform == 'Windows':
print('ERROR: kiwix-build is not intented to run on Windows platform.\n'
'There is no backend for Windows, so we can\'t launch any commands.')
sys.exit(0)
if _platform == 'Linux':
_platform, _, _ = platform.linux_distribution()
_platform = _platform.lower()
_backend = backends.Linux()
return _backend

168
kiwixbuild/buildenv.py
View File

@ -1,4 +1,3 @@
import os, sys, shutil import os, sys, shutil
import subprocess import subprocess
import platform import platform
@ -8,62 +7,60 @@ from .utils import pj, download_remote, escape_path
from ._global import neutralEnv, option from ._global import neutralEnv, option
class PlatformNeutralEnv: class NeutralEnv:
def __init__(self): def __init__(self):
self.working_dir = option('working_dir') self.working_dir = option("working_dir")
self.source_dir = pj(self.working_dir, "SOURCE") self.source_dir = pj(self.working_dir, "SOURCE")
self.archive_dir = pj(self.working_dir, "ARCHIVE") self.archive_dir = pj(self.working_dir, "ARCHIVE")
self.toolchain_dir = pj(self.working_dir, "TOOLCHAINS") self.toolchain_dir = pj(self.working_dir, "TOOLCHAINS")
self.log_dir = pj(self.working_dir, 'LOGS') self.log_dir = pj(self.working_dir, "LOGS")
for d in (self.source_dir, for d in (self.source_dir, self.archive_dir, self.toolchain_dir, self.log_dir):
self.archive_dir,
self.toolchain_dir,
self.log_dir):
os.makedirs(d, exist_ok=True) os.makedirs(d, exist_ok=True)
self.detect_platform() self.detect_platform()
self.ninja_command = self._detect_command( self.ninja_command = self._detect_command(
'ninja', "ninja", default=[["ninja"], ["ninja-build"]]
default=[['ninja'], ['ninja-build']]) )
self.meson_command = self._detect_command( self.meson_command = self._detect_command(
'meson', "meson", default=[["meson.py"], ["meson"]]
default= [['meson.py'], ['meson']]) )
self.mesontest_command = [*self.meson_command, "test"] self.mesontest_command = [*self.meson_command, "test"]
self.patch_command = self._detect_command('patch') self.patch_command = self._detect_command("patch")
self.git_command = self._detect_command('git') self.git_command = self._detect_command("git")
self.make_command = self._detect_command('make') self.make_command = self._detect_command("make")
self.cmake_command = self._detect_command('cmake') self.cmake_command = self._detect_command("cmake")
self.qmake_command = self._detect_command('qmake', required=False) self.qmake_command = self._detect_command("qmake", required=False)
def detect_platform(self): def detect_platform(self):
_platform = platform.system() _platform = platform.system()
self.distname = _platform self.distname = _platform
if _platform == 'Windows': if _platform == "Windows":
print('ERROR: kiwix-build is not intented to run on Windows platform.\n' print(
'It should probably not work, but well, you still can have a try.') "ERROR: kiwix-build is not intented to run on Windows platform.\n"
cont = input('Do you want to continue ? [y/N]') "It should probably not work, but well, you still can have a try."
if cont.lower() != 'y': )
cont = input("Do you want to continue ? [y/N]")
if cont.lower() != "y":
sys.exit(0) sys.exit(0)
if _platform == 'Linux': if _platform == "Linux":
self.distname = distro.id() self.distname = distro.id()
if self.distname == 'ubuntu': if self.distname == "ubuntu":
self.distname = 'debian' self.distname = "debian"
def download(self, what, where=None): def download(self, what, where=None):
where = where or self.archive_dir where = where or self.archive_dir
download_remote(what, where) download_remote(what, where)
def _detect_command(self, name, default=None, options=["--version"], required=True):
def _detect_command(self, name, default=None, options=['--version'], required=True):
if default is None: if default is None:
default = [[name]] default = [[name]]
env_key = 'KBUILD_{}_COMMAND'.format(name.upper()) env_key = "KBUILD_{}_COMMAND".format(name.upper())
if env_key in os.environ: if env_key in os.environ:
default = [os.environ[env_key].split()] + default default = [os.environ[env_key].split()] + default
for command in default: for command in default:
try: try:
retcode = subprocess.check_call(command + options, retcode = subprocess.check_call(
stdout=subprocess.DEVNULL) command + options, stdout=subprocess.DEVNULL
)
except (FileNotFoundError, PermissionError, OSError): except (FileNotFoundError, PermissionError, OSError):
# Doesn't exist in PATH or isn't executable # Doesn't exist in PATH or isn't executable
continue continue
@ -76,22 +73,20 @@ class PlatformNeutralEnv:
print("WARNING: {} command not found".format(name)) print("WARNING: {} command not found".format(name))
return ["{}_NOT_FOUND".format(name.upper())] return ["{}_NOT_FOUND".format(name.upper())]
class BuildEnv: class BuildEnv:
def __init__(self, platformInfo): def __init__(self, configInfo):
build_dir = "BUILD_{}".format(platformInfo.name) build_dir = "BUILD_{}".format(configInfo.name)
self.platformInfo = platformInfo self.configInfo = configInfo
self.base_build_dir = pj(option('working_dir'), option('build_dir')) self.base_build_dir = pj(option("working_dir"), option("build_dir"))
self.build_dir = pj(self.base_build_dir, build_dir) self.build_dir = pj(self.base_build_dir, build_dir)
self.install_dir = pj(self.build_dir, "INSTALL") self.install_dir = pj(self.build_dir, "INSTALL")
self.toolchain_dir = pj(self.build_dir, "TOOLCHAINS") self.toolchain_dir = pj(self.build_dir, "TOOLCHAINS")
self.log_dir = pj(self.build_dir, 'LOGS') self.log_dir = pj(self.build_dir, "LOGS")
for d in (self.build_dir, for d in (self.build_dir, self.install_dir, self.toolchain_dir, self.log_dir):
self.install_dir,
self.toolchain_dir,
self.log_dir):
os.makedirs(d, exist_ok=True) os.makedirs(d, exist_ok=True)
self.libprefix = option('libprefix') or self._detect_libdir() self.libprefix = option("libprefix") or self._detect_libdir()
def clean_intermediate_directories(self): def clean_intermediate_directories(self):
for subdir in os.listdir(self.build_dir): for subdir in os.listdir(self.build_dir):
@ -104,79 +99,82 @@ class BuildEnv:
os.remove(subpath) os.remove(subpath)
def _is_debianlike(self): def _is_debianlike(self):
return os.path.isfile('/etc/debian_version') return os.path.isfile("/etc/debian_version")
def _detect_libdir(self): def _detect_libdir(self):
if self.platformInfo.libdir is not None: if self.configInfo.libdir is not None:
return self.platformInfo.libdir return self.configInfo.libdir
if self._is_debianlike(): if self._is_debianlike():
try: try:
pc = subprocess.Popen(['dpkg-architecture', '-qDEB_HOST_MULTIARCH'], pc = subprocess.Popen(
["dpkg-architecture", "-qDEB_HOST_MULTIARCH"],
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL) stderr=subprocess.DEVNULL,
)
(stdo, _) = pc.communicate() (stdo, _) = pc.communicate()
if pc.returncode == 0: if pc.returncode == 0:
archpath = stdo.decode().strip() archpath = stdo.decode().strip()
return 'lib/' + archpath return "lib/" + archpath
except Exception: except Exception:
pass pass
if os.path.isdir('/usr/lib64') and not os.path.islink('/usr/lib64'): if os.path.isdir("/usr/lib64") and not os.path.islink("/usr/lib64"):
return 'lib64' return "lib64"
return 'lib' return "lib"
def get_env(self, *, cross_comp_flags, cross_compilers, cross_path): def get_env(self, *, cross_comp_flags, cross_compilers, cross_path):
env = self.platformInfo.get_env() env = self.configInfo.get_env()
pkgconfig_path = pj(self.install_dir, self.libprefix, 'pkgconfig') pkgconfig_path = pj(self.install_dir, self.libprefix, "pkgconfig")
env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path]) env["PKG_CONFIG_PATH"] = ":".join([env["PKG_CONFIG_PATH"], pkgconfig_path])
env['PATH'] = ':'.join([ env["PATH"] = ":".join([escape_path(pj(self.install_dir, "bin")), env["PATH"]])
escape_path(pj(self.install_dir, 'bin')),
env['PATH']
])
env['LD_LIBRARY_PATH'] = ':'.join([env['LD_LIBRARY_PATH'], env["LD_LIBRARY_PATH"] = ":".join(
pj(self.install_dir, 'lib'), [
pj(self.install_dir, self.libprefix) env["LD_LIBRARY_PATH"],
]) pj(self.install_dir, "lib"),
pj(self.install_dir, self.libprefix),
]
)
env['QMAKE_CXXFLAGS'] = " ".join([ env["QMAKE_CXXFLAGS"] = " ".join(
escape_path('-I'+pj(self.install_dir, 'include')), [escape_path("-I" + pj(self.install_dir, "include")), env["QMAKE_CXXFLAGS"]]
env['QMAKE_CXXFLAGS'] )
]) env["CPPFLAGS"] = " ".join(
env['CPPFLAGS'] = " ".join([ [escape_path("-I" + pj(self.install_dir, "include")), env["CPPFLAGS"]]
escape_path('-I'+pj(self.install_dir, 'include')), )
env['CPPFLAGS'] env["QMAKE_LFLAGS"] = " ".join(
]) [
env['QMAKE_LFLAGS'] = " ".join([ escape_path("-L" + pj(self.install_dir, "lib")),
escape_path('-L'+pj(self.install_dir, 'lib')), escape_path("-L" + pj(self.install_dir, self.libprefix)),
escape_path('-L'+pj(self.install_dir, self.libprefix)), env["QMAKE_LFLAGS"],
env['QMAKE_LFLAGS'] ]
]) )
env['LDFLAGS'] = " ".join([ env["LDFLAGS"] = " ".join(
escape_path('-L'+pj(self.install_dir, 'lib')), [
escape_path('-L'+pj(self.install_dir, self.libprefix)), escape_path("-L" + pj(self.install_dir, "lib")),
env['LDFLAGS'] escape_path("-L" + pj(self.install_dir, self.libprefix)),
]) env["LDFLAGS"],
]
)
if cross_comp_flags: if cross_comp_flags:
self.platformInfo.set_comp_flags(env) self.configInfo.set_comp_flags(env)
if cross_compilers: if cross_compilers:
self.platformInfo.set_compiler(env) self.configInfo.set_compiler(env)
if cross_path: if cross_path:
env['PATH'] = ':'.join(self.platformInfo.get_bin_dir() + [env['PATH']]) env["PATH"] = ":".join(self.configInfo.get_bin_dir() + [env["PATH"]])
return env return env
@property @property
def configure_wrapper(self): def configure_wrapper(self):
try: try:
yield self.platformInfo.configure_wrapper yield self.configInfo.configure_wrapper
except AttributeError: except AttributeError:
pass pass
@property @property
def make_wrapper(self): def make_wrapper(self):
try: try:
yield self.platformInfo.make_wrapper yield self.configInfo.make_wrapper
except AttributeError: except AttributeError:
pass pass

137
kiwixbuild/builder.py
View File

@ -1,30 +1,36 @@
import sys import sys
from collections import OrderedDict from collections import OrderedDict
from .buildenv import * from .buildenv import *
from .platforms import PlatformInfo from .configs import ConfigInfo
from .utils import remove_duplicates, StopBuild, colorize from .utils import remove_duplicates, StopBuild, colorize
from .dependencies import Dependency from .dependencies import Dependency
from .packages import PACKAGE_NAME_MAPPERS from .packages import PACKAGE_NAME_MAPPERS
from ._global import ( from ._global import (
neutralEnv, option, neutralEnv,
add_target_step, get_target_step, target_steps, option,
backend) add_target_step,
get_target_step,
target_steps,
)
from . import _global from . import _global
class Builder: class Builder:
def __init__(self): def __init__(self):
self._targets = {} self._targets = {}
PlatformInfo.get_platform('neutral', self._targets) ConfigInfo.get_config("neutral", self._targets)
target_platform = option('target_platform') config_name = option("config")
platform = PlatformInfo.get_platform(target_platform, self._targets) config = ConfigInfo.get_config(config_name, self._targets)
if neutralEnv('distname') not in platform.compatible_hosts: if neutralEnv("distname") not in config.compatible_hosts:
print((colorize('ERROR')+': The target platform {} cannot be build on host {}.\n' print(
'Select another target platform or change your host system.' (
).format(platform.name, neutralEnv('distname'))) colorize("ERROR") + ": The config {} cannot be build on host {}.\n"
self.targetDefs = platform.add_targets(option('target'), self._targets) "Select another config or change your host system."
).format(config.name, neutralEnv("distname"))
)
self.targetDefs = config.add_targets(option("target"), self._targets)
def finalize_target_steps(self): def finalize_target_steps(self):
steps = [] steps = []
@ -32,36 +38,36 @@ class Builder:
steps += self.order_steps(targetDef) steps += self.order_steps(targetDef)
steps = list(remove_duplicates(steps)) steps = list(remove_duplicates(steps))
if option('build_nodeps'): if option("build_nodeps"):
# add all platform steps # add all config steps
for dep in steps: for dep in steps:
stepClass = Dependency.all_deps[dep[1]] stepClass = Dependency.all_deps[dep[1]]
if stepClass.dont_skip: if stepClass.dont_skip:
add_target_step(dep, self._targets[dep]) add_target_step(dep, self._targets[dep])
src_targetDef = ('source', targetDef[1]) src_targetDef = ("source", targetDef[1])
add_target_step(src_targetDef, self._targets[src_targetDef]) add_target_step(src_targetDef, self._targets[src_targetDef])
add_target_step(targetDef, self._targets[targetDef]) add_target_step(targetDef, self._targets[targetDef])
else: else:
for dep in steps: for dep in steps:
if option('build_deps_only') and dep[1] == targetDef[1]: if option("build_deps_only") and dep[1] == targetDef[1]:
continue continue
add_target_step(dep, self._targets[dep]) add_target_step(dep, self._targets[dep])
self.instanciate_steps() self.instanciate_steps()
def order_steps(self, targetDef): def order_steps(self, targetDef):
for pltName in PlatformInfo.all_running_platforms: for cfgName in ConfigInfo.all_running_configs:
plt = PlatformInfo.all_platforms[pltName] cfg = ConfigInfo.all_configs[cfgName]
for tlcName in plt.toolchain_names: for tlcName in cfg.toolchain_names:
tlc = Dependency.all_deps[tlcName] tlc = Dependency.all_deps[tlcName]
yield('source', tlcName) yield ("source", tlcName)
yield('neutral' if tlc.neutral else pltName, tlcName) yield ("neutral" if tlc.neutral else cfgName, tlcName)
_targets = dict(self._targets) _targets = dict(self._targets)
yield from self.order_dependencies(targetDef, _targets) yield from self.order_dependencies(targetDef, _targets)
def order_dependencies(self, targetDef, targets): def order_dependencies(self, targetDef, targets):
targetPlatformName, targetName = targetDef targetConfigName, targetName = targetDef
if targetPlatformName == 'source': if targetConfigName == "source":
# Do not try to order sources, they will be added as dep by the # Do not try to order sources, they will be added as dep by the
# build step two lines later. # build step two lines later.
return return
@ -70,43 +76,45 @@ class Builder:
except KeyError: except KeyError:
return return
targetPlatform = PlatformInfo.get_platform(targetPlatformName) targetConfig = ConfigInfo.get_config(targetConfigName)
for dep in target.get_dependencies(targetPlatform, True): for dep in target.get_dependencies(targetConfig, True):
depPlatform, depName = targetPlatform.get_fully_qualified_dep(dep) depConfig, depName = targetConfig.get_fully_qualified_dep(dep)
if (depPlatform, depName) in targets: if (depConfig, depName) in targets:
yield from self.order_dependencies((depPlatform, depName), targets) yield from self.order_dependencies((depConfig, depName), targets)
yield ('source', targetName) yield ("source", targetName)
yield targetDef yield targetDef
def instanciate_steps(self): def instanciate_steps(self):
for stepDef in list(target_steps()): for stepDef in list(target_steps()):
stepPlatform, stepName = stepDef stepConfig, stepName = stepDef
stepClass = Dependency.all_deps[stepName] stepClass = Dependency.all_deps[stepName]
if stepPlatform == 'source': if stepConfig == "source":
source = get_target_step(stepDef)(stepClass) source = get_target_step(stepDef)(stepClass)
add_target_step(stepDef, source) add_target_step(stepDef, source)
else: else:
source = get_target_step(stepName, 'source') source = get_target_step(stepName, "source")
env = PlatformInfo.get_platform(stepPlatform).buildEnv env = ConfigInfo.get_config(stepConfig).buildEnv
builder = get_target_step(stepDef)(stepClass, source, env) builder = get_target_step(stepDef)(stepClass, source, env)
add_target_step(stepDef, builder) add_target_step(stepDef, builder)
def prepare_sources(self): def prepare_sources(self):
if option('skip_source_prepare'): if option("skip_source_prepare"):
print(colorize("SKIP")) print(colorize("SKIP"))
return return
sourceDefs = remove_duplicates(tDef for tDef in target_steps() if tDef[0]=='source') sourceDefs = remove_duplicates(
tDef for tDef in target_steps() if tDef[0] == "source"
)
for sourceDef in sourceDefs: for sourceDef in sourceDefs:
print("prepare sources {} :".format(sourceDef[1])) print("prepare sources {} :".format(sourceDef[1]))
source = get_target_step(sourceDef) source = get_target_step(sourceDef)
source.prepare() source.prepare()
def build(self): def build(self):
builderDefs = (tDef for tDef in target_steps() if tDef[0] != 'source') builderDefs = (tDef for tDef in target_steps() if tDef[0] != "source")
for builderDef in builderDefs: for builderDef in builderDefs:
builder = get_target_step(builderDef) builder = get_target_step(builderDef)
if option('make_dist') and builderDef[1] == option('target'): if option("make_dist") and builderDef[1] == option("target"):
print("make dist {} ({}):".format(builder.name, builderDef[0])) print("make dist {} ({}):".format(builder.name, builderDef[0]))
builder.make_dist() builder.make_dist()
continue continue
@ -116,19 +124,19 @@ class Builder:
def _get_packages(self): def _get_packages(self):
packages_list = [] packages_list = []
for platform in PlatformInfo.all_running_platforms.values(): for config in ConfigInfo.all_running_configs.values():
mapper_name = "{host}_{target}".format( mapper_name = "{host}_{config}".format(
host=neutralEnv('distname'), host=neutralEnv("distname"), config=config
target=platform) )
package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {}) package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {})
packages_list += package_name_mapper.get('COMMON', []) packages_list += package_name_mapper.get("COMMON", [])
to_drop = [] to_drop = []
for builderDef in self._targets: for builderDef in self._targets:
platformName, builderName = builderDef configName, builderName = builderDef
mapper_name = "{host}_{target}".format( mapper_name = "{host}_{config}".format(
host=neutralEnv('distname'), host=neutralEnv("distname"), config=configName
target=platformName) )
package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {}) package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {})
packages = package_name_mapper.get(builderName) packages = package_name_mapper.get(builderName)
if packages: if packages:
@ -144,20 +152,20 @@ class Builder:
packages_to_have = self._get_packages() packages_to_have = self._get_packages()
packages_to_have = remove_duplicates(packages_to_have) packages_to_have = remove_duplicates(packages_to_have)
if option('assume_packages_installed'): if option("assume_packages_installed"):
print(colorize("SKIP") + ", Assume package installed") print(colorize("SKIP") + ", Assume package installed")
return return
distname = neutralEnv('distname') distname = neutralEnv("distname")
if distname in ('fedora', 'redhat', 'centos'): if distname in ("fedora", "redhat", "centos"):
package_installer = 'sudo dnf install {}' package_installer = "sudo dnf install {}"
package_checker = 'rpm -q --quiet {}' package_checker = "rpm -q --quiet {}"
elif distname in ('debian', 'Ubuntu'): elif distname in ("debian", "Ubuntu"):
package_installer = 'sudo apt-get install {}' package_installer = "sudo apt-get install {}"
package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1' package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1'
elif distname == 'Darwin': elif distname == "Darwin":
package_installer = 'brew install {}' package_installer = "brew install {}"
package_checker = 'brew ls --version {} > /dev/null' package_checker = "brew ls --version {} > /dev/null"
packages_to_install = [] packages_to_install = []
for package in packages_to_have: for package in packages_to_have:
@ -181,26 +189,25 @@ class Builder:
def run(self): def run(self):
try: try:
print("[INSTALL PACKAGES]") print("[INSTALL PACKAGES]")
if option('dont_install_packages'): if option("dont_install_packages"):
print(colorize("SKIP")) print(colorize("SKIP"))
else: else:
self.install_packages() self.install_packages()
self.finalize_target_steps() self.finalize_target_steps()
print("[SETUP PLATFORMS]") print("[SETUP TOOLCHAINS]")
for platform in PlatformInfo.all_running_platforms.values(): for config in ConfigInfo.all_running_configs.values():
platform.finalize_setup() config.finalize_setup()
print("[PREPARE]") print("[PREPARE]")
self.prepare_sources() self.prepare_sources()
print("[BUILD]") print("[BUILD]")
self.build() self.build()
# No error, clean intermediate file at end of build if needed. # No error, clean intermediate file at end of build if needed.
print("[CLEAN]") print("[CLEAN]")
if option('clean_at_end'): if option("clean_at_end"):
for platform in PlatformInfo.all_running_platforms.values(): for config in ConfigInfo.all_running_configs.values():
platform.clean_intermediate_directories() config.clean_intermediate_directories()
else: else:
print(colorize("SKIP")) print(colorize("SKIP"))
except StopBuild as e: except StopBuild as e:
print(e) print(e)
sys.exit("Stopping build due to errors") sys.exit("Stopping build due to errors")

3
kiwixbuild/configs/__init__.py Normal file
View File

@ -0,0 +1,3 @@
from .base import *
from . import android, armhf, musl, flatpak, i586, ios, native, neutral, win32, wasm

161
kiwixbuild/configs/android.py Normal file
View File

@ -0,0 +1,161 @@
from .base import ConfigInfo, MetaConfigInfo
from kiwixbuild.utils import pj
from kiwixbuild._global import get_target_step, option
class AndroidConfigInfo(ConfigInfo):
build = "android"
static = True
toolchain_names = ["android-ndk"]
compatible_hosts = ["fedora", "debian"]
def __str__(self):
return "android"
@property
def libdir(self):
return "lib/{}".format(self.arch_full)
@property
def binaries_name(self):
arch_full = self.arch_full
return {
"CC": "{}-{}".format(arch_full, "clang"),
"CXX": "{}-{}".format(arch_full, "clang++"),
"AR": "{}-{}".format(arch_full, "ar"),
"STRIP": "{}-{}".format(arch_full, "strip"),
"RANLIB": "{}-{}".format(arch_full, "ranlib"),
"LD": "{}-{}".format(arch_full, "ld"),
}
def binaries(self):
install_path = self.install_path
binaries = {
k: pj(install_path, "bin", v) for k, v in self.binaries_name.items()
}
binaries["PKGCONFIG"] = "pkg-config"
return binaries
@property
def ndk_builder(self):
return get_target_step("android-ndk", self.name)
@property
def install_path(self):
return self.ndk_builder.install_path
def get_cross_config(self):
extra_libs = ["-llog"]
extra_cflags = [
"-I{}".format(include_dir) for include_dir in self.get_include_dirs()
]
if hasattr(self, "march"):
extra_libs.append("-march={}".format(self.march))
extra_cflags.append("-march={}".format(self.march))
return {
"exe_wrapper_def": "",
"install_path": self.install_path,
"binaries": self.binaries(),
"root_path": pj(self.install_path, "sysroot"),
"extra_libs": extra_libs,
"extra_cflags": extra_cflags,
"host_machine": {
"system": "Android",
"lsystem": "android",
"cpu_family": self.arch,
"cpu": self.cpu,
"endian": "little",
"abi": self.abi,
},
}
def get_env(self):
env = super().get_env()
root_path = pj(self.install_path, "sysroot")
env["PKG_CONFIG_LIBDIR"] = pj(root_path, "lib", "pkgconfig")
env["NDK_DEBUG"] = "0"
return env
def get_bin_dir(self):
return [pj(self.install_path, "bin")]
def set_comp_flags(self, env):
super().set_comp_flags(env)
root_path = pj(self.install_path, "sysroot")
march = "-march={}".format(self.march) if hasattr(self, "march") else ""
env["CFLAGS"] = (
"-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} {} ".format(
root_path, march
)
+ env["CFLAGS"]
)
env["CXXFLAGS"] = (
"-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} {} ".format(
root_path, march
)
+ env["CXXFLAGS"]
)
env["LDFLAGS"] = "--sysroot={} {} ".format(root_path, march) + env["LDFLAGS"]
def set_compiler(self, env):
binaries = self.binaries()
for k, v in binaries.items():
env[k] = v
@property
def configure_options(self):
yield "--host={}".format(self.arch_full)
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile(
"cmake_android_cross_file.txt", "cmake_cross_file.txt"
)
self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
class AndroidArm(AndroidConfigInfo):
name = "android_arm"
arch = cpu = "arm"
arch_full = "arm-linux-androideabi"
abi = "armeabi-v7a"
march = "armv7-a"
class AndroidArm64(AndroidConfigInfo):
name = "android_arm64"
arch = "arm64"
arch_full = "aarch64-linux-android"
cpu = "aarch64"
abi = "arm64-v8a"
class AndroidX86(AndroidConfigInfo):
name = "android_x86"
arch = abi = "x86"
arch_full = "i686-linux-android"
cpu = "i686"
class AndroidX8664(AndroidConfigInfo):
name = "android_x86_64"
arch = cpu = abi = "x86_64"
arch_full = "x86_64-linux-android"
class Android(MetaConfigInfo):
name = "android"
compatible_hosts = ["fedora", "debian"]
@property
def subConfigNames(self):
return ["android_{}".format(arch) for arch in option("android_arch")]
def add_targets(self, targetName, targets):
return super().add_targets(targetName, targets)
def __str__(self):
return self.name
def set_comp_flags(self, env):
pass

186
kiwixbuild/configs/armhf.py Normal file
View File

@ -0,0 +1,186 @@
from .base import ConfigInfo, MixedMixin
from kiwixbuild.utils import pj
from kiwixbuild._global import get_target_step
# Base config for arm
class ArmConfigInfo(ConfigInfo):
compatible_hosts = ["fedora", "debian"]
def get_cross_config(self):
return {
"binaries": self.binaries,
"exe_wrapper_def": "",
"root_path": self.root_path,
"extra_libs": [],
"extra_cflags": [
"-I{}".format(include_dir) for include_dir in self.get_include_dirs()
],
"host_machine": {
"system": "linux",
"lsystem": "linux",
"cpu_family": self.cpu_family,
"cpu": self.cpu,
"endian": "little",
"abi": "",
},
}
@property
def libdir(self):
return "lib/{}".format(self.arch_full)
@property
def toolchain(self):
return get_target_step(self.build, "neutral")
@property
def root_path(self):
return self.toolchain.build_path
@property
def binaries(self):
binaries = (
(k, "{}-{}".format(self.arch_full, v))
for k, v in (
("CC", "gcc"),
("CXX", "g++"),
("AR", "ar"),
("STRIP", "strip"),
("WINDRES", "windres"),
("RANLIB", "ranlib"),
("LD", "ld"),
("LDSHARED", "g++ -shared"),
)
)
binaries = {k: pj(self.root_path, "bin", v) for k, v in binaries}
binaries["PKGCONFIG"] = "pkg-config"
return binaries
@property
def exe_wrapper_def(self):
try:
which("qemu-arm")
except subprocess.CalledProcessError:
return ""
else:
return "exe_wrapper = 'qemu-arm'"
@property
def configure_options(self):
yield "--host={}".format(self.arch_full)
def get_bin_dir(self):
return [pj(self.root_path, "bin")]
def get_env(self):
env = super().get_env()
env["LD_LIBRARY_PATH"] = ":".join(
[
pj(self.root_path, self.arch_full, "lib64"),
pj(self.root_path, "lib"),
env["LD_LIBRARY_PATH"],
]
)
env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig")
env["QEMU_LD_PREFIX"] = pj(self.root_path, self.arch_full, "libc")
env["QEMU_SET_ENV"] = "LD_LIBRARY_PATH={}".format(
":".join(
[pj(self.root_path, self.arch_full, "lib"), env["LD_LIBRARY_PATH"]]
)
)
return env
def set_comp_flags(self, env):
super().set_comp_flags(env)
env["CFLAGS"] = (
" -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
+ env["CFLAGS"]
)
env["CXXFLAGS"] = (
" -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
+ env["CXXFLAGS"]
)
def set_compiler(self, env):
for k, v in self.binaries.items():
env[k] = v
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt")
self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
class Armv6(ArmConfigInfo):
build = "armv6"
arch_full = "armv6-rpi-linux-gnueabihf"
toolchain_names = ["armv6"]
cpu_family = "arm"
cpu = "armv6"
class Armv6Dyn(Armv6):
name = "armv6_dyn"
static = False
class Armv6Static(Armv6):
name = "armv6_static"
static = True
class Armv6Mixed(MixedMixin("armv6_static"), Armv6):
name = "armv6_mixed"
static = False
class Armv8(ArmConfigInfo):
build = "armv8"
arch_full = "armv8-rpi3-linux-gnueabihf"
toolchain_names = ["armv8"]
cpu_family = "arm"
cpu = "armv8"
class Armv8Dyn(Armv8):
name = "armv8_dyn"
static = False
class Armv8Static(Armv8):
name = "armv8_static"
static = True
class Armv8Mixed(MixedMixin("armv8_static"), Armv8):
name = "armv8_mixed"
static = False
class Aarch64(ArmConfigInfo):
build = "aarch64"
arch_full = "aarch64-linux-gnu"
toolchain_names = ["aarch64"]
cpu_family = "aarch64"
cpu = "aarch64"
@property
def root_path(self):
return self.toolchain.build_path
class Aarch64Dyn(Aarch64):
name = "aarch64_dyn"
static = False
class Aarch64Static(Aarch64):
name = "aarch64_static"
static = True
class Aarch64Mixed(MixedMixin("aarch64_static"), Aarch64):
name = "aarch64_mixed"
static = False

114
kiwixbuild/platforms/base.py → kiwixbuild/configs/base.py
View File

@ -1,4 +1,3 @@
import os, sys import os, sys
import subprocess import subprocess
@ -8,63 +7,64 @@ from kiwixbuild.buildenv import BuildEnv
from kiwixbuild._global import neutralEnv, option, target_steps from kiwixbuild._global import neutralEnv, option, target_steps
_SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) _SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
TEMPLATES_DIR = pj(os.path.dirname(_SCRIPT_DIR), 'templates') TEMPLATES_DIR = pj(os.path.dirname(_SCRIPT_DIR), "templates")
class _MetaPlatform(type):
class _MetaConfig(type):
def __new__(cls, name, bases, dct): def __new__(cls, name, bases, dct):
_class = type.__new__(cls, name, bases, dct) _class = type.__new__(cls, name, bases, dct)
if name not in ('PlatformInfo', 'MetaPlatformInfo') and 'name' in dct: if name not in ("ConfigInfo", "MetaConfigInfo") and "name" in dct:
dep_name = dct['name'] dep_name = dct["name"]
PlatformInfo.all_platforms[dep_name] = _class ConfigInfo.all_configs[dep_name] = _class
return _class return _class
class PlatformInfo(metaclass=_MetaPlatform): class ConfigInfo(metaclass=_MetaConfig):
all_platforms = {} all_configs = {}
all_running_platforms = {} all_running_configs = {}
toolchain_names = [] toolchain_names = []
configure_options = [] configure_options = []
mixed = False mixed = False
libdir = None libdir = None
@classmethod @classmethod
def get_platform(cls, name, targets=None): def get_config(cls, name, targets=None):
if name not in cls.all_running_platforms: if name not in cls.all_running_configs:
if targets is None: if targets is None:
print("Should not got there.") print("Should not got there.")
print(cls.all_running_platforms) print(cls.all_running_configs)
raise KeyError(name) raise KeyError(name)
cls.all_running_platforms[name] = cls.all_platforms[name](targets) cls.all_running_configs[name] = cls.all_configs[name](targets)
return cls.all_running_platforms[name] return cls.all_running_configs[name]
def __init__(self, targets): def __init__(self, targets):
self.all_running_platforms[self.name] = self self.all_running_configs[self.name] = self
self.buildEnv = BuildEnv(self) self.buildEnv = BuildEnv(self)
self.setup_toolchains(targets) self.setup_toolchains(targets)
def __str__(self): def __str__(self):
return "{}_{}".format(self.build, 'static' if self.static else 'dyn') return "{}_{}".format(self.build, "static" if self.static else "dyn")
def setup_toolchains(self, targets): def setup_toolchains(self, targets):
for tlc_name in self.toolchain_names: for tlc_name in self.toolchain_names:
ToolchainClass = Dependency.all_deps[tlc_name] ToolchainClass = Dependency.all_deps[tlc_name]
targets[('source', tlc_name)] = ToolchainClass.Source targets[("source", tlc_name)] = ToolchainClass.Source
plt_name = 'neutral' if ToolchainClass.neutral else self.name cfg_name = "neutral" if ToolchainClass.neutral else self.name
targets[(plt_name, tlc_name)] = ToolchainClass.Builder targets[(cfg_name, tlc_name)] = ToolchainClass.Builder
def add_targets(self, targetName, targets): def add_targets(self, targetName, targets):
if (self.name, targetName) in targets: if (self.name, targetName) in targets:
return [] return []
targetClass = Dependency.all_deps[targetName] targetClass = Dependency.all_deps[targetName]
targets[('source', targetName)] = targetClass.Source targets[("source", targetName)] = targetClass.Source
targets[(self.name, targetName)] = targetClass.Builder targets[(self.name, targetName)] = targetClass.Builder
for dep in targetClass.Builder.get_dependencies(self, False): for dep in targetClass.Builder.get_dependencies(self, False):
if isinstance(dep, tuple): if isinstance(dep, tuple):
depPlatformName, depName = dep depConfigName, depName = dep
else: else:
depPlatformName, depName = self.name, dep depConfigName, depName = self.name, dep
depPlatform = self.get_platform(depPlatformName, targets) depConfig = self.get_config(depConfigName, targets)
depPlatform.add_targets(depName, targets) depConfig.add_targets(depName, targets)
return [(self.name, targetName)] return [(self.name, targetName)]
def get_fully_qualified_dep(self, dep): def get_fully_qualified_dep(self, dep):
@ -73,42 +73,35 @@ class PlatformInfo(metaclass=_MetaPlatform):
else: else:
return self.name, dep return self.name, dep
def get_cross_config(self): def get_cross_config(self):
return {} return {}
def get_include_dirs(self): def get_include_dirs(self):
return [pj(self.buildEnv.install_dir, 'include')] return [pj(self.buildEnv.install_dir, "include")]
def get_env(self): def get_env(self):
return DefaultEnv() return DefaultEnv()
def get_bin_dir(self): def get_bin_dir(self):
return [] return []
def set_compiler(self, env): def set_compiler(self, env):
pass pass
def set_comp_flags(self, env): def set_comp_flags(self, env):
if self.static: if self.static:
env['CFLAGS'] = env['CFLAGS'] + ' -fPIC' env["CFLAGS"] = env["CFLAGS"] + " -fPIC"
env['CXXFLAGS'] = env['CXXFLAGS'] + ' -fPIC' env["CXXFLAGS"] = env["CXXFLAGS"] + " -fPIC"
def _gen_crossfile(self, name, outname=None): def _gen_crossfile(self, name, outname=None):
if outname is None: if outname is None:
outname = name outname = name
crossfile = pj(self.buildEnv.build_dir, outname) crossfile = pj(self.buildEnv.build_dir, outname)
template_file = pj(TEMPLATES_DIR, name) template_file = pj(TEMPLATES_DIR, name)
with open(template_file, 'r') as f: with open(template_file, "r") as f:
template = f.read() template = f.read()
content = template.format( content = template.format(**self.get_cross_config())
**self.get_cross_config() with open(crossfile, "w") as outfile:
)
with open(crossfile, 'w') as outfile:
outfile.write(content) outfile.write(content)
return crossfile return crossfile
@ -121,19 +114,17 @@ class PlatformInfo(metaclass=_MetaPlatform):
self.buildEnv.clean_intermediate_directories() self.buildEnv.clean_intermediate_directories()
class MetaConfigInfo(ConfigInfo):
class MetaPlatformInfo(PlatformInfo): subConfigNames = []
subPlatformNames = []
def add_targets(self, targetName, targets): def add_targets(self, targetName, targets):
targetDefs = [] targetDefs = []
for platformName in self.subPlatformNames: for configName in self.subConfigNames:
platform = self.get_platform(platformName, targets) config = self.get_config(configName, targets)
targetDefs += platform.add_targets(targetName, targets) targetDefs += config.add_targets(targetName, targets)
return targetDefs return targetDefs
def MixedMixin(static_name): def MixedMixin(static_name):
class MixedMixinClass: class MixedMixinClass:
mixed = True mixed = True
@ -141,36 +132,47 @@ def MixedMixin(static_name):
def add_targets(self, targetName, targets): def add_targets(self, targetName, targets):
print(targetName) print(targetName)
if option('target') == targetName: if option("target") == targetName:
return super().add_targets(targetName, targets) return super().add_targets(targetName, targets)
else: else:
static_platform = self.get_platform(static_name, targets) static_config = self.get_config(static_name, targets)
return static_platform.add_targets(targetName, targets) return static_config.add_targets(targetName, targets)
def get_fully_qualified_dep(self, dep): def get_fully_qualified_dep(self, dep):
if isinstance(dep, tuple): if isinstance(dep, tuple):
return dep return dep
if option('target') == dep: if option("target") == dep:
return self.name, dep return self.name, dep
return static_name, dep return static_name, dep
@property @property
def static_buildEnv(self): def static_buildEnv(self):
static_platform = self.get_platform(static_name) static_config = self.get_config(static_name)
return static_platform.buildEnv return static_config.buildEnv
def get_include_dirs(self): def get_include_dirs(self):
return [ return [
pj(self.buildEnv.install_dir, 'include'), pj(self.buildEnv.install_dir, "include"),
pj(self.static_buildEnv.install_dir, 'include') pj(self.static_buildEnv.install_dir, "include"),
] ]
def get_env(self): def get_env(self):
env = super().get_env() env = super().get_env()
env['PATH'] = ':'.join([pj(self.static_buildEnv.install_dir, 'bin')] + [env['PATH']]) env["PATH"] = ":".join(
pkgconfig_path = pj(self.static_buildEnv.install_dir, self.static_buildEnv.libprefix, 'pkgconfig') [pj(self.static_buildEnv.install_dir, "bin")] + [env["PATH"]]
env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path]) )
env['CPPFLAGS'] = " ".join(['-I'+pj(self.static_buildEnv.install_dir, 'include'), env['CPPFLAGS']]) pkgconfig_path = pj(
self.static_buildEnv.install_dir,
self.static_buildEnv.libprefix,
"pkgconfig",
)
env["PKG_CONFIG_PATH"] = ":".join([env["PKG_CONFIG_PATH"], pkgconfig_path])
env["CPPFLAGS"] = " ".join(
[
"-I" + pj(self.static_buildEnv.install_dir, "include"),
env["CPPFLAGS"],
]
)
return env return env
return MixedMixinClass return MixedMixinClass

18
kiwixbuild/configs/flatpak.py Normal file
View File

@ -0,0 +1,18 @@
from .base import ConfigInfo
from kiwixbuild._global import option, neutralEnv
class FlatpakConfigInfo(ConfigInfo):
name = "flatpak"
build = "flatpak"
static = ""
toolchain_names = ["org.kde", "io.qt.qtwebengine"]
compatible_hosts = ["debian", "fedora"]
def __str__(self):
return "flatpak"
def get_env(self):
env = super().get_env()
env["FLATPAK_USER_DIR"] = self.buildEnv.build_dir
return env

79
kiwixbuild/configs/i586.py Normal file
View File

@ -0,0 +1,79 @@
import os
from .base import ConfigInfo
from kiwixbuild.utils import which, pj
class I586ConfigInfo(ConfigInfo):
build = "i586"
arch_full = "i586-linux-gnu"
compatible_hosts = ["fedora", "debian"]
def get_cross_config(self):
return {
"binaries": self.binaries,
"exe_wrapper_def": "",
"extra_libs": ["-m32", "-march=i586", "-mno-sse"],
"extra_cflags": [
"-m32",
"-march=i586",
"-mno-sse",
*(
"-I{}".format(include_dir)
for include_dir in self.get_include_dirs()
),
],
"host_machine": {
"system": "linux",
"lsystem": "linux",
"cpu_family": "x86",
"cpu": "i586",
"endian": "little",
"abi": "",
},
}
@property
def configure_options(self):
yield f"--host={self.arch_full}"
@property
def binaries(self):
return {
k: which(v)
for k, v in (
("CC", os.environ.get("CC", "gcc")),
("CXX", os.environ.get("CXX", "g++")),
("AR", "ar"),
("STRIP", "strip"),
("RANLIB", "ranlib"),
("LD", "ld"),
("PKGCONFIG", "pkg-config"),
)
}
def set_comp_flags(self, env):
super().set_comp_flags(env)
env["CFLAGS"] = "-m32 -march=i586 -mno-sse " + env["CFLAGS"]
env["CXXFLAGS"] = "-m32 -march=i586 -mno-sse " + env["CXXFLAGS"]
env["LDFLAGS"] = "-m32 -march=i586 -mno-sse " + env["LDFLAGS"]
def get_bin_dir(self):
return []
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile(
"cmake_i586_cross_file.txt", "cmake_cross_file.txt"
)
self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
class I586Dyn(I586ConfigInfo):
name = "i586_dyn"
static = False
class I586Static(I586ConfigInfo):
name = "i586_static"
static = True

237
kiwixbuild/configs/ios.py Normal file
View File

@ -0,0 +1,237 @@
import subprocess
from kiwixbuild._global import option
from kiwixbuild.utils import pj, xrun_find
from .base import ConfigInfo, MetaConfigInfo, MixedMixin
from kiwixbuild.dependencies.apple_xcframework import AppleXCFramework
MIN_MACOS_VERSION = "12.0"
class AppleConfigInfo(ConfigInfo):
build = "iOS"
static = True
compatible_hosts = ["Darwin"]
arch = None
host = None
target = None
sdk_name = None
min_iphoneos_version = None
min_macos_version = None
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._root_path = None
@property
def root_path(self):
if self._root_path is None:
command = "xcrun --sdk {} --show-sdk-path".format(self.sdk_name)
self._root_path = subprocess.check_output(command, shell=True)[:-1].decode()
return self._root_path
def __str__(self):
return "iOS"
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile(
"cmake_ios_cross_file.txt", "cmake_cross_file.txt"
)
self.buildEnv.meson_crossfile = self._gen_crossfile(
"meson_ios_cross_file.txt", "meson_cross_file.txt"
)
def get_cross_config(self):
config = {
"root_path": self.root_path,
"binaries": self.binaries,
"exe_wrapper_def": "",
"extra_libs": [
"-isysroot",
self.root_path,
"-arch",
self.arch,
"-target",
self.target,
],
"extra_cflags": [
"-isysroot",
self.root_path,
"-arch",
self.arch,
"-target",
self.target,
*(
"-I{}".format(include_dir)
for include_dir in self.get_include_dirs()
),
],
"host_machine": {
"system": "Darwin",
"lsystem": "darwin",
"cpu_family": self.arch,
"cpu": self.cpu,
"endian": "",
"abi": "",
},
}
if self.min_iphoneos_version:
config["extra_libs"].append(
"-miphoneos-version-min={}".format(self.min_iphoneos_version)
)
config["extra_cflags"].append(
"-miphoneos-version-min={}".format(self.min_iphoneos_version)
)
if self.min_macos_version:
config["extra_libs"].append(
"-mmacosx-version-min={}".format(self.min_macos_version)
)
config["extra_cflags"].append(
"-mmacosx-version-min={}".format(self.min_macos_version)
)
return config
def get_env(self):
env = super().get_env()
cflags = [env["CFLAGS"]]
if self.min_iphoneos_version:
cflags.append("-miphoneos-version-min={}".format(self.min_iphoneos_version))
if self.min_macos_version:
cflags.append("-mmacosx-version-min={}".format(self.min_macos_version))
env["CFLAGS"] = " ".join(cflags)
return env
def set_comp_flags(self, env):
super().set_comp_flags(env)
cflags = [
"-isysroot {}".format(self.root_path),
"-arch {}".format(self.arch),
"-target {}".format(self.target),
env["CFLAGS"],
]
if self.min_iphoneos_version:
cflags.append("-miphoneos-version-min={}".format(self.min_iphoneos_version))
env["CFLAGS"] = " ".join(cflags)
env["CXXFLAGS"] = " ".join(
[
env["CFLAGS"],
"-std=c++11",
env["CXXFLAGS"],
]
)
env["LDFLAGS"] = " ".join(
[
" -arch {}".format(self.arch),
"-isysroot {}".format(self.root_path),
]
)
def get_bin_dir(self):
return [pj(self.root_path, "bin")]
@property
def binaries(self):
return {
"CC": xrun_find("clang"),
"CXX": xrun_find("clang++"),
"AR": xrun_find("ar"),
"STRIP": xrun_find("strip"),
"RANLIB": xrun_find("ranlib"),
"LD": xrun_find("ld"),
"PKGCONFIG": "pkg-config",
}
@property
def configure_options(self):
yield f"--host={self.host}"
class iOSArm64(AppleConfigInfo):
name = "iOS_arm64"
arch = cpu = "arm64"
host = "arm-apple-darwin"
target = "aarch64-apple-ios"
sdk_name = "iphoneos"
min_iphoneos_version = "15.0"
class iOSx64Simulator(AppleConfigInfo):
name = "iOSSimulator_x86_64"
arch = cpu = "x86_64"
host = "x86_64-apple-darwin"
target = "x86-apple-ios-simulator"
sdk_name = "iphonesimulator"
min_iphoneos_version = "15.0"
class iOSArm64Simulator(AppleConfigInfo):
name = "iOSSimulator_arm64"
arch = cpu = "arm64"
host = "arm-apple-darwin"
target = "aarch64-apple-ios-simulator"
sdk_name = "iphonesimulator"
min_iphoneos_version = "15.0"
class macOSArm64(AppleConfigInfo):
name = "macOS_arm64_static"
arch = cpu = "arm64"
host = "aarch64-apple-darwin"
target = "arm64-apple-macos"
sdk_name = "macosx"
min_iphoneos_version = None
min_macos_version = MIN_MACOS_VERSION
class macOSArm64Mixed(MixedMixin("macOS_arm64_static"), AppleConfigInfo):
name = "macOS_arm64_mixed"
arch = cpu = "arm64"
host = "aarch64-apple-darwin"
target = "arm64-apple-macos"
sdk_name = "macosx"
min_iphoneos_version = None
min_macos_version = MIN_MACOS_VERSION
class macOSx64(AppleConfigInfo):
name = "macOS_x86_64"
arch = cpu = "x86_64"
host = "x86_64-apple-darwin"
target = "x86_64-apple-macos"
sdk_name = "macosx"
min_iphoneos_version = None
min_macos_version = MIN_MACOS_VERSION
class IOS(MetaConfigInfo):
name = "iOS_multi"
compatible_hosts = ["Darwin"]
@property
def subConfigNames(self):
return ["iOS_{}".format(arch) for arch in option("ios_arch")]
def add_targets(self, targetName, targets):
super().add_targets(targetName, targets)
return ConfigInfo.add_targets(self, "_ios_fat_lib", targets)
def __str__(self):
return self.name
class AppleStaticAll(MetaConfigInfo):
name = "apple_all_static"
compatible_hosts = ["Darwin"]
@property
def subConfigNames(self):
return AppleXCFramework.subConfigNames
def add_targets(self, targetName, targets):
super().add_targets(targetName, targets)
return ConfigInfo.add_targets(self, "apple_xcframework", targets)
def __str__(self):
return self.name

162
kiwixbuild/configs/musl.py Normal file
View File

@ -0,0 +1,162 @@
from .base import ConfigInfo, MixedMixin
from kiwixbuild.utils import pj
from kiwixbuild._global import get_target_step
class MuslConfigInfo(ConfigInfo):
compatible_hosts = ["fedora", "debian"]
def get_cross_config(self):
return {
"binaries": self.binaries,
"exe_wrapper_def": "",
"root_path": self.root_path,
"extra_libs": [],
"extra_cflags": [
"-I{}".format(include_dir) for include_dir in self.get_include_dirs()
],
"host_machine": {
"system": "linux",
"lsystem": "linux",
"cpu_family": self.cpu_family,
"cpu": self.cpu,
"endian": "little",
"abi": "",
},
}
@property
def toolchain(self):
return get_target_step(self.build, "neutral")
@property
def root_path(self):
return self.toolchain.build_path
@property
def binaries(self):
binaries = (
(k, "{}-{}".format(self.arch_full, v))
for k, v in (
("CC", "gcc"),
("CXX", "g++"),
("AR", "ar"),
("STRIP", "strip"),
("WINDRES", "windres"),
("RANLIB", "ranlib"),
("LD", "ld"),
("LDSHARED", "g++ -shared"),
)
)
binaries = {k: pj(self.root_path, "bin", v) for k, v in binaries}
binaries["PKGCONFIG"] = "pkg-config"
return binaries
@property
def exe_wrapper_def(self):
try:
which(self.qemu)
except subprocess.CalledProcessError:
return ""
except AttributeError:
return ""
else:
return f"exe_wrapper = '{self.qemu}'"
@property
def configure_options(self):
return [f"--host={self.arch_full}"]
def get_bin_dir(self):
return [pj(self.root_path, "bin")]
def get_env(self):
env = super().get_env()
env["LD_LIBRARY_PATH"] = ":".join(
[
pj(self.root_path, self.arch_full, "lib64"),
pj(self.root_path, "lib"),
env["LD_LIBRARY_PATH"],
]
)
env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig")
env["QEMU_LD_PREFIX"] = pj(self.root_path, self.arch_full, "libc")
env["QEMU_SET_ENV"] = "LD_LIBRARY_PATH={}".format(
":".join(
[pj(self.root_path, self.arch_full, "lib"), env["LD_LIBRARY_PATH"]]
)
)
return env
def set_comp_flags(self, env):
super().set_comp_flags(env)
env["LD_LIBRARY_PATH"] = ":".join(
[pj(self.root_path, self.arch_full, "lib"), env["LD_LIBRARY_PATH"]]
)
env["CFLAGS"] = (
" -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
+ env["CFLAGS"]
)
env["CXXFLAGS"] = (
" -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
+ env["CXXFLAGS"]
)
def set_compiler(self, env):
for k, v in self.binaries.items():
env[k] = v
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt")
self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
class Aarch64MuslConfigInfo(MuslConfigInfo):
build = "aarch64_musl"
arch_full = "aarch64-linux-musl"
toolchain_names = ["aarch64_musl"]
libdir = "lib/aarch64-linux-musl"
cpu_family = "arm"
cpu = "armhf"
qemu = "qemu-arm"
class Aarch64MuslDyn(Aarch64MuslConfigInfo):
name = "aarch64_musl_dyn"
static = False
class Aarch64MuslStatic(Aarch64MuslConfigInfo):
name = "aarch64_musl_static"
static = True
class Aarch64MuslMixed(MixedMixin("aarch64_musl_static"), Aarch64MuslConfigInfo):
name = "aarch64_musl_mixed"
static = False
class X86_64MuslConfigInfo(MuslConfigInfo):
build = "x86-64_musl"
arch_full = "x86_64-linux-musl"
toolchain_names = ["x86-64_musl"]
libdir = "lib/x86_64-linux-musl"
cpu_family = "x86_64"
cpu = "x86_64"
class X86_64MuslDyn(X86_64MuslConfigInfo):
name = "x86-64_musl_dyn"
static = False
class X86_64MuslStatic(X86_64MuslConfigInfo):
name = "x86-64_musl_static"
static = True
class x86_64MuslMixed(MixedMixin("x86-64_musl_static"), X86_64MuslConfigInfo):
name = "x86-64_musl_mixed"
static = False

35
kiwixbuild/configs/native.py Normal file
View File

@ -0,0 +1,35 @@
from .base import ConfigInfo, MixedMixin
from kiwixbuild.utils import pj
from kiwixbuild._global import option, neutralEnv
from kiwixbuild.configs.ios import MIN_MACOS_VERSION
class NativeConfigInfo(ConfigInfo):
build = "native"
def get_env(self):
env = super().get_env()
if neutralEnv("distname") == "fedora":
env["QT_SELECT"] = "5-64"
if neutralEnv("distname") == "Darwin":
env["CFLAGS"] += f"-mmacosx-version-min={MIN_MACOS_VERSION}"
return env
class NativeDyn(NativeConfigInfo):
name = "native_dyn"
static = False
compatible_hosts = ["fedora", "debian", "Darwin"]
class NativeStatic(NativeConfigInfo):
name = "native_static"
static = True
compatible_hosts = ["fedora", "debian"]
class NativeMixed(MixedMixin("native_static"), NativeConfigInfo):
name = "native_mixed"
static = False
compatible_hosts = ["fedora", "debian", "Darwin"]

10
kiwixbuild/configs/neutral.py Normal file
View File

@ -0,0 +1,10 @@
from .base import ConfigInfo
class NeutralConfigInfo(ConfigInfo):
name = "neutral"
static = ""
compatible_hosts = ["fedora", "debian", "Darwin"]
def __str__(self):
return "neutral"

114
kiwixbuild/configs/wasm.py Normal file
View File

@ -0,0 +1,114 @@
from .base import ConfigInfo
from kiwixbuild.utils import pj
from kiwixbuild._global import get_target_step
class WasmConfigInfo(ConfigInfo):
name = "wasm"
static = True
build = "wasm"
arch_full = "wasm64-emscripten"
libdir = "lib"
# arch_full = 'wasm64-linux'
toolchain_names = ["emsdk"]
compatible_hosts = ["fedora", "debian"]
exe_wrapper_def = ""
def get_cross_config(self):
return {
"binaries": self.binaries,
"exe_wrapper_def": "",
"root_path": self.root_path,
"extra_libs": [],
"extra_cflags": [],
"host_machine": {
"system": "emscripten",
"lsystem": "emscripten",
"cpu_family": "wasm64",
"cpu": "wasm64",
"endian": "little",
"abi": "",
},
}
@property
def wasm_sdk(self):
return get_target_step("emsdk", self.name)
@property
def install_path(self):
return self.wasm_sdk.install_path
@property
def root_path(self):
return self.install_path
@property
def binaries(self):
binaries = (
("CC", "emcc"),
("CXX", "em++"),
("AR", "emar"),
("STRIP", "emstrip"),
("WINDRES", "windres"),
("RANLIB", "emranlib"),
("LD", "wasm-ld"),
)
binaries = {
k: pj(self.install_path, "upstream", "emscripten", v) for k, v in binaries
}
binaries["PKGCONFIG"] = "pkg-config"
return binaries
@property
def configure_options(self):
# return ""
return [f"--host={self.arch_full}"]
@property
def configure_wrapper(self):
return "emconfigure"
@property
def make_wrapper(self):
return "emmake"
def get_bin_dir(self):
return [pj(self.install_path, "bin")]
def get_env(self):
env = super().get_env()
env["PATH"] = ":".join(
[
env["PATH"],
self.install_path,
pj(self.install_path, "upstream", "emscripten"),
pj(self.install_path, "node", "14.18.2_64bit", "bin"),
]
)
env["EMSDK"] = self.install_path
env["EMSDK_NODE"] = pj(
self.install_path, "node", "14.18.2_64bit", "bin", "node"
)
return env
def set_comp_flags(self, env):
super().set_comp_flags(env)
env["CFLAGS"] = (
" -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
+ env["CFLAGS"]
)
env["CXXFLAGS"] = (
" -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
+ env["CXXFLAGS"]
)
def set_compiler(self, env):
for k, v in self.binaries.items():
env[k] = v
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt")
self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")

99
kiwixbuild/configs/win32.py Normal file
View File

@ -0,0 +1,99 @@
import subprocess
from .base import ConfigInfo
from kiwixbuild.utils import which, pj
from kiwixbuild._global import neutralEnv
class Win32ConfigInfo(ConfigInfo):
build = "win32"
compatible_hosts = ["fedora", "debian"]
arch_full = "i686-w64-mingw32"
extra_libs = ["-lwinmm", "-lshlwapi", "-lws2_32", "-lssp"]
def get_cross_config(self):
return {
"exe_wrapper_def": self.exe_wrapper_def,
"binaries": self.binaries,
"root_path": self.root_path,
"extra_libs": self.extra_libs,
"extra_cflags": [
"-DWIN32",
*(
"-I{}".format(include_dir)
for include_dir in self.get_include_dirs()
),
],
"host_machine": {
"system": "Windows",
"lsystem": "windows",
"cpu_family": "x86",
"cpu": "i686",
"endian": "little",
"abi": "",
},
}
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt")
self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
@property
def root_path(self):
root_paths = {
"fedora": "/usr/i686-w64-mingw32/sys-root/mingw",
"debian": "/usr/i686-w64-mingw32",
}
return root_paths[neutralEnv("distname")]
@property
def binaries(self):
return {
k: which("{}-{}".format(self.arch_full, v))
for k, v in (
("CC", "gcc"),
("CXX", "g++"),
("AR", "ar"),
("STRIP", "strip"),
("WINDRES", "windres"),
("RANLIB", "ranlib"),
("PKGCONFIG", "pkg-config"),
)
}
@property
def exe_wrapper_def(self):
try:
which("wine")
except subprocess.CalledProcessError:
return ""
else:
return "exe_wrapper = 'wine'"
@property
def configure_options(self):
yield f"--host={self.arch_full}"
def set_compiler(self, env):
for k, v in self.binaries.items():
env[k] = v
def get_bin_dir(self):
return [pj(self.root_path, "bin")]
def get_env(self):
env = super().get_env()
env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig")
env["LIBS"] = " ".join(self.extra_libs) + " " + env["LIBS"]
return env
class Win32Dyn(Win32ConfigInfo):
name = "win32_dyn"
static = False
class Win32Static(Win32ConfigInfo):
name = "win32_static"
static = True

103
kiwixbuild/configs/win64.py Normal file
View File

@ -0,0 +1,103 @@
import subprocess
from .base import ConfigInfo
from kiwixbuild.utils import which, pj
from kiwixbuild._global import neutralEnv
class Win64ConfigInfo(ConfigInfo):
extra_libs = [
"-lmingw32",
"-lwinmm",
"-lws2_32",
"-lshlwapi",
"-lrpcrt4",
"-lmsvcr100",
"-liphlpapi",
"-lshell32",
"-lkernel32",
]
build = "win64"
compatible_hosts = ["fedora", "debian"]
arch_full = "x86_64-w64-mingw32"
def get_cross_config(self):
return {
"exe_wrapper_def": self.exe_wrapper_def,
"binaries": self.binaries,
"root_path": self.root_path,
"extra_libs": self.extra_libs,
"extra_cflags": ["-DWIN32"],
"host_machine": {
"system": "Windows",
"lsystem": "windows",
"cpu_family": "x86_64",
"cpu": "x86_64",
"endian": "little",
"abi": "",
},
}
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt")
self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
@property
def root_path(self):
root_paths = {
"fedora": "/usr/x86_64-w64-mingw32/sys-root/mingw",
"debian": "/usr/x86_64-w64-mingw32",
}
return root_paths[neutralEnv("distname")]
@property
def binaries(self):
return {
k: which("{}-{}".format(self.arch_full, v))
for k, v in (
("CC", "gcc"),
("CXX", "g++"),
("AR", "ar"),
("STRIP", "strip"),
("WINDRES", "windres"),
("RANLIB", "ranlib"),
("PKGCONFIG", "pkg-config"),
)
}
@property
def exe_wrapper_def(self):
try:
which("wine")
except subprocess.CalledProcessError:
return ""
else:
return "exe_wrapper = 'wine'"
@property
def configure_options(self):
return [f"--host={self.arch_full}"]
def set_compiler(self, env):
for k, v in self.binaries.items():
env[k] = v
def get_bin_dir(self):
return [pj(self.root_path, "bin")]
def get_env(self):
env = super().get_env()
env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig")
env["LIBS"] = " ".join(self.extra_libs) + " " + env["LIBS"]
return env
class Win64Dyn(Win64ConfigInfo):
name = "win64_dyn"
static = False
class Win64Static(Win64ConfigInfo):
name = "win64_static"
static = True

3
kiwixbuild/dependencies/__init__.py
View File

@ -1,4 +1,3 @@
from .base import * from .base import *
from . import ( from . import (
all_dependencies, all_dependencies,
@ -28,5 +27,5 @@ from . import (
zim_tools, zim_tools,
zim_testing_suite, zim_testing_suite,
zlib, zlib,
zstd zstd,
) )

40
kiwixbuild/dependencies/all_dependencies.py
View File

@ -1,32 +1,42 @@
from os import environ from os import environ
from .base import ( from .base import Dependency, NoopSource, NoopBuilder
Dependency,
NoopSource,
NoopBuilder)
from kiwixbuild._global import neutralEnv from kiwixbuild._global import neutralEnv
class AllBaseDependencies(Dependency): class AllBaseDependencies(Dependency):
name = "alldependencies" name = "alldependencies"
Source = NoopSource Source = NoopSource
class Builder(NoopBuilder): class Builder(NoopBuilder):
@classmethod @classmethod
def get_dependencies(cls, platformInfo, allDeps): def get_dependencies(cls, configInfo, allDeps):
if platformInfo.build == "wasm" or environ.get('OS_NAME') == 'bionic': if configInfo.build == "wasm" or environ.get("OS_NAME") == "bionic":
return ['zlib', 'lzma', 'zstd', 'icu4c', 'xapian-core'] return ["zlib", "lzma", "zstd", "icu4c", "xapian-core"]
base_deps = ['zlib', 'lzma', 'zstd', 'xapian-core', 'pugixml', 'libcurl', 'icu4c', 'mustache', 'libmicrohttpd', 'zim-testing-suite'] base_deps = [
# Add specific dependencies depending of the platform "zlib",
if platformInfo.build not in ('android', 'iOS'): "lzma",
"zstd",
"xapian-core",
"pugixml",
"libcurl",
"icu4c",
"mustache",
"libmicrohttpd",
"zim-testing-suite",
]
# Add specific dependencies depending of the config
if configInfo.build not in ("android", "iOS"):
# For zimtools # For zimtools
base_deps += ['docoptcpp'] base_deps += ["docoptcpp"]
if platformInfo.build != 'win32': if configInfo.build != "win32":
# zimwriterfs # zimwriterfs
base_deps += ['libmagic', 'gumbo'] base_deps += ["libmagic", "gumbo"]
if platformInfo.build == 'native' and neutralEnv('distname') != 'Darwin': if configInfo.build == "native" and neutralEnv("distname") != "Darwin":
# We compile kiwix-desktop only on native and not on `Darwin` # We compile kiwix-desktop only on native and not on `Darwin`
# So we need aria2 only there # So we need aria2 only there
base_deps += ['aria2'] base_deps += ["aria2"]
return base_deps return base_deps

52
kiwixbuild/dependencies/apple_xcframework.py
View File

@ -2,14 +2,14 @@ import os
import shutil import shutil
from pathlib import Path from pathlib import Path
from kiwixbuild.platforms import PlatformInfo from kiwixbuild.configs import ConfigInfo
from kiwixbuild.utils import pj, run_command from kiwixbuild.utils import pj, run_command
from .base import Dependency, NoopSource, Builder as BaseBuilder from .base import Dependency, NoopSource, Builder as BaseBuilder
class AppleXCFramework(Dependency): class AppleXCFramework(Dependency):
name = "apple_xcframework" name = "apple_xcframework"
subPlatformNames = [ subConfigNames = [
"macOS_x86_64", "macOS_x86_64",
"macOS_arm64_static", "macOS_arm64_static",
"iOS_arm64", "iOS_arm64",
@ -20,34 +20,32 @@ class AppleXCFramework(Dependency):
class Builder(BaseBuilder): class Builder(BaseBuilder):
@property @property
def all_subplatforms(self): def all_subconfigs(self):
return self.buildEnv.platformInfo.subPlatformNames return self.buildEnv.configInfo.subConfigNames
@property @property
def macos_subplatforms(self): def macos_subconfigs(self):
return [ return [
target for target in self.all_subplatforms if target.startswith("macOS") target for target in self.all_subconfigs if target.startswith("macOS")
] ]
@property @property
def iossimulator_subplatforms(self): def iossimulator_subconfigs(self):
return [ return [
target target
for target in self.all_subplatforms for target in self.all_subconfigs
if target.startswith("iOSSimulator") if target.startswith("iOSSimulator")
] ]
@property @property
def ios_subplatforms(self): def ios_subconfigs(self):
return [ return [
target for target in self.all_subplatforms if target.startswith("iOS_") target for target in self.all_subconfigs if target.startswith("iOS_")
] ]
@classmethod @classmethod
def get_dependencies(cls, platfomInfo, alldeps): def get_dependencies(cls, configInfo, alldeps):
return [ return [(target, "libkiwix") for target in AppleXCFramework.subConfigNames]
(target, "libkiwix") for target in AppleXCFramework.subPlatformNames
]
@property @property
def final_path(self): def final_path(self):
@ -62,11 +60,11 @@ class AppleXCFramework(Dependency):
def _merge_libs(self, context): def _merge_libs(self, context):
"""create merged.a in all targets to bundle all static archives""" """create merged.a in all targets to bundle all static archives"""
xcf_libs = [] xcf_libs = []
for target in self.all_subplatforms: for target in self.all_subconfigs:
static_ars = [] static_ars = []
plt = PlatformInfo.get_platform(target) cfg = ConfigInfo.get_config(target)
lib_dir = pj(plt.buildEnv.install_dir, "lib") lib_dir = pj(cfg.buildEnv.install_dir, "lib")
static_ars = [str(f) for f in Path(lib_dir).glob("*.a")] static_ars = [str(f) for f in Path(lib_dir).glob("*.a")]
# create merged.a from all *.a in install_dir/lib # create merged.a from all *.a in install_dir/lib
@ -74,17 +72,17 @@ class AppleXCFramework(Dependency):
run_command(command, lib_dir, context) run_command(command, lib_dir, context)
# will be included in xcframework # will be included in xcframework
if target in self.ios_subplatforms: if target in self.ios_subconfigs:
xcf_libs.append(pj(lib_dir, "merged.a")) xcf_libs.append(pj(lib_dir, "merged.a"))
return xcf_libs return xcf_libs
def make_fat_with(self, platforms, folder_name, context): def make_fat_with(self, configs, folder_name, context):
"""create fat merged.a in {folder_name} install/lib with {platforms} archs""" """create fat merged.a in {folder_name} install/lib with {configs}"""
libs = [] libs = []
for target in platforms: for target in configs:
plt = PlatformInfo.get_platform(target) cfg = ConfigInfo.get_config(target)
libs.append(pj(plt.buildEnv.install_dir, "lib", "merged.a")) libs.append(pj(cfg.buildEnv.install_dir, "lib", "merged.a"))
fat_dir = pj(self.buildEnv.build_dir, folder_name) fat_dir = pj(self.buildEnv.build_dir, folder_name)
os.makedirs(fat_dir, exist_ok=True) os.makedirs(fat_dir, exist_ok=True)
@ -97,14 +95,14 @@ class AppleXCFramework(Dependency):
def _build_xcframework(self, xcf_libs, context): def _build_xcframework(self, xcf_libs, context):
# create xcframework # create xcframework
ref_plat = PlatformInfo.get_platform(self.macos_subplatforms[0]) ref_conf = ConfigInfo.get_config(self.macos_subconfigs[0])
command = ["xcodebuild", "-create-xcframework"] command = ["xcodebuild", "-create-xcframework"]
for lib in xcf_libs: for lib in xcf_libs:
command += [ command += [
"-library", "-library",
lib, lib,
"-headers", "-headers",
pj(ref_plat.buildEnv.install_dir, "include"), pj(ref_conf.buildEnv.install_dir, "include"),
] ]
command += ["-output", self.final_path] command += ["-output", self.final_path]
run_command(command, self.buildEnv.build_dir, context) run_command(command, self.buildEnv.build_dir, context)
@ -116,13 +114,13 @@ class AppleXCFramework(Dependency):
xcf_libs += self.command( xcf_libs += self.command(
"make_macos_fat", "make_macos_fat",
self.make_fat_with, self.make_fat_with,
self.macos_subplatforms, self.macos_subconfigs,
"macOS_fat", "macOS_fat",
) )
xcf_libs += self.command( xcf_libs += self.command(
"make_simulator_fat", "make_simulator_fat",
self.make_fat_with, self.make_fat_with,
self.iossimulator_subplatforms, self.iossimulator_subconfigs,
"iOS-simulator_fat", "iOS-simulator_fat",
) )
self.command("build_xcframework", self._build_xcframework, xcf_libs) self.command("build_xcframework", self._build_xcframework, xcf_libs)

23
kiwixbuild/dependencies/aria2.py
View File

@ -1,18 +1,17 @@
from .base import ( from .base import Dependency, ReleaseDownload, MakeBuilder
Dependency,
ReleaseDownload,
MakeBuilder
)
from kiwixbuild.utils import Remotefile, run_command from kiwixbuild.utils import Remotefile, run_command
class Aria2(Dependency): class Aria2(Dependency):
name = "aria2" name = "aria2"
class Source(ReleaseDownload): class Source(ReleaseDownload):
archive = Remotefile('aria2-1.37.0.tar.xz', archive = Remotefile(
'60a420ad7085eb616cb6e2bdf0a7206d68ff3d37fb5a956dc44242eb2f79b66b', "aria2-1.37.0.tar.xz",
'https://github.com/aria2/aria2/releases/download/release-1.37.0/aria2-1.37.0.tar.xz') "60a420ad7085eb616cb6e2bdf0a7206d68ff3d37fb5a956dc44242eb2f79b66b",
"https://github.com/aria2/aria2/releases/download/release-1.37.0/aria2-1.37.0.tar.xz",
)
def _post_prepare_script(self, context): def _post_prepare_script(self, context):
context.try_skip(self.extract_path) context.try_skip(self.extract_path)
@ -20,5 +19,9 @@ class Aria2(Dependency):
run_command(command, self.extract_path, context) run_command(command, self.extract_path, context)
class Builder(MakeBuilder): class Builder(MakeBuilder):
dependencies = ['zlib'] dependencies = ["zlib"]
configure_options = ["--disable-libaria2", "--disable-websocket", "--without-sqlite3"] configure_options = [
"--disable-libaria2",
"--disable-websocket",
"--without-sqlite3",
]

302
kiwixbuild/dependencies/base.py
View File

@ -3,7 +3,17 @@ import os
import shutil import shutil
import time import time
from kiwixbuild.utils import pj, Context, SkipCommand, WarningMessage, extract_archive, StopBuild, run_command, colorize, copy_tree from kiwixbuild.utils import (
pj,
Context,
SkipCommand,
WarningMessage,
extract_archive,
StopBuild,
run_command,
colorize,
copy_tree,
)
from kiwixbuild.versions import main_project_versions, base_deps_versions from kiwixbuild.versions import main_project_versions, base_deps_versions
from kiwixbuild._global import neutralEnv, option, get_target_step from kiwixbuild._global import neutralEnv, option, get_target_step
@ -13,8 +23,8 @@ SCRIPT_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
class _MetaDependency(type): class _MetaDependency(type):
def __new__(cls, name, bases, dct): def __new__(cls, name, bases, dct):
_class = type.__new__(cls, name, bases, dct) _class = type.__new__(cls, name, bases, dct)
if name != 'Dependency': if name != "Dependency":
dep_name = dct['name'] dep_name = dct["name"]
Dependency.all_deps[dep_name] = _class Dependency.all_deps[dep_name] = _class
return _class return _class
@ -29,7 +39,7 @@ class Dependency(metaclass=_MetaDependency):
def version(cls): def version(cls):
if cls.name in base_deps_versions: if cls.name in base_deps_versions:
return base_deps_versions[cls.name] return base_deps_versions[cls.name]
elif option('make_release'): elif option("make_release"):
return main_project_versions.get(cls.name, None) return main_project_versions.get(cls.name, None)
return None return None
@ -44,6 +54,7 @@ class Source:
"""Base Class to the real preparator """Base Class to the real preparator
A source preparator must install source in the self.source_dir attribute A source preparator must install source in the self.source_dir attribute
inside the neutralEnv.source_dir.""" inside the neutralEnv.source_dir."""
def __init__(self, target): def __init__(self, target):
self.target = target self.target = target
@ -61,22 +72,22 @@ class Source:
@property @property
def source_path(self): def source_path(self):
return pj(neutralEnv('source_dir'), self.source_dir) return pj(neutralEnv("source_dir"), self.source_dir)
@property @property
def _log_dir(self): def _log_dir(self):
return neutralEnv('log_dir') return neutralEnv("log_dir")
def _patch(self, context): def _patch(self, context):
context.try_skip(self.source_path) context.try_skip(self.source_path)
for p in self.patches: for p in self.patches:
patch_file_path = pj(SCRIPT_DIR, 'patches', p) patch_file_path = pj(SCRIPT_DIR, "patches", p)
patch_command = [*neutralEnv('patch_command'), "-p1", "-i", patch_file_path] patch_command = [*neutralEnv("patch_command"), "-p1", "-i", patch_file_path]
run_command(patch_command, self.source_path, context) run_command(patch_command, self.source_path, context)
def command(self, name, function, *args): def command(self, name, function, *args):
print(" {} {} : ".format(name, self.name), end="", flush=True) print(" {} {} : ".format(name, self.name), end="", flush=True)
log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name)) log = pj(self._log_dir, "cmd_{}_{}.log".format(name, self.name))
context = Context(name, log, True) context = Context(name, log, True)
try: try:
start_time = time.time() start_time = time.time()
@ -92,7 +103,7 @@ class Source:
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
print(colorize("ERROR")) print(colorize("ERROR"))
try: try:
with open(log, 'r') as f: with open(log, "r") as f:
print(f.read()) print(f.read())
except: except:
pass pass
@ -116,30 +127,32 @@ class ReleaseDownload(Source):
@property @property
def extract_path(self): def extract_path(self):
return pj(neutralEnv('source_dir'), self.source_dir) return pj(neutralEnv("source_dir"), self.source_dir)
def _download(self, context): def _download(self, context):
context.try_skip(neutralEnv('archive_dir'), self.full_name) context.try_skip(neutralEnv("archive_dir"), self.full_name)
for archive in self.archives: for archive in self.archives:
neutralEnv('download')(archive) neutralEnv("download")(archive)
def _extract(self, context): def _extract(self, context):
context.try_skip(self.extract_path) context.try_skip(self.extract_path)
if os.path.exists(self.extract_path): if os.path.exists(self.extract_path):
shutil.rmtree(self.extract_path) shutil.rmtree(self.extract_path)
for archive in self.archives: for archive in self.archives:
extract_archive(pj(neutralEnv('archive_dir'), archive.name), extract_archive(
neutralEnv('source_dir'), pj(neutralEnv("archive_dir"), archive.name),
neutralEnv("source_dir"),
topdir=self.archive_top_dir, topdir=self.archive_top_dir,
name=self.source_dir) name=self.source_dir,
)
def prepare(self): def prepare(self):
self.command('download', self._download) self.command("download", self._download)
self.command('extract', self._extract) self.command("extract", self._extract)
if hasattr(self, 'patches'): if hasattr(self, "patches"):
self.command('patch', self._patch) self.command("patch", self._patch)
if hasattr(self, '_post_prepare_script'): if hasattr(self, "_post_prepare_script"):
self.command('post_prepare_script', self._post_prepare_script) self.command("post_prepare_script", self._post_prepare_script)
class GitClone(Source): class GitClone(Source):
@ -152,48 +165,66 @@ class GitClone(Source):
@property @property
def source_dir(self): def source_dir(self):
if option('make_release'): if option("make_release"):
return "{}_release".format(self.git_dir) return "{}_release".format(self.git_dir)
else: else:
return self.git_dir return self.git_dir
@property @property
def git_path(self): def git_path(self):
return pj(neutralEnv('source_dir'), self.source_dir) return pj(neutralEnv("source_dir"), self.source_dir)
@property @property
def git_ref(self): def git_ref(self):
if option('make_release'): if option("make_release"):
return self.release_git_ref return self.release_git_ref
else: else:
return self.base_git_ref return self.base_git_ref
def _git_init(self, context): def _git_init(self, context):
if option('fast_clone') and self.force_full_clone == False: if option("fast_clone") and self.force_full_clone == False:
command = [*neutralEnv('git_command'), "clone" , "--depth=1", "--branch", self.git_ref, self.git_remote, self.source_dir] command = [
run_command(command, neutralEnv('source_dir'), context) *neutralEnv("git_command"),
"clone",
"--depth=1",
"--branch",
self.git_ref,
self.git_remote,
self.source_dir,
]
run_command(command, neutralEnv("source_dir"), context)
else: else:
command = [*neutralEnv('git_command'), "clone", self.git_remote, self.source_dir] command = [
run_command(command, neutralEnv('source_dir'), context) *neutralEnv("git_command"),
command = [*neutralEnv('git_command'), "checkout", self.git_ref] "clone",
self.git_remote,
self.source_dir,
]
run_command(command, neutralEnv("source_dir"), context)
command = [*neutralEnv("git_command"), "checkout", self.git_ref]
run_command(command, self.git_path, context) run_command(command, self.git_path, context)
def _git_update(self, context): def _git_update(self, context):
command = [*neutralEnv('git_command'), "fetch", "origin", self.git_ref] command = [*neutralEnv("git_command"), "fetch", "origin", self.git_ref]
run_command(command, self.git_path, context) run_command(command, self.git_path, context)
try: try:
command = [*neutralEnv('git_command'), "merge", "--ff-only", f"origin/{self.git_ref}"] command = [
*neutralEnv("git_command"),
"merge",
"--ff-only",
f"origin/{self.git_ref}",
]
run_command(command, self.git_path, context) run_command(command, self.git_path, context)
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
raise WarningMessage("Cannot update, please check log for information") raise WarningMessage("Cannot update, please check log for information")
def prepare(self): def prepare(self):
if not os.path.exists(self.git_path): if not os.path.exists(self.git_path):
self.command('gitinit', self._git_init) self.command("gitinit", self._git_init)
else: else:
self.command('gitupdate', self._git_update) self.command("gitupdate", self._git_update)
if hasattr(self, '_post_prepare_script'): if hasattr(self, "_post_prepare_script"):
self.command('post_prepare_script', self._post_prepare_script) self.command("post_prepare_script", self._post_prepare_script)
class Builder: class Builder:
@ -206,7 +237,7 @@ class Builder:
self.buildEnv = buildEnv self.buildEnv = buildEnv
@classmethod @classmethod
def get_dependencies(cls, platformInfo, allDeps): def get_dependencies(cls, configInfo, allDeps):
return cls.dependencies return cls.dependencies
@property @property
@ -230,7 +261,7 @@ class Builder:
def command(self, name, function, *args): def command(self, name, function, *args):
print(" {} {} : ".format(name, self.name), end="", flush=True) print(" {} {} : ".format(name, self.name), end="", flush=True)
log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name)) log = pj(self._log_dir, "cmd_{}_{}.log".format(name, self.name))
context = Context(name, log, self.target.force_native_build) context = Context(name, log, self.target.force_native_build)
if self.target.force_build: if self.target.force_build:
context.no_skip = True context.no_skip = True
@ -248,7 +279,7 @@ class Builder:
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
print(colorize("ERROR")) print(colorize("ERROR"))
try: try:
with open(log, 'r') as f: with open(log, "r") as f:
print(f.read()) print(f.read())
except: except:
pass pass
@ -258,39 +289,43 @@ class Builder:
raise raise
def build(self): def build(self):
if hasattr(self, '_pre_build_script'): if hasattr(self, "_pre_build_script"):
self.command('pre_build_script', self._pre_build_script) self.command("pre_build_script", self._pre_build_script)
self.command('configure', self._configure) self.command("configure", self._configure)
if hasattr(self, '_post_configure_script'): if hasattr(self, "_post_configure_script"):
self.command('post_configure_script', self._post_configure_script) self.command("post_configure_script", self._post_configure_script)
self.command('compile', self._compile) self.command("compile", self._compile)
if hasattr(self, '_test'): if hasattr(self, "_test"):
self.command('test', self._test) self.command("test", self._test)
self.command('install', self._install) self.command("install", self._install)
if hasattr(self, '_post_build_script'): if hasattr(self, "_post_build_script"):
self.command('post_build_script', self._post_build_script) self.command("post_build_script", self._post_build_script)
def make_dist(self): def make_dist(self):
if hasattr(self, '_pre_build_script'): if hasattr(self, "_pre_build_script"):
self.command('pre_build_script', self._pre_build_script) self.command("pre_build_script", self._pre_build_script)
self.command('configure', self._configure) self.command("configure", self._configure)
self.command('make_dist', self._make_dist) self.command("make_dist", self._make_dist)
def set_flatpak_buildsystem(self, module): def set_flatpak_buildsystem(self, module):
if getattr(self, 'flatpak_buildsystem', None): if getattr(self, "flatpak_buildsystem", None):
module['buildsystem'] = self.flatpak_buildsystem module["buildsystem"] = self.flatpak_buildsystem
if getattr(self, 'subsource_dir', None): if getattr(self, "subsource_dir", None):
module['subdir'] = self.subsource_dir module["subdir"] = self.subsource_dir
if getattr(self, 'flatpack_build_options', None): if getattr(self, "flatpack_build_options", None):
module['build-options'] = self.flatpack_build_options module["build-options"] = self.flatpack_build_options
if getattr(self, 'configure_option', ''): if getattr(self, "configure_option", ""):
module['config-opts'] = self.configure_option.split(' ') module["config-opts"] = self.configure_option.split(" ")
def get_env(self, *, cross_comp_flags, cross_compilers, cross_path): def get_env(self, *, cross_comp_flags, cross_compilers, cross_path):
env = self.buildEnv.get_env(cross_comp_flags=cross_comp_flags, cross_compilers=cross_compilers, cross_path=cross_path) env = self.buildEnv.get_env(
for dep in self.get_dependencies(self.buildEnv.platformInfo, False): cross_comp_flags=cross_comp_flags,
cross_compilers=cross_compilers,
cross_path=cross_path,
)
for dep in self.get_dependencies(self.buildEnv.configInfo, False):
try: try:
builder = get_target_step(dep, self.buildEnv.platformInfo.name) builder = get_target_step(dep, self.buildEnv.configInfo.name)
builder.set_env(env) builder.set_env(env)
except KeyError: except KeyError:
# Some target may be missing (installed by a package, ...) # Some target may be missing (installed by a package, ...)
@ -317,7 +352,7 @@ class TcCopyBuilder(Builder):
return pj(self.buildEnv.toolchain_dir, self.target.full_name()) return pj(self.buildEnv.toolchain_dir, self.target.full_name())
def build(self): def build(self):
self.command('copy', self._copy) self.command("copy", self._copy)
def _copy(self, context): def _copy(self, context):
context.try_skip(self.build_path) context.try_skip(self.build_path)
@ -339,49 +374,48 @@ class MakeBuilder(Builder):
install_options = [] install_options = []
configure_script = "configure" configure_script = "configure"
configure_env = { configure_env = {
'_format_CFLAGS' : '{env[CFLAGS]} -O3', "_format_CFLAGS": "{env[CFLAGS]} -O3",
'_format_CXXFLAGS': '{env[CXXFLAGS]} -O3' "_format_CXXFLAGS": "{env[CXXFLAGS]} -O3",
} }
make_targets = [] make_targets = []
flatpak_buildsystem = None flatpak_buildsystem = None
@property @property
def make_install_targets(self): def make_install_targets(self):
if self.buildEnv.platformInfo.build in ('iOS', "wasm"): if self.buildEnv.configInfo.build in ("iOS", "wasm"):
yield 'install' yield "install"
else: else:
yield 'install-strip' yield "install-strip"
@property @property
def all_configure_options(self): def all_configure_options(self):
yield from self.configure_options yield from self.configure_options
if self.buildEnv.platformInfo.static: if self.buildEnv.configInfo.static:
yield from self.static_configure_options yield from self.static_configure_options
else: else:
yield from self.dynamic_configure_options yield from self.dynamic_configure_options
if not self.target.force_native_build: if not self.target.force_native_build:
yield from self.buildEnv.platformInfo.configure_options yield from self.buildEnv.configInfo.configure_options
yield from ('--prefix', self.buildEnv.install_dir) yield from ("--prefix", self.buildEnv.install_dir)
yield from ('--libdir', pj(self.buildEnv.install_dir, self.buildEnv.libprefix)) yield from ("--libdir", pj(self.buildEnv.install_dir, self.buildEnv.libprefix))
def set_configure_env(self, env): def set_configure_env(self, env):
dep_conf_env = self.configure_env dep_conf_env = self.configure_env
if not dep_conf_env: if not dep_conf_env:
return return
for k, v in dep_conf_env.items(): for k, v in dep_conf_env.items():
if k.startswith('_format_'): if k.startswith("_format_"):
v = v.format(buildEnv=self.buildEnv, env=env) v = v.format(buildEnv=self.buildEnv, env=env)
env[k[8:]] = v env[k[8:]] = v
else: else:
env[k] = v env[k] = v
def _configure(self, context): def _configure(self, context):
context.try_skip(self.build_path) context.try_skip(self.build_path)
command = [ command = [
*self.buildEnv.configure_wrapper, *self.buildEnv.configure_wrapper,
pj(self.source_path, self.configure_script), pj(self.source_path, self.configure_script),
*self.all_configure_options *self.all_configure_options,
] ]
env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
self.set_configure_env(env) self.set_configure_env(env)
@ -391,10 +425,10 @@ class MakeBuilder(Builder):
context.try_skip(self.build_path) context.try_skip(self.build_path)
command = [ command = [
*self.buildEnv.make_wrapper, *self.buildEnv.make_wrapper,
*neutralEnv('make_command'), *neutralEnv("make_command"),
"-j4", "-j4",
*self.make_targets, *self.make_targets,
*self.make_options *self.make_options,
] ]
env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
run_command(command, self.build_path, context, env=env) run_command(command, self.build_path, context, env=env)
@ -403,26 +437,22 @@ class MakeBuilder(Builder):
context.try_skip(self.build_path) context.try_skip(self.build_path)
command = [ command = [
*self.buildEnv.make_wrapper, *self.buildEnv.make_wrapper,
*neutralEnv('make_command'), *neutralEnv("make_command"),
*self.make_install_targets, *self.make_install_targets,
*self.make_options *self.make_options,
] ]
env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
run_command(command, self.build_path, context, env=env) run_command(command, self.build_path, context, env=env)
def _make_dist(self, context): def _make_dist(self, context):
context.try_skip(self.build_path) context.try_skip(self.build_path)
command = [ command = [*self.buildEnv.make_wrapper, *neutralEnv("make_command"), "dist"]
*self.buildEnv.make_wrapper,
*neutralEnv('make_command'),
"dist"
]
env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
run_command(command, self.build_path, context, env=env) run_command(command, self.build_path, context, env=env)
class CMakeBuilder(MakeBuilder): class CMakeBuilder(MakeBuilder):
flatpak_buildsystem = 'cmake' flatpak_buildsystem = "cmake"
def _configure(self, context): def _configure(self, context):
context.try_skip(self.build_path) context.try_skip(self.build_path)
@ -430,32 +460,34 @@ class CMakeBuilder(MakeBuilder):
if not self.target.force_native_build and self.buildEnv.cmake_crossfile: if not self.target.force_native_build and self.buildEnv.cmake_crossfile:
cross_options += [f"-DCMAKE_TOOLCHAIN_FILE={self.buildEnv.cmake_crossfile}"] cross_options += [f"-DCMAKE_TOOLCHAIN_FILE={self.buildEnv.cmake_crossfile}"]
command = [ command = [
*neutralEnv('cmake_command'), *neutralEnv("cmake_command"),
*self.configure_options, *self.configure_options,
"-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON", "-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON",
f"-DCMAKE_INSTALL_PREFIX={self.buildEnv.install_dir}", f"-DCMAKE_INSTALL_PREFIX={self.buildEnv.install_dir}",
f"-DCMAKE_INSTALL_LIBDIR={self.buildEnv.libprefix}", f"-DCMAKE_INSTALL_LIBDIR={self.buildEnv.libprefix}",
self.source_path, self.source_path,
*cross_options *cross_options,
] ]
env = self.get_env(cross_comp_flags=True, cross_compilers=False, cross_path=True) env = self.get_env(
cross_comp_flags=True, cross_compilers=False, cross_path=True
)
self.set_configure_env(env) self.set_configure_env(env)
run_command(command, self.build_path, context, env=env) run_command(command, self.build_path, context, env=env)
def set_flatpak_buildsystem(self, module): def set_flatpak_buildsystem(self, module):
super().set_flatpak_buildsystem(module) super().set_flatpak_buildsystem(module)
module['buildir'] = True module["buildir"] = True
class QMakeBuilder(MakeBuilder): class QMakeBuilder(MakeBuilder):
qmake_targets = [] qmake_targets = []
flatpak_buildsystem = 'qmake' flatpak_buildsystem = "qmake"
@property @property
def env_options(self): def env_options(self):
if 'QMAKE_CC' in os.environ: if "QMAKE_CC" in os.environ:
yield f"QMAKE_CC={os.environ['QMAKE_CC']}" yield f"QMAKE_CC={os.environ['QMAKE_CC']}"
if 'QMAKE_CXX' in os.environ: if "QMAKE_CXX" in os.environ:
yield f"QMAKE_CXX={os.environ['QMAKE_CXX']}" yield f"QMAKE_CXX={os.environ['QMAKE_CXX']}"
def _configure(self, context): def _configure(self, context):
@ -464,40 +496,43 @@ class QMakeBuilder(MakeBuilder):
"qmake", "qmake",
*self.configure_options, *self.configure_options,
*self.env_options, *self.env_options,
self.source_path self.source_path,
] ]
env = self.get_env(cross_comp_flags=True, cross_compilers=False, cross_path=True) env = self.get_env(
cross_comp_flags=True, cross_compilers=False, cross_path=True
)
self.set_configure_env(env) self.set_configure_env(env)
run_command(command, self.build_path, context, env=env) run_command(command, self.build_path, context, env=env)
def _make_dist(self, context): def _make_dist(self, context):
command = [ command = [
*neutralEnv('git_command'), "archive", *neutralEnv("git_command"),
"-o", f"{self.build_path}/{self.target_full_name()}.tar.gz", "archive",
"-o",
f"{self.build_path}/{self.target_full_name()}.tar.gz",
f"--prefix={self.target_full_name()}/", f"--prefix={self.target_full_name()}/",
"HEAD" "HEAD",
] ]
run_command(command, self.source_path, context) run_command(command, self.source_path, context)
class MesonBuilder(Builder): class MesonBuilder(Builder):
configure_options = [] configure_options = []
test_options = [] test_options = []
flatpak_buildsystem = 'meson' flatpak_buildsystem = "meson"
@property @property
def build_type(self): def build_type(self):
return 'release' if option('make_release') else 'debug' return "release" if option("make_release") else "debug"
@property @property
def strip_options(self): def strip_options(self):
if option('make_release'): if option("make_release"):
yield '--strip' yield "--strip"
@property @property
def library_type(self): def library_type(self):
return 'static' if self.buildEnv.platformInfo.static else 'shared' return "static" if self.buildEnv.configInfo.static else "shared"
def _configure(self, context): def _configure(self, context):
context.try_skip(self.build_path) context.try_skip(self.build_path)
@ -508,47 +543,54 @@ class MesonBuilder(Builder):
if not self.target.force_native_build and self.buildEnv.meson_crossfile: if not self.target.force_native_build and self.buildEnv.meson_crossfile:
cross_options += ["--cross-file", self.buildEnv.meson_crossfile] cross_options += ["--cross-file", self.buildEnv.meson_crossfile]
command = [ command = [
*neutralEnv('meson_command'), *neutralEnv("meson_command"),
'.', self.build_path, ".",
f'--buildtype={self.build_type}', self.build_path,
f"--buildtype={self.build_type}",
*self.strip_options, *self.strip_options,
f'--default-library={self.library_type}', f"--default-library={self.library_type}",
*self.configure_options, *self.configure_options,
f'--prefix={self.buildEnv.install_dir}', f"--prefix={self.buildEnv.install_dir}",
f'--libdir={self.buildEnv.libprefix}', f"--libdir={self.buildEnv.libprefix}",
*cross_options *cross_options,
] ]
env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) env = self.get_env(
cross_comp_flags=False, cross_compilers=False, cross_path=True
)
run_command(command, self.source_path, context, env=env) run_command(command, self.source_path, context, env=env)
def _compile(self, context): def _compile(self, context):
context.try_skip(self.build_path) context.try_skip(self.build_path)
command = [*neutralEnv('ninja_command'), "-v"] command = [*neutralEnv("ninja_command"), "-v"]
env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) env = self.get_env(
cross_comp_flags=False, cross_compilers=False, cross_path=True
)
run_command(command, self.build_path, context, env=env) run_command(command, self.build_path, context, env=env)
def _test(self, context): def _test(self, context):
context.try_skip(self.build_path) context.try_skip(self.build_path)
if ( self.buildEnv.platformInfo.build == 'android' if self.buildEnv.configInfo.build == "android" or (
or (self.buildEnv.platformInfo.build != 'native' self.buildEnv.configInfo.build != "native"
and not self.buildEnv.platformInfo.static) and not self.buildEnv.configInfo.static
): ):
raise SkipCommand() raise SkipCommand()
command = [ command = [*neutralEnv("mesontest_command"), "--verbose", *self.test_options]
*neutralEnv('mesontest_command'), env = self.get_env(
'--verbose', cross_comp_flags=False, cross_compilers=False, cross_path=True
*self.test_options )
]
env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True)
run_command(command, self.build_path, context, env=env) run_command(command, self.build_path, context, env=env)
def _install(self, context): def _install(self, context):
context.try_skip(self.build_path) context.try_skip(self.build_path)
command = [*neutralEnv('ninja_command'), '-v', 'install'] command = [*neutralEnv("ninja_command"), "-v", "install"]
env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) env = self.get_env(
cross_comp_flags=False, cross_compilers=False, cross_path=True
)
run_command(command, self.build_path, context, env=env) run_command(command, self.build_path, context, env=env)
def _make_dist(self, context): def _make_dist(self, context):
command = [*neutralEnv('ninja_command'), "-v", "dist"] command = [*neutralEnv("ninja_command"), "-v", "dist"]
env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) env = self.get_env(
cross_comp_flags=False, cross_compilers=False, cross_path=True
)
run_command(command, self.build_path, context, env=env) run_command(command, self.build_path, context, env=env)

11
kiwixbuild/dependencies/docoptcpp.py
View File

@ -1,14 +1,10 @@
from .base import ( from .base import Dependency, GitClone, CMakeBuilder
Dependency,
GitClone,
CMakeBuilder)
from kiwixbuild.utils import Remotefile from kiwixbuild.utils import Remotefile
class docoptcpp(Dependency): class docoptcpp(Dependency):
name = 'docoptcpp' name = "docoptcpp"
class Source(GitClone): class Source(GitClone):
git_remote = "https://github.com/docopt/docopt.cpp.git" git_remote = "https://github.com/docopt/docopt.cpp.git"
@ -17,5 +13,4 @@ class docoptcpp(Dependency):
git_ref = "3dd23e3280f213bacefdf5fcb04857bf52e90917" git_ref = "3dd23e3280f213bacefdf5fcb04857bf52e90917"
class Builder(CMakeBuilder): class Builder(CMakeBuilder):
make_install_targets = ['install'] make_install_targets = ["install"]

14
kiwixbuild/dependencies/gumbo.py
View File

@ -1,8 +1,4 @@
from .base import ( from .base import Dependency, ReleaseDownload, MakeBuilder
Dependency,
ReleaseDownload,
MakeBuilder
)
from kiwixbuild.utils import Remotefile, run_command from kiwixbuild.utils import Remotefile, run_command
@ -11,9 +7,11 @@ class Gumbo(Dependency):
name = "gumbo" name = "gumbo"
class Source(ReleaseDownload): class Source(ReleaseDownload):
archive = Remotefile('gumbo-0.10.1.tar.gz', archive = Remotefile(
'28463053d44a5dfbc4b77bcf49c8cee119338ffa636cc17fc3378421d714efad', "gumbo-0.10.1.tar.gz",
'https://github.com/google/gumbo-parser/archive/v0.10.1.tar.gz') "28463053d44a5dfbc4b77bcf49c8cee119338ffa636cc17fc3378421d714efad",
"https://github.com/google/gumbo-parser/archive/v0.10.1.tar.gz",
)
def _post_prepare_script(self, context): def _post_prepare_script(self, context):
context.try_skip(self.extract_path) context.try_skip(self.extract_path)

70
kiwixbuild/dependencies/icu4c.py
View File

@ -1,25 +1,25 @@
from .base import ( from .base import Dependency, ReleaseDownload, MakeBuilder
Dependency,
ReleaseDownload,
MakeBuilder
)
from kiwixbuild.utils import pj, SkipCommand, Remotefile, extract_archive from kiwixbuild.utils import pj, SkipCommand, Remotefile, extract_archive
from kiwixbuild._global import get_target_step, neutralEnv from kiwixbuild._global import get_target_step, neutralEnv
import os, shutil import os, shutil
import fileinput import fileinput
class Icu(Dependency): class Icu(Dependency):
name = "icu4c" name = "icu4c"
class Source(ReleaseDownload): class Source(ReleaseDownload):
archive_src = Remotefile('icu4c-73_2-src.tgz', archive_src = Remotefile(
'818a80712ed3caacd9b652305e01afc7fa167e6f2e94996da44b90c2ab604ce1', "icu4c-73_2-src.tgz",
'https://github.com/unicode-org/icu/releases/download/release-73-2/icu4c-73_2-src.tgz') "818a80712ed3caacd9b652305e01afc7fa167e6f2e94996da44b90c2ab604ce1",
archive_data = Remotefile('icu4c-73_2-data.zip', "https://github.com/unicode-org/icu/releases/download/release-73-2/icu4c-73_2-src.tgz",
'ca1ee076163b438461e484421a7679fc33a64cd0a54f9d4b401893fa1eb42701', )
'https://github.com/unicode-org/icu/releases/download/release-73-2/icu4c-73_2-data.zip') archive_data = Remotefile(
"icu4c-73_2-data.zip",
"ca1ee076163b438461e484421a7679fc33a64cd0a54f9d4b401893fa1eb42701",
"https://github.com/unicode-org/icu/releases/download/release-73-2/icu4c-73_2-data.zip",
)
archives = [archive_src, archive_data] archives = [archive_src, archive_data]
@ -28,16 +28,19 @@ class Icu(Dependency):
if os.path.exists(self.extract_path): if os.path.exists(self.extract_path):
shutil.rmtree(self.extract_path) shutil.rmtree(self.extract_path)
extract_archive( extract_archive(
pj(neutralEnv('archive_dir'), self.archive_src.name), pj(neutralEnv("archive_dir"), self.archive_src.name),
neutralEnv('source_dir'), neutralEnv("source_dir"),
topdir=None, topdir=None,
name=self.source_dir) name=self.source_dir,
shutil.rmtree(pj(neutralEnv('source_dir'), self.source_dir, 'source', 'data')) )
shutil.rmtree(
pj(neutralEnv("source_dir"), self.source_dir, "source", "data")
)
extract_archive( extract_archive(
pj(neutralEnv('archive_dir'), self.archive_data.name), pj(neutralEnv("archive_dir"), self.archive_data.name),
pj(neutralEnv('source_dir'), self.source_dir, 'source'), pj(neutralEnv("source_dir"), self.source_dir, "source"),
topdir='data', topdir="data",
name='data' name="data",
) )
patches = [ patches = [
@ -47,18 +50,17 @@ class Icu(Dependency):
# "icu4c_noxlocale.patch", # "icu4c_noxlocale.patch",
"icu4c_rpath.patch", "icu4c_rpath.patch",
# "icu4c_build_config.patch", # "icu4c_build_config.patch",
"icu4c_wasm.patch" "icu4c_wasm.patch",
] ]
class Builder(MakeBuilder): class Builder(MakeBuilder):
subsource_dir = "source" subsource_dir = "source"
make_install_targets = ["install"] make_install_targets = ["install"]
@classmethod @classmethod
def get_dependencies(cls, platformInfo, allDeps): def get_dependencies(cls, configInfo, allDeps):
plt = 'native_static' if platformInfo.static else 'native_dyn' plt = "native_static" if configInfo.static else "native_dyn"
return [(plt, 'icu4c')] return [(plt, "icu4c")]
@property @property
def configure_options(self): def configure_options(self):
@ -69,24 +71,26 @@ class Icu(Dependency):
yield "--enable-rpath" yield "--enable-rpath"
yield "--disable-icuio" yield "--disable-icuio"
yield "--disable-layoutex" yield "--disable-layoutex"
platformInfo = self.buildEnv.platformInfo configInfo = self.buildEnv.configInfo
if platformInfo.build != 'native': if configInfo.build != "native":
icu_native_builder = get_target_step( icu_native_builder = get_target_step(
'icu4c', "icu4c", "native_static" if configInfo.static else "native_dyn"
'native_static' if platformInfo.static else 'native_dyn') )
yield f"--with-cross-build={icu_native_builder.build_path}" yield f"--with-cross-build={icu_native_builder.build_path}"
yield "--disable-tools" yield "--disable-tools"
if platformInfo.build in ('android', 'wasm'): if configInfo.build in ("android", "wasm"):
yield "--with-data-packaging=archive" yield "--with-data-packaging=archive"
def set_env(self, env): def set_env(self, env):
env['ICU_DATA_FILTER_FILE'] = pj(os.path.dirname(os.path.realpath(__file__)), "icu4c_data_filter.json") env["ICU_DATA_FILTER_FILE"] = pj(
os.path.dirname(os.path.realpath(__file__)), "icu4c_data_filter.json"
)
def _post_configure_script(self, context): def _post_configure_script(self, context):
if self.buildEnv.platformInfo.build != "wasm": if self.buildEnv.configInfo.build != "wasm":
context.skip() context.skip()
context.try_skip(self.build_path) context.try_skip(self.build_path)
for line in fileinput.input(pj(self.build_path, 'Makefile'), inplace=True): for line in fileinput.input(pj(self.build_path, "Makefile"), inplace=True):
if line == "#DATASUBDIR = data\n": if line == "#DATASUBDIR = data\n":
print("DATASUBDIR = data") print("DATASUBDIR = data")
else: else:

41
kiwixbuild/dependencies/ios_fat_lib.py
View File

@ -1,13 +1,9 @@
import os import os
from kiwixbuild.platforms import PlatformInfo from kiwixbuild.configs import ConfigInfo
from kiwixbuild.utils import pj, copy_tree, run_command from kiwixbuild.utils import pj, copy_tree, run_command
from kiwixbuild._global import option from kiwixbuild._global import option
from .base import ( from .base import Dependency, NoopSource, Builder as BaseBuilder
Dependency,
NoopSource,
Builder as BaseBuilder)
class IOSFatLib(Dependency): class IOSFatLib(Dependency):
@ -16,38 +12,39 @@ class IOSFatLib(Dependency):
Source = NoopSource Source = NoopSource
class Builder(BaseBuilder): class Builder(BaseBuilder):
@classmethod @classmethod
def get_dependencies(self, platfomInfo, alldeps): def get_dependencies(self, platfomInfo, alldeps):
base_target = option('target') base_target = option("target")
return [('iOS_{}'.format(arch), base_target) for arch in option('ios_arch')] return [("iOS_{}".format(arch), base_target) for arch in option("ios_arch")]
def _copy_headers(self, context): def _copy_headers(self, context):
plt = PlatformInfo.get_platform('iOS_{}'.format(option('ios_arch')[0])) plt = ConfigInfo.get_config("iOS_{}".format(option("ios_arch")[0]))
include_src = pj(plt.buildEnv.install_dir, 'include') include_src = pj(plt.buildEnv.install_dir, "include")
include_dst = pj(self.buildEnv.install_dir, 'include') include_dst = pj(self.buildEnv.install_dir, "include")
copy_tree(include_src, include_dst) copy_tree(include_src, include_dst)
def _merge_libs(self, context): def _merge_libs(self, context):
lib_dirs = [] lib_dirs = []
for arch in option('ios_arch'): for arch in option("ios_arch"):
plt = PlatformInfo.get_platform('iOS_{}'.format(arch)) plt = ConfigInfo.get_config("iOS_{}".format(arch))
lib_dirs.append(pj(plt.buildEnv.install_dir, 'lib')) lib_dirs.append(pj(plt.buildEnv.install_dir, "lib"))
libs = [] libs = []
for f in os.listdir(lib_dirs[0]): for f in os.listdir(lib_dirs[0]):
if os.path.islink(pj(lib_dirs[0], f)): if os.path.islink(pj(lib_dirs[0], f)):
continue continue
if f.endswith('.a') or f.endswith('.dylib'): if f.endswith(".a") or f.endswith(".dylib"):
libs.append(f) libs.append(f)
os.makedirs(pj(self.buildEnv.install_dir, 'lib'), exist_ok=True) os.makedirs(pj(self.buildEnv.install_dir, "lib"), exist_ok=True)
for l in libs: for l in libs:
command = [ command = [
'lipo', "lipo",
'-create', *[pj(d, l) for d in lib_dirs], "-create",
'-output', pj(self.buildEnv.install_dir, 'lib', l) *[pj(d, l) for d in lib_dirs],
"-output",
pj(self.buildEnv.install_dir, "lib", l),
] ]
run_command(command, self.buildEnv.install_dir, context) run_command(command, self.buildEnv.install_dir, context)
def build(self): def build(self):
self.command('copy_headers', self._copy_headers) self.command("copy_headers", self._copy_headers)
self.command('merge_libs', self._merge_libs) self.command("merge_libs", self._merge_libs)

26
kiwixbuild/dependencies/kiwix_desktop.py
View File

@ -1,7 +1,5 @@
from .base import ( from .base import Dependency, GitClone, QMakeBuilder
Dependency,
GitClone,
QMakeBuilder)
class KiwixDesktop(Dependency): class KiwixDesktop(Dependency):
name = "kiwix-desktop" name = "kiwix-desktop"
@ -13,22 +11,18 @@ class KiwixDesktop(Dependency):
class Builder(QMakeBuilder): class Builder(QMakeBuilder):
dependencies = ["qt", "qtwebengine", "libkiwix", "aria2"] dependencies = ["qt", "qtwebengine", "libkiwix", "aria2"]
make_install_targets = ['install'] make_install_targets = ["install"]
configure_env = None configure_env = None
flatpack_build_options = { flatpack_build_options = {"env": ["QMAKEPATH=/app/lib"]}
"env": [
"QMAKEPATH=/app/lib"
]
}
@property @property
def configure_options(self): def configure_options(self):
if self.buildEnv.platformInfo.name == 'flatpak': if self.buildEnv.configInfo.name == "flatpak":
yield 'QMAKE_INCDIR+=/app/include/QtWebEngine' yield "QMAKE_INCDIR+=/app/include/QtWebEngine"
yield 'QMAKE_INCDIR+=/app/include/QtWebEngineCore' yield "QMAKE_INCDIR+=/app/include/QtWebEngineCore"
yield 'QMAKE_INCDIR+=/app/include/QtWebEngineWidgets' yield "QMAKE_INCDIR+=/app/include/QtWebEngineWidgets"
else: else:
yield f"PREFIX={self.buildEnv.install_dir}" yield f"PREFIX={self.buildEnv.install_dir}"
if self.buildEnv.platformInfo.static: if self.buildEnv.configInfo.static:
yield 'CONFIG+=static' yield "CONFIG+=static"

8
kiwixbuild/dependencies/kiwix_tools.py
View File

@ -1,7 +1,5 @@
from .base import ( from .base import Dependency, GitClone, MesonBuilder
Dependency,
GitClone,
MesonBuilder)
class KiwixTools(Dependency): class KiwixTools(Dependency):
name = "kiwix-tools" name = "kiwix-tools"
@ -16,5 +14,5 @@ class KiwixTools(Dependency):
@property @property
def configure_options(self): def configure_options(self):
if self.buildEnv.platformInfo.static: if self.buildEnv.configInfo.static:
yield "-Dstatic-linkage=true" yield "-Dstatic-linkage=true"

46
kiwixbuild/dependencies/libcurl.py
View File

@ -9,22 +9,50 @@ from .base import (
from kiwixbuild.utils import Remotefile, pj, Defaultdict, SkipCommand, run_command from kiwixbuild.utils import Remotefile, pj, Defaultdict, SkipCommand, run_command
from kiwixbuild._global import get_target_step from kiwixbuild._global import get_target_step
class LibCurl(Dependency): class LibCurl(Dependency):
name = "libcurl" name = "libcurl"
class Source(ReleaseDownload): class Source(ReleaseDownload):
name = "libcurl" name = "libcurl"
archive = Remotefile('curl-7.67.0.tar.xz', archive = Remotefile(
'f5d2e7320379338c3952dcc7566a140abb49edb575f9f99272455785c40e536c', "curl-7.67.0.tar.xz",
'https://curl.haxx.se/download/curl-7.67.0.tar.xz') "f5d2e7320379338c3952dcc7566a140abb49edb575f9f99272455785c40e536c",
"https://curl.haxx.se/download/curl-7.67.0.tar.xz",
)
class Builder(MakeBuilder): class Builder(MakeBuilder):
dependencies = ['zlib'] dependencies = ["zlib"]
configure_options = [ configure_options = [
*[f"--without-{p}" for p in *[
('libssh2', 'ssl', 'libmetalink', 'librtmp', 'nghttp2', 'libidn2', 'brotli') f"--without-{p}"
for p in (
"libssh2",
"ssl",
"libmetalink",
"librtmp",
"nghttp2",
"libidn2",
"brotli",
)
],
*[
f"--disable-{p}"
for p in (
"ftp",
"file",
"ldap",
"ldaps",
"rtsp",
"dict",
"telnet",
"tftp",
"pop3",
"imap",
"smb",
"smtp",
"gopher",
"manual",
)
], ],
*[f"--disable-{p}" for p in
('ftp', 'file', 'ldap', 'ldaps', 'rtsp', 'dict', 'telnet',
'tftp', 'pop3', 'imap', 'smb', 'smtp', 'gopher', 'manual')]
] ]

42
kiwixbuild/dependencies/libkiwix.py
View File

@ -1,12 +1,10 @@
import shutil, os import shutil, os
from .base import ( from .base import Dependency, GitClone, MesonBuilder
Dependency,
GitClone,
MesonBuilder)
from kiwixbuild.utils import pj, copy_tree from kiwixbuild.utils import pj, copy_tree
from kiwixbuild._global import option, get_target_step, neutralEnv from kiwixbuild._global import option, get_target_step, neutralEnv
class Libkiwix(Dependency): class Libkiwix(Dependency):
name = "libkiwix" name = "libkiwix"
force_build = True force_build = True
@ -16,30 +14,40 @@ class Libkiwix(Dependency):
git_dir = "libkiwix" git_dir = "libkiwix"
class Builder(MesonBuilder): class Builder(MesonBuilder):
dependencies = ["pugixml", "libzim", "zlib", "lzma", "libcurl", "libmicrohttpd", "icu4c", "mustache", "xapian-core"] dependencies = [
"pugixml",
"libzim",
"zlib",
"lzma",
"libcurl",
"libmicrohttpd",
"icu4c",
"mustache",
"xapian-core",
]
strip_options = [] strip_options = []
@property @property
def build_type(self): def build_type(self):
if self.buildEnv.platformInfo.build == "android": if self.buildEnv.configInfo.build == "android":
return "debug" return "debug"
return super().build_type return super().build_type
@property @property
def configure_options(self): def configure_options(self):
platformInfo = self.buildEnv.platformInfo configInfo = self.buildEnv.configInfo
if platformInfo.build == 'android': if configInfo.build == "android":
yield '-Dstatic-linkage=true' yield "-Dstatic-linkage=true"
yield '-Dwerror=false' yield "-Dwerror=false"
if platformInfo.build == 'iOS': if configInfo.build == "iOS":
yield '-Db_bitcode=true' yield "-Db_bitcode=true"
if platformInfo.name == 'flatpak': if configInfo.name == "flatpak":
yield '--wrap-mode=nodownload' yield "--wrap-mode=nodownload"
if platformInfo.mixed and option('target') == 'libkiwix': if configInfo.mixed and option("target") == "libkiwix":
yield "-Dstatic-linkage=true" yield "-Dstatic-linkage=true"
@property @property
def library_type(self): def library_type(self):
if self.buildEnv.platformInfo.build == 'android': if self.buildEnv.configInfo.build == "android":
return 'shared' return "shared"
return super().library_type return super().library_type

36
kiwixbuild/dependencies/libmagic.py
View File

@ -9,15 +9,18 @@ from .base import (
from kiwixbuild.utils import Remotefile, pj, SkipCommand, run_command from kiwixbuild.utils import Remotefile, pj, SkipCommand, run_command
from kiwixbuild._global import get_target_step from kiwixbuild._global import get_target_step
class LibMagic(Dependency): class LibMagic(Dependency):
name = "libmagic" name = "libmagic"
class Source(ReleaseDownload): class Source(ReleaseDownload):
name = "libmagic" name = "libmagic"
source_dir = "libmagic" source_dir = "libmagic"
archive_top_dir = 'file-5.44' archive_top_dir = "file-5.44"
archive = Remotefile('file-5.44.tar.gz', archive = Remotefile(
'3751c7fba8dbc831cb8d7cc8aff21035459b8ce5155ef8b0880a27d028475f3b') "file-5.44.tar.gz",
"3751c7fba8dbc831cb8d7cc8aff21035459b8ce5155ef8b0880a27d028475f3b",
)
class Builder(MakeBuilder): class Builder(MakeBuilder):
@property @property
@ -28,23 +31,22 @@ class LibMagic(Dependency):
yield "--disable-lzlib" yield "--disable-lzlib"
@classmethod @classmethod
def get_dependencies(cls, platformInfo, allDeps): def get_dependencies(cls, configInfo, allDeps):
if platformInfo.build != 'native': if configInfo.build != "native":
return [('native_static', 'libmagic')] return [("native_static", "libmagic")]
return [] return []
def _compile(self, context): def _compile(self, context):
platformInfo = self.buildEnv.platformInfo configInfo = self.buildEnv.configInfo
if platformInfo.build == 'native': if configInfo.build == "native":
return super()._compile(context) return super()._compile(context)
context.try_skip(self.build_path) context.try_skip(self.build_path)
command = [ command = ["make", "-j4", *self.make_targets, *self.make_options]
"make", env = self.buildEnv.get_env(
"-j4", cross_comp_flags=True, cross_compilers=True, cross_path=True
*self.make_targets, )
*self.make_options libmagic_native_builder = get_target_step("libmagic", "native_static")
] env["PATH"] = ":".join(
env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) [pj(libmagic_native_builder.build_path, "src"), env["PATH"]]
libmagic_native_builder = get_target_step('libmagic', 'native_static') )
env['PATH'] = ':'.join([pj(libmagic_native_builder.build_path, 'src'), env['PATH']])
run_command(command, self.build_path, context, env=env) run_command(command, self.build_path, context, env=env)

22
kiwixbuild/dependencies/libmicrohttpd.py
View File

@ -1,17 +1,23 @@
from .base import ( from .base import Dependency, ReleaseDownload, MakeBuilder
Dependency,
ReleaseDownload,
MakeBuilder)
from kiwixbuild.utils import Remotefile from kiwixbuild.utils import Remotefile
class MicroHttpd(Dependency): class MicroHttpd(Dependency):
name = "libmicrohttpd" name = "libmicrohttpd"
class Source(ReleaseDownload): class Source(ReleaseDownload):
archive = Remotefile('libmicrohttpd-0.9.76.tar.gz', archive = Remotefile(
'f0b1547b5a42a6c0f724e8e1c1cb5ce9c4c35fb495e7d780b9930d35011ceb4c', "libmicrohttpd-0.9.76.tar.gz",
'https://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.76.tar.gz') "f0b1547b5a42a6c0f724e8e1c1cb5ce9c4c35fb495e7d780b9930d35011ceb4c",
"https://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.76.tar.gz",
)
class Builder(MakeBuilder): class Builder(MakeBuilder):
configure_options = ["--disable-https", "--without-libgcrypt", "--without-libcurl", "--disable-doc", "--disable-examples"] configure_options = [
"--disable-https",
"--without-libgcrypt",
"--without-libcurl",
"--disable-doc",
"--disable-examples",
]

36
kiwixbuild/dependencies/libzim.py
View File

@ -1,9 +1,7 @@
from .base import ( from .base import Dependency, GitClone, MesonBuilder
Dependency,
GitClone,
MesonBuilder)
from kiwixbuild._global import option, get_target_step from kiwixbuild._global import option, get_target_step
class Libzim(Dependency): class Libzim(Dependency):
name = "libzim" name = "libzim"
force_build = True force_build = True
@ -18,37 +16,37 @@ class Libzim(Dependency):
@property @property
def build_type(self): def build_type(self):
if self.buildEnv.platformInfo.build == "android": if self.buildEnv.configInfo.build == "android":
return "debug" return "debug"
return super().build_type return super().build_type
@classmethod @classmethod
def get_dependencies(cls, platformInfo, allDeps): def get_dependencies(cls, configInfo, allDeps):
deps = ['lzma', 'zstd', 'xapian-core', 'icu4c'] deps = ["lzma", "zstd", "xapian-core", "icu4c"]
if platformInfo.name not in ('flatpak', 'wasm'): if configInfo.name not in ("flatpak", "wasm"):
deps.append('zim-testing-suite') deps.append("zim-testing-suite")
return deps return deps
@property @property
def configure_options(self): def configure_options(self):
platformInfo = self.buildEnv.platformInfo configInfo = self.buildEnv.configInfo
if platformInfo.build == 'android': if configInfo.build == "android":
yield "-DUSE_BUFFER_HEADER=false" yield "-DUSE_BUFFER_HEADER=false"
yield "-Dstatic-linkage=true" yield "-Dstatic-linkage=true"
if platformInfo.mixed and option('target') == 'libzim': if configInfo.mixed and option("target") == "libzim":
yield "-Dstatic-linkage=true" yield "-Dstatic-linkage=true"
if platformInfo.name == "flatpak": if configInfo.name == "flatpak":
yield "--wrap-mode=nodownload" yield "--wrap-mode=nodownload"
yield "-Dtest_data_dir=none" yield "-Dtest_data_dir=none"
if platformInfo.name == "wasm": if configInfo.name == "wasm":
yield "-Dexamples=false" yield "-Dexamples=false"
yield "-DUSE_MMAP=false" yield "-DUSE_MMAP=false"
if platformInfo.name not in ("flatpak", "wasm"): if configInfo.name not in ("flatpak", "wasm"):
zim_testing_suite = get_target_step('zim-testing-suite', 'source') zim_testing_suite = get_target_step("zim-testing-suite", "source")
yield '-Dtest_data_dir={}'.format(zim_testing_suite.source_path) yield "-Dtest_data_dir={}".format(zim_testing_suite.source_path)
@property @property
def library_type(self): def library_type(self):
if self.buildEnv.platformInfo.build == 'android': if self.buildEnv.configInfo.build == "android":
return 'shared' return "shared"
return super().library_type return super().library_type

18
kiwixbuild/dependencies/lzma.py
View File

@ -1,23 +1,23 @@
from .base import ( from .base import Dependency, ReleaseDownload, MakeBuilder
Dependency,
ReleaseDownload,
MakeBuilder)
from kiwixbuild.utils import Remotefile from kiwixbuild.utils import Remotefile
class lzma(Dependency): class lzma(Dependency):
name = 'lzma' name = "lzma"
class Source(ReleaseDownload): class Source(ReleaseDownload):
archive = Remotefile('xz-5.2.6.tar.gz', archive = Remotefile(
'a2105abee17bcd2ebd15ced31b4f5eda6e17efd6b10f921a01cda4a44c91b3a0', "xz-5.2.6.tar.gz",
'https://altushost-swe.dl.sourceforge.net/project/lzmautils/xz-5.2.6.tar.gz' "a2105abee17bcd2ebd15ced31b4f5eda6e17efd6b10f921a01cda4a44c91b3a0",
"https://altushost-swe.dl.sourceforge.net/project/lzmautils/xz-5.2.6.tar.gz",
) )
class Builder(MakeBuilder): class Builder(MakeBuilder):
@property @property
def configure_options(self): def configure_options(self):
return ["--disable-xz", return [
"--disable-xz",
"--disable-xzdec", "--disable-xzdec",
"--disable-lzmadec", "--disable-lzmadec",
"--disable-lzmainfo", "--disable-lzmainfo",

26
kiwixbuild/dependencies/mustache.py
View File

@ -1,28 +1,30 @@
from .base import ( from .base import Dependency, ReleaseDownload, Builder as BaseBuilder
Dependency,
ReleaseDownload,
Builder as BaseBuilder)
from kiwixbuild.utils import Remotefile, pj from kiwixbuild.utils import Remotefile, pj
from shutil import copy2 from shutil import copy2
class Mustache(Dependency): class Mustache(Dependency):
name = "mustache" name = "mustache"
class Source(ReleaseDownload): class Source(ReleaseDownload):
archive = Remotefile('Mustache-4.1.tar.gz', archive = Remotefile(
'acd66359feb4318b421f9574cfc5a511133a77d916d0b13c7caa3783c0bfe167', "Mustache-4.1.tar.gz",
'https://github.com/kainjow/Mustache/archive/v4.1.tar.gz') "acd66359feb4318b421f9574cfc5a511133a77d916d0b13c7caa3783c0bfe167",
"https://github.com/kainjow/Mustache/archive/v4.1.tar.gz",
)
class Builder(BaseBuilder): class Builder(BaseBuilder):
def build(self): def build(self):
self.command('copy_header', self._copy_header) self.command("copy_header", self._copy_header)
def _copy_header(self, context): def _copy_header(self, context):
context.try_skip(self.build_path) context.try_skip(self.build_path)
copy2(pj(self.source_path, 'mustache.hpp'), copy2(
pj(self.buildEnv.install_dir, 'include')) pj(self.source_path, "mustache.hpp"),
pj(self.buildEnv.install_dir, "include"),
)
def set_flatpak_buildsystem(self, module): def set_flatpak_buildsystem(self, module):
module['buildsystem'] = 'simple' module["buildsystem"] = "simple"
module['build-commands'] = ['cp mustache.hpp /app/include'] module["build-commands"] = ["cp mustache.hpp /app/include"]

14
kiwixbuild/dependencies/pugixml.py
View File

@ -1,19 +1,19 @@
from .base import ( from .base import Dependency, ReleaseDownload, MesonBuilder
Dependency,
ReleaseDownload,
MesonBuilder)
from kiwixbuild.utils import Remotefile from kiwixbuild.utils import Remotefile
class Pugixml(Dependency): class Pugixml(Dependency):
name = "pugixml" name = "pugixml"
class Source(ReleaseDownload): class Source(ReleaseDownload):
archive = Remotefile('pugixml-1.2.tar.gz', archive = Remotefile(
'0f422dad86da0a2e56a37fb2a88376aae6e931f22cc8b956978460c9db06136b') "pugixml-1.2.tar.gz",
"0f422dad86da0a2e56a37fb2a88376aae6e931f22cc8b956978460c9db06136b",
)
patches = ["pugixml_meson.patch"] patches = ["pugixml_meson.patch"]
flatpak_dest = "src" flatpak_dest = "src"
class Builder(MesonBuilder): class Builder(MesonBuilder):
build_type = 'release' build_type = "release"
strip_options = [] strip_options = []

91
kiwixbuild/dependencies/qt.py
View File

@ -1,79 +1,80 @@
import shutil import shutil
from .base import ( from .base import Dependency, ReleaseDownload, MakeBuilder, QMakeBuilder
Dependency,
ReleaseDownload,
MakeBuilder,
QMakeBuilder)
from kiwixbuild.utils import Remotefile, pj, SkipCommand from kiwixbuild.utils import Remotefile, pj, SkipCommand
class Qt(Dependency): class Qt(Dependency):
name = 'qt' name = "qt"
class Source(ReleaseDownload): class Source(ReleaseDownload):
name = "qt" name = "qt"
source_dir = "qt-5.10.1" source_dir = "qt-5.10.1"
archive = Remotefile('qt-everywhere-src-5.10.1.tar.xz', archive = Remotefile(
'', "qt-everywhere-src-5.10.1.tar.xz",
'http://ftp.oregonstate.edu/.1/blfs/conglomeration/qt5/qt-everywhere-src-5.10.1.tar.xz') "",
"http://ftp.oregonstate.edu/.1/blfs/conglomeration/qt5/qt-everywhere-src-5.10.1.tar.xz",
)
class Builder(MakeBuilder): class Builder(MakeBuilder):
dependencies = ['icu4c', 'zlib'] dependencies = ["icu4c", "zlib"]
dynamic_configure_options = ["-shared"] dynamic_configure_options = ["-shared"]
static_configure_options = ["-static"] static_configure_options = ["-static"]
@property @property
def all_configure_options(self): def all_configure_options(self):
yield from self.configure_options yield from self.configure_options
if self.buildEnv.platformInfo.static: if self.buildEnv.configInfo.static:
yield from self.static_configure_options yield from self.static_configure_options
else: else:
yield from self.dynamic_configure_options yield from self.dynamic_configure_options
if not self.target.force_native_build: if not self.target.force_native_build:
yield from self.buildEnv.platformInfo.configure_options yield from self.buildEnv.configInfo.configure_options
yield from ('-prefix', self.buildEnv.install_dir) yield from ("-prefix", self.buildEnv.install_dir)
yield from ('-libdir', pj(self.buildEnv.install_dir, self.buildEnv.libprefix)) yield from (
"-libdir",
pj(self.buildEnv.install_dir, self.buildEnv.libprefix),
)
@property @property
def configure_options(self): def configure_options(self):
skip_modules = [ skip_modules = [
'qt3d', "qt3d",
'qtcanvas3d', "qtcanvas3d",
'qtcharts', "qtcharts",
'qtconnectivity', "qtconnectivity",
'qtdatavis3d', "qtdatavis3d",
# 'qtdeclarative', # 'qtdeclarative',
'qtdoc', "qtdoc",
'qtgamepad', "qtgamepad",
'qtgraphicaleffects', "qtgraphicaleffects",
'qtlocation', "qtlocation",
'qtmultimedia', "qtmultimedia",
'qtnetworkauth', "qtnetworkauth",
'qtpurchasing', "qtpurchasing",
# 'qtquickcontrols', # 'qtquickcontrols',
'qtquickcontrols2', "qtquickcontrols2",
'qtremoteobjects', "qtremoteobjects",
'qtscript', "qtscript",
'qtscxml', "qtscxml",
'qtsensors', "qtsensors",
'qtserialbus', "qtserialbus",
'qtserialport', "qtserialport",
'qtspeech', "qtspeech",
'qtvirtualkeyboard', "qtvirtualkeyboard",
'qtwayland', "qtwayland",
'qtwebglplugin', "qtwebglplugin",
'qtwebsockets', "qtwebsockets",
# 'qtwebview', # 'qtwebview',
] ]
yield '-recheck' yield "-recheck"
yield '-opensource' yield "-opensource"
yield '-confirm-license' yield "-confirm-license"
yield '-ccache' yield "-ccache"
yield from ('-make', 'libs') yield from ("-make", "libs")
for module in skip_modules: for module in skip_modules:
yield from ('-skip', module) yield from ("-skip", module)
class QtWebEngine(Dependency): class QtWebEngine(Dependency):
@ -82,5 +83,5 @@ class QtWebEngine(Dependency):
Source = Qt.Source Source = Qt.Source
class Builder(QMakeBuilder): class Builder(QMakeBuilder):
dependencies = ['qt'] dependencies = ["qt"]
subsource_dir = "qtwebengine" subsource_dir = "qtwebengine"

58
kiwixbuild/dependencies/tc_android_ndk.py
View File

@ -5,23 +5,25 @@ from kiwixbuild.utils import Remotefile, add_execution_right, run_command
pj = os.path.join pj = os.path.join
class android_ndk(Dependency): class android_ndk(Dependency):
dont_skip = True dont_skip = True
neutral = False neutral = False
name = 'android-ndk' name = "android-ndk"
gccver = '4.9.x' gccver = "4.9.x"
api = '24' api = "24"
class Source(ReleaseDownload): class Source(ReleaseDownload):
archive = Remotefile('android-ndk-r21e-linux-x86_64.zip', archive = Remotefile(
'ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e', "android-ndk-r21e-linux-x86_64.zip",
'https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip') "ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e",
"https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip",
)
@property @property
def source_dir(self): def source_dir(self):
return self.target.full_name() return self.target.full_name()
class Builder(Builder): class Builder(Builder):
@property @property
def install_path(self): def install_path(self):
@ -32,36 +34,45 @@ class android_ndk(Dependency):
return self.target.api return self.target.api
@property @property
def platform(self): def config(self):
return 'android-'+self.api return "android-" + self.api
@property @property
def arch(self): def arch(self):
return self.buildEnv.platformInfo.arch return self.buildEnv.configInfo.arch
@property @property
def arch_full(self): def arch_full(self):
return self.buildEnv.platformInfo.arch_full return self.buildEnv.configInfo.arch_full
def _build_platform(self, context): def _build_toolchain(self, context):
context.try_skip(self.build_path) context.try_skip(self.build_path)
script = pj(self.source_path, 'build/tools/make_standalone_toolchain.py') script = pj(self.source_path, "build/tools/make_standalone_toolchain.py")
add_execution_right(script) add_execution_right(script)
command = [ command = [
script, script,
f'--arch={self.arch}', f"--arch={self.arch}",
f'--api={self.api}', f"--api={self.api}",
f'--install-dir={self.install_path}', f"--install-dir={self.install_path}",
'--force' "--force",
] ]
env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False) env = self.buildEnv.get_env(
cross_comp_flags=False, cross_compilers=False, cross_path=False
)
run_command(command, self.build_path, context, env=env) run_command(command, self.build_path, context, env=env)
def _fix_permission_right(self, context): def _fix_permission_right(self, context):
context.try_skip(self.build_path) context.try_skip(self.build_path)
bin_dirs = [pj(self.install_path, 'bin'), bin_dirs = [
pj(self.install_path, self.arch_full, 'bin'), pj(self.install_path, "bin"),
pj(self.install_path, 'libexec', 'gcc', self.arch_full, self.target.gccver) pj(self.install_path, self.arch_full, "bin"),
pj(
self.install_path,
"libexec",
"gcc",
self.arch_full,
self.target.gccver,
),
] ]
for root, dirs, files in os.walk(self.install_path): for root, dirs, files in os.walk(self.install_path):
if not root in bin_dirs: if not root in bin_dirs:
@ -74,6 +85,5 @@ class android_ndk(Dependency):
add_execution_right(file_path) add_execution_right(file_path)
def build(self): def build(self):
self.command('build_platform', self._build_platform) self.command("build_toolchain", self._build_toolchain)
self.command('fix_permission_right', self._fix_permission_right) self.command("fix_permission_right", self._fix_permission_right)

37
kiwixbuild/dependencies/tc_armhf.py
View File

@ -4,21 +4,25 @@ from kiwixbuild.utils import Remotefile
# The arm toolchains # The arm toolchains
# This is based on toolchains published here : https://github.com/tttapa/docker-arm-cross-toolchain # This is based on toolchains published here : https://github.com/tttapa/docker-arm-cross-toolchain
base_url = "https://github.com/tttapa/docker-arm-cross-toolchain/releases/download/0.1.0/" base_url = (
"https://github.com/tttapa/docker-arm-cross-toolchain/releases/download/0.1.0/"
)
aarch_base_url = 'https://master.dl.sourceforge.net/project/raspberry-pi-cross-compilers/Bonus%20Raspberry%20Pi%20GCC%2064-Bit%20Toolchains/Raspberry%20Pi%20GCC%2064-Bit%20Cross-Compiler%20Toolchains/Stretch/GCC%206.3.0/' aarch_base_url = "https://master.dl.sourceforge.net/project/raspberry-pi-cross-compilers/Bonus%20Raspberry%20Pi%20GCC%2064-Bit%20Toolchains/Raspberry%20Pi%20GCC%2064-Bit%20Cross-Compiler%20Toolchains/Stretch/GCC%206.3.0/"
class armv6_toolchain(Dependency): class armv6_toolchain(Dependency):
dont_skip = True dont_skip = True
neutral = True neutral = True
name = 'armv6' name = "armv6"
class Source(ReleaseDownload): class Source(ReleaseDownload):
archive = Remotefile('x-tools-armv6-rpi-linux-gnueabihf.tar.xz', archive = Remotefile(
'4c371c4c5b55ebd1f3d7dd26b14703632d9ba47423f901bcd9303d83ad444434', "x-tools-armv6-rpi-linux-gnueabihf.tar.xz",
base_url + 'x-tools-armv6-rpi-linux-gnueabihf.tar.xz') "4c371c4c5b55ebd1f3d7dd26b14703632d9ba47423f901bcd9303d83ad444434",
base_url + "x-tools-armv6-rpi-linux-gnueabihf.tar.xz",
)
class Builder(TcCopyBuilder): class Builder(TcCopyBuilder):
src_subdir = "armv6-rpi-linux-gnueabihf" src_subdir = "armv6-rpi-linux-gnueabihf"
@ -27,24 +31,29 @@ class armv6_toolchain(Dependency):
class armv8_toolchain(Dependency): class armv8_toolchain(Dependency):
dont_skip = True dont_skip = True
neutral = True neutral = True
name = 'armv8' name = "armv8"
class Source(ReleaseDownload): class Source(ReleaseDownload):
archive = Remotefile('x-tools-armv8-rpi-linux-gnueabihf.tar.xz', archive = Remotefile(
'cc28f5c3f6a3e7d9985f98779c4e72224b4eb5a7e4dc2bcdefd90cb241fb94a5', "x-tools-armv8-rpi-linux-gnueabihf.tar.xz",
base_url + 'x-tools-armv8-rpi3-linux-gnueabihf.tar.xz') "cc28f5c3f6a3e7d9985f98779c4e72224b4eb5a7e4dc2bcdefd90cb241fb94a5",
base_url + "x-tools-armv8-rpi3-linux-gnueabihf.tar.xz",
)
class Builder(TcCopyBuilder): class Builder(TcCopyBuilder):
src_subdir = "armv8-rpi3-linux-gnueabihf" src_subdir = "armv8-rpi3-linux-gnueabihf"
class aarch64_toolchain(Dependency): class aarch64_toolchain(Dependency):
dont_skip = True dont_skip = True
neutral = True neutral = True
name = "aarch64" name = "aarch64"
class Source(ReleaseDownload): class Source(ReleaseDownload):
archive = Remotefile('cross-gcc-6.3.0-pi_64.tar.gz', archive = Remotefile(
'1b048bb8886ad63d21797cd9129fc37b9ea0dfaac7e3c36f888aa16fbec1d320', "cross-gcc-6.3.0-pi_64.tar.gz",
aarch_base_url + 'cross-gcc-6.3.0-pi_64.tar.gz') "1b048bb8886ad63d21797cd9129fc37b9ea0dfaac7e3c36f888aa16fbec1d320",
aarch_base_url + "cross-gcc-6.3.0-pi_64.tar.gz",
)
Builder = TcCopyBuilder Builder = TcCopyBuilder

20
kiwixbuild/dependencies/tc_emsdk.py
View File

@ -5,21 +5,23 @@ from kiwixbuild.utils import Remotefile, run_command, copy_tree
pj = os.path.join pj = os.path.join
class emsdk(Dependency): class emsdk(Dependency):
dont_skip = True dont_skip = True
neutral = False neutral = False
name = 'emsdk' name = "emsdk"
class Source(ReleaseDownload): class Source(ReleaseDownload):
archive = Remotefile('emsdk-3.1.41.tar.gz', archive = Remotefile(
'147a2d72df34227bdb4ffedc587a8cb674a42269c40458f3f69ae37e8966cdc6', "emsdk-3.1.41.tar.gz",
'https://codeload.github.com/emscripten-core/emsdk/tar.gz/refs/tags/3.1.41') "147a2d72df34227bdb4ffedc587a8cb674a42269c40458f3f69ae37e8966cdc6",
"https://codeload.github.com/emscripten-core/emsdk/tar.gz/refs/tags/3.1.41",
)
@property @property
def source_dir(self): def source_dir(self):
return self.target.full_name() return self.target.full_name()
class Builder(Builder): class Builder(Builder):
@property @property
def install_path(self): def install_path(self):
@ -39,9 +41,7 @@ class emsdk(Dependency):
command = ["./emsdk", "activate", "3.1.24"] command = ["./emsdk", "activate", "3.1.24"]
run_command(command, self.install_path, context) run_command(command, self.install_path, context)
def build(self): def build(self):
self.command('copy_source', self._copy_source) self.command("copy_source", self._copy_source)
self.command('install', self._install) self.command("install", self._install)
self.command('activate', self._activate) self.command("activate", self._activate)

55
kiwixbuild/dependencies/tc_flatpak.py
View File

@ -5,9 +5,10 @@ from kiwixbuild.utils import Remotefile, add_execution_right, run_command
pj = os.path.join pj = os.path.join
class org_kde(Dependency): class org_kde(Dependency):
neutral = False neutral = False
name = 'org.kde' name = "org.kde"
Source = NoopSource Source = NoopSource
@ -15,32 +16,42 @@ class org_kde(Dependency):
def _setup_remote(self, context): def _setup_remote(self, context):
command = [ command = [
"flatpak", "flatpak",
"--user", "remote-add", "--if-not-exists", "--user",
"remote-add",
"--if-not-exists",
"flathub", "flathub",
"https://flathub.org/repo/flathub.flatpakrepo" "https://flathub.org/repo/flathub.flatpakrepo",
] ]
env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False) env = self.buildEnv.get_env(
cross_comp_flags=False, cross_compilers=False, cross_path=False
)
run_command(command, self.buildEnv.build_dir, context, env=env) run_command(command, self.buildEnv.build_dir, context, env=env)
def _install_sdk(self, context): def _install_sdk(self, context):
command = [ command = [
"flatpak", "flatpak",
"--user", "install", "--noninteractive", "--verbose", "-y", "--user",
"install",
"--noninteractive",
"--verbose",
"-y",
"flathub", "flathub",
f"{self.target.name}.Sdk//{self.target.version()}", f"{self.target.name}.Sdk//{self.target.version()}",
f"{self.target.name}.Platform//{self.target.version()}" f"{self.target.name}.Platform//{self.target.version()}",
] ]
env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False) env = self.buildEnv.get_env(
cross_comp_flags=False, cross_compilers=False, cross_path=False
)
run_command(command, self.buildEnv.build_dir, context, env=env) run_command(command, self.buildEnv.build_dir, context, env=env)
def build(self): def build(self):
self.command('setup_remote', self._setup_remote) self.command("setup_remote", self._setup_remote)
self.command('install_sdk', self._install_sdk) self.command("install_sdk", self._install_sdk)
class io_qt_qtwebengine(Dependency): class io_qt_qtwebengine(Dependency):
neutral = False neutral = False
name = 'io.qt.qtwebengine' name = "io.qt.qtwebengine"
Source = NoopSource Source = NoopSource
@ -48,23 +59,31 @@ class io_qt_qtwebengine(Dependency):
def _setup_remote(self, context): def _setup_remote(self, context):
command = [ command = [
"flatpak", "flatpak",
"--user", "remote-add", "--if-not-exists", "--user",
"remote-add",
"--if-not-exists",
"flathub", "flathub",
"https://flathub.org/repo/flathub.flatpakrepo" "https://flathub.org/repo/flathub.flatpakrepo",
] ]
env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False) env = self.buildEnv.get_env(
cross_comp_flags=False, cross_compilers=False, cross_path=False
)
run_command(command, self.buildEnv.build_dir, context, env=env) run_command(command, self.buildEnv.build_dir, context, env=env)
def _install_sdk(self, context): def _install_sdk(self, context):
command = [ command = [
"flatpak", "flatpak",
"--user", "install", "-y", "--user",
"install",
"-y",
"flathub", "flathub",
f"{self.target.name}.BaseApp//{self.target.version()}" f"{self.target.name}.BaseApp//{self.target.version()}",
] ]
env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False) env = self.buildEnv.get_env(
cross_comp_flags=False, cross_compilers=False, cross_path=False
)
run_command(command, self.buildEnv.build_dir, context, env=env) run_command(command, self.buildEnv.build_dir, context, env=env)
def build(self): def build(self):
self.command('setup_remote', self._setup_remote) self.command("setup_remote", self._setup_remote)
self.command('install_sdk', self._install_sdk) self.command("install_sdk", self._install_sdk)

17
kiwixbuild/dependencies/tc_musl.py
View File

@ -1,15 +1,18 @@
from .base import Dependency, ReleaseDownload, TcCopyBuilder from .base import Dependency, ReleaseDownload, TcCopyBuilder
from kiwixbuild.utils import Remotefile from kiwixbuild.utils import Remotefile
class aarch64_musl_toolchain(Dependency): class aarch64_musl_toolchain(Dependency):
dont_skip = True dont_skip = True
neutral = True neutral = True
name = "aarch64_musl" name = "aarch64_musl"
class Source(ReleaseDownload): class Source(ReleaseDownload):
archive = Remotefile('aarch64-linux-musl-cross.tgz', archive = Remotefile(
'0f18a885b161815520bbb5757a4b4ab40d0898c29bebee58d0cddd6112e59cc6', "aarch64-linux-musl-cross.tgz",
'https://more.musl.cc/10/x86_64-linux-musl/aarch64-linux-musl-cross.tgz') "0f18a885b161815520bbb5757a4b4ab40d0898c29bebee58d0cddd6112e59cc6",
"https://more.musl.cc/10/x86_64-linux-musl/aarch64-linux-musl-cross.tgz",
)
Builder = TcCopyBuilder Builder = TcCopyBuilder
@ -20,8 +23,10 @@ class x86_64_musl_toolchain(Dependency):
name = "x86-64_musl" name = "x86-64_musl"
class Source(ReleaseDownload): class Source(ReleaseDownload):
archive = Remotefile('x86_64-linux-musl-cross.tgz', archive = Remotefile(
'a3d55de8105739fcfb8b10eaa72cdb5d779319726bacff24149388d7608d1ed8', "x86_64-linux-musl-cross.tgz",
'https://more.musl.cc/10/x86_64-linux-musl/x86_64-linux-musl-cross.tgz') "a3d55de8105739fcfb8b10eaa72cdb5d779319726bacff24149388d7608d1ed8",
"https://more.musl.cc/10/x86_64-linux-musl/x86_64-linux-musl-cross.tgz",
)
Builder = TcCopyBuilder Builder = TcCopyBuilder

30
kiwixbuild/dependencies/uuid.py
View File

@ -1,30 +1,30 @@
from .base import ( from .base import Dependency, ReleaseDownload, MakeBuilder
Dependency,
ReleaseDownload,
MakeBuilder
)
from kiwixbuild.utils import Remotefile from kiwixbuild.utils import Remotefile
class UUID(Dependency): class UUID(Dependency):
name = 'uuid' name = "uuid"
class Source(ReleaseDownload): class Source(ReleaseDownload):
archive = Remotefile('e2fsprogs-libs-1.43.4.tar.gz', archive = Remotefile(
'eed4516325768255c9745e7b82c9d7d0393abce302520a5b2cde693204b0e419', "e2fsprogs-libs-1.43.4.tar.gz",
'https://www.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v1.43.4/e2fsprogs-libs-1.43.4.tar.gz') "eed4516325768255c9745e7b82c9d7d0393abce302520a5b2cde693204b0e419",
extract_dir = 'e2fsprogs-libs-1.43.4' "https://www.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v1.43.4/e2fsprogs-libs-1.43.4.tar.gz",
)
extract_dir = "e2fsprogs-libs-1.43.4"
class Builder(MakeBuilder): class Builder(MakeBuilder):
configure_options = ["--enable-libuuid", configure_options = [
"--enable-libuuid",
"--disable-debugfs", "--disable-debugfs",
"--disable-imager", "--disable-imager",
"--disable-resizer", "--disable-resizer",
"--disable-defrag", "--disable-defrag",
"--enable-fsck", "--enable-fsck",
"--disable-uuidd"] "--disable-uuidd",
configure_env = {'_format_CFLAGS': "{env.CFLAGS} -O3 -fPIC"} ]
configure_env = {"_format_CFLAGS": "{env.CFLAGS} -O3 -fPIC"}
static_configure_options = dynamic_configure_options = [] static_configure_options = dynamic_configure_options = []
make_targets = ['libs'] make_targets = ["libs"]
make_install_targets = ['install-libs'] make_install_targets = ["install-libs"]

34
kiwixbuild/dependencies/xapian.py
View File

@ -1,8 +1,4 @@
from .base import ( from .base import Dependency, ReleaseDownload, MakeBuilder
Dependency,
ReleaseDownload,
MakeBuilder
)
from kiwixbuild.utils import Remotefile from kiwixbuild.utils import Remotefile
from kiwixbuild._global import neutralEnv from kiwixbuild._global import neutralEnv
@ -12,23 +8,29 @@ class Xapian(Dependency):
name = "xapian-core" name = "xapian-core"
class Source(ReleaseDownload): class Source(ReleaseDownload):
archive = Remotefile('xapian-core-1.4.23.tar.xz', archive = Remotefile(
'30d3518172084f310dab86d262b512718a7f9a13635aaa1a188e61dc26b2288c') "xapian-core-1.4.23.tar.xz",
"30d3518172084f310dab86d262b512718a7f9a13635aaa1a188e61dc26b2288c",
)
class Builder(MakeBuilder): class Builder(MakeBuilder):
configure_options = [ configure_options = [
"--disable-sse", "--disable-sse",
"--disable-backend-chert", "--disable-backend-chert",
"--disable-backend-remote", "--disable-backend-remote",
"--disable-documentation"] "--disable-documentation",
configure_env = {'_format_LDFLAGS': "{env.LDFLAGS} -L{buildEnv.install_dir}/{buildEnv.libprefix}", ]
'_format_CXXFLAGS': "{env.CXXFLAGS} -O3 -I{buildEnv.install_dir}/include"} configure_env = {
"_format_LDFLAGS": "{env.LDFLAGS} -L{buildEnv.install_dir}/{buildEnv.libprefix}",
"_format_CXXFLAGS": "{env.CXXFLAGS} -O3 -I{buildEnv.install_dir}/include",
}
@classmethod @classmethod
def get_dependencies(cls, platformInfo, allDeps): def get_dependencies(cls, configInfo, allDeps):
deps = ['zlib', 'lzma'] deps = ["zlib", "lzma"]
if (platformInfo.build in ('win32', 'wasm') if (
or neutralEnv('distname') == 'Darwin'): configInfo.build in ("win32", "wasm")
or neutralEnv("distname") == "Darwin"
):
return deps return deps
return deps + ['uuid'] return deps + ["uuid"]

14
kiwixbuild/dependencies/zim_testing_suite.py
View File

@ -1,8 +1,4 @@
from .base import ( from .base import Dependency, ReleaseDownload, NoopBuilder
Dependency,
ReleaseDownload,
NoopBuilder
)
from kiwixbuild.utils import Remotefile from kiwixbuild.utils import Remotefile
@ -12,8 +8,10 @@ class ZimTestingSuite(Dependency):
dont_skip = True dont_skip = True
class Source(ReleaseDownload): class Source(ReleaseDownload):
archive = Remotefile('zim-testing-suite-0.3.tar.gz', archive = Remotefile(
'cd7d1ccc48af3783af9156cb6bf3c18d9a3319a73fdeefe65f0b4cae402d3d66', "zim-testing-suite-0.3.tar.gz",
'https://github.com/openzim/zim-testing-suite/releases/download/v0.3/zim-testing-suite-0.3.tar.gz') "cd7d1ccc48af3783af9156cb6bf3c18d9a3319a73fdeefe65f0b4cae402d3d66",
"https://github.com/openzim/zim-testing-suite/releases/download/v0.3/zim-testing-suite-0.3.tar.gz",
)
Builder = NoopBuilder Builder = NoopBuilder

18
kiwixbuild/dependencies/zim_tools.py
View File

@ -1,7 +1,5 @@
from .base import ( from .base import Dependency, GitClone, MesonBuilder
Dependency,
GitClone,
MesonBuilder)
class ZimTools(Dependency): class ZimTools(Dependency):
name = "zim-tools" name = "zim-tools"
@ -13,16 +11,16 @@ class ZimTools(Dependency):
class Builder(MesonBuilder): class Builder(MesonBuilder):
@classmethod @classmethod
def get_dependencies(cls, platformInfo, allDeps): def get_dependencies(cls, configInfo, allDeps):
base_deps = ['libzim', 'docoptcpp', 'mustache'] base_deps = ["libzim", "docoptcpp", "mustache"]
if platformInfo.build != 'win32': if configInfo.build != "win32":
base_deps += ['libmagic', 'gumbo'] base_deps += ["libmagic", "gumbo"]
return base_deps return base_deps
@property @property
def configure_options(self): def configure_options(self):
# We don't build zimwriterfs on win32, and so we don't have magic # We don't build zimwriterfs on win32, and so we don't have magic
if self.buildEnv.platformInfo.build != 'win32': if self.buildEnv.configInfo.build != "win32":
yield f"-Dmagic-install-prefix={self.buildEnv.install_dir}" yield f"-Dmagic-install-prefix={self.buildEnv.install_dir}"
if self.buildEnv.platformInfo.static: if self.buildEnv.configInfo.static:
yield "-Dstatic-linkage=true" yield "-Dstatic-linkage=true"

46
kiwixbuild/dependencies/zlib.py
View File

@ -1,59 +1,63 @@
import shutil import shutil
from .base import ( from .base import Dependency, ReleaseDownload, MakeBuilder
Dependency,
ReleaseDownload,
MakeBuilder)
from kiwixbuild.utils import Remotefile, pj, SkipCommand from kiwixbuild.utils import Remotefile, pj, SkipCommand
class zlib(Dependency): class zlib(Dependency):
name = 'zlib' name = "zlib"
class Source(ReleaseDownload): class Source(ReleaseDownload):
archive = Remotefile('zlib-1.2.12.tar.gz', archive = Remotefile(
'91844808532e5ce316b3c010929493c0244f3d37593afd6de04f71821d5136d9') "zlib-1.2.12.tar.gz",
patches = ['zlib_std_libname.patch'] "91844808532e5ce316b3c010929493c0244f3d37593afd6de04f71821d5136d9",
)
patches = ["zlib_std_libname.patch"]
class Builder(MakeBuilder): class Builder(MakeBuilder):
dynamic_configure_options = ["--shared"] dynamic_configure_options = ["--shared"]
static_configure_options = ["--static"] static_configure_options = ["--static"]
make_install_targets = ['install'] make_install_targets = ["install"]
def _pre_build_script(self, context): def _pre_build_script(self, context):
context.try_skip(self.build_path) context.try_skip(self.build_path)
shutil.copytree(self.source_path, self.build_path) shutil.copytree(self.source_path, self.build_path)
def _configure(self, context): def _configure(self, context):
if self.buildEnv.platformInfo.build == 'win32': if self.buildEnv.configInfo.build == "win32":
raise SkipCommand() raise SkipCommand()
return super()._configure(context) return super()._configure(context)
@property @property
def all_configure_options(self): def all_configure_options(self):
yield from self.configure_options yield from self.configure_options
yield from self.static_configure_options if self.buildEnv.platformInfo.static else self.dynamic_configure_options yield from self.static_configure_options if self.buildEnv.configInfo.static else self.dynamic_configure_options
yield from ('--prefix', self.buildEnv.install_dir) yield from ("--prefix", self.buildEnv.install_dir)
yield from ('--libdir', pj(self.buildEnv.install_dir, self.buildEnv.libprefix)) yield from (
"--libdir",
pj(self.buildEnv.install_dir, self.buildEnv.libprefix),
)
@property @property
def make_options(self): def make_options(self):
if self.buildEnv.platformInfo.build != 'win32': if self.buildEnv.configInfo.build != "win32":
return return
yield "--makefile" yield "--makefile"
yield "win32/Makefile.gcc" yield "win32/Makefile.gcc"
yield "PREFIX=i686-w64-mingw32-", yield "PREFIX=i686-w64-mingw32-",
yield "SHARED_MODE={}".format("0" if self.buildEnv.platformInfo.static else "1"), yield "SHARED_MODE={}".format(
yield "INCLUDE_PATH={}".format(pj(self.buildEnv.install_dir, 'include')), "0" if self.buildEnv.configInfo.static else "1"
yield "LIBRARY_PATH={}".format(pj(self.buildEnv.install_dir, self.buildEnv.libprefix)), ),
yield "BINARY_PATH={}".format(pj(self.buildEnv.install_dir, 'bin')) yield "INCLUDE_PATH={}".format(pj(self.buildEnv.install_dir, "include")),
yield "LIBRARY_PATH={}".format(
pj(self.buildEnv.install_dir, self.buildEnv.libprefix)
),
yield "BINARY_PATH={}".format(pj(self.buildEnv.install_dir, "bin"))
@property @property
def make_targets(self): def make_targets(self):
if self.buildEnv.platformInfo.static: if self.buildEnv.configInfo.static:
return ["static"] return ["static"]
else: else:
return ["shared"] return ["shared"]

22
kiwixbuild/dependencies/zstd.py
View File

@ -1,22 +1,20 @@
from .base import ( from .base import Dependency, ReleaseDownload, MesonBuilder
Dependency,
ReleaseDownload,
MesonBuilder)
from kiwixbuild.utils import Remotefile from kiwixbuild.utils import Remotefile
class zstd(Dependency): class zstd(Dependency):
name = 'zstd' name = "zstd"
class Source(ReleaseDownload): class Source(ReleaseDownload):
archive = Remotefile('zstd-1.5.2.tar.gz', archive = Remotefile(
'f7de13462f7a82c29ab865820149e778cbfe01087b3a55b5332707abf9db4a6e', "zstd-1.5.2.tar.gz",
'https://github.com/facebook/zstd/archive/refs/tags/v1.5.2.tar.gz') "f7de13462f7a82c29ab865820149e778cbfe01087b3a55b5332707abf9db4a6e",
"https://github.com/facebook/zstd/archive/refs/tags/v1.5.2.tar.gz",
)
class Builder(MesonBuilder): class Builder(MesonBuilder):
subsource_dir = 'build/meson' subsource_dir = "build/meson"
build_type = 'release' build_type = "release"
strip_options = [] strip_options = []
configure_options = ['-Dbin_programs=false', '-Dbin_contrib=false'] configure_options = ["-Dbin_programs=false", "-Dbin_contrib=false"]

295
kiwixbuild/flatpak_builder.py
View File

@ -1,17 +1,19 @@
import sys import sys
from collections import OrderedDict from collections import OrderedDict
from .buildenv import * from .buildenv import *
from .platforms import PlatformInfo from .configs import ConfigInfo
from .utils import remove_duplicates, run_command, StopBuild, Context from .utils import remove_duplicates, run_command, StopBuild, Context
from .dependencies import Dependency from .dependencies import Dependency
from .packages import PACKAGE_NAME_MAPPERS from .packages import PACKAGE_NAME_MAPPERS
from .versions import base_deps_versions from .versions import base_deps_versions
from ._global import ( from ._global import (
neutralEnv, option, neutralEnv,
add_target_step, get_target_step, target_steps, option,
backend) add_target_step,
get_target_step,
target_steps,
)
from . import _global from . import _global
from .dependencies.base import ( from .dependencies.base import (
Source, Source,
@ -22,7 +24,8 @@ from .dependencies.base import (
CMakeBuilder, CMakeBuilder,
QMakeBuilder, QMakeBuilder,
MakeBuilder, MakeBuilder,
SCRIPT_DIR) SCRIPT_DIR,
)
import json import json
from shutil import copyfile from shutil import copyfile
from urllib.parse import urlparse from urllib.parse import urlparse
@ -30,54 +33,60 @@ from urllib.request import urlopen
import json import json
MANIFEST = { MANIFEST = {
'app-id': 'org.kiwix.desktop', "app-id": "org.kiwix.desktop",
'runtime': 'org.kde.Platform', "runtime": "org.kde.Platform",
'runtime-version': base_deps_versions['org.kde'], "runtime-version": base_deps_versions["org.kde"],
'base': 'io.qt.qtwebengine.BaseApp', "base": "io.qt.qtwebengine.BaseApp",
'base-version': base_deps_versions['org.kde'], # keep BaseApp (qwebengine) in sync with org.kde "base-version": base_deps_versions[
'sdk': 'org.kde.Sdk', "org.kde"
'command': 'kiwix-desktop', ], # keep BaseApp (qwebengine) in sync with org.kde
'rename-icon': 'kiwix-desktop', "sdk": "org.kde.Sdk",
'finish-args': [ "command": "kiwix-desktop",
'--socket=wayland', "rename-icon": "kiwix-desktop",
'--socket=x11', "finish-args": [
'--share=network', "--socket=wayland",
'--share=ipc', "--socket=x11",
'--device=dri', "--share=network",
'--socket=pulseaudio', "--share=ipc",
"--device=dri",
"--socket=pulseaudio",
],
"cleanup": [
"/include",
"/lib/pkgconfig",
"/lib/cmake",
"/lib/*.la",
"/bin/curl",
"/bin/copydatabase",
"/bin/kiwix-compile-resources",
"/bin/kiwix-manage",
"/bin/kiwix-read",
"/bin/kiwix-search",
"/bin/quest",
"/bin/simple*",
"/bin/xapian-*",
"/share/aclocal",
"/share/doc",
"/share/man",
], ],
'cleanup': [
'/include',
'/lib/pkgconfig',
'/lib/cmake',
'/lib/*.la',
'/bin/curl',
'/bin/copydatabase',
'/bin/kiwix-compile-resources',
'/bin/kiwix-manage',
'/bin/kiwix-read',
'/bin/kiwix-search',
'/bin/quest',
'/bin/simple*',
'/bin/xapian-*',
'/share/aclocal',
'/share/doc',
'/share/man'
]
} }
GET_REF_URL_API_TEMPLATE = 'https://api.github.com/repos{repo}/git/refs/tags/{ref}' GET_REF_URL_API_TEMPLATE = "https://api.github.com/repos{repo}/git/refs/tags/{ref}"
class FlatpakBuilder: class FlatpakBuilder:
def __init__(self): def __init__(self):
self._targets = {} self._targets = {}
PlatformInfo.get_platform('neutral', self._targets) ConfigInfo.get_config("neutral", self._targets)
self.platform = PlatformInfo.get_platform('flatpak', self._targets) self.config = ConfigInfo.get_config("flatpak", self._targets)
if neutralEnv('distname') not in self.platform.compatible_hosts: if neutralEnv("distname") not in self.config.compatible_hosts:
print(('ERROR: The target platform {} cannot be build on host {}.\n' print(
'Select another target platform or change your host system.' (
).format(self.platform.name, neutralEnv('distname'))) "ERROR: The config {} cannot be build on host {}.\n"
self.targetDefs = self.platform.add_targets(option('target'), self._targets) "Select another config or change your host system."
).format(self.config.name, neutralEnv("distname"))
)
self.targetDefs = self.config.add_targets(option("target"), self._targets)
def finalize_target_steps(self): def finalize_target_steps(self):
steps = [] steps = []
@ -85,14 +94,14 @@ class FlatpakBuilder:
steps += self.order_steps(targetDef) steps += self.order_steps(targetDef)
steps = list(remove_duplicates(steps)) steps = list(remove_duplicates(steps))
for pltName in PlatformInfo.all_running_platforms: for cfgName in ConfigInfo.all_running_configs:
plt = PlatformInfo.all_platforms[pltName] cfg = ConfigInfo.all_configs[cfgName]
for tlcName in plt.toolchain_names: for tlcName in cfg.toolchain_names:
tlc = Dependency.all_deps[tlcName] tlc = Dependency.all_deps[tlcName]
src_plt_step = ('source', tlcName) src_cfg_step = ("source", tlcName)
add_target_step(src_plt_step, self._targets[src_plt_step]) add_target_step(src_cfg_step, self._targets[src_cfg_step])
blt_plt_step = ('neutral' if tlc.neutral else pltName, tlcName) blt_cfg_step = ("neutral" if tlc.neutral else cfgName, tlcName)
add_target_step(blt_plt_step, self._targets[blt_plt_step]) add_target_step(blt_cfg_step, self._targets[blt_cfg_step])
for dep in steps: for dep in steps:
add_target_step(dep, self._targets[dep]) add_target_step(dep, self._targets[dep])
@ -103,8 +112,8 @@ class FlatpakBuilder:
yield from self.order_dependencies(targetDef, _targets) yield from self.order_dependencies(targetDef, _targets)
def order_dependencies(self, targetDef, targets): def order_dependencies(self, targetDef, targets):
targetPlatformName, targetName = targetDef targetConfigName, targetName = targetDef
if targetPlatformName == 'source': if targetConfigName == "source":
# Do not try to order sources, they will be added as dep by the # Do not try to order sources, they will be added as dep by the
# build step two lines later. # build step two lines later.
return return
@ -113,27 +122,27 @@ class FlatpakBuilder:
except KeyError: except KeyError:
return return
targetPlatform = PlatformInfo.get_platform(targetPlatformName) targetConfig = ConfigInfo.get_config(targetConfigName)
for dep in target.get_dependencies(targetPlatform, True): for dep in target.get_dependencies(targetConfig, True):
if isinstance(dep, tuple): if isinstance(dep, tuple):
depPlatform, depName = dep depConfig, depName = dep
else: else:
depPlatform, depName = targetPlatformName, dep depConfig, depName = targetConfigName, dep
if (depPlatform, depName) in targets: if (depConfig, depName) in targets:
yield from self.order_dependencies((depPlatform, depName), targets) yield from self.order_dependencies((depConfig, depName), targets)
yield ('source', targetName) yield ("source", targetName)
yield targetDef yield targetDef
def instanciate_steps(self): def instanciate_steps(self):
for stepDef in list(target_steps()): for stepDef in list(target_steps()):
stepPlatform, stepName = stepDef stepConfig, stepName = stepDef
stepClass = Dependency.all_deps[stepName] stepClass = Dependency.all_deps[stepName]
if stepPlatform == 'source': if stepConfig == "source":
source = get_target_step(stepDef)(stepClass) source = get_target_step(stepDef)(stepClass)
add_target_step(stepDef, source) add_target_step(stepDef, source)
else: else:
source = get_target_step(stepName, 'source') source = get_target_step(stepName, "source")
env = PlatformInfo.get_platform(stepPlatform).buildEnv env = ConfigInfo.get_config(stepConfig).buildEnv
builder = get_target_step(stepDef)(stepClass, source, env) builder = get_target_step(stepDef)(stepClass, source, env)
add_target_step(stepDef, builder) add_target_step(stepDef, builder)
@ -142,125 +151,130 @@ class FlatpakBuilder:
modules = OrderedDict() modules = OrderedDict()
for stepDef in steps: for stepDef in steps:
module = modules.setdefault(stepDef[1], {}) module = modules.setdefault(stepDef[1], {})
module['name'] = stepDef[1] module["name"] = stepDef[1]
if stepDef[0] == 'source': if stepDef[0] == "source":
source = get_target_step(stepDef) source = get_target_step(stepDef)
if getattr(source, 'flatpak_no_autogen', False): if getattr(source, "flatpak_no_autogen", False):
module['no-autogen'] = True module["no-autogen"] = True
module_sources = module.setdefault('sources', []) module_sources = module.setdefault("sources", [])
if isinstance(source, ReleaseDownload): if isinstance(source, ReleaseDownload):
src = { src = {
'type': 'archive', "type": "archive",
'sha256': source.archive.sha256, "sha256": source.archive.sha256,
'url': source.archive.url "url": source.archive.url,
} }
if hasattr(source, 'flatpak_dest'): if hasattr(source, "flatpak_dest"):
src['dest'] = source.flatpak_dest src["dest"] = source.flatpak_dest
module_sources.append(src) module_sources.append(src)
elif isinstance(source, GitClone): elif isinstance(source, GitClone):
src = { src = {
'type': 'git', "type": "git",
'url': source.git_remote, "url": source.git_remote,
'tag': source.git_ref "tag": source.git_ref,
} }
try: try:
parsed = urlparse(source.git_remote) parsed = urlparse(source.git_remote)
if parsed.hostname == 'github.com': if parsed.hostname == "github.com":
repo = parsed.path[:-4] repo = parsed.path[:-4]
api_url = GET_REF_URL_API_TEMPLATE.format( api_url = GET_REF_URL_API_TEMPLATE.format(
repo = repo, repo=repo, ref=source.git_ref
ref = source.git_ref) )
with urlopen(api_url) as r: with urlopen(api_url) as r:
ret = json.loads(r.read()) ret = json.loads(r.read())
src['commit'] = ret['object']['sha'] src["commit"] = ret["object"]["sha"]
except: except:
pass pass
module_sources.append(src) module_sources.append(src)
for p in getattr(source, 'patches', []): for p in getattr(source, "patches", []):
patch = { patch = {"type": "patch", "path": "patches/" + p}
'type': 'patch',
'path': 'patches/' + p
}
module_sources.append(patch) module_sources.append(patch)
if hasattr(source, 'flatpak_command'): if hasattr(source, "flatpak_command"):
command = { command = {"type": "shell", "commands": [source.flatpak_command]}
'type': 'shell',
'commands': [source.flatpak_command]
}
module_sources.append(command) module_sources.append(command)
else: else:
builder = get_target_step(stepDef) builder = get_target_step(stepDef)
builder.set_flatpak_buildsystem(module) builder.set_flatpak_buildsystem(module)
print(module['name']) print(module["name"])
manifest = MANIFEST.copy() manifest = MANIFEST.copy()
modules = [m for m in modules.values() if m.get('sources')] modules = [m for m in modules.values() if m.get("sources")]
for m in modules: for m in modules:
temp = m['sources'] temp = m["sources"]
del m['sources'] del m["sources"]
m['sources'] = temp m["sources"] = temp
manifest['modules'] = modules manifest["modules"] = modules
manifest_name = "{}.json".format(MANIFEST['app-id']) manifest_name = "{}.json".format(MANIFEST["app-id"])
manifest_path = pj(self.platform.buildEnv.build_dir, manifest_name) manifest_path = pj(self.config.buildEnv.build_dir, manifest_name)
with open(manifest_path, 'w') as f: with open(manifest_path, "w") as f:
f.write(json.dumps(manifest, indent=4)) f.write(json.dumps(manifest, indent=4))
def copy_patches(self): def copy_patches(self):
sourceDefs = (tDef for tDef in target_steps() if tDef[0] == 'source') sourceDefs = (tDef for tDef in target_steps() if tDef[0] == "source")
for sourceDef in sourceDefs: for sourceDef in sourceDefs:
source = get_target_step(sourceDef) source = get_target_step(sourceDef)
if not hasattr(source, 'patches'): if not hasattr(source, "patches"):
continue continue
for p in source.patches: for p in source.patches:
path = pj(SCRIPT_DIR, 'patches', p) path = pj(SCRIPT_DIR, "patches", p)
os.makedirs(pj(self.platform.buildEnv.build_dir, 'patches'), exist_ok=True) os.makedirs(
dest = pj(self.platform.buildEnv.build_dir, 'patches', p) pj(self.config.buildEnv.build_dir, "patches"), exist_ok=True
)
dest = pj(self.config.buildEnv.build_dir, "patches", p)
copyfile(path, dest) copyfile(path, dest)
def build(self): def build(self):
log = pj(self.platform.buildEnv.log_dir, 'cmd_build_flatpak.log') log = pj(self.config.buildEnv.log_dir, "cmd_build_flatpak.log")
context = Context('build', log, False) context = Context("build", log, False)
command = [ command = [
"flatpak-builder", "flatpak-builder",
"--user", "--ccache", "--force-clean", "--keep-build-dirs", "--user",
"--disable-rofiles-fuse", "--repo=repo", "builddir", "--ccache",
f"{MANIFEST['app-id']}.json" "--force-clean",
"--keep-build-dirs",
"--disable-rofiles-fuse",
"--repo=repo",
"builddir",
f"{MANIFEST['app-id']}.json",
] ]
try: try:
run_command(command, self.platform.buildEnv.build_dir, context, env=self.platform.get_env()) run_command(
command,
self.config.buildEnv.build_dir,
context,
env=self.config.get_env(),
)
context._finalise() context._finalise()
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
with open(log, 'r') as f: with open(log, "r") as f:
print(f.read()) print(f.read())
raise StopBuild() raise StopBuild()
def bundle(self): def bundle(self):
log = pj(self.platform.buildEnv.log_dir, 'cmd_bundle_flatpak.log') log = pj(self.config.buildEnv.log_dir, "cmd_bundle_flatpak.log")
context = Context('bundle', log, False) context = Context("bundle", log, False)
app_id = MANIFEST['app-id'] app_id = MANIFEST["app-id"]
command = [ command = ["flatpak", "build-bundle", "repo", f"{app_id}.flatpak", app_id]
"flatpak", "build-bundle", "repo",
f"{app_id}.flatpak",
app_id
]
try: try:
run_command(command, self.platform.buildEnv.build_dir, context, env=self.platform.get_env()) run_command(
command,
self.config.buildEnv.build_dir,
context,
env=self.config.get_env(),
)
context._finalise() context._finalise()
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
with open(log, 'r') as f: with open(log, "r") as f:
print(f.read()) print(f.read())
raise StopBuild() raise StopBuild()
def _get_packages(self): def _get_packages(self):
package_name_mapper = PACKAGE_NAME_MAPPERS.get('flatpak', {}) package_name_mapper = PACKAGE_NAME_MAPPERS.get("flatpak", {})
to_drop = [] to_drop = []
for builderDef in self._targets: for builderDef in self._targets:
platformName, builderName = builderDef configName, builderName = builderDef
packages = package_name_mapper.get(builderName) packages = package_name_mapper.get(builderName)
if packages: if packages:
to_drop.append(builderDef) to_drop.append(builderDef)
@ -274,16 +288,16 @@ class FlatpakBuilder:
# dependencies we already have in the sdk. # dependencies we already have in the sdk.
self._get_packages() self._get_packages()
self.finalize_target_steps() self.finalize_target_steps()
print("[SETUP PLATFORMS]") print("[SETUP TOOLCHAINS]")
for platform in PlatformInfo.all_running_platforms.values(): for config in ConfigInfo.all_running_configs.values():
platform.finalize_setup() config.finalize_setup()
for pltName in PlatformInfo.all_running_platforms: for cfgName in ConfigInfo.all_running_configs:
plt = PlatformInfo.all_platforms[pltName] cfg = ConfigInfo.all_configs[cfgName]
for tlcName in plt.toolchain_names: for tlcName in cfg.toolchain_names:
tlc = Dependency.all_deps[tlcName] tlc = Dependency.all_deps[tlcName]
builderDef = (pltName, tlcName) builderDef = (cfgName, tlcName)
builder = get_target_step(builderDef) builder = get_target_step(builderDef)
print("build {} ({}):".format(builder.name, pltName[0])) print("build {} ({}):".format(builder.name, cfgName[0]))
add_target_step(builderDef, builder) add_target_step(builderDef, builder)
builder.build() builder.build()
print("[GENERATE FLATPAK MANIFEST]") print("[GENERATE FLATPAK MANIFEST]")
@ -295,11 +309,10 @@ class FlatpakBuilder:
self.bundle() self.bundle()
# No error, clean intermediate file at end of build if needed. # No error, clean intermediate file at end of build if needed.
print("[CLEAN]") print("[CLEAN]")
if option('clean_at_end'): if option("clean_at_end"):
for platform in PlatformInfo.all_running_platforms.values(): for config in ConfigInfo.all_running_configs.values():
platform.clean_intermediate_directories() config.clean_intermediate_directories()
else: else:
print("SKIP") print("SKIP")
except StopBuild: except StopBuild:
sys.exit("Stopping build due to errors") sys.exit("Stopping build due to errors")

210
kiwixbuild/packages.py
View File

@ -1,107 +1,147 @@
_fedora_common = [
"automake",
_fedora_common = ['automake', 'libtool', 'cmake', 'git', 'subversion', 'ccache', 'pkgconf-pkg-config', 'gcc-c++', 'gettext-devel'] "libtool",
_debian_common = ['automake', 'libtool', 'cmake', 'git', 'subversion', 'ccache', 'pkg-config', 'gcc', 'autopoint'] "cmake",
"git",
"subversion",
"ccache",
"pkgconf-pkg-config",
"gcc-c++",
"gettext-devel",
]
_debian_common = [
"automake",
"libtool",
"cmake",
"git",
"subversion",
"ccache",
"pkg-config",
"gcc",
"autopoint",
]
PACKAGE_NAME_MAPPERS = { PACKAGE_NAME_MAPPERS = {
'flatpak': { "flatpak": {
'zlib': True, "zlib": True,
'lzma': True, "lzma": True,
'icu4c': True, "icu4c": True,
'qt': True, "qt": True,
'qtwebengine': True, "qtwebengine": True,
'uuid': True, "uuid": True,
'libxml2': True, "libxml2": True,
'libssl': True, "libssl": True,
'libcurl': True, "libcurl": True,
}, },
'fedora_native_dyn': { "fedora_native_dyn": {
'COMMON': _fedora_common, "COMMON": _fedora_common,
'uuid': ['libuuid-devel'], "uuid": ["libuuid-devel"],
'xapian-core': None, # Not the right version on fedora 25 "xapian-core": None, # Not the right version on fedora 25
'pugixml': None, # ['pugixml-devel'] but package doesn't provide pkg-config file "pugixml": None, # ['pugixml-devel'] but package doesn't provide pkg-config file
'libmicrohttpd': ['libmicrohttpd-devel'], "libmicrohttpd": ["libmicrohttpd-devel"],
'zlib': ['zlib-devel'], "zlib": ["zlib-devel"],
'lzma': ['xz-devel'], "lzma": ["xz-devel"],
'icu4c': None, "icu4c": None,
'zimlib': None, "zimlib": None,
'file' : ['file-devel'], "file": ["file-devel"],
'gumbo' : ['gumbo-parser-devel'], "gumbo": ["gumbo-parser-devel"],
'aria2': ['aria2'], "aria2": ["aria2"],
'qt': ['qt5-qtbase-devel', 'qt5-qtsvg'], "qt": ["qt5-qtbase-devel", "qt5-qtsvg"],
'qtwebengine': ['qt5-qtwebengine-devel'] "qtwebengine": ["qt5-qtwebengine-devel"],
}, },
'fedora_native_static': { "fedora_native_static": {
'COMMON': _fedora_common + ['glibc-static', 'libstdc++-static'], "COMMON": _fedora_common + ["glibc-static", "libstdc++-static"],
'lzma': ['xz-devel', 'xz-static'] "lzma": ["xz-devel", "xz-static"]
# Either there is no packages, or no static or too old # Either there is no packages, or no static or too old
}, },
'fedora_i586_dyn': { "fedora_i586_dyn": {
'COMMON': _fedora_common + ['glibc-devel.i686', 'libstdc++-devel.i686'], "COMMON": _fedora_common + ["glibc-devel.i686", "libstdc++-devel.i686"],
}, },
'fedora_i586_static': { "fedora_i586_static": {
'COMMON': _fedora_common + ['glibc-devel.i686'], "COMMON": _fedora_common + ["glibc-devel.i686"],
}, },
'fedora_win32_dyn': { "fedora_win32_dyn": {
'COMMON': _fedora_common + ['mingw32-gcc-c++', 'mingw32-bzip2', 'mingw32-win-iconv', 'mingw32-winpthreads', 'wine'], "COMMON": _fedora_common
'zlib': ['mingw32-zlib'], + [
'lzma': ['mingw32-xz-libs'], "mingw32-gcc-c++",
'libmicrohttpd': ['mingw32-libmicrohttpd'], "mingw32-bzip2",
"mingw32-win-iconv",
"mingw32-winpthreads",
"wine",
],
"zlib": ["mingw32-zlib"],
"lzma": ["mingw32-xz-libs"],
"libmicrohttpd": ["mingw32-libmicrohttpd"],
}, },
'fedora_win32_static': { "fedora_win32_static": {
'COMMON': _fedora_common + ['mingw32-gcc-c++', 'mingw32-bzip2-static', 'mingw32-win-iconv-static', 'mingw32-winpthreads-static', 'wine'], "COMMON": _fedora_common
'zlib': ['mingw32-zlib-static'], + [
'lzma': ['mingw32-xz-libs-static'], "mingw32-gcc-c++",
'libmicrohttpd': None, # ['mingw32-libmicrohttpd-static'] packaging dependecy seems buggy, and some static lib are name libfoo.dll.a and "mingw32-bzip2-static",
"mingw32-win-iconv-static",
"mingw32-winpthreads-static",
"wine",
],
"zlib": ["mingw32-zlib-static"],
"lzma": ["mingw32-xz-libs-static"],
"libmicrohttpd": None, # ['mingw32-libmicrohttpd-static'] packaging dependecy seems buggy, and some static lib are name libfoo.dll.a and
# gcc cannot found them. # gcc cannot found them.
}, },
'fedora_armhf_static': { "fedora_armhf_static": {"COMMON": _fedora_common},
'COMMON': _fedora_common "fedora_armhf_dyn": {"COMMON": _fedora_common},
"fedora_android": {"COMMON": _fedora_common},
"debian_native_dyn": {
"COMMON": _debian_common + ["libbz2-dev", "libmagic-dev"],
"zlib": ["zlib1g-dev"],
"uuid": ["uuid-dev"],
"libmicrohttpd": ["libmicrohttpd-dev", "ccache"],
"qt": ["libqt5gui5", "qtbase5-dev", "qt5-default"],
"qtwebengine": ["qtwebengine5-dev"],
"aria2": ["aria2"],
}, },
'fedora_armhf_dyn': { "debian_native_static": {
'COMMON': _fedora_common "COMMON": _debian_common + ["libbz2-dev", "libmagic-dev"],
}, },
'fedora_android': { "debian_i586_dyn": {
'COMMON': _fedora_common "COMMON": _debian_common
+ ["libc6-dev-i386", "lib32stdc++6", "gcc-multilib", "g++-multilib"],
}, },
'debian_native_dyn': { "debian_i586_static": {
'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'], "COMMON": _debian_common
'zlib': ['zlib1g-dev'], + ["libc6-dev-i386", "lib32stdc++6", "gcc-multilib", "g++-multilib"],
'uuid': ['uuid-dev'],
'libmicrohttpd': ['libmicrohttpd-dev', 'ccache'],
'qt' : ['libqt5gui5', 'qtbase5-dev', 'qt5-default'],
'qtwebengine' : ['qtwebengine5-dev'],
'aria2': ['aria2'],
}, },
'debian_native_static': { "debian_win32_dyn": {
'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'], "COMMON": _debian_common
+ [
"g++-mingw-w64-i686",
"gcc-mingw-w64-i686",
"gcc-mingw-w64-base",
"mingw-w64-tools",
],
}, },
'debian_i586_dyn': { "debian_win32_static": {
'COMMON': _debian_common + ['libc6-dev-i386', 'lib32stdc++6', 'gcc-multilib', 'g++-multilib'], "COMMON": _debian_common
+ [
"g++-mingw-w64-i686",
"gcc-mingw-w64-i686",
"gcc-mingw-w64-base",
"mingw-w64-tools",
],
}, },
'debian_i586_static': { "debian_armhf_static": {
'COMMON': _debian_common + ['libc6-dev-i386', 'lib32stdc++6', 'gcc-multilib', 'g++-multilib'], "COMMON": _debian_common,
}, },
'debian_win32_dyn': { "debian_armhf_dyn": {
'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'], "COMMON": _debian_common,
}, },
'debian_win32_static': { "debian_android": {
'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'], "COMMON": _debian_common,
}, },
'debian_armhf_static': { "Darwin_native_dyn": {
'COMMON': _debian_common, "COMMON": ["autoconf", "automake", "libtool", "cmake", "pkg-config"],
"file": ["libmagic"],
}, },
'debian_armhf_dyn': { "Darwin_iOS": {
'COMMON': _debian_common, "COMMON": ["autoconf", "automake", "libtool", "cmake", "pkg-config"],
}, "file": ["libmagic"],
'debian_android': {
'COMMON': _debian_common,
},
'Darwin_native_dyn': {
'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'],
'file': ['libmagic']
},
'Darwin_iOS': {
'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'],
'file': ['libmagic']
}, },
} }

15
kiwixbuild/platforms/__init__.py
View File

@ -1,15 +0,0 @@
from .base import *
from . import (
android,
armhf,
musl,
flatpak,
i586,
ios,
native,
neutral,
win32,
wasm
)

146
kiwixbuild/platforms/android.py
View File

@ -1,146 +0,0 @@
from .base import PlatformInfo, MetaPlatformInfo
from kiwixbuild.utils import pj
from kiwixbuild._global import get_target_step, option
class AndroidPlatformInfo(PlatformInfo):
build = 'android'
static = True
toolchain_names = ['android-ndk']
compatible_hosts = ['fedora', 'debian']
def __str__(self):
return "android"
@property
def libdir(self):
return 'lib/{}'.format(self.arch_full)
@property
def binaries_name(self):
arch_full = self.arch_full
return {
'CC': '{}-{}'.format(arch_full, 'clang'),
'CXX': '{}-{}'.format(arch_full, 'clang++'),
'AR': '{}-{}'.format(arch_full, 'ar'),
'STRIP': '{}-{}'.format(arch_full, 'strip'),
'RANLIB': '{}-{}'.format(arch_full, 'ranlib'),
'LD': '{}-{}'.format(arch_full, 'ld')
}
def binaries(self):
install_path = self.install_path
binaries = {k:pj(install_path, 'bin', v)
for k,v in self.binaries_name.items()}
binaries['PKGCONFIG'] = 'pkg-config'
return binaries
@property
def ndk_builder(self):
return get_target_step('android-ndk', self.name)
@property
def install_path(self):
return self.ndk_builder.install_path
def get_cross_config(self):
extra_libs = ['-llog']
extra_cflags = ['-I{}'.format(include_dir) for include_dir in self.get_include_dirs()]
if hasattr(self, 'march'):
extra_libs.append('-march={}'.format(self.march))
extra_cflags.append('-march={}'.format(self.march))
return {
'exe_wrapper_def': '',
'install_path': self.install_path,
'binaries': self.binaries(),
'root_path': pj(self.install_path, 'sysroot'),
'extra_libs': extra_libs,
'extra_cflags': extra_cflags,
'host_machine': {
'system': 'Android',
'lsystem': 'android',
'cpu_family': self.arch,
'cpu': self.cpu,
'endian': 'little',
'abi': self.abi
}
}
def get_env(self):
env = super().get_env()
root_path = pj(self.install_path, 'sysroot')
env['PKG_CONFIG_LIBDIR'] = pj(root_path, 'lib', 'pkgconfig')
env['NDK_DEBUG'] = '0'
return env
def get_bin_dir(self):
return [pj(self.install_path, 'bin')]
def set_comp_flags(self, env):
super().set_comp_flags(env)
root_path = pj(self.install_path, 'sysroot')
march = '-march={}'.format(self.march) if hasattr(self,'march') else ''
env['CFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} {} '.format(root_path, march) + env['CFLAGS']
env['CXXFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} {} '.format(root_path, march) + env['CXXFLAGS']
env['LDFLAGS'] = '--sysroot={} {} '.format(root_path, march) + env['LDFLAGS']
def set_compiler(self, env):
binaries = self.binaries()
for k,v in binaries.items():
env[k] = v
@property
def configure_options(self):
yield '--host={}'.format(self.arch_full)
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_android_cross_file.txt', 'cmake_cross_file.txt')
self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
class AndroidArm(AndroidPlatformInfo):
name = 'android_arm'
arch = cpu = 'arm'
arch_full = 'arm-linux-androideabi'
abi = 'armeabi-v7a'
march = 'armv7-a'
class AndroidArm64(AndroidPlatformInfo):
name = 'android_arm64'
arch = 'arm64'
arch_full = 'aarch64-linux-android'
cpu = 'aarch64'
abi = 'arm64-v8a'
class AndroidX86(AndroidPlatformInfo):
name = 'android_x86'
arch = abi = 'x86'
arch_full = 'i686-linux-android'
cpu = 'i686'
class AndroidX8664(AndroidPlatformInfo):
name = 'android_x86_64'
arch = cpu = abi = 'x86_64'
arch_full = 'x86_64-linux-android'
class Android(MetaPlatformInfo):
name = "android"
compatible_hosts = ['fedora', 'debian']
@property
def subPlatformNames(self):
return ['android_{}'.format(arch) for arch in option('android_arch')]
def add_targets(self, targetName, targets):
return super().add_targets(targetName, targets)
def __str__(self):
return self.name
def set_comp_flags(self, env):
pass

166
kiwixbuild/platforms/armhf.py
View File

@ -1,166 +0,0 @@
from .base import PlatformInfo, MixedMixin
from kiwixbuild.utils import pj
from kiwixbuild._global import get_target_step
# Base platform
class ArmPlatformInfo(PlatformInfo):
compatible_hosts = ['fedora', 'debian']
def get_cross_config(self):
return {
'binaries': self.binaries,
'exe_wrapper_def': '',
'root_path': self.root_path,
'extra_libs': [],
'extra_cflags': ['-I{}'.format(include_dir) for include_dir in self.get_include_dirs()],
'host_machine': {
'system': 'linux',
'lsystem': 'linux',
'cpu_family': self.cpu_family,
'cpu': self.cpu,
'endian': 'little',
'abi': ''
}
}
@property
def libdir(self):
return "lib/{}".format(self.arch_full)
@property
def toolchain(self):
return get_target_step(self.build, 'neutral')
@property
def root_path(self):
return self.toolchain.build_path
@property
def binaries(self):
binaries = ((k,'{}-{}'.format(self.arch_full, v))
for k, v in (('CC', 'gcc'),
('CXX', 'g++'),
('AR', 'ar'),
('STRIP', 'strip'),
('WINDRES', 'windres'),
('RANLIB', 'ranlib'),
('LD', 'ld'),
('LDSHARED', 'g++ -shared')
)
)
binaries = {k:pj(self.root_path, 'bin', v)
for k,v in binaries}
binaries['PKGCONFIG'] = 'pkg-config'
return binaries
@property
def exe_wrapper_def(self):
try:
which('qemu-arm')
except subprocess.CalledProcessError:
return ""
else:
return "exe_wrapper = 'qemu-arm'"
@property
def configure_options(self):
yield '--host={}'.format(self.arch_full)
def get_bin_dir(self):
return [pj(self.root_path, 'bin')]
def get_env(self):
env = super().get_env()
env['LD_LIBRARY_PATH'] = ':'.join([
pj(self.root_path, self.arch_full, 'lib64'),
pj(self.root_path, 'lib'),
env['LD_LIBRARY_PATH']
])
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
env['QEMU_LD_PREFIX'] = pj(self.root_path, self.arch_full, "libc")
env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format(
':'.join([
pj(self.root_path, self.arch_full, "lib"),
env['LD_LIBRARY_PATH']
]))
return env
def set_comp_flags(self, env):
super().set_comp_flags(env)
env['CFLAGS'] = " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS']
env['CXXFLAGS'] = " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
def set_compiler(self, env):
for k, v in self.binaries.items():
env[k] = v
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
class Armv6(ArmPlatformInfo):
build = "armv6"
arch_full = "armv6-rpi-linux-gnueabihf"
toolchain_names = ['armv6']
cpu_family = 'arm'
cpu = 'armv6'
class Armv6Dyn(Armv6):
name = 'armv6_dyn'
static = False
class Armv6Static(Armv6):
name = 'armv6_static'
static = True
class Armv6Mixed(MixedMixin('armv6_static'), Armv6):
name = 'armv6_mixed'
static = False
class Armv8(ArmPlatformInfo):
build = "armv8"
arch_full = "armv8-rpi3-linux-gnueabihf"
toolchain_names = ['armv8']
cpu_family = 'arm'
cpu = 'armv8'
class Armv8Dyn(Armv8):
name = 'armv8_dyn'
static = False
class Armv8Static(Armv8):
name = 'armv8_static'
static = True
class Armv8Mixed(MixedMixin('armv8_static'), Armv8):
name = 'armv8_mixed'
static = False
class Aarch64(ArmPlatformInfo):
build = 'aarch64'
arch_full = 'aarch64-linux-gnu'
toolchain_names = ['aarch64']
cpu_family = 'aarch64'
cpu = 'aarch64'
@property
def root_path(self):
return self.toolchain.build_path
class Aarch64Dyn(Aarch64):
name = 'aarch64_dyn'
static = False
class Aarch64Static(Aarch64):
name = 'aarch64_static'
static = True
class Aarch64Mixed(MixedMixin('aarch64_static'), Aarch64):
name = 'aarch64_mixed'
static = False

18
kiwixbuild/platforms/flatpak.py
View File

@ -1,18 +0,0 @@
from .base import PlatformInfo
from kiwixbuild._global import option, neutralEnv
class FlatpakPlatformInfo(PlatformInfo):
name = 'flatpak'
build = 'flatpak'
static = ''
toolchain_names = ['org.kde', 'io.qt.qtwebengine']
compatible_hosts = ['debian', 'fedora']
def __str__(self):
return "flatpak"
def get_env(self):
env = super().get_env()
env['FLATPAK_USER_DIR'] = self.buildEnv.build_dir
return env

70
kiwixbuild/platforms/i586.py
View File

@ -1,70 +0,0 @@
import os
from .base import PlatformInfo
from kiwixbuild.utils import which, pj
class I586PlatformInfo(PlatformInfo):
build = 'i586'
arch_full = 'i586-linux-gnu'
compatible_hosts = ['fedora', 'debian']
def get_cross_config(self):
return {
'binaries': self.binaries,
'exe_wrapper_def': '',
'extra_libs': ['-m32', '-march=i586', '-mno-sse'],
'extra_cflags': [
'-m32',
'-march=i586',
'-mno-sse',
*('-I{}'.format(include_dir) for include_dir in self.get_include_dirs())
],
'host_machine': {
'system': 'linux',
'lsystem': 'linux',
'cpu_family': 'x86',
'cpu': 'i586',
'endian': 'little',
'abi': ''
}
}
@property
def configure_options(self):
yield f'--host={self.arch_full}'
@property
def binaries(self):
return {k:which(v)
for k, v in (('CC', os.environ.get('CC', 'gcc')),
('CXX', os.environ.get('CXX', 'g++')),
('AR', 'ar'),
('STRIP', 'strip'),
('RANLIB', 'ranlib'),
('LD', 'ld'),
('PKGCONFIG', 'pkg-config'))
}
def set_comp_flags(self, env):
super().set_comp_flags(env)
env['CFLAGS'] = "-m32 -march=i586 -mno-sse "+env['CFLAGS']
env['CXXFLAGS'] = "-m32 -march=i586 -mno-sse "+env['CXXFLAGS']
env['LDFLAGS'] = "-m32 -march=i586 -mno-sse "+env['LDFLAGS']
def get_bin_dir(self):
return []
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_i586_cross_file.txt', 'cmake_cross_file.txt')
self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
class I586Dyn(I586PlatformInfo):
name = 'i586_dyn'
static = False
class I586Static(I586PlatformInfo):
name = 'i586_static'
static = True

210
kiwixbuild/platforms/ios.py
View File

@ -1,210 +0,0 @@
import subprocess
from kiwixbuild._global import option
from kiwixbuild.utils import pj, xrun_find
from .base import PlatformInfo, MetaPlatformInfo, MixedMixin
from kiwixbuild.dependencies.apple_xcframework import AppleXCFramework
MIN_MACOS_VERSION = '12.0'
class ApplePlatformInfo(PlatformInfo):
build = 'iOS'
static = True
compatible_hosts = ['Darwin']
arch = None
host = None
target = None
sdk_name = None
min_iphoneos_version = None
min_macos_version = None
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._root_path = None
@property
def root_path(self):
if self._root_path is None:
command = "xcrun --sdk {} --show-sdk-path".format(self.sdk_name)
self._root_path = subprocess.check_output(command, shell=True)[:-1].decode()
return self._root_path
def __str__(self):
return "iOS"
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_ios_cross_file.txt', 'cmake_cross_file.txt')
self.buildEnv.meson_crossfile = self._gen_crossfile('meson_ios_cross_file.txt', 'meson_cross_file.txt')
def get_cross_config(self):
config = {
'root_path': self.root_path,
'binaries': self.binaries,
'exe_wrapper_def': '',
'extra_libs': [
'-isysroot', self.root_path,
'-arch', self.arch,
'-target', self.target,
],
'extra_cflags': [
'-isysroot', self.root_path,
'-arch', self.arch,
'-target', self.target,
*('-I{}'.format(include_dir) for include_dir in self.get_include_dirs())
],
'host_machine': {
'system': 'Darwin',
'lsystem': 'darwin',
'cpu_family': self.arch,
'cpu': self.cpu,
'endian': '',
'abi': ''
}
}
if self.min_iphoneos_version:
config['extra_libs'].append('-miphoneos-version-min={}'.format(self.min_iphoneos_version))
config['extra_cflags'].append('-miphoneos-version-min={}'.format(self.min_iphoneos_version))
if self.min_macos_version:
config['extra_libs'].append('-mmacosx-version-min={}'.format(self.min_macos_version))
config['extra_cflags'].append('-mmacosx-version-min={}'.format(self.min_macos_version))
return config
def get_env(self):
env = super().get_env()
cflags = [env['CFLAGS']]
if self.min_iphoneos_version:
cflags.append('-miphoneos-version-min={}'.format(self.min_iphoneos_version))
if self.min_macos_version:
cflags.append('-mmacosx-version-min={}'.format(self.min_macos_version))
env['CFLAGS'] = ' '.join(cflags)
return env
def set_comp_flags(self, env):
super().set_comp_flags(env)
cflags = [
'-isysroot {}'.format(self.root_path),
'-arch {}'.format(self.arch),
'-target {}'.format(self.target),
env['CFLAGS'],
]
if self.min_iphoneos_version:
cflags.append('-miphoneos-version-min={}'.format(self.min_iphoneos_version))
env['CFLAGS'] = ' '.join(cflags)
env['CXXFLAGS'] = ' '.join([
env['CFLAGS'],
'-std=c++11',
env['CXXFLAGS'],
])
env['LDFLAGS'] = ' '.join([
' -arch {}'.format(self.arch),
'-isysroot {}'.format(self.root_path),
])
def get_bin_dir(self):
return [pj(self.root_path, 'bin')]
@property
def binaries(self):
return {
'CC': xrun_find('clang'),
'CXX': xrun_find('clang++'),
'AR': xrun_find('ar'),
'STRIP': xrun_find('strip'),
'RANLIB': xrun_find('ranlib'),
'LD': xrun_find('ld'),
'PKGCONFIG': 'pkg-config',
}
@property
def configure_options(self):
yield f'--host={self.host}'
class iOSArm64(ApplePlatformInfo):
name = 'iOS_arm64'
arch = cpu = 'arm64'
host = 'arm-apple-darwin'
target = 'aarch64-apple-ios'
sdk_name = 'iphoneos'
min_iphoneos_version = '15.0'
class iOSx64Simulator(ApplePlatformInfo):
name = 'iOSSimulator_x86_64'
arch = cpu = 'x86_64'
host = 'x86_64-apple-darwin'
target = 'x86-apple-ios-simulator'
sdk_name = 'iphonesimulator'
min_iphoneos_version = '15.0'
class iOSArm64Simulator(ApplePlatformInfo):
name = 'iOSSimulator_arm64'
arch = cpu = 'arm64'
host = 'arm-apple-darwin'
target = 'aarch64-apple-ios-simulator'
sdk_name = 'iphonesimulator'
min_iphoneos_version = '15.0'
class macOSArm64(ApplePlatformInfo):
name = 'macOS_arm64_static'
arch = cpu = 'arm64'
host = 'aarch64-apple-darwin'
target = 'arm64-apple-macos'
sdk_name = 'macosx'
min_iphoneos_version = None
min_macos_version = MIN_MACOS_VERSION
class macOSArm64Mixed(MixedMixin('macOS_arm64_static'), ApplePlatformInfo):
name = 'macOS_arm64_mixed'
arch = cpu = 'arm64'
host = 'aarch64-apple-darwin'
target = 'arm64-apple-macos'
sdk_name = 'macosx'
min_iphoneos_version = None
min_macos_version = MIN_MACOS_VERSION
class macOSx64(ApplePlatformInfo):
name = 'macOS_x86_64'
arch = cpu = 'x86_64'
host = 'x86_64-apple-darwin'
target = 'x86_64-apple-macos'
sdk_name = 'macosx'
min_iphoneos_version = None
min_macos_version = MIN_MACOS_VERSION
class IOS(MetaPlatformInfo):
name = "iOS_multi"
compatible_hosts = ['Darwin']
@property
def subPlatformNames(self):
return ['iOS_{}'.format(arch) for arch in option('ios_arch')]
def add_targets(self, targetName, targets):
super().add_targets(targetName, targets)
return PlatformInfo.add_targets(self, '_ios_fat_lib', targets)
def __str__(self):
return self.name
class AppleStaticAll(MetaPlatformInfo):
name = "apple_all_static"
compatible_hosts = ['Darwin']
@property
def subPlatformNames(self):
return AppleXCFramework.subPlatformNames
def add_targets(self, targetName, targets):
super().add_targets(targetName, targets)
return PlatformInfo.add_targets(self, 'apple_xcframework', targets)
def __str__(self):
return self.name

148
kiwixbuild/platforms/musl.py
View File

@ -1,148 +0,0 @@
from .base import PlatformInfo, MixedMixin
from kiwixbuild.utils import pj
from kiwixbuild._global import get_target_step
class MuslPlatformInfo(PlatformInfo):
compatible_hosts = ['fedora', 'debian']
def get_cross_config(self):
return {
'binaries': self.binaries,
'exe_wrapper_def': '',
'root_path': self.root_path,
'extra_libs': [],
'extra_cflags': ['-I{}'.format(include_dir) for include_dir in self.get_include_dirs()],
'host_machine': {
'system': 'linux',
'lsystem': 'linux',
'cpu_family': self.cpu_family,
'cpu': self.cpu,
'endian': 'little',
'abi': ''
}
}
@property
def toolchain(self):
return get_target_step(self.build, 'neutral')
@property
def root_path(self):
return self.toolchain.build_path
@property
def binaries(self):
binaries = ((k,'{}-{}'.format(self.arch_full, v))
for k, v in (('CC', 'gcc'),
('CXX', 'g++'),
('AR', 'ar'),
('STRIP', 'strip'),
('WINDRES', 'windres'),
('RANLIB', 'ranlib'),
('LD', 'ld'),
('LDSHARED', 'g++ -shared')
)
)
binaries = {k:pj(self.root_path, 'bin', v)
for k,v in binaries}
binaries['PKGCONFIG'] = 'pkg-config'
return binaries
@property
def exe_wrapper_def(self):
try:
which(self.qemu)
except subprocess.CalledProcessError:
return ""
except AttributeError:
return ""
else:
return f"exe_wrapper = '{self.qemu}'"
@property
def configure_options(self):
return [f'--host={self.arch_full}']
def get_bin_dir(self):
return [pj(self.root_path, 'bin')]
def get_env(self):
env = super().get_env()
env['LD_LIBRARY_PATH'] = ':'.join([
pj(self.root_path, self.arch_full, 'lib64'),
pj(self.root_path, 'lib'),
env['LD_LIBRARY_PATH']
])
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
env['QEMU_LD_PREFIX'] = pj(self.root_path, self.arch_full, "libc")
env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format(
':'.join([
pj(self.root_path, self.arch_full, "lib"),
env['LD_LIBRARY_PATH']
]))
return env
def set_comp_flags(self, env):
super().set_comp_flags(env)
env['LD_LIBRARY_PATH'] = ':'.join([
pj(self.root_path, self.arch_full, 'lib'),
env['LD_LIBRARY_PATH']
])
env['CFLAGS'] = " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS']
env['CXXFLAGS'] = " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
def set_compiler(self, env):
for k, v in self.binaries.items():
env[k] = v
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
class Aarch64MuslPlatformInfo(MuslPlatformInfo):
build = 'aarch64_musl'
arch_full = 'aarch64-linux-musl'
toolchain_names = ['aarch64_musl']
libdir = "lib/aarch64-linux-musl"
cpu_family = 'arm'
cpu = 'armhf'
qemu = 'qemu-arm'
class Aarch64MuslDyn(Aarch64MuslPlatformInfo):
name = 'aarch64_musl_dyn'
static = False
class Aarch64MuslStatic(Aarch64MuslPlatformInfo):
name = 'aarch64_musl_static'
static = True
class Aarch64MuslMixed(MixedMixin('aarch64_musl_static'), Aarch64MuslPlatformInfo):
name = 'aarch64_musl_mixed'
static = False
class X86_64MuslPlatformInfo(MuslPlatformInfo):
build = 'x86-64_musl'
arch_full = 'x86_64-linux-musl'
toolchain_names = ['x86-64_musl']
libdir = "lib/x86_64-linux-musl"
cpu_family = 'x86_64'
cpu = 'x86_64'
class X86_64MuslDyn(X86_64MuslPlatformInfo):
name = 'x86-64_musl_dyn'
static = False
class X86_64MuslStatic(X86_64MuslPlatformInfo):
name = 'x86-64_musl_static'
static = True
class x86_64MuslMixed(MixedMixin('x86-64_musl_static'), X86_64MuslPlatformInfo):
name = 'x86-64_musl_mixed'
static = False

32
kiwixbuild/platforms/native.py
View File

@ -1,32 +0,0 @@
from .base import PlatformInfo, MixedMixin
from kiwixbuild.utils import pj
from kiwixbuild._global import option, neutralEnv
from kiwixbuild.platforms.ios import MIN_MACOS_VERSION
class NativePlatformInfo(PlatformInfo):
build = 'native'
def get_env(self):
env = super().get_env()
if neutralEnv('distname') == 'fedora':
env['QT_SELECT'] = "5-64"
if neutralEnv('distname') == 'Darwin':
env['CFLAGS'] += f'-mmacosx-version-min={MIN_MACOS_VERSION}'
return env
class NativeDyn(NativePlatformInfo):
name = 'native_dyn'
static = False
compatible_hosts = ['fedora', 'debian', 'Darwin']
class NativeStatic(NativePlatformInfo):
name = 'native_static'
static = True
compatible_hosts = ['fedora', 'debian']
class NativeMixed(MixedMixin('native_static'), NativePlatformInfo):
name = 'native_mixed'
static = False
compatible_hosts = ['fedora', 'debian', 'Darwin']

9
kiwixbuild/platforms/neutral.py
View File

@ -1,9 +0,0 @@
from .base import PlatformInfo
class NeutralPlatformInfo(PlatformInfo):
name = 'neutral'
static = ''
compatible_hosts = ['fedora', 'debian', 'Darwin']
def __str__(self):
return "neutral"

101
kiwixbuild/platforms/wasm.py
View File

@ -1,101 +0,0 @@
from .base import PlatformInfo
from kiwixbuild.utils import pj
from kiwixbuild._global import get_target_step
class WasmPlatformInfo(PlatformInfo):
name = 'wasm'
static = True
build = 'wasm'
arch_full = 'wasm64-emscripten'
libdir = "lib"
#arch_full = 'wasm64-linux'
toolchain_names = ['emsdk']
compatible_hosts = ['fedora', 'debian']
exe_wrapper_def = ""
def get_cross_config(self):
return {
'binaries': self.binaries,
'exe_wrapper_def': '',
'root_path': self.root_path,
'extra_libs': [],
'extra_cflags': [],
'host_machine': {
'system': 'emscripten',
'lsystem': 'emscripten',
'cpu_family': 'wasm64',
'cpu': 'wasm64',
'endian': 'little',
'abi': ''
}
}
@property
def wasm_sdk(self):
return get_target_step('emsdk', self.name)
@property
def install_path(self):
return self.wasm_sdk.install_path
@property
def root_path(self):
return self.install_path
@property
def binaries(self):
binaries = (('CC', 'emcc'),
('CXX', 'em++'),
('AR', 'emar'),
('STRIP', 'emstrip'),
('WINDRES', 'windres'),
('RANLIB', 'emranlib'),
('LD', 'wasm-ld'))
binaries = {k:pj(self.install_path, 'upstream', 'emscripten', v)
for k,v in binaries}
binaries['PKGCONFIG'] = 'pkg-config'
return binaries
@property
def configure_options(self):
#return ""
return [f'--host={self.arch_full}']
@property
def configure_wrapper(self):
return "emconfigure"
@property
def make_wrapper(self):
return "emmake"
def get_bin_dir(self):
return [pj(self.install_path, 'bin')]
def get_env(self):
env = super().get_env()
env['PATH'] = ':'.join([
env['PATH'],
self.install_path,
pj(self.install_path, 'upstream', 'emscripten'),
pj(self.install_path, 'node', '14.18.2_64bit', 'bin')
])
env['EMSDK'] = self.install_path
env['EMSDK_NODE'] = pj(self.install_path, 'node', '14.18.2_64bit', 'bin', 'node')
return env
def set_comp_flags(self, env):
super().set_comp_flags(env)
env['CFLAGS'] = " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS']
env['CXXFLAGS'] = " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
def set_compiler(self, env):
for k, v in self.binaries.items():
env[k] = v
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')

88
kiwixbuild/platforms/win32.py
View File

@ -1,88 +0,0 @@
import subprocess
from .base import PlatformInfo
from kiwixbuild.utils import which, pj
from kiwixbuild._global import neutralEnv
class Win32PlatformInfo(PlatformInfo):
build = 'win32'
compatible_hosts = ['fedora', 'debian']
arch_full = 'i686-w64-mingw32'
extra_libs = ['-lwinmm', '-lshlwapi', '-lws2_32', '-lssp']
def get_cross_config(self):
return {
'exe_wrapper_def': self.exe_wrapper_def,
'binaries': self.binaries,
'root_path': self.root_path,
'extra_libs': self.extra_libs,
'extra_cflags': ['-DWIN32', *('-I{}'.format(include_dir) for include_dir in self.get_include_dirs())],
'host_machine': {
'system': 'Windows',
'lsystem': 'windows',
'cpu_family': 'x86',
'cpu': 'i686',
'endian': 'little',
'abi': ''
}
}
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
@property
def root_path(self):
root_paths = {
'fedora': '/usr/i686-w64-mingw32/sys-root/mingw',
'debian': '/usr/i686-w64-mingw32'
}
return root_paths[neutralEnv('distname')]
@property
def binaries(self):
return {k:which('{}-{}'.format(self.arch_full, v))
for k, v in (('CC', 'gcc'),
('CXX', 'g++'),
('AR', 'ar'),
('STRIP', 'strip'),
('WINDRES', 'windres'),
('RANLIB', 'ranlib'),
('PKGCONFIG', 'pkg-config'))
}
@property
def exe_wrapper_def(self):
try:
which('wine')
except subprocess.CalledProcessError:
return ""
else:
return "exe_wrapper = 'wine'"
@property
def configure_options(self):
yield f'--host={self.arch_full}'
def set_compiler(self, env):
for k, v in self.binaries.items():
env[k] = v
def get_bin_dir(self):
return [pj(self.root_path, 'bin')]
def get_env(self):
env = super().get_env()
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
env['LIBS'] = " ".join(self.extra_libs) + " " +env['LIBS']
return env
class Win32Dyn(Win32PlatformInfo):
name = 'win32_dyn'
static = False
class Win32Static(Win32PlatformInfo):
name = 'win32_static'
static = True

88
kiwixbuild/platforms/win64.py
View File

@ -1,88 +0,0 @@
import subprocess
from .base import PlatformInfo
from kiwixbuild.utils import which, pj
from kiwixbuild._global import neutralEnv
class Win64PlatformInfo(PlatformInfo):
extra_libs = ['-lmingw32', '-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4', '-lmsvcr100', '-liphlpapi', '-lshell32', '-lkernel32']
build = 'win64'
compatible_hosts = ['fedora', 'debian']
arch_full = 'x86_64-w64-mingw32'
def get_cross_config(self):
return {
'exe_wrapper_def': self.exe_wrapper_def,
'binaries': self.binaries,
'root_path': self.root_path,
'extra_libs': self.extra_libs,
'extra_cflags': ['-DWIN32'],
'host_machine': {
'system': 'Windows',
'lsystem': 'windows',
'cpu_family': 'x86_64',
'cpu': 'x86_64',
'endian': 'little',
'abi': ''
}
}
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
@property
def root_path(self):
root_paths = {
'fedora': '/usr/x86_64-w64-mingw32/sys-root/mingw',
'debian': '/usr/x86_64-w64-mingw32'
}
return root_paths[neutralEnv('distname')]
@property
def binaries(self):
return {k:which('{}-{}'.format(self.arch_full, v))
for k, v in (('CC', 'gcc'),
('CXX', 'g++'),
('AR', 'ar'),
('STRIP', 'strip'),
('WINDRES', 'windres'),
('RANLIB', 'ranlib'),
('PKGCONFIG', 'pkg-config'))
}
@property
def exe_wrapper_def(self):
try:
which('wine')
except subprocess.CalledProcessError:
return ""
else:
return "exe_wrapper = 'wine'"
@property
def configure_options(self):
return [f'--host={self.arch_full}']
def set_compiler(self, env):
for k, v in self.binaries.items():
env[k] = v
def get_bin_dir(self):
return [pj(self.root_path, 'bin')]
def get_env(self):
env = super().get_env()
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
env['LIBS'] = " ".join(self.extra_libs) + " " +env['LIBS']
return env
class Win64Dyn(Win64PlatformInfo):
name = 'win64_dyn'
static = False
class Win64Static(Win64PlatformInfo):
name = 'win64_static'
static = True

78
kiwixbuild/utils.py
View File

@ -17,16 +17,16 @@ pj = os.path.join
COLORS = { COLORS = {
'OK': '\033[92m', "OK": "\033[92m",
'WARNING': '\033[93m', "WARNING": "\033[93m",
'NEEDED': '\033[93m', "NEEDED": "\033[93m",
'SKIP': '\033[34m', "SKIP": "\033[34m",
'ERROR': '\033[91m', "ERROR": "\033[91m",
'': '\033[0m', "": "\033[0m",
} }
REMOTE_PREFIX = 'http://mirror.download.kiwix.org/dev/kiwix-build/' REMOTE_PREFIX = "http://mirror.download.kiwix.org/dev/kiwix-build/"
def which(name): def which(name):
@ -34,16 +34,19 @@ def which(name):
output = subprocess.check_output(command, shell=True) output = subprocess.check_output(command, shell=True)
return output[:-1].decode() return output[:-1].decode()
def xrun_find(name): def xrun_find(name):
command = "xcrun -find {}".format(name) command = "xcrun -find {}".format(name)
output = subprocess.check_output(command, shell=True) output = subprocess.check_output(command, shell=True)
return output[:-1].decode() return output[:-1].decode()
regex_space = re.compile(r'((?<!\\) )') regex_space = re.compile(r"((?<!\\) )")
def escape_path(path): def escape_path(path):
path = str(path) path = str(path)
return regex_space.sub(r'\ ', path) return regex_space.sub(r"\ ", path)
class Defaultdict(defaultdict): class Defaultdict(defaultdict):
@ -56,7 +59,7 @@ class DefaultEnv(Defaultdict):
super().__init__(str, os.environ) super().__init__(str, os.environ)
def __getitem__(self, name): def __getitem__(self, name):
if name == b'PATH': if name == b"PATH":
raise KeyError raise KeyError
return super().__getitem__(name) return super().__getitem__(name)
@ -78,7 +81,7 @@ def get_sha256(path):
current = 0 current = 0
batch_size = 1024 * 8 batch_size = 1024 * 8
sha256 = hashlib.sha256() sha256 = hashlib.sha256()
with open(path, 'br') as f: with open(path, "br") as f:
while True: while True:
batch = f.read(batch_size) batch = f.read(batch_size)
if not batch: if not batch:
@ -92,11 +95,11 @@ def get_sha256(path):
def colorize(text, color=None): def colorize(text, color=None):
if color is None: if color is None:
color = text color = text
return "{}{}{}".format(COLORS[color], text, COLORS['']) return "{}{}{}".format(COLORS[color], text, COLORS[""])
def print_progress(progress): def print_progress(progress):
if option('show_progress'): if option("show_progress"):
text = "{}\033[{}D".format(progress, len(progress)) text = "{}\033[{}D".format(progress, len(progress))
print(text, end="") print(text, end="")
@ -126,18 +129,20 @@ def download_remote(what, where):
raise SkipCommand() raise SkipCommand()
os.remove(file_path) os.remove(file_path)
if option('no_cert_check'): if option("no_cert_check"):
context = ssl.create_default_context() context = ssl.create_default_context()
context.check_hostname = False context.check_hostname = False
context.verify_mode = ssl.CERT_NONE context.verify_mode = ssl.CERT_NONE
else: else:
context = None context = None
batch_size = 1024 * 8 batch_size = 1024 * 8
extra_args = {'context':context} if sys.version_info >= (3, 4, 3) else {} extra_args = {"context": context} if sys.version_info >= (3, 4, 3) else {}
progress_chars = "/-\|" progress_chars = "/-\|"
try: try:
with urllib.request.urlopen(what.url, **extra_args) as resource, open(file_path, 'wb') as file: with urllib.request.urlopen(what.url, **extra_args) as resource, open(
tsize = resource.info().get('Content-Length', None) file_path, "wb"
) as file:
tsize = resource.info().get("Content-Length", None)
if tsize is not None: if tsize is not None:
tsize = int(tsize) tsize = int(tsize)
current = 0 current = 0
@ -157,7 +162,7 @@ def download_remote(what, where):
raise StopBuild() raise StopBuild()
if not what.sha256: if not what.sha256:
print('Sha256 for {} not set, do no verify download'.format(what.name)) print("Sha256 for {} not set, do no verify download".format(what.name))
elif what.sha256 != get_sha256(file_path): elif what.sha256 != get_sha256(file_path):
os.remove(file_path) os.remove(file_path)
raise StopBuild("Sha 256 doesn't correspond") raise StopBuild("Sha 256 doesn't correspond")
@ -187,7 +192,7 @@ class StopBuild(BaseCommandResult):
pass pass
class Remotefile(namedtuple('Remotefile', ('name', 'sha256', 'url'))): class Remotefile(namedtuple("Remotefile", ("name", "sha256", "url"))):
def __new__(cls, name, sha256, url=None): def __new__(cls, name, sha256, url=None):
if url is None: if url is None:
url = REMOTE_PREFIX + name url = REMOTE_PREFIX + name
@ -217,28 +222,28 @@ class Context:
def _finalise(self): def _finalise(self):
if self.autoskip_file is not None: if self.autoskip_file is not None:
os.makedirs(os.path.dirname(self.autoskip_file), exist_ok=True) os.makedirs(os.path.dirname(self.autoskip_file), exist_ok=True)
with open(self.autoskip_file, 'w'): with open(self.autoskip_file, "w"):
pass pass
def extract_archive(archive_path, dest_dir, topdir=None, name=None): def extract_archive(archive_path, dest_dir, topdir=None, name=None):
is_zip_archive = archive_path.endswith('.zip') is_zip_archive = archive_path.endswith(".zip")
archive = None archive = None
try: try:
if is_zip_archive: if is_zip_archive:
archive = zipfile.ZipFile(archive_path) archive = zipfile.ZipFile(archive_path)
members = archive.infolist() members = archive.infolist()
getname = lambda info: info.filename getname = lambda info: info.filename
isdir = lambda info: info.filename.endswith('/') isdir = lambda info: info.filename.endswith("/")
else: else:
archive = tarfile.open(archive_path) archive = tarfile.open(archive_path)
members = archive.getmembers() members = archive.getmembers()
getname = lambda info : getattr(info, 'name') getname = lambda info: getattr(info, "name")
isdir = lambda info: info.isdir() isdir = lambda info: info.isdir()
if not topdir: if not topdir:
for d in (m for m in members if isdir(m)): for d in (m for m in members if isdir(m)):
_name = getname(d) _name = getname(d)
if _name.endswith('/'): if _name.endswith("/"):
_name = _name[:-1] _name = _name[:-1]
if not os.path.dirname(_name): if not os.path.dirname(_name):
if topdir: if topdir:
@ -249,9 +254,13 @@ def extract_archive(archive_path, dest_dir, topdir=None, name=None):
break break
topdir = _name topdir = _name
if topdir: if topdir:
members_to_extract = [m for m in members if getname(m).startswith(topdir+'/')] members_to_extract = [
m for m in members if getname(m).startswith(topdir + "/")
]
os.makedirs(dest_dir, exist_ok=True) os.makedirs(dest_dir, exist_ok=True)
with tempfile.TemporaryDirectory(prefix=os.path.basename(archive_path), dir=dest_dir) as tmpdir: with tempfile.TemporaryDirectory(
prefix=os.path.basename(archive_path), dir=dest_dir
) as tmpdir:
if is_zip_archive: if is_zip_archive:
_members_to_extract = [getname(m) for m in members_to_extract] _members_to_extract = [getname(m) for m in members_to_extract]
else: else:
@ -281,8 +290,8 @@ def run_command(command, cwd, context, *, env=None, input=None):
env = DefaultEnv() env = DefaultEnv()
log = None log = None
try: try:
if not option('verbose'): if not option("verbose"):
log = open(context.log_file, 'w') log = open(context.log_file, "w")
print("run command '{}'".format(command), file=log) print("run command '{}'".format(command), file=log)
print("current directory is '{}'".format(cwd), file=log) print("current directory is '{}'".format(cwd), file=log)
print("env is :", file=log) print("env is :", file=log)
@ -293,8 +302,15 @@ def run_command(command, cwd, context, *, env=None, input=None):
log.flush() log.flush()
kwargs = dict() kwargs = dict()
if input: if input:
kwargs['stdin'] = subprocess.PIPE kwargs["stdin"] = subprocess.PIPE
process = subprocess.Popen(command, cwd=cwd, env=env, stdout=log or sys.stdout, stderr=subprocess.STDOUT, **kwargs) process = subprocess.Popen(
command,
cwd=cwd,
env=env,
stdout=log or sys.stdout,
stderr=subprocess.STDOUT,
**kwargs
)
if input: if input:
input = input.encode() input = input.encode()
while True: while True:
@ -308,7 +324,7 @@ def run_command(command, cwd, context, *, env=None, input=None):
# `communicate` timeout (and we must set `input` to None # `communicate` timeout (and we must set `input` to None
# to not communicate again). # to not communicate again).
input = None input = None
print('.', end='', flush=True) print(".", end="", flush=True)
else: else:
break break
if process.returncode: if process.returncode:

63
kiwixbuild/versions.py
View File

@ -1,12 +1,11 @@
# This file reference all the versions of the depedencies we use in kiwix-build. # This file reference all the versions of the depedencies we use in kiwix-build.
main_project_versions = { main_project_versions = {
'libzim': '9.1.0', "libzim": "9.1.0",
'libkiwix': '13.0.0', "libkiwix": "13.0.0",
'kiwix-tools': '3.6.0', "kiwix-tools": "3.6.0",
'zim-tools': '3.4.0', "zim-tools": "3.4.0",
'kiwix-desktop': '2.3.1' # Also change KIWIX_DESKTOP_VERSION and KIWIX_DESKTOP_RELEASE in appveyor.yml "kiwix-desktop": "2.3.1", # Also change KIWIX_DESKTOP_VERSION and KIWIX_DESKTOP_RELEASE in appveyor.yml
} }
# This dictionnary specify what we need to build at each release process. # This dictionnary specify what we need to build at each release process.
@ -30,37 +29,37 @@ main_project_versions = {
# - set KIWIX_DESKTOP_RELEASE to 0 # - set KIWIX_DESKTOP_RELEASE to 0
release_versions = { release_versions = {
'libzim': None, # Depends of base deps (was 0) "libzim": None, # Depends of base deps (was 0)
'libkiwix': None, # Depends of libzim (was 1) "libkiwix": None, # Depends of libzim (was 1)
'kiwix-tools': None, # Depends of libkiwix and libzim (was 0) "kiwix-tools": None, # Depends of libkiwix and libzim (was 0)
'zim-tools': 0, # Depends of libzim (was None) "zim-tools": 0, # Depends of libzim (was None)
'kiwix-desktop': None # Depends of libkiwix and libzim (was 4) "kiwix-desktop": None, # Depends of libkiwix and libzim (was 4)
} }
# This is the "version" of the whole base_deps_versions dict. # This is the "version" of the whole base_deps_versions dict.
# Change this when you change base_deps_versions. # Change this when you change base_deps_versions.
base_deps_meta_version = '96' base_deps_meta_version = "97"
base_deps_versions = { base_deps_versions = {
'zlib' : '1.2.12', "zlib": "1.2.12",
'lzma' : '5.2.6', "lzma": "5.2.6",
'zstd' : '1.5.2', "zstd": "1.5.2",
'docoptcpp' : '0.6.2', "docoptcpp": "0.6.2",
'uuid' : '1.43.4', "uuid": "1.43.4",
'xapian-core' : '1.4.23', "xapian-core": "1.4.23",
'mustache' : '4.1', "mustache": "4.1",
'pugixml' : '1.2', "pugixml": "1.2",
'libmicrohttpd' : '0.9.76', "libmicrohttpd": "0.9.76",
'gumbo' : '0.10.1', "gumbo": "0.10.1",
'icu4c' : '73.2', "icu4c": "73.2",
'libaria2' : '1.37.0', "libaria2": "1.37.0",
'libmagic' : '5.44', "libmagic": "5.44",
'android-ndk' : 'r21e', "android-ndk": "r21e",
'qt' : '5.10.1', "qt": "5.10.1",
'qtwebengine' : '5.10.1', "qtwebengine": "5.10.1",
'org.kde' : '5.15-21.08', "org.kde": "5.15-21.08",
'io.qt.qtwebengine' : '5.15-21.08', "io.qt.qtwebengine": "5.15-21.08",
'zim-testing-suite': '0.3', "zim-testing-suite": "0.3",
'emsdk': '3.1.41', "emsdk": "3.1.41",
} }