Compare commits

..

No commits in common. "main" and "r_49" have entirely different histories.
main ... r_49

135 changed files with 6730 additions and 35341 deletions

12
.github/FUNDING.yml vendored Normal file
View File

@ -0,0 +1,12 @@
# These are supported funding model platforms
github: kiwix # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
patreon: # Replace with a single Patreon username
open_collective: # Replace with a single Open Collective username
ko_fi: # Replace with a single Ko-fi username
tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
liberapay: # Replace with a single Liberapay username
issuehunt: # Replace with a single IssueHunt username
otechie: # Replace with a single Otechie username
custom: # https://kiwix.org/support-us/

View File

@ -1,46 +0,0 @@
[DEFAULTS]
[aqt]
concurrency: 4
baseurl: https://qt.mirror.constant.com/
7zcmd: 7z
print_stacktrace_on_error: False
always_keep_archives: False
archive_download_location: .
min_module_size: 41
[requests]
connection_timeout: 3.5
response_timeout: 30
max_retries_on_connection_error: 5
retry_backoff: 0.1
max_retries_on_checksum_error: 5
max_retries_to_retrieve_hash: 5
hash_algorithm: sha256
INSECURE_NOT_FOR_PRODUCTION_ignore_hash: False
[mirrors]
trusted_mirrors:
https://download.qt.io
https://qt.mirror.constant.com/
https://ftp.fau.de/qtproject/
blacklist:
http://mirrors.ocf.berkeley.edu
http://mirrors.tuna.tsinghua.edu.cn
http://mirrors.geekpie.club
fallbacks:
https://qtproject.mirror.liquidtelecom.com/
https://mirrors.aliyun.com/qt/
https://mirrors.ustc.edu.cn/qtproject/
https://ftp.jaist.ac.jp/pub/qtproject/
https://ftp.yz.yamagata-u.ac.jp/pub/qtproject/
https://qt-mirror.dannhauer.de/
https://ftp.fau.de/qtproject/
https://mirror.netcologne.de/qtproject/
https://mirrors.dotsrc.org/qtproject/
https://www.nic.funet.fi/pub/mirrors/download.qt-project.org/
https://master.qt.io/
https://mirrors.ukfast.co.uk/sites/qt.io/
https://ftp2.nluug.nl/languages/qt/
https://ftp1.nluug.nl/languages/qt/
https://qt.mirror.constant.com/

View File

@ -3,7 +3,7 @@ import csv, io, re
# Definition of what to build. # Definition of what to build.
# Array is read line by line. # Array is read line by line.
# Empty cells under (OS_NAME, COMPILE_CONFIG) mean "always match" (catch all, or `.*` regex) # Empty cells under (OS_NAME, PLATFORM_TARGET) mean "always match" (catch all, or `.*` regex)
# Once a cell doesn't match, skip to the next line. # Once a cell doesn't match, skip to the next line.
# Once a line matches, other lines are not read, so put more specific combination first. # Once a line matches, other lines are not read, so put more specific combination first.
# Lines composed of `-` , or `=`, or starting by `#` are ignored. # Lines composed of `-` , or `=`, or starting by `#` are ignored.
@ -15,56 +15,49 @@ import csv, io, re
# 'D' letter means we trigger the docker forkflow to build the docker image. # 'D' letter means we trigger the docker forkflow to build the docker image.
# If a cell contains several letters, all are done. # If a cell contains several letters, all are done.
BUILD_DEF = """ BUILD_DEF = """
| OS_NAME | COMPILE_CONFIG | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | platform_name | dependency_name | | OS_NAME | PLATFORM_TARGET | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | platform_name |
============================================================================================================================================== =====================================================================================================================
# manylinux is a special case as we need to compile libzim on old arch for python # Bionic is a special case as we need to compile libzim on old arch for python
| manylinux | native_mixed | BP | | | | | linux-x86_64-manylinux | | | bionic | native_mixed | BP | | | | | linux-x86_64-bionic |
| manylinux | aarch64_mixed | BP | | | | | linux-aarch64-manylinux | | | bionic | aarch64_mixed | BP | | | | | linux-aarch64-bionic |
---------------------------------------------------------------------------------------------------------------------------------------------- --------------------------------------------------------------------------------------------------------------------
# On Windows, we build only libzim for now. And only native_mixed as xapian doesn't compile as dll
| windows | native_static | Bd | d | BPd | BPd | | win-x86_64 | win-x86_64-static |
| windows | native_dyn | Bd | | | | | win-x86_64 | win-x86_64-dyn |
| windows | native_mixed | BPd | d | | | BPd | win-x86_64 | win-x86_64-mixed |
----------------------------------------------------------------------------------------------------------------------------------------------
# Osx builds, build binaries on native_dyn and native_static. On anyother things, build only the libraries # Osx builds, build binaries on native_dyn and native_static. On anyother things, build only the libraries
| macos | native_dyn | d | d | dB | B | | | macos-x86_64-dyn | | macos | native_dyn | d | d | dB | B | | |
| macos | native_static | | | BP | BP | | macos-x86_64 | | | macos | native_static | | | BP | BP | | macos-x86_64 |
| macos | native_mixed | BP | BP | | | | macos-x86_64 | | | macos | native_mixed | BP | BP | | | | macos-x86_64 |
| macos | iOS_arm64 | dB | dB | | | | | ios-arm64-dyn | | macos | iOS_arm64 | dB | B | | | | |
| macos | iOSSimulator_x86_64| dB | dB | | | | | ios-x86_64-dyn | | macos | iOS_x86_64 | dB | B | | | | |
| macos | iOSSimulator_arm64 | B | B | | | | | | | macos | iOS_Mac_ABI | B | B | | | | |
| macos | macOS_arm64_static | | | BP | BP | | macos-arm64 | | | macos | macOS_arm64_static | | | | | | |
| macos | macOS_arm64_mixed | dBP | dBP | d | | | macos-arm64 | macos-aarch64-dyn | | macos | macOS_arm64_mixed | BP | BP | | | | macos-arm64 |
| macos | macOS_x86_64 | B | B | | | | | | | macos | macOS_x86_64 | B | B | | | | |
| macos | apple_all_static | | BP | | | | xcframework | | ----------------------------------------------------------------------------------------------
---------------------------------------------------------------------------------------------------------------------------------------------- | | flatpak | | | | | BP | |
| focal | flatpak | | | | | BP | | | | | native_static | d | d | dBPSD | dBPSD | | linux-x86_64 |
| focal | native_static | d | d | dBPSD | dBPSD | | linux-x86_64 | linux-x86_64-static | | | native_mixed | BPS | BPS | | | | linux-x86_64 |
| focal | native_mixed | BPS | BPS | | | | linux-x86_64 | | | | native_dyn | d | d | dB | dB | dBPS | |
| focal | native_dyn | d | d | dB | dB | | | linux-x86_64-dyn |
| jammy | native_dyn | | | | | dBPS | | linux-x86_64-dyn |
# libzim CI is building alpine_dyn but not us # libzim CI is building alpine_dyn but not us
| focal | android_arm | dBP | dBP | | | | android-arm | android-arm | | | android_arm | dBP | dBP | | | | android-arm |
| focal | android_arm64 | dBP | dBP | | | | android-arm64 | android-arm64 | | | android_arm64 | dBP | dBP | | | | android-arm64 |
| focal | android_x86 | BP | BP | | | | android-x86 | | | | android_x86 | BP | BP | | | | android-x86 |
| focal | android_x86_64 | BP | BP | | | | android-x86_64 | | | | android_x86_64 | BP | BP | | | | android-x86_64 |
| focal | armv6_static | | | BP | BP | | linux-armv6 | | | | armv6_static | | | BP | BP | | linux-armv6 |
| focal | armv6_mixed | BP | | | | | linux-armv6 | | | | armv6_mixed | BP | | | | | linux-armv6 |
| focal | armv6_dyn | | | B | B | | | | | | armv6_dyn | | | B | B | | |
| focal | armv8_static | | | BP | BP | | linux-armv8 | | | | armv8_static | | | BP | BP | | linux-armv8 |
| focal | armv8_mixed | BP | | | | | linux-armv8 | | | | armv8_mixed | BP | | | | | linux-armv8 |
| focal | armv8_dyn | | | B | B | | | | | | armv8_dyn | | | B | B | | |
| focal | aarch64_static | | | BP | BP | | linux-aarch64 | | | | aarch64_static | | | BP | BP | | linux-aarch64 |
| focal | aarch64_mixed | BP | | | | | linux-aarch64 | | | | aarch64_mixed | BP | | | | | linux-aarch64 |
| focal | aarch64_dyn | d | | B | B | | | linux-aarch64-dyn | | | aarch64_dyn | d | | B | B | | |
| focal | aarch64_musl_static| | | BP | BP | | linux-aarch64-musl | | | | aarch64_musl_static| | | BP | BP | | linux-aarch64-musl |
| focal | aarch64_musl_mixed | BP | | | | | linux-aarch64-musl | | | | aarch64_musl_mixed | BP | | | | | linux-aarch64-musl |
| focal | aarch64_musl_dyn | d | | B | B | | | linux-aarch64-musl-dyn | | | aarch64_musl_dyn | d | | B | B | | |
| focal | x86-64_musl_static | | | BP | BP | | linux-x86_64-musl | | | | win32_static | d | dB | dBP | dBP | | win-i686 |
| focal | x86-64_musl_mixed | BP | | | | | linux-x86_64-musl | | | | win32_dyn | d | dB | dB | dB | | |
| focal | i586_static | | | BP | BP | | linux-i586 | | | | i586_static | | | BP | BP | | linux-i586 |
| focal | i586_dyn | | | B | B | | | | | | i586_dyn | | | B | B | | |
| focal | wasm | dBP | | | | | wasm-emscripten | wasm | | | wasm | dBP | | | | | wasm-emscripten |
""" """
@ -96,10 +89,10 @@ def selector_match(selector, value):
class Context(NamedTuple): class Context(NamedTuple):
OS_NAME: str OS_NAME: str
COMPILE_CONFIG: str PLATFORM_TARGET: str
def match(self, row): def match(self, row):
for key in ["OS_NAME", "COMPILE_CONFIG"]: for key in ["OS_NAME", "PLATFORM_TARGET"]:
context_value = getattr(self, key) context_value = getattr(self, key)
selector = row[key] selector = row[key]
if not selector_match(selector, context_value): if not selector_match(selector, context_value):
@ -113,11 +106,10 @@ SOURCE_PUBLISH = "S"
DEPS = "d" DEPS = "d"
DOCKER = "D" DOCKER = "D"
def select_build_targets(criteria): def select_build_targets(criteria):
from common import COMPILE_CONFIG, OS_NAME from common import PLATFORM_TARGET, OS_NAME
context = Context(COMPILE_CONFIG=COMPILE_CONFIG, OS_NAME=OS_NAME) context = Context(PLATFORM_TARGET=PLATFORM_TARGET, OS_NAME=OS_NAME)
reader = csv.DictReader(strip_array(BUILD_DEF), dialect=TableDialect()) reader = csv.DictReader(strip_array(BUILD_DEF), dialect=TableDialect())
for row in reader: for row in reader:
@ -136,26 +128,17 @@ def select_build_targets(criteria):
print(build_order) print(build_order)
return build_order return build_order
raise ValueError("No definition match with current context.") raise "No definition match with current context."
def get_platform_name():
from common import PLATFORM_TARGET, OS_NAME
def get_column_value(column_name): context = Context(PLATFORM_TARGET=PLATFORM_TARGET, OS_NAME=OS_NAME)
from common import COMPILE_CONFIG, OS_NAME
context = Context(COMPILE_CONFIG=COMPILE_CONFIG, OS_NAME=OS_NAME)
reader = csv.DictReader(strip_array(BUILD_DEF), dialect=TableDialect()) reader = csv.DictReader(strip_array(BUILD_DEF), dialect=TableDialect())
for row in reader: for row in reader:
if context.match(row): if context.match(row):
name = row[column_name] name = row["platform_name"]
return name or None return name or None
raise ValueError("No definition match with current context.") raise "No definition match with current context."
def get_platform_name():
return get_column_value("platform_name")
def get_dependency_archive_name():
return get_column_value("dependency_name")

View File

@ -8,16 +8,16 @@ from common import (
fix_macos_rpath, fix_macos_rpath,
upload_archive, upload_archive,
OS_NAME, OS_NAME,
COMPILE_CONFIG, PLATFORM_TARGET,
DEV_BRANCH, DEV_BRANCH,
) )
for target in select_build_targets(BUILD): for target in select_build_targets(BUILD):
run_kiwix_build(target, config=COMPILE_CONFIG) run_kiwix_build(target, platform=PLATFORM_TARGET)
if target == "kiwix-desktop": if target == "kiwix-desktop":
archive = create_desktop_image(make_release=False) archive = create_desktop_image(make_release=False)
else: else:
if COMPILE_CONFIG == "native_mixed" and OS_NAME == "macos": if PLATFORM_TARGET == "native_mixed" and OS_NAME == "macos":
fix_macos_rpath(target) fix_macos_rpath(target)
archive = make_archive(target, make_release=False) archive = make_archive(target, make_release=False)
if archive and DEV_BRANCH: if archive and DEV_BRANCH:

View File

@ -13,7 +13,7 @@ from common import (
fix_macos_rpath, fix_macos_rpath,
BASE_DIR, BASE_DIR,
OS_NAME, OS_NAME,
COMPILE_CONFIG, PLATFORM_TARGET,
MAKE_RELEASE, MAKE_RELEASE,
notarize_macos_build, notarize_macos_build,
) )
@ -21,22 +21,23 @@ from common import (
from build_definition import select_build_targets, BUILD, PUBLISH, SOURCE_PUBLISH from build_definition import select_build_targets, BUILD, PUBLISH, SOURCE_PUBLISH
def release_filter(project):
return release_versions.get(project) is not None
# Filter what to build if we are doing a release. # Filter what to build if we are doing a release.
TARGETS = select_build_targets(PUBLISH)
if MAKE_RELEASE: if MAKE_RELEASE:
TARGETS = select_build_targets(PUBLISH)
def release_filter(project):
return release_versions.get(project) is not None
TARGETS = tuple(filter(release_filter, TARGETS)) TARGETS = tuple(filter(release_filter, TARGETS))
else:
TARGETS = select_build_targets(BUILD)
for target in TARGETS: for target in TARGETS:
run_kiwix_build(target, config=COMPILE_CONFIG, make_release=MAKE_RELEASE) run_kiwix_build(target, platform=PLATFORM_TARGET, make_release=MAKE_RELEASE)
if target == "kiwix-desktop": if target == "kiwix-desktop":
archive = create_desktop_image(make_release=MAKE_RELEASE) archive = create_desktop_image(make_release=MAKE_RELEASE)
else: else:
if OS_NAME == "macos" and COMPILE_CONFIG.endswith("_mixed"): if OS_NAME == "macos" and PLATFORM_TARGET.endswith("_mixed"):
fix_macos_rpath(target) fix_macos_rpath(target)
notarize_macos_build(target) notarize_macos_build(target)
archive = make_archive(target, make_release=MAKE_RELEASE) archive = make_archive(target, make_release=MAKE_RELEASE)
@ -56,11 +57,13 @@ if MAKE_RELEASE:
if target not in source_published_targets: if target not in source_published_targets:
continue continue
run_kiwix_build( run_kiwix_build(
target, config=COMPILE_CONFIG, make_release=MAKE_RELEASE, make_dist=True target, platform=PLATFORM_TARGET, make_release=MAKE_RELEASE, make_dist=True
) )
full_target_name = "{}-{}".format(target, main_project_versions[target]) full_target_name = "{}-{}".format(target, main_project_versions[target])
if target == "kiwix-desktop": if target == "kiwix-desktop":
archive = BASE_DIR / full_target_name / "{}.tar.gz".format(full_target_name) archive = (
BASE_DIR / full_target_name / "{}.tar.gz".format(full_target_name)
)
else: else:
archive = ( archive = (
BASE_DIR BASE_DIR
@ -71,5 +74,5 @@ if MAKE_RELEASE:
upload_archive(archive, target, make_release=MAKE_RELEASE) upload_archive(archive, target, make_release=MAKE_RELEASE)
# Publish flathub # Publish flathub
if COMPILE_CONFIG == "flatpak" and "kiwix-desktop" in TARGETS: if PLATFORM_TARGET == "flatpak" and "kiwix-desktop" in TARGETS:
update_flathub_git() update_flathub_git()

View File

@ -1,19 +1,17 @@
import os import os
from os import environ as _environ from os import environ as _environ
from pathlib import Path, PurePosixPath from pathlib import Path
from datetime import date from datetime import date
import tarfile import tarfile
import zipfile import zipfile
import subprocess import subprocess
import re import re
import shutil import shutil
import platform
import requests import requests
from build_definition import get_platform_name, get_dependency_archive_name from build_definition import get_platform_name
from kiwixbuild.dependencies.apple_xcframework import AppleXCFramework
from kiwixbuild.versions import ( from kiwixbuild.versions import (
main_project_versions, main_project_versions,
release_versions, release_versions,
@ -21,40 +19,21 @@ from kiwixbuild.versions import (
) )
def get_build_dir(config) -> Path:
command = ["kiwix-build"]
command.extend(["--config", config])
command.append("--get-build-dir")
command.append("--use-target-arch-name")
return Path(
subprocess.run(command, cwd=str(HOME), check=True, stdout=subprocess.PIPE)
.stdout.strip()
.decode("utf8")
)
PLATFORM_TARGET = _environ["PLATFORM_TARGET"]
COMPILE_CONFIG = _environ["COMPILE_CONFIG"]
OS_NAME = _environ["OS_NAME"] OS_NAME = _environ["OS_NAME"]
HOME = Path(os.path.expanduser("~")) HOME = Path(os.path.expanduser("~"))
BASE_DIR = get_build_dir(COMPILE_CONFIG) BASE_DIR = HOME / "BUILD_{}".format(PLATFORM_TARGET)
SOURCE_DIR = HOME / "SOURCE" SOURCE_DIR = HOME / "SOURCE"
ARCHIVE_DIR = HOME / "ARCHIVE" ARCHIVE_DIR = HOME / "ARCHIVE"
TOOLCHAIN_DIR = BASE_DIR / "TOOLCHAINS"
INSTALL_DIR = BASE_DIR / "INSTALL" INSTALL_DIR = BASE_DIR / "INSTALL"
default_tmp_dir = os.getenv("TEMP") if platform.system() == "Windows" else "/tmp" TMP_DIR = Path("/tmp")
TMP_DIR = Path(os.getenv("TMP_DIR", default_tmp_dir)) KBUILD_SOURCE_DIR = HOME / "kiwix-build"
if platform.system() == "Windows":
KBUILD_SOURCE_DIR = Path(_environ["GITHUB_WORKSPACE"])
BIN_EXT = ".exe"
else:
KBUILD_SOURCE_DIR = HOME / "kiwix-build"
BIN_EXT = ""
_ref = _environ.get("GITHUB_REF", "").split("/")[-1] _ref = _environ.get("GITHUB_REF", "").split("/")[-1]
MAKE_RELEASE = re.fullmatch(r"r_[0-9]+", _ref) is not None MAKE_RELEASE = re.fullmatch(r"r_[0-9]+", _ref) is not None
MAKE_RELEASE = MAKE_RELEASE and (_environ.get("GITHUB_EVENT_NAME") != "schedule") MAKE_RELEASE = MAKE_RELEASE and (_environ.get('GITHUB_EVENT_NAME') != 'schedule')
if not MAKE_RELEASE and _ref != "main": if not MAKE_RELEASE and _ref != "main":
DEV_BRANCH = _ref DEV_BRANCH = _ref
@ -64,25 +43,19 @@ else:
FLATPAK_HTTP_GIT_REMOTE = "https://github.com/flathub/org.kiwix.desktop.git" FLATPAK_HTTP_GIT_REMOTE = "https://github.com/flathub/org.kiwix.desktop.git"
FLATPAK_GIT_REMOTE = "git@github.com:flathub/org.kiwix.desktop.git" FLATPAK_GIT_REMOTE = "git@github.com:flathub/org.kiwix.desktop.git"
BIN_EXT = ".exe" if PLATFORM_TARGET.startswith("win32_") else ""
def major_version(version: str) -> str: def major_version(version: str) -> str:
return version.split(".")[0] return version.split(".")[0]
# Depending of base distribution, libraries are in "lib64" (redhat base) or "lib/<arch>" (debian base).
# On top of that, when cross-compiling, libraries are always put in `lib/<arch>`.
# As we use this as glob regex to select which files to add to archive, this is not a problem to have both.
def lib_prefix(file):
yield "lib64/" + file
yield "lib/*/" + file
# We have build everything. Now create archives for public deployement. # We have build everything. Now create archives for public deployement.
EXPORT_FILES = { EXPORT_FILES = {
"kiwix-tools": ( "kiwix-tools": (
INSTALL_DIR / "bin", INSTALL_DIR / "bin",
[f + BIN_EXT for f in ("kiwix-manage", "kiwix-search", "kiwix-serve")] [
+ ["icu*.dll"], f + BIN_EXT
for f in ("kiwix-manage", "kiwix-search", "kiwix-serve")
],
), ),
"zim-tools": ( "zim-tools": (
INSTALL_DIR / "bin", INSTALL_DIR / "bin",
@ -97,52 +70,33 @@ EXPORT_FILES = {
"zimpatch", "zimpatch",
"zimsplit", "zimsplit",
"zimwriterfs", "zimwriterfs",
"zimrecreate", "zimrecreate"
) )
] ],
+ ["icu*.dll"],
), ),
"libzim": ( "libzim": (
INSTALL_DIR, INSTALL_DIR,
( (
## Linux
# We need to package all dependencies (`*.a`) on wasm # We need to package all dependencies (`*.a`) on wasm
*lib_prefix("libzim.a" if COMPILE_CONFIG != "wasm" else "*.a"), "lib/*/libzim.a" if PLATFORM_TARGET != "wasm" else "lib/*.a",
*lib_prefix("libzim.so"), "lib/*/libzim.so",
*lib_prefix( "lib/*/libzim.so.{version}".format(
"libzim.so.{version}".format(version=main_project_versions["libzim"]) version=main_project_versions["libzim"]
), ),
*lib_prefix( "lib/*/libzim.so.{version}".format(
"libzim.so.{version}".format( version=major_version(main_project_versions["libzim"])
version=major_version(main_project_versions["libzim"])
)
), ),
## MacOS
"lib/libzim.{}.dylib".format( "lib/libzim.{}.dylib".format(
major_version(main_project_versions["libzim"]) major_version(main_project_versions["libzim"])
), ),
"lib/libzim.dylib", "lib/libzim.dylib",
"lib/*/libzim.pc", "lib/*/libzim.pc",
## Windows
"bin/zim-{version}.dll".format(
version=major_version(main_project_versions["libzim"])
),
"bin/icu*.dll",
"bin/zim-{version}.pdb".format(
version=major_version(main_project_versions["libzim"])
),
"lib/zim.lib",
## Includes and others
"include/zim/**/*.h", "include/zim/**/*.h",
"share/icu/{}/icudt{}l.dat".format(
base_deps_versions["icu4c"], major_version(base_deps_versions["icu4c"])
),
), ),
), ),
"libkiwix": ( "libkiwix": (
INSTALL_DIR, INSTALL_DIR,
( (
"lib/CoreKiwix.xcframework/",
"lib/*/libkiwix.so", "lib/*/libkiwix.so",
"lib/*/libkiwix.so.{version}".format( "lib/*/libkiwix.so.{version}".format(
version=main_project_versions["libkiwix"] version=main_project_versions["libkiwix"]
@ -153,27 +107,15 @@ EXPORT_FILES = {
"lib/libkiwix.{}.dylib".format( "lib/libkiwix.{}.dylib".format(
major_version(main_project_versions["libkiwix"]) major_version(main_project_versions["libkiwix"])
), ),
"bin/kiwix-{version}.dll".format(
version=major_version(main_project_versions["libkiwix"])
),
"bin/icu*.dll",
"bin/kiwix-{version}.pdb".format(
version=major_version(main_project_versions["libkiwix"])
),
"lib/kiwix.lib",
"lib/libkiwix.dylib", "lib/libkiwix.dylib",
"lib/*/libkiwix.pc", "lib/*/libkiwix.pc",
"include/kiwix/**/*.h", "include/kiwix/**/*.h"
"share/icu/{}/icudt{}l.dat".format(
base_deps_versions["icu4c"], major_version(base_deps_versions["icu4c"])
),
), ),
), ),
} }
DATE = date.today().isoformat() DATE = date.today().isoformat()
def print_message(message, *args, **kwargs): def print_message(message, *args, **kwargs):
message = message.format(*args, **kwargs) message = message.format(*args, **kwargs)
message = "{0} {1} {0}".format("-" * 3, message) message = "{0} {1} {0}".format("-" * 3, message)
@ -183,26 +125,23 @@ def print_message(message, *args, **kwargs):
MANIFEST_TEMPLATE = """{archive_name} MANIFEST_TEMPLATE = """{archive_name}
*************************** ***************************
Dependencies archive for {target} using config {config} Dependencies archive for {target} on platform {platform}
Generated at {date} Generated at {date}
""" """
def write_manifest(manifest_file, archive_name, target, config): def write_manifest(manifest_file, archive_name, target, platform):
with manifest_file.open(mode="w") as f: with manifest_file.open(mode="w") as f:
f.write( f.write(
MANIFEST_TEMPLATE.format( MANIFEST_TEMPLATE.format(
archive_name=archive_name, archive_name=archive_name, target=target, platform=platform, date=DATE,
target=target,
config=config,
date=DATE,
) )
) )
def run_kiwix_build( def run_kiwix_build(
target, target,
config, platform,
build_deps_only=False, build_deps_only=False,
target_only=False, target_only=False,
make_release=False, make_release=False,
@ -213,8 +152,7 @@ def run_kiwix_build(
command.append("--hide-progress") command.append("--hide-progress")
command.append("--fast-clone") command.append("--fast-clone")
command.append("--assume-packages-installed") command.append("--assume-packages-installed")
command.append("--use-target-arch-name") command.extend(["--target-platform", platform])
command.extend(["--config", config])
if build_deps_only: if build_deps_only:
command.append("--build-deps-only") command.append("--build-deps-only")
if target_only: if target_only:
@ -236,123 +174,54 @@ def run_kiwix_build(
print_message("Build ended") print_message("Build ended")
try: def upload(file_to_upload, host, dest_path):
import paramiko if not file_to_upload.exists():
print_message("No {} to upload!", file_to_upload)
return
def upload(file_to_upload, host, dest_path): if ":" in host:
if not file_to_upload.exists(): host, port = host.split(":", 1)
print_message("No {} to upload!", file_to_upload) else:
return port = "22"
if ":" in host: # sending SFTP mkdir command to the sftp interactive mode and not batch (-b) mode
host, port = host.split(":", 1) # as the latter would exit on any mkdir error while it is most likely
else: # the first parts of the destination is already present and thus can't be created
port = "22" sftp_commands = "\n".join(
if "@" in host: [
user, host = host.split("@", 1) f"mkdir {part}"
else: for part in list(reversed(Path(dest_path).parents)) + [dest_path]
user = None
from contextlib import contextmanager
@contextmanager
def get_client():
client = paramiko.client.SSHClient()
client.set_missing_host_key_policy(paramiko.client.WarningPolicy)
print_message(f"Connect to {host}:{port}")
client.connect(
host,
port=port,
username=user,
key_filename=_environ.get("SSH_KEY"),
look_for_keys=False,
compress=True,
)
try:
yield client
finally:
client.close()
@contextmanager
def get_sftp():
with get_client() as client:
sftp = client.open_sftp()
try:
yield sftp
finally:
sftp.close()
dest_path = PurePosixPath(dest_path)
remote_file = dest_path.joinpath(file_to_upload.name)
with get_sftp() as sftp:
for part in list(reversed(dest_path.parents)) + [dest_path]:
part = str(part)
try:
sftp.stat(part)
except FileNotFoundError:
sftp.mkdir(part)
print_message(f"Sending archive {file_to_upload} to {remote_file}")
sftp.put(str(file_to_upload), str(remote_file), confirm=True)
except ModuleNotFoundError:
# On old system (bionic) paramiko is really complex to install
# Keep the old implementaion on sush system.
def upload(file_to_upload, host, dest_path):
if not file_to_upload.exists():
print_message("No {} to upload!", file_to_upload)
return
if ":" in host:
host, port = host.split(":", 1)
else:
port = "22"
# Using SFTP to create the directory hierarchy because we can not
# use SSH (no shell for this user); and then scp to upload the file.
#
# Sending SFTP mkdir command to the SFTP interactive mode and not batch (-b) mode
# as the latter would exit on any mkdir error while it is most likely
# the first parts of the destination is already present and thus can't be created
sftp_commands = "\n".join(
[
f"mkdir {part}"
for part in list(reversed(Path(dest_path).parents)) + [dest_path]
]
)
command = [
"sftp",
"-c",
"aes128-ctr",
"-i",
_environ.get("SSH_KEY"),
"-P",
port,
"-o",
"StrictHostKeyChecking=no",
host,
] ]
print_message("Creating dest path {}", dest_path) )
subprocess.run(command, input=sftp_commands.encode("utf-8"), check=True) command = [
"sftp",
"-i",
_environ.get("SSH_KEY"),
"-P",
port,
"-o",
"StrictHostKeyChecking=no",
host,
]
print_message("Creating dest path {}", dest_path)
subprocess.run(command, input=sftp_commands.encode("utf-8"), check=True)
command = [ command = [
"scp", "scp",
"-c", "-c",
"aes128-ctr", "aes128-ctr",
"-rp", "-rp",
"-P", "-P",
port, port,
"-i", "-i",
_environ.get("SSH_KEY"), _environ.get("SSH_KEY"),
"-o", "-o",
"StrictHostKeyChecking=no", "StrictHostKeyChecking=no",
str(file_to_upload), str(file_to_upload),
"{}:{}".format(host, dest_path), "{}:{}".format(host, dest_path),
] ]
print_message("Sending archive with command {}", command) print_message("Sending archive with command {}", command)
subprocess.check_call(command) subprocess.check_call(command)
def upload_archive(archive, project, make_release, dev_branch=None): def upload_archive(archive, project, make_release, dev_branch=None):
@ -360,7 +229,7 @@ def upload_archive(archive, project, make_release, dev_branch=None):
print_message("No archive {} to upload!", archive) print_message("No archive {} to upload!", archive)
return return
if project.startswith("kiwix-") or project in ["libkiwix"]: if project.startswith("kiwix-") or project in ['libkiwix']:
host = "ci@master.download.kiwix.org:30022" host = "ci@master.download.kiwix.org:30022"
dest_path = "/data/download/" dest_path = "/data/download/"
else: else:
@ -384,47 +253,43 @@ def upload_archive(archive, project, make_release, dev_branch=None):
# This remove "share/doc" and "share/man" from the thing to copy in the deps archive # This remove "share/doc" and "share/man" from the thing to copy in the deps archive
def filter_install_dir(path): def filter_install_dir(path):
for dir in path.glob("*"): for dir in path.glob('*'):
if dir.name not in ["share"]: if dir.name not in ['share']:
yield dir yield dir
else: else:
for sub_dir in dir.glob("*"): for sub_dir in dir.glob('*'):
if sub_dir.name not in ["doc", "man"]: if sub_dir.name not in ['doc', 'man']:
yield sub_dir yield sub_dir
# Full: True if we are creating a full archive to be used as cache by kiwix-build (base_deps2_{os}_{platform}_{base_deps_version}.tar.xz)
# Full: True if we are creating a full archive to be used as cache by kiwix-build (base_deps_{os}_{config}_{base_deps_version}.tar.gz) # Full: False if we are creating a archive to be used as pre-cached dependencies for project's CI (deps2_{os}_{platform}_{target}.tar.xz)
# Full: False if we are creating a archive to be used as pre-cached dependencies for project's CI (deps_{config}_{target}.tar.gz)
def make_deps_archive(target=None, name=None, full=False): def make_deps_archive(target=None, name=None, full=False):
archive_name = name or "deps_{}_{}.tar.gz".format( archive_name = name or "deps2_{}_{}_{}.tar.xz".format(
get_dependency_archive_name(), target OS_NAME, PLATFORM_TARGET, target
) )
print_message("Create archive {}.", archive_name) print_message("Create archive {}.", archive_name)
files_to_archive = list(filter_install_dir(INSTALL_DIR)) files_to_archive = list(filter_install_dir(INSTALL_DIR))
files_to_archive += HOME.glob("BUILD_*/LOGS") files_to_archive += HOME.glob("BUILD_*/LOGS")
if COMPILE_CONFIG == "apple_all_static": if PLATFORM_TARGET.endswith("_mixed"):
for subconfig in AppleXCFramework.subConfigNames: static_platform = PLATFORM_TARGET.replace("_mixed", "_static")
base_dir = get_build_dir(subconfig) files_to_archive += filter_install_dir(HOME / ("BUILD_" + static_platform) / "INSTALL")
files_to_archive += filter_install_dir(base_dir / "INSTALL") if PLATFORM_TARGET.startswith("android_"):
if (base_dir / "meson_cross_file.txt").exists():
files_to_archive.append(base_dir / "meson_cross_file.txt")
if COMPILE_CONFIG.endswith("_mixed"):
static_config = COMPILE_CONFIG.replace("_mixed", "_static")
files_to_archive += filter_install_dir(get_build_dir(static_config) / "INSTALL")
if COMPILE_CONFIG.startswith("android_"):
files_to_archive += filter_install_dir(HOME / "BUILD_neutral" / "INSTALL") files_to_archive += filter_install_dir(HOME / "BUILD_neutral" / "INSTALL")
base_dir = get_build_dir(COMPILE_CONFIG) base_dir = HOME / "BUILD_{}".format(PLATFORM_TARGET)
if (base_dir / "meson_cross_file.txt").exists(): if (base_dir / "meson_cross_file.txt").exists():
files_to_archive.append(base_dir / "meson_cross_file.txt") files_to_archive.append(base_dir / "meson_cross_file.txt")
# Copy any toolchain # Add ndk/sdk/toolchains to allow project's CI to find them and compile
files_to_archive += [TOOLCHAIN_DIR] files_to_archive += HOME.glob("BUILD_*/android-ndk*")
files_to_archive += HOME.glob("BUILD_neutral/TOOLCHAINS/*") files_to_archive += HOME.glob("BUILD_*/emsdk*")
if PLATFORM_TARGET.startswith("aarch64"):
files_to_archive += SOURCE_DIR.glob("aarch64*/*")
if PLATFORM_TARGET.startswith("armv"):
files_to_archive += SOURCE_DIR.glob("armv*/*")
if (BASE_DIR / "meson_cross_file.txt").exists(): if (BASE_DIR / "meson_cross_file.txt").exists():
files_to_archive.append(BASE_DIR / "meson_cross_file.txt") files_to_archive.append(BASE_DIR / "meson_cross_file.txt")
manifest_file = BASE_DIR / "manifest.txt" manifest_file = BASE_DIR / "manifest.txt"
write_manifest(manifest_file, archive_name, target, COMPILE_CONFIG) write_manifest(manifest_file, archive_name, target, PLATFORM_TARGET)
files_to_archive.append(manifest_file) files_to_archive.append(manifest_file)
relative_path = HOME relative_path = HOME
@ -432,20 +297,23 @@ def make_deps_archive(target=None, name=None, full=False):
files_to_archive += ARCHIVE_DIR.glob(".*_ok") files_to_archive += ARCHIVE_DIR.glob(".*_ok")
files_to_archive += BASE_DIR.glob("*/.*_ok") files_to_archive += BASE_DIR.glob("*/.*_ok")
# Add also static build for mixed target # Add also static build for mixed target
if COMPILE_CONFIG.endswith("_mixed"): if PLATFORM_TARGET.endswith("_mixed"):
static_config = COMPILE_CONFIG.replace("_mixed", "_static") static_platform = PLATFORM_TARGET.replace("_mixed", "_static")
files_to_archive += get_build_dir(static_config).glob("*/.*_ok") files_to_archive += (HOME / ("BUILD_" + static_platform)).glob("*/.*_ok")
# Native dyn and static is needed for potential cross compilation that use native tools (icu) # Native dyn and static is needed for potential cross compilation that use native tools (icu)
files_to_archive += get_build_dir("native_dyn").glob("*/.*_ok") files_to_archive += (HOME / "BUILD_native_dyn").glob("*/.*_ok")
files_to_archive += get_build_dir("native_static").glob("*/.*_ok") files_to_archive += (HOME / "BUILD_native_static").glob("*/.*_ok")
files_to_archive += HOME.glob("BUILD_*android*/**/.*_ok") files_to_archive += HOME.glob("BUILD_android*/**/.*_ok")
files_to_archive += HOME.glob("BUILD_*apple-macos*/**/.*_ok")
files_to_archive += HOME.glob("BUILD_*apple-ios*/**/.*_ok")
files_to_archive += SOURCE_DIR.glob("*/.*_ok") files_to_archive += SOURCE_DIR.glob("*/.*_ok")
files_to_archive += SOURCE_DIR.glob("zim-testing-suite-*/*") files_to_archive += SOURCE_DIR.glob("zim-testing-suite-*/*")
toolchains_subdirs = HOME.glob("BUILD_*/TOOLCHAINS/*/*")
for subdir in toolchains_subdirs:
if not subdir.match("tools"):
files_to_archive.append(subdir)
archive_file = TMP_DIR / archive_name archive_file = TMP_DIR / archive_name
with tarfile.open(str(archive_file), "w:gz") as tar: with tarfile.open(str(archive_file), "w:xz") as tar:
for name in set(files_to_archive): for name in set(files_to_archive):
print(".{}".format(name), flush=True) print(".{}".format(name), flush=True)
tar.add(str(name), arcname=str(name.relative_to(relative_path))) tar.add(str(name), arcname=str(name.relative_to(relative_path)))
@ -461,25 +329,6 @@ def get_postfix(project):
return postfix return postfix
def sign_binary(path):
# We assume here that signtool and certificate are properly configured.
# Env var `SIGNTOOL_THUMBPRINT` must contain thumbprint of the certificate to use.
command = [
os.getenv("SIGNTOOL_PATH", "signtool.exe"),
"sign",
"/fd",
"sha256",
"/tr",
"http://ts.ssl.com",
"/td",
"sha256",
"/sha1",
os.environ["SIGNTOOL_THUMBPRINT"],
str(path),
]
subprocess.run(command, check=True)
def make_archive(project, make_release): def make_archive(project, make_release):
platform_name = get_platform_name() platform_name = get_platform_name()
if not platform_name: if not platform_name:
@ -501,13 +350,7 @@ def make_archive(project, make_release):
files_to_archive = [] files_to_archive = []
for export_file in export_files: for export_file in export_files:
files_to_archive.extend(base_dir.glob(export_file)) files_to_archive.extend(base_dir.glob(export_file))
if platform_name == "win-i686":
if make_release and platform.system() == "Windows":
for file in files_to_archive:
if str(file).endswith(".exe"):
sign_binary(file)
if platform_name == "win-i686" or platform.system() == "Windows":
open_archive = lambda a: zipfile.ZipFile( open_archive = lambda a: zipfile.ZipFile(
str(a), "w", compression=zipfile.ZIP_DEFLATED str(a), "w", compression=zipfile.ZIP_DEFLATED
) )
@ -537,31 +380,15 @@ def create_desktop_image(make_release):
postfix = DATE postfix = DATE
src_dir = SOURCE_DIR / "kiwix-desktop" src_dir = SOURCE_DIR / "kiwix-desktop"
if COMPILE_CONFIG == "flatpak": if PLATFORM_TARGET == "flatpak":
build_path = BASE_DIR / "org.kiwix.desktop.flatpak" build_path = BASE_DIR / "org.kiwix.desktop.flatpak"
app_name = "org.kiwix.desktop.{}.flatpak".format(postfix) app_name = "org.kiwix.desktop.{}.flatpak".format(postfix)
print_message("archive is {}", build_path) print_message("archive is {}", build_path)
elif platform.system() == "Windows":
archive_basename = "kiwix-desktop_windows_x64_{}".format(postfix)
working_dir = INSTALL_DIR / archive_basename
build_path = Path(str(working_dir) + ".zip")
app_name = build_path.name
command = [
"python",
KBUILD_SOURCE_DIR / "scripts" / "package_kiwix-desktop_windows.py",
str(INSTALL_DIR),
str(working_dir),
str(build_path),
]
if make_release:
command += ["-s"]
print_message("Package archive of kiwix-desktop")
subprocess.check_call(command, cwd=str(HOME))
else: else:
build_path = HOME / "Kiwix-{}-x86_64.AppImage".format(postfix) build_path = HOME / "Kiwix-{}-x86_64.AppImage".format(postfix)
app_name = "kiwix-desktop_x86_64_{}.appimage".format(postfix) app_name = "kiwix-desktop_x86_64_{}.appimage".format(postfix)
command = [ command = [
KBUILD_SOURCE_DIR / "scripts" / "create_kiwix-desktop_appImage.sh", "kiwix-build/scripts/create_kiwix-desktop_appImage.sh",
str(INSTALL_DIR), str(INSTALL_DIR),
str(src_dir), str(src_dir),
str(HOME / "AppDir"), str(HOME / "AppDir"),
@ -595,7 +422,7 @@ def update_flathub_git():
call(command) call(command)
shutil.copy(str(BASE_DIR / "org.kiwix.desktop.json"), str(git_repo_dir)) shutil.copy(str(BASE_DIR / "org.kiwix.desktop.json"), str(git_repo_dir))
patch_dir = KBUILD_SOURCE_DIR / "kiwixbuild" / "patches" patch_dir = KBUILD_SOURCE_DIR / "kiwixbuild" / "patches"
for dep in ["pugixml", "libmicrohttpd"]: for dep in ["pugixml"]:
for f in patch_dir.glob("{}_*.patch".format(dep)): for f in patch_dir.glob("{}_*.patch".format(dep)):
shutil.copy(str(f), str(git_repo_dir / "patches")) shutil.copy(str(f), str(git_repo_dir / "patches"))
command = ["git", "add", "-A", "."] command = ["git", "add", "-A", "."]
@ -621,6 +448,7 @@ def update_flathub_git():
def fix_macos_rpath(project): def fix_macos_rpath(project):
base_dir, export_files = EXPORT_FILES[project] base_dir, export_files = EXPORT_FILES[project]
for file in filter(lambda f: f.endswith(".dylib"), export_files): for file in filter(lambda f: f.endswith(".dylib"), export_files):
lib = base_dir / file lib = base_dir / file
@ -639,35 +467,22 @@ def trigger_workflow(repo, workflow="docker.yml", ref="main", inputs=None):
ref: branch or tag name ref: branch or tag name
inputs: dict of inputs to pass to the workflow""" inputs: dict of inputs to pass to the workflow"""
print_message( print_message(
"triggering workflow `{workflow}` on {repo}@{ref} " "with inputs={inputs}", "triggering workflow `{workflow}` on {repo}@{ref} "
workflow=workflow, "with inputs={inputs}", workflow=workflow, repo=repo, ref=ref, inputs=inputs)
repo=repo,
ref=ref,
inputs=inputs,
)
url = "{base_url}/repos/{repo}/actions/workflows/{workflow}/dispatches".format( url = "{base_url}/repos/{repo}/actions/workflows/{workflow}/dispatches".format(
base_url=os.getenv("GITHUB_API_URL", "https://api.github.com"), base_url=os.getenv("GITHUB_API_URL", "https://api.github.com"),
repo=repo, repo=repo, workflow=workflow)
workflow=workflow,
)
resp = requests.post( resp = requests.post(url, headers={
url,
headers={
"Content-Type": "application/json", "Content-Type": "application/json",
"Authorization": "token {token}".format(token=os.getenv("GITHUB_PAT", "")), "Authorization": "token {token}".format(
token=os.getenv('GITHUB_PAT', '')),
"Accept": "application/vnd.github.v3+json", "Accept": "application/vnd.github.v3+json",
}, }, json={"ref": ref, "inputs": inputs}, timeout=5)
json={"ref": ref, "inputs": inputs},
timeout=5,
)
if resp.status_code != 204: if resp.status_code != 204:
raise ValueError( raise ValueError("Unexpected HTTP {code}: {reason}".format(
"Unexpected HTTP {code}: {reason}".format( code=resp.status_code, reason=resp.reason))
code=resp.status_code, reason=resp.reason
)
)
def trigger_docker_publish(target): def trigger_docker_publish(target):
@ -675,14 +490,13 @@ def trigger_docker_publish(target):
return return
version = get_postfix(target) version = get_postfix(target)
repo = {"zim-tools": "openzim/zim-tools", "kiwix-tools": "kiwix/kiwix-tools"}.get( repo = {
target "zim-tools": "openzim/zim-tools",
) "kiwix-tools": "kiwix/kiwix-tools"}.get(target)
try: try:
trigger_workflow( trigger_workflow(repo, workflow="docker.yml", ref="main",
repo, workflow="docker.yml", ref="main", inputs={"version": version} inputs={"version": version})
)
print_message("triggered docker workflow on {repo}", repo=repo) print_message("triggered docker workflow on {repo}", repo=repo)
except Exception as exc: except Exception as exc:
print_message("Error triggering workflow: {exc}", exc=exc) print_message("Error triggering workflow: {exc}", exc=exc)
@ -690,52 +504,39 @@ def trigger_docker_publish(target):
def notarize_macos_build(project): def notarize_macos_build(project):
"""sign and notarize files for macOS """ sign and notarize files for macOS
Expects the following environment: Expects the following environment:
- `SIGNING_IDENTITY` environ with Certificate name/identity - `SIGNING_IDENTITY` environ with Certificate name/identity
- `KEYCHAIN` environ with path to the keychain storing credentials - `KEYCHAIN` environ with path to the keychain storing credentials
- `KEYCHAIN_PROFILE` environ with name of the profile in that keychain - `KEYCHAIN_PROFILE` environ with name of the profile in that keychain
- `KEYCHAIN_PASSWORD` environ with password to unlock the keychain - `KEYCHAIN_PASSWORD` environ with password to unlock the keychain
""" """
if project != "libzim": if project != "libzim":
return return
# currently only supports libzim use case: sign every dylib # currently only supports libzim use case: sign every dylib
base_dir, export_files = EXPORT_FILES[project] base_dir, export_files = EXPORT_FILES[project]
filepaths = [ filepaths = [base_dir.joinpath(file)
base_dir.joinpath(file) for file in filter(lambda f: f.endswith(".dylib"), export_files)
for file in filter(lambda f: f.endswith(".dylib"), export_files) if not base_dir.joinpath(file).is_symlink()]
if not base_dir.joinpath(file).is_symlink()
]
if not filepaths: if not filepaths:
return return
for filepath in filepaths: for filepath in filepaths:
subprocess.check_call( subprocess.check_call(["/usr/bin/codesign", "--force", "--sign",
[ os.getenv("SIGNING_IDENTITY", "no-signing-ident"),
"/usr/bin/codesign", "--keychain",
"--force", os.getenv("KEYCHAIN", "no-keychain-path"),
"--sign", str(filepath), "--deep", "--timestamp"], env=os.environ)
os.getenv("SIGNING_IDENTITY", "no-signing-ident"),
"--keychain",
os.getenv("KEYCHAIN", "no-keychain-path"),
str(filepath),
"--deep",
"--timestamp",
],
env=os.environ,
)
# create a zip of the dylibs and upload for notarization # create a zip of the dylibs and upload for notarization
zip_name = "{}.zip".format(project) zip_name = "{}.zip".format(project)
subprocess.check_call( subprocess.check_call(
["/usr/bin/ditto", "-c", "-k", "--keepParent"] ["/usr/bin/ditto", "-c", "-k", "--keepParent"]
+ [str(f) for f in filepaths] + [str(f) for f in filepaths] + [zip_name],
+ [zip_name], env=os.environ)
env=os.environ,
)
# make sure keychain is unlocked # make sure keychain is unlocked
subprocess.check_call( subprocess.check_call(

View File

@ -6,17 +6,13 @@ from common import (
run_kiwix_build, run_kiwix_build,
make_deps_archive, make_deps_archive,
upload, upload,
COMPILE_CONFIG, OS_NAME,
DEV_BRANCH, PLATFORM_TARGET,
) )
from build_definition import select_build_targets, DEPS from build_definition import select_build_targets, DEPS
for target in select_build_targets(DEPS): for target in select_build_targets(DEPS):
run_kiwix_build(target, config=COMPILE_CONFIG, build_deps_only=True) run_kiwix_build(target, platform=PLATFORM_TARGET, build_deps_only=True)
archive_file = make_deps_archive(target=target) archive_file = make_deps_archive(target=target)
if DEV_BRANCH: upload(archive_file, "ci@tmp.kiwix.org:30022", "/data/tmp/ci")
destination = "/data/tmp/ci/dev_preview/" + DEV_BRANCH
else:
destination = "/data/tmp/ci"
upload(archive_file, "ci@tmp.kiwix.org:30022", destination)
os.remove(str(archive_file)) os.remove(str(archive_file))

View File

@ -13,12 +13,10 @@ from common import (
upload, upload,
make_deps_archive, make_deps_archive,
HOME, HOME,
COMPILE_CONFIG, PLATFORM_TARGET,
OS_NAME, OS_NAME,
MAKE_RELEASE,
) )
def download_base_archive(base_name): def download_base_archive(base_name):
url = "http://tmp.kiwix.org/ci/{}".format(base_name) url = "http://tmp.kiwix.org/ci/{}".format(base_name)
file_path = str(HOME / base_name) file_path = str(HOME / base_name)
@ -32,41 +30,29 @@ def download_base_archive(base_name):
file.write(batch) file.write(batch)
return file_path return file_path
ARCHIVE_NAME_TEMPLATE = "base_deps2_{os}_{platform}_{version}.tar.xz"
def get_archive_name(): if PLATFORM_TARGET == 'flatpak':
ARCHIVE_NAME_TEMPLATE = "base_deps_{os}_{config}_{version}.tar.gz" base_dep_archive_name = "base_deps2_flatpak.tar.xz"
else:
if COMPILE_CONFIG == "flatpak": base_dep_archive_name = ARCHIVE_NAME_TEMPLATE.format(
return "base_deps_flatpak.tar.gz"
return ARCHIVE_NAME_TEMPLATE.format(
os=OS_NAME, os=OS_NAME,
config=COMPILE_CONFIG, platform=PLATFORM_TARGET,
version=base_deps_meta_version, version=base_deps_meta_version,
) )
print_message("Getting archive {}", base_dep_archive_name)
def main(): try:
base_dep_archive_name = get_archive_name() local_filename = download_base_archive(base_dep_archive_name)
print_message("Getting archive {}", base_dep_archive_name) with tarfile.open(local_filename) as f:
try: f.extractall(str(HOME))
local_filename = download_base_archive(base_dep_archive_name) os.remove(str(local_filename))
with tarfile.open(local_filename) as f: except URLError:
f.extractall(str(HOME)) if PLATFORM_TARGET == "flatpak":
os.remove(str(local_filename)) print_message("Cannot get archive. Move on")
except URLError:
if COMPILE_CONFIG == "flatpak":
print_message("Cannot get archive. Move on")
else:
print_message("Cannot get archive. Build dependencies")
run_kiwix_build("alldependencies", config=COMPILE_CONFIG)
archive_file = make_deps_archive(name=base_dep_archive_name, full=True)
upload(archive_file, "ci@tmp.kiwix.org:30022", "/data/tmp/ci")
os.remove(str(archive_file))
if __name__ == "__main__":
if MAKE_RELEASE:
print_message("We are building release. Don't download deps archive.")
else: else:
main() print_message("Cannot get archive. Build dependencies")
run_kiwix_build("alldependencies", platform=PLATFORM_TARGET)
archive_file = make_deps_archive(name=base_dep_archive_name, full=True)
upload(archive_file, "ci@tmp.kiwix.org:30022", "/data/tmp/ci")
os.remove(str(archive_file))

View File

@ -1,18 +0,0 @@
#!/usr/bin/env python3
import tarfile
from pathlib import Path
from common import upload, OS_NAME, COMPILE_CONFIG, HOME
ARCHIVE_NAME = Path(f"fail_log_{OS_NAME}_{COMPILE_CONFIG}.tar.gz")
files_to_archive = []
files_to_archive += HOME.glob("BUILD_*")
files_to_archive += [HOME / "SOURCE", HOME / "LOGS", HOME / "TOOLCHAINS"]
with tarfile.open(ARCHIVE_NAME, "w:xz") as tar:
for name in set(files_to_archive):
tar.add(str(name))
upload(ARCHIVE_NAME, "ci@tmp.kiwix.org:30022", "/data/tmp/ci")

16
.github/scripts/upload_failure_logs.sh vendored Executable file
View File

@ -0,0 +1,16 @@
#!/usr/bin/env bash
set -e
cd $HOME
ARCHIVE_NAME=fail_log_${OS_NAME}_${PLATFORM_TARGET}.tar.gz
tar -czf ${ARCHIVE_NAME} $HOME/BUILD_* $HOME/SOURCE $HOME/LOGS $HOME/TOOLCHAINS
echo "Uploading archive $ARCHIVE_NAME"
scp -c aes128-ctr -P 30022 -p -i ${SSH_KEY} \
-o PasswordAuthentication=no \
-o StrictHostKeyChecking=no \
$ARCHIVE_NAME \
ci@tmp.kiwix.org:/data/tmp/ci

15
.github/stale.yml vendored Normal file
View File

@ -0,0 +1,15 @@
daysUntilClose: false
staleLabel: stale
issues:
daysUntilStale: 60
markComment: >
This issue has been automatically marked as stale because it has not had
recent activity. It will be now be reviewed manually. Thank you
for your contributions.
pulls:
daysUntilStale: 7
markComment: >
This pull request has been automatically marked as stale because it has not had
recent activity. It will be now be reviewed manually. Thank you
for your contributions.

View File

@ -6,80 +6,11 @@ on:
- cron: '0 1 * * *' - cron: '0 1 * * *'
jobs: jobs:
Windows:
strategy:
fail-fast: false
matrix:
config:
- native_mixed
- native_dyn
- native_static
runs-on: windows-2022
env:
OS_NAME: windows
COMPILE_CONFIG: ${{matrix.config}}
HOME: 'C:\\Users\\runneradmin'
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Setup python 3.12
uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install packages
run: |
choco.exe install pkgconfiglite ninja
- name: Install python modules
shell: bash
run: |
pip3 install meson pytest requests distro paramiko
pip3 install --no-deps $GITHUB_WORKSPACE
- name: Install QT
if: ${{ matrix.config == 'native_mixed' }}
uses: jurplel/install-qt-action@v4
with:
version: 6.4.3
modules: "qtwebengine qtwebchannel qtpositioning"
setup-python: false
env:
AQT_CONFIG: ${{ github.workspace }}/.github/configs/aqt.ini
- name: Setup MSVC compiler
uses: bus1/cabuild/action/msdevshell@v1
with:
architecture: x64
- name: secret
shell: bash
run: |
echo "${{secrets.ssh_key}}" > $SSH_KEY
env:
SSH_KEY: ${{ runner.temp }}/id_rsa
- name: Ensure base deps
run: |
python .github\\scripts\\ensure_base_deps.py
env:
SSH_KEY: ${{ runner.temp }}/id_rsa
- name: Compile all deps
run: |
python .github\\scripts\\compile_all_deps.py
env:
SSH_KEY: ${{ runner.temp }}/id_rsa
- name: Build projects
run: |
python .github\\scripts\\build_projects.py
env:
SSH_KEY: ${{ runner.temp }}/id_rsa
- name: Upload failure logs
if: failure()
run: |
python .github\\scripts\\upload_failure_logs.py
env:
SSH_KEY: ${{ runner.temp }}/id_rsa
Linux: Linux:
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
config: target:
- native_static - native_static
- native_dyn - native_dyn
- native_mixed - native_mixed
@ -96,8 +27,6 @@ jobs:
- aarch64_musl_static - aarch64_musl_static
- aarch64_musl_dyn - aarch64_musl_dyn
- aarch64_musl_mixed - aarch64_musl_mixed
- x86-64_musl_static
- x86-64_musl_mixed
- i586_static - i586_static
- i586_dyn - i586_dyn
- android_arm - android_arm
@ -106,19 +35,20 @@ jobs:
- android_x86_64 - android_x86_64
image_variant: ['focal'] image_variant: ['focal']
include: include:
- config: native_mixed - target: native_mixed
image_variant: manylinux image_variant: bionic
- config: aarch64_mixed - target: aarch64_mixed
image_variant: manylinux image_variant: bionic
- config: native_dyn - target: win32_static
image_variant: jammy image_variant: f35
- target: win32_dyn
image_variant: f35
env: env:
HOME: /home/runner HOME: /home/runner
SSH_KEY: /tmp/id_rsa SSH_KEY: /tmp/id_rsa
OS_NAME: ${{matrix.image_variant}}
runs-on: ubuntu-22.04 runs-on: ubuntu-22.04
container: container:
image: "ghcr.io/kiwix/kiwix-build_ci_${{matrix.image_variant}}:2024-11-30" image: "ghcr.io/kiwix/kiwix-build_ci_${{matrix.image_variant}}:dev"
options: "--device /dev/fuse --privileged" options: "--device /dev/fuse --privileged"
steps: steps:
- name: Checkout code - name: Checkout code
@ -131,10 +61,6 @@ jobs:
pip3 install --user --no-deps . pip3 install --user --no-deps .
env: env:
REP: ${{github.repository}} REP: ${{github.repository}}
- name: Install paramiko
if: ${{matrix.image_variant != 'bionic' }}
shell: bash
run: pip3 install --user paramiko
- name: secret - name: secret
shell: bash shell: bash
run: | run: |
@ -146,26 +72,26 @@ jobs:
cd $HOME cd $HOME
kiwix-build/.github/scripts/ensure_base_deps.py kiwix-build/.github/scripts/ensure_base_deps.py
env: env:
COMPILE_CONFIG: ${{matrix.config}} PLATFORM_TARGET: ${{matrix.target}}
- name: Compile all deps - name: Compile all deps
shell: bash shell: bash
run: | run: |
cd $HOME cd $HOME
kiwix-build/.github/scripts/compile_all_deps.py kiwix-build/.github/scripts/compile_all_deps.py
env: env:
COMPILE_CONFIG: ${{matrix.config}} PLATFORM_TARGET: ${{matrix.target}}
- name: Build projects - name: Build projects
shell: bash shell: bash
run: | run: |
cd $HOME cd $HOME
kiwix-build/.github/scripts/build_projects.py kiwix-build/.github/scripts/build_projects.py
env: env:
COMPILE_CONFIG: ${{matrix.config}} PLATFORM_TARGET: ${{matrix.target}}
- name: Upload failure logs - name: Upload failure logs
if: failure() if: failure()
run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh
env: env:
COMPILE_CONFIG: ${{matrix.config}} PLATFORM_TARGET: ${{matrix.target}}
Flatpak: Flatpak:
strategy: strategy:
@ -173,7 +99,7 @@ jobs:
env: env:
HOME: /home/runner HOME: /home/runner
SSH_KEY: /tmp/id_rsa SSH_KEY: /tmp/id_rsa
COMPILE_CONFIG: flatpak PLATFORM_TARGET: flatpak
OS_NAME: focal OS_NAME: focal
runs-on: ubuntu-22.04 runs-on: ubuntu-22.04
steps: steps:
@ -184,7 +110,6 @@ jobs:
git clone https://github.com/${REP} git clone https://github.com/${REP}
cd ./${REP##*/} cd ./${REP##*/}
git checkout --force ${GITHUB_SHA} git checkout --force ${GITHUB_SHA}
pip3 install --user paramiko
pip3 install --user --no-deps . pip3 install --user --no-deps .
env: env:
REP: ${{github.repository}} REP: ${{github.repository}}
@ -209,43 +134,39 @@ jobs:
kiwix-build/.github/scripts/build_projects.py kiwix-build/.github/scripts/build_projects.py
- name: Upload failure logs - name: Upload failure logs
if: failure() if: failure()
run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh
Macos: Macos:
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
config: target:
- native_dyn - native_dyn
- native_static - native_static
- native_mixed - native_mixed
- iOS_arm64 - iOS_arm64
- iOSSimulator_x86_64 - iOS_x86_64
- iOSSimulator_arm64 - iOS_Mac_ABI
- macOS_arm64_static - macOS_arm64_static
- macOS_arm64_mixed - macOS_arm64_mixed
- macOS_x86_64 - macOS_x86_64
- apple_all_static runs-on: macos-11
runs-on: macos-13
env: env:
SSH_KEY: /tmp/id_rsa SSH_KEY: /tmp/id_rsa
OS_NAME: macos OS_NAME: macos
steps: steps:
- name: Set Xcode version (15.0.1)
# https://github.com/actions/runner-images/blob/main/images/macos/macos-13-Readme.md#xcode
run: sudo xcode-select -s /Applications/Xcode_15.0.1.app
- name: Checkout code - name: Checkout code
uses: actions/checkout@v4 uses: actions/checkout@v3
- name: Setup Python 3.10 - name: Setup python 3.8
uses: actions/setup-python@v5 uses: actions/setup-python@v3
with: with:
python-version: '3.10' python-version: '3.8'
- name: Install packages - name: Install packages
run: run:
brew install automake # ninja brew install pkg-config ninja automake autoconf
- name: Install python modules - name: Install python modules
run: | run: |
pip3 install meson pytest requests distro paramiko pip3 install meson pytest requests distro
pip3 install --no-deps $GITHUB_WORKSPACE pip3 install --no-deps $GITHUB_WORKSPACE
- name: secret - name: secret
shell: bash shell: bash
@ -258,23 +179,23 @@ jobs:
cd $HOME cd $HOME
$GITHUB_WORKSPACE/.github/scripts/ensure_base_deps.py $GITHUB_WORKSPACE/.github/scripts/ensure_base_deps.py
env: env:
COMPILE_CONFIG: ${{matrix.config}} PLATFORM_TARGET: ${{matrix.target}}
- name: Compile all deps - name: Compile all deps
shell: bash shell: bash
run: | run: |
cd $HOME cd $HOME
$GITHUB_WORKSPACE/.github/scripts/compile_all_deps.py $GITHUB_WORKSPACE/.github/scripts/compile_all_deps.py
env: env:
COMPILE_CONFIG: ${{matrix.config}} PLATFORM_TARGET: ${{matrix.target}}
- name: Build projects - name: Build projects
shell: bash shell: bash
run: | run: |
cd $HOME cd $HOME
$GITHUB_WORKSPACE/.github/scripts/build_projects.py $GITHUB_WORKSPACE/.github/scripts/build_projects.py
env: env:
COMPILE_CONFIG: ${{matrix.config}} PLATFORM_TARGET: ${{matrix.target}}
- name: Upload failure logs - name: Upload failure logs
if: failure() if: failure()
run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.py run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.sh
env: env:
COMPILE_CONFIG: ${{matrix.config}} PLATFORM_TARGET: ${{matrix.target}}

43
.github/workflows/container_images.yml vendored Normal file
View File

@ -0,0 +1,43 @@
name: Container images CD
on:
push:
paths:
- 'ci_images/**'
release:
types: [published]
env:
IMAGE_PREFIX: ghcr.io/kiwix/kiwix-build_ci_
jobs:
Container:
strategy:
fail-fast: false
matrix:
variant: [f35, focal, bionic, alpine]
runs-on: ubuntu-22.04
steps:
- name: Retrieve the code
uses: actions/checkout@v3
- name: Setup container image name
id: env
run: |
echo "IMAGE_NAME=${{ env.IMAGE_PREFIX }}${{ matrix.variant }}" >> $GITHUB_OUTPUT
- name: Build container image
run: |
docker build -t ${{ steps.env.outputs.IMAGE_NAME }}:dev - < ci_images/${{ matrix.variant }}_builder.dockerfile
- name: Tag release container image
if: github.event.action == 'published'
run: |
docker tag ${{ steps.env.outputs.IMAGE_NAME }}:dev ${{ steps.env.outputs.IMAGE_NAME }}:$GITHUB_REF_NAME
docker tag ${{ steps.env.outputs.IMAGE_NAME }}:dev ${{ steps.env.outputs.IMAGE_NAME }}:latest
- name: Upload container image
run: |
echo "${{ secrets.GHCR_TOKEN }}" | docker login ghcr.io -u "${{ secrets.GHCR_USERNAME }}" --password-stdin
docker push --all-tags ${{ steps.env.outputs.IMAGE_NAME }}

View File

@ -1,116 +1,19 @@
name: CD name: Release&Nigthly
on: on:
push: push:
tags: tags:
- r_[0-9]+ - r_[0-9]+
schedule: schedule:
- cron: '0 3 * * *' - cron: '0 1 * * *'
jobs: jobs:
Windows:
strategy:
fail-fast: false
matrix:
config:
- native_mixed
- native_dyn
- native_static
runs-on: windows-2022
env:
OS_NAME: windows
COMPILE_CONFIG: ${{matrix.config}}
HOME: 'C:\\Users\\runneradmin'
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Setup python 3.12
uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install packages
run: |
choco.exe install pkgconfiglite ninja
- name: Install python modules
shell: bash
run: |
pip3 install meson pytest requests distro paramiko
pip3 install --no-deps $GITHUB_WORKSPACE
- name: Install QT
if: ${{ matrix.config == 'native_mixed' }}
uses: jurplel/install-qt-action@v4
with:
version: 5.15.2
modules: "qtwebengine"
setup-python: false
env:
AQT_CONFIG: ${{ github.workspace }}/.github/configs/aqt.ini
- name: Setup MSVC compiler
uses: bus1/cabuild/action/msdevshell@v1
with:
architecture: x64
- name: secret
shell: bash
run: |
echo "${{secrets.ssh_key}}" > $SSH_KEY
env:
SSH_KEY: ${{ runner.temp }}/id_rsa
- name: Install and configure eSigner CKA and Windows SDK
if: github.event_name == 'push'
env:
ESIGNER_URL: https://github.com/SSLcom/eSignerCKA/releases/download/v1.0.7/SSL.COM-eSigner-CKA_1.0.7.zip
run: |
Set-StrictMode -Version 'Latest'
# Download and Unzip eSignerCKA Setup
Invoke-WebRequest -OutFile eSigner_CKA_Setup.zip "$env:ESIGNER_URL"
Expand-Archive -Force eSigner_CKA_Setup.zip
Remove-Item eSigner_CKA_Setup.zip
Move-Item -Destination “eSigner_CKA_Installer.exe” -Path “eSigner_CKA_*\*.exe”
# Install eSignerCKA
New-Item -ItemType Directory -Force -Path "C:\esigner"
./eSigner_CKA_Installer.exe /CURRENTUSER /VERYSILENT /SUPPRESSMSGBOXES /DIR=”C:\esigner” /TYPE=automatic | Out-Null
Remove-Item "eSigner_CKA_Installer.exe"
# Configure the CKA with SSL.com credentials
C:\esigner\eSignerCKATool.exe config -mode product -user "${{ secrets.ESIGNER_USERNAME }}" -pass "${{ secrets.ESIGNER_PASSWORD }}" -totp "${{ secrets.ESIGNER_TOTP_SECRET }}" -key "C:\esigner\master.key" -r
C:\esigner\eSignerCKATool.exe unload
C:\esigner\eSignerCKATool.exe load
# Find certificate
$CodeSigningCert = Get-ChildItem Cert:\CurrentUser\My -CodeSigningCert | Select-Object -First 1
echo Certificate: $CodeSigningCert
# Extract thumbprint and subject name
$Thumbprint = $CodeSigningCert.Thumbprint
echo "SIGNTOOL_THUMBPRINT=$Thumbprint" >> $env:GITHUB_ENV
- name: Ensure base deps
run: |
python .github\\scripts\\ensure_base_deps.py
env:
SSH_KEY: ${{ runner.temp }}/id_rsa
- name: Build Release
run: |
python .github\\scripts\\build_release_nightly.py
env:
SIGNTOOL_PATH: "C:/Program Files (x86)/Windows Kits/10/bin/10.0.22621.0/x86/signtool.exe"
SSH_KEY: ${{ runner.temp }}/id_rsa
- name: Upload failure logs
if: failure()
run: |
python .github\\scripts\\upload_failure_logs.py
env:
SSH_KEY: ${{ runner.temp }}/id_rsa
Linux: Linux:
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
config: target:
- native_static - native_static
- native_mixed
- native_dyn - native_dyn
- wasm - wasm
- armv6_static - armv6_static
@ -121,27 +24,59 @@ jobs:
- aarch64_mixed - aarch64_mixed
- aarch64_musl_static - aarch64_musl_static
- aarch64_musl_mixed - aarch64_musl_mixed
- x86-64_musl_static - win32_static
- x86-64_musl_mixed
- i586_static - i586_static
- android_arm - android_arm
- android_arm64 - android_arm64
- android_x86 - android_x86
- android_x86_64 - android_x86_64
image_variant: ['focal']
include: include:
- config: native_mixed - target: native_static
image_variant: manylinux image_variant: focal
- config: aarch64_mixed - target: native_dyn
image_variant: manylinux image_variant: focal
- config: native_dyn - target: native_mixed
image_variant: jammy image_variant: focal
- target: native_mixed
image_variant: bionic
- target: wasm
image_variant: focal
- target: armv6_static
image_variant: focal
- target: armv6_mixed
image_variant: focal
- target: armv8_static
image_variant: focal
- target: armv8_mixed
image_variant: focal
- target: aarch64_static
image_variant: focal
- target: aarch64_mixed
image_variant: focal
- target: aarch64_mixed
image_variant: bionic
- target: aarch64_musl_static
image_variant: focal
- target: aarch64_musl_mixed
image_variant: focal
- target: win32_static
image_variant: f35
- target: i586_static
image_variant: focal
- target: android_arm
image_variant: focal
- target: android_arm64
image_variant: focal
- target: android_x86
image_variant: focal
- target: android_x86_64
image_variant: focal
env: env:
HOME: /home/runner HOME: /home/runner
SSH_KEY: /tmp/id_rsa SSH_KEY: /tmp/id_rsa
runs-on: ubuntu-22.04 runs-on: ubuntu-22.04
container: container:
image: "ghcr.io/kiwix/kiwix-build_ci_${{matrix.image_variant}}:2024-11-30" image: "ghcr.io/kiwix/kiwix-build_ci_${{matrix.image_variant}}:38"
options: "--device /dev/fuse --privileged" options: "--device /dev/fuse --privileged"
steps: steps:
- name: Checkout code - name: Checkout code
@ -165,19 +100,21 @@ jobs:
cd $HOME cd $HOME
kiwix-build/.github/scripts/ensure_base_deps.py kiwix-build/.github/scripts/ensure_base_deps.py
env: env:
COMPILE_CONFIG: ${{matrix.config}} PLATFORM_TARGET: ${{matrix.target}}
- name: Build release - name: Build release
shell: bash shell: bash
run: | run: |
cd $HOME cd $HOME
kiwix-build/.github/scripts/build_release_nightly.py kiwix-build/.github/scripts/build_release_nightly.py
env: env:
COMPILE_CONFIG: ${{matrix.config}} PLATFORM_TARGET: ${{matrix.target}}
BINTRAY_USER: kiwix
BINTRAY_PASS: ${{secrets.bintray_pass}}
- name: Upload failure logs - name: Upload failure logs
if: failure() if: failure()
run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh
env: env:
COMPILE_CONFIG: ${{matrix.config}} PLATFORM_TARGET: ${{matrix.target}}
Flatpak: Flatpak:
strategy: strategy:
@ -185,7 +122,7 @@ jobs:
env: env:
HOME: /home/runner HOME: /home/runner
SSH_KEY: /tmp/id_rsa SSH_KEY: /tmp/id_rsa
COMPILE_CONFIG: flatpak PLATFORM_TARGET: flatpak
OS_NAME: focal OS_NAME: focal
runs-on: ubuntu-22.04 runs-on: ubuntu-22.04
steps: steps:
@ -220,20 +157,19 @@ jobs:
kiwix-build/.github/scripts/build_release_nightly.py kiwix-build/.github/scripts/build_release_nightly.py
- name: Upload failure logs - name: Upload failure logs
if: failure() if: failure()
run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh
Macos: Macos:
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
config: target:
- native_dyn - native_dyn
- native_static - native_static
- native_mixed - native_mixed
- macOS_arm64_static - macOS_arm64_static
- macOS_arm64_mixed - macOS_arm64_mixed
- apple_all_static runs-on: macos-11
runs-on: macos-13
env: env:
SSH_KEY: /tmp/id_rsa SSH_KEY: /tmp/id_rsa
OS_NAME: macos OS_NAME: macos
@ -243,18 +179,15 @@ jobs:
KEYCHAIN_PASSWORD: mysecretpassword KEYCHAIN_PASSWORD: mysecretpassword
KEYCHAIN_PROFILE: build-profile KEYCHAIN_PROFILE: build-profile
steps: steps:
- name: Set Xcode version (15.0.1)
# https://github.com/actions/runner-images/blob/main/images/macos/macos-13-Readme.md#xcode
run: sudo xcode-select -s /Applications/Xcode_15.0.1.app
- name: Checkout code - name: Checkout code
uses: actions/checkout@v4 uses: actions/checkout@v3
- name: Setup python 3.10 - name: Setup python 3.8
uses: actions/setup-python@v5 uses: actions/setup-python@v3
with: with:
python-version: '3.10' python-version: '3.8'
- name: Install packages - name: Install packages
run: run:
brew install automake # ninja brew install pkg-config ninja automake autoconf
- name: Install python modules - name: Install python modules
run: | run: |
pip3 install meson pytest requests distro pip3 install meson pytest requests distro
@ -289,30 +222,30 @@ jobs:
cd $HOME cd $HOME
$GITHUB_WORKSPACE/.github/scripts/ensure_base_deps.py $GITHUB_WORKSPACE/.github/scripts/ensure_base_deps.py
env: env:
COMPILE_CONFIG: ${{matrix.config}} PLATFORM_TARGET: ${{matrix.target}}
- name: Build release - name: Build release
shell: bash shell: bash
run: | run: |
cd $HOME cd $HOME
$GITHUB_WORKSPACE/.github/scripts/build_release_nightly.py $GITHUB_WORKSPACE/.github/scripts/build_release_nightly.py
env: env:
COMPILE_CONFIG: ${{matrix.config}} PLATFORM_TARGET: ${{matrix.target}}
- name: Upload failure logs - name: Upload failure logs
if: failure() if: failure()
run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.py run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.sh
env: env:
COMPILE_CONFIG: ${{matrix.config}} PLATFORM_TARGET: ${{matrix.target}}
Trigger_Docker: Trigger_Docker:
needs: [Linux] needs: [Linux]
runs-on: ubuntu-22.04 runs-on: ubuntu-22.04
env: env:
COMPILE_CONFIG: native_static PLATFORM_TARGET: LINUX_DOCKER_TRIGGER
OS_NAME: focal OS_NAME: LINUX_DOCKER_TRIGGER
steps: steps:
- name: Checkout code - name: Checkout code
uses: actions/checkout@v4 uses: actions/checkout@v3
- name: Install python modules - name: Install python modules
shell: bash shell: bash
run: | run: |

View File

@ -1,3 +1,2 @@
include kiwixbuild/templates/*_cross_file.txt include kiwixbuild/templates/*_cross_file.txt
include kiwixbuild/patches/*.patch include kiwixbuild/patches/*.patch
include kiwixbuild/dependencies/icu4c_data_filter.json

View File

@ -11,8 +11,7 @@ Kiwix Build audience is:
compilations manually compilations manually
* Kiwix developer team for its own CI/CD * Kiwix developer team for its own CI/CD
[![CI Build Status](https://github.com/kiwix/kiwix-build/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/kiwix/kiwix-build/actions/workflows/ci.yml?query=branch%3Amain) [![Build Status](https://github.com/kiwix/kiwix-build/workflows/CI/badge.svg?query=branch%3Amain)](https://github.com/kiwix/kiwix-build/actions?query=branch%3Amain)
[![CD Build Status](https://github.com/kiwix/kiwix-build/actions/workflows/cd.yml/badge.svg?branch=main)](https://github.com/kiwix/kiwix-build/actions/workflows/cd.yml?query=branch%3Amain)
[![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0) [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
Prerequisites Prerequisites
@ -78,17 +77,19 @@ invalid choice: 'not-existing-target' (choose from 'alldependencies', 'android-n
... ...
``` ```
#### Config #### Target platform
If no config is specified, the default will be `native_dyn`. If no target platform is specified, the default will be `native_dyn`.
You can select another config using the option You can select another target platform using the option
`--config`. For now, there is ten different supported `--target-platform`. For now, there is ten different supported
platforms: platforms:
- native_dyn - native_dyn
- native_mixed - native_mixed
- native_static - native_static
- win32_dyn
- win32_static
- android - android
- android_arm - android_arm
- android_arm64 - android_arm64
@ -96,31 +97,42 @@ platforms:
- android_x86_64 - android_x86_64
- flatpak - flatpak
All `native_*` config means using the native compiler without any cross-compilation option. So, if you want to compile `kiwix-tools` for win32 using static linkage:
Other may simply use cross-compilation or may download a specific toolchain to use. ```bash
kiwix-build --target-platform win32_dyn
```
Android Android
------- -------
`kiwix-android` (https://github.com/kiwix/kiwix-android) depends of `kiwix-android` (https://github.com/kiwix/kiwix-android) depends of
the `libkiwix` project. the `libkiwix` project.
It uses a special `.aar` file that represent (and embed) the libkiwix for
all supported android arch. This is a kind of fat archive we have for MacOs.
The `.aar` file is build using the `libkiwix-app` project.
`libkiwix-app` itself is architecture independent (it is just a packaging of
other archives) but it use `libkiwix` who is architecture dependent.
When building `libkiwix`, you should directly use the When building `libkiwix`, you should directly use the
target-platform `android_<arch>`: target-platform `android_<arch>`:
```bash ```bash
kiwix-build libkiwix --config android_arm kiwix-build libkiwix --target-platform android_arm
``` ```
You may directly use the special config `android` which will build different android architectures But, `libkiwix-app` is mainly multi arch.
To compile `libkiwix-app`, you must use the `android` platform:
```bash ```bash
kiwix-build --config android libkiwix kiwix-build --target-platform android libkiwix-app
kiwix-build libkiwix-app # because `android` platform is the default for `libkiwix-app`
``` ```
By default, it will build for all android architecture, By default, when using platform `android`, `libkiwix` will be build for
you can limit this with option `--android-arch`: all architectures. This can be changed by using the option `--android-arch`:
```bash ```bash
kiwix-build libkiwix --config android --android-arch arm # aar with arm architecture kiwix-build libkiwix-app # aar with all architectures
kiwix-build libkiwix --config android --android-arch arm --android-arch arm64 # aan with arm and arm64 architectures kiwix-build libkiwix-app --android-arch arm # aar with arm architecture
kiwix-build libkiwix-app --android-arch arm --android-arch arm64 # aan with arm and arm64 architectures
``` ```
To build `kiwix-android` itself, you should see the documentation of `kiwix-android`. To build `kiwix-android` itself, you should see the documentation of `kiwix-android`.
@ -135,13 +147,13 @@ To do so, you should directly use the target-platfrom `ios_multi`.
As for `android`, `kiwix-build` will build the library several times As for `android`, `kiwix-build` will build the library several times
(once for each platform) and then create the fat library. (once for each platform) and then create the fat library.
```bash ```bash
kiwix-build --config iOS_multi libkiwix kiwix-build --target-platform iOS_multi libkiwix
``` ```
You can specify the supported architectures with the option `--ios-arch`: You can specify the supported architectures with the option `--ios-arch`:
```bash ```bash
kiwix-build --config iOS_multi libkiwix # all architetures kiwix-build --target-platform iOS_multi libkiwix # all architetures
kiwix-build --config iOS_multi --ios-arch arm --ios-arch arm64 # arm and arm64 arch only kiwix-build --target-platform iOS_multi --ios-arch arm --ios-arch arm64 # arm and arm64 arch only
``` ```
Outputs Outputs
@ -150,9 +162,9 @@ Outputs
Kiwix-build.py will create several directories: Kiwix-build.py will create several directories:
- `ARCHIVES`: All the downloaded archives go there. - `ARCHIVES`: All the downloaded archives go there.
- `SOURCES`: All the sources (extracted from archives and patched) go there. - `SOURCES`: All the sources (extracted from archives and patched) go there.
- `BUILD_<config>`: All the build files go there. - `BUILD_<target_platform>`: All the build files go there.
- `BUILD_<config>/INSTALL`: The installed files go there. - `BUILD_<target_platform>/INSTALL`: The installed files go there.
- `BUILD_<config>/LOGS`: The logs files of the build. - `BUILD_<target_platform>/LOGS`: The logs files of the build.
If you want to install all those directories elsewhere, you can pass the If you want to install all those directories elsewhere, you can pass the
`--working-dir` option to `kiwix-build`: `--working-dir` option to `kiwix-build`:

View File

@ -1 +0,0 @@
node_modules

View File

@ -1,55 +0,0 @@
# Download dependencies archive action
This action download dependencies archive made by kiwix-build.
It is intended to be used in projects using theses dependencies
## Inputs
### `base_url`
The base url where we can download the archive.
This input is provided for greater customization but you probably shouldn't set it.
### `os_name`
The os "name" on which the compilation is done.
By default this use the `OS_NAME` env var, which is set in the docker file.
### `target_platform`
**Required** The targeted platform. Must be provided. Values are kind of :
- native_dyn
- android_arm
- ...
### `project`
The name of the project being compiled.
By default, the name of the repository.
### `branch`
The name of the "branch" to try to download (`/dev_preview/<branch>`).
By default, the current branch on which the action is run.
### `extract_dir`
Where to extract the dependencies archive. By default it is `$HOME`
## Example usage
```yaml
uses: kiwix/kiwix-build/actions/dl_deps_archive@main
with:
target_platform: ${{ matrix.target_platform }}
```
```
uses: kiwix/kiwix-build/actions/dl_deps_archive@main
with:
target_platform: native_mixed
os_name: windows
```

View File

@ -1,25 +0,0 @@
name: 'DL deps archive'
description: 'Download the dependencies archive of the project'
inputs:
base_url: # id of input
description: 'What is the base url to download the archive'
required: false
default: 'https://tmp.kiwix.org/ci'
os_name:
description: "On which os are we running. [Default to env var `OS_NAME`]"
required: false
target_platform:
description: "What platform are we targetting"
required: true
project:
description: "What project are we building"
required: false
branch:
description: "On which branch are we ?"
required: false
extract_dir:
description: "Where to extract our dependencies. [Default to env var `HOME`]"
required: false
runs:
using: 'node20'
main: 'dist/index.js'

File diff suppressed because one or more lines are too long

View File

@ -1,59 +0,0 @@
const tc = require("@actions/tool-cache");
const core = require("@actions/core");
const path = require("path");
const os = require("os");
function getInput(name, dflt) {
const val = process.env[`INPUT_${name.replace(/ /g, "_").toUpperCase()}`];
if (!val) {
return dflt;
}
return val;
}
function addLocalPath(inputPath) {
process.env["PATH"] = `${inputPath}${path.delimiter}${process.env["PATH"]}`;
}
async function run() {
try {
const base_url = core.getInput("base_url");
const target = core.getInput("target_platform");
const project = getInput(
"project",
process.env["GITHUB_REPOSITORY"].split("/")[1],
);
const branch = getInput(
"branch",
process.env["GITHUB_HEAD_REF"] || process.env["GITHUB_REF_NAME"],
);
const extract_dir = getInput(
"extract_dir",
process.env["HOME"] || process.env["GITHUB_WORKSPACE"],
);
let archivePath;
try {
const archive_url = `${base_url}/dev_preview/${branch}/deps_${target}_${project}.tar.gz`;
process.stdout.write("Downloading " + archive_url + "\n");
archivePath = await tc.downloadTool(archive_url);
} catch (error) {
const archive_url = `${base_url}/deps_${target}_${project}.tar.gz`;
process.stdout.write("Downloading " + archive_url + "\n");
archivePath = await tc.downloadTool(archive_url);
}
process.stdout.write("Extracting " + archivePath + " to " + extract_dir);
const archive_dir = await tc.extractTar(archivePath, extract_dir);
process.stdout.write("Extracted to " + archive_dir);
} catch (error) {
core.setFailed(error.message);
}
}
if (os.platform() === "win32") {
addLocalPath("C:\\Program Files\\Git\\usr\\bin");
}
core.setCommandEcho(true);
run();

View File

@ -1,144 +0,0 @@
{
"name": "dl_deps_archive",
"version": "1.0.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "dl_deps_archive",
"version": "1.0.0",
"license": "ISC",
"dependencies": {
"@actions/core": "^1.10.1",
"@actions/tool-cache": "^2.0.1",
"path": "^0.12.7"
}
},
"node_modules/@actions/core": {
"version": "1.10.1",
"resolved": "https://registry.npmjs.org/@actions/core/-/core-1.10.1.tgz",
"integrity": "sha512-3lBR9EDAY+iYIpTnTIXmWcNbX3T2kCkAEQGIQx4NVQ0575nk2k3GRZDTPQG+vVtS2izSLmINlxXf0uLtnrTP+g==",
"dependencies": {
"@actions/http-client": "^2.0.1",
"uuid": "^8.3.2"
}
},
"node_modules/@actions/exec": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/@actions/exec/-/exec-1.1.1.tgz",
"integrity": "sha512-+sCcHHbVdk93a0XT19ECtO/gIXoxvdsgQLzb2fE2/5sIZmWQuluYyjPQtrtTHdU1YzTZ7bAPN4sITq2xi1679w==",
"dependencies": {
"@actions/io": "^1.0.1"
}
},
"node_modules/@actions/http-client": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.2.0.tgz",
"integrity": "sha512-q+epW0trjVUUHboliPb4UF9g2msf+w61b32tAkFEwL/IwP0DQWgbCMM0Hbe3e3WXSKz5VcUXbzJQgy8Hkra/Lg==",
"dependencies": {
"tunnel": "^0.0.6",
"undici": "^5.25.4"
}
},
"node_modules/@actions/io": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/@actions/io/-/io-1.1.3.tgz",
"integrity": "sha512-wi9JjgKLYS7U/z8PPbco+PvTb/nRWjeoFlJ1Qer83k/3C5PHQi28hiVdeE2kHXmIL99mQFawx8qt/JPjZilJ8Q=="
},
"node_modules/@actions/tool-cache": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/@actions/tool-cache/-/tool-cache-2.0.1.tgz",
"integrity": "sha512-iPU+mNwrbA8jodY8eyo/0S/QqCKDajiR8OxWTnSk/SnYg0sj8Hp4QcUEVC1YFpHWXtrfbQrE13Jz4k4HXJQKcA==",
"dependencies": {
"@actions/core": "^1.2.6",
"@actions/exec": "^1.0.0",
"@actions/http-client": "^2.0.1",
"@actions/io": "^1.1.1",
"semver": "^6.1.0",
"uuid": "^3.3.2"
}
},
"node_modules/@actions/tool-cache/node_modules/uuid": {
"version": "3.4.0",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz",
"integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==",
"deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.",
"bin": {
"uuid": "bin/uuid"
}
},
"node_modules/@fastify/busboy": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.0.tgz",
"integrity": "sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==",
"engines": {
"node": ">=14"
}
},
"node_modules/inherits": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
"integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw=="
},
"node_modules/path": {
"version": "0.12.7",
"resolved": "https://registry.npmjs.org/path/-/path-0.12.7.tgz",
"integrity": "sha512-aXXC6s+1w7otVF9UletFkFcDsJeO7lSZBPUQhtb5O0xJe8LtYhj/GxldoL09bBj9+ZmE2hNoHqQSFMN5fikh4Q==",
"dependencies": {
"process": "^0.11.1",
"util": "^0.10.3"
}
},
"node_modules/process": {
"version": "0.11.10",
"resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz",
"integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==",
"engines": {
"node": ">= 0.6.0"
}
},
"node_modules/semver": {
"version": "6.3.1",
"resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
"integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
"bin": {
"semver": "bin/semver.js"
}
},
"node_modules/tunnel": {
"version": "0.0.6",
"resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz",
"integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==",
"engines": {
"node": ">=0.6.11 <=0.7.0 || >=0.7.3"
}
},
"node_modules/undici": {
"version": "5.28.4",
"resolved": "https://registry.npmjs.org/undici/-/undici-5.28.4.tgz",
"integrity": "sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g==",
"dependencies": {
"@fastify/busboy": "^2.0.0"
},
"engines": {
"node": ">=14.0"
}
},
"node_modules/util": {
"version": "0.10.4",
"resolved": "https://registry.npmjs.org/util/-/util-0.10.4.tgz",
"integrity": "sha512-0Pm9hTQ3se5ll1XihRic3FDIku70C+iHUdT/W926rSgHV5QgXsYbKZN8MSC3tJtSkhuROzvsQjAaFENRXr+19A==",
"dependencies": {
"inherits": "2.0.3"
}
},
"node_modules/uuid": {
"version": "8.3.2",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
"integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
"bin": {
"uuid": "dist/bin/uuid"
}
}
}
}

View File

@ -1,17 +0,0 @@
{
"name": "dl_deps_archive",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"keywords": [],
"author": "",
"license": "ISC",
"dependencies": {
"@actions/core": "^1.10.1",
"@actions/tool-cache": "^2.0.1",
"path": "^0.12.7"
}
}

69
appveyor.yml Normal file
View File

@ -0,0 +1,69 @@
image: Visual Studio 2019
version: '0.1.{build}'
environment:
EXTRA_DIR: 'C:\extra'
MINGW64_EXTRA_DIR: '/c/extra'
PATH: 'C:\\Python36\\Scripts;%PATH%;%EXTRA_DIR%\bin;C:\\Program Files (x86)\\Windows Kits\\10\\bin\\%platform%'
PKG_CONFIG_PATH: '%EXTRA_DIR%\lib\pkgconfig'
MSYS2_PATH_TYPE: 'inherit'
MSYS2_ARG_CONV_EXCL: '-Tp'
MINGW64_RUN: 'C:\msys64\usr\bin\bash.exe --login -c'
MSYSTEM: 'MINGW64'
KIWIX_DESKTOP_VERSION: '2.3.1-3'
KIWIX_DESKTOP_RELEASE: 0
init:
- cmd: |
echo %TEST_VAR%
REM Set VC 2019
"C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars64.bat"
mkdir C:\extra\bin
install:
- cmd: |
cd C:\projects
REM Install meson
C:\Python36\Scripts\pip install meson
REM Install ninja
curl -o ninja-win.zip -L https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-win.zip
7z e ninja-win.zip -o%EXTRA_DIR%\bin
REM Install pkg-config-lite
curl --insecure -o pkg-config-lite-0.28-1.zip https://netix.dl.sourceforge.net/project/pkgconfiglite/0.28-1/pkg-config-lite-0.28-1_bin-win32.zip
7z e pkg-config-lite-0.28-1.zip -o%EXTRA_DIR%\bin pkg-config-lite-0.28-1/bin/pkg-config.exe
cd kiwix-build
REM Install openssh (scp) for deployement
choco install openssh openssl.light
- cmd: openssl enc -d -aes-256-cbc -md sha256 -pass pass:%ENC_PASSWD% -in appveyor\nightlybot_id_key.enc -out appveyor\nightlybot_id_key
build_script:
- IF "%USE_CACHE%" EQU "1" appveyor\setup_from_cache.cmd
- IF "%USE_CACHE%" NEQ "1" appveyor\install_zlib.cmd
- IF "%USE_CACHE%" NEQ "1" appveyor\install_lzma.cmd
- IF "%USE_CACHE%" NEQ "1" appveyor\install_zstd.cmd
- IF "%USE_CACHE%" NEQ "1" appveyor\install_libcurl.cmd
- IF "%USE_CACHE%" NEQ "1" appveyor\install_pthread.cmd
- IF "%USE_CACHE%" NEQ "1" appveyor\install_dirent.cmd
- IF "%USE_CACHE%" NEQ "1" appveyor\install_xapian.cmd
- IF "%USE_CACHE%" NEQ "1" appveyor\install_icu.cmd
- IF "%USE_CACHE%" NEQ "1" appveyor\install_pugixml.cmd
- IF "%USE_CACHE%" NEQ "1" appveyor\install_aria2.cmd
- IF "%USE_CACHE%" NEQ "1" appveyor\install_mustache.cmd
- IF "%USE_CACHE%" NEQ "1" appveyor\install_libmicrohttpd.cmd
- IF "%USE_CACHE%" NEQ "1" %MINGW64_RUN% "cd /c/projects/kiwix-build && appveyor/create_cache.sh"
- appveyor\install_libzim.cmd
- appveyor\install_libkiwix.cmd
- appveyor\install_kiwix-desktop.cmd
before_deploy:
- cmd: openssl enc -d -pbkdf2 -aes-256-cbc -md sha256 -pass pass:%win_certificate_secret% -in appveyor\kiwix.pfx.enc -out appveyor\kiwix.pfx
- cmd: '%MINGW64_RUN% "cd /c/projects/kiwix-build && appveyor/package_kiwix-desktop.sh"'
deploy_script:
- cmd: '%MINGW64_RUN% "cd /c/projects/kiwix-build && appveyor/deploy.sh"'

16
appveyor/build_xapian.sh Normal file
View File

@ -0,0 +1,16 @@
XAPIAN_SOURCE=$(pwd)/..
$XAPIAN_SOURCE/configure \
CC="cl -nologo" \
CXX="$XAPIAN_SOURCE/compile cl -nologo" \
CXXFLAGS="-EHsc -MD" AR=lib \
CPPFLAGS="-I${MINGW64_EXTRA_DIR}/include" \
LDFLAGS="-L${MINGW64_EXTRA_DIR}/lib" \
--disable-backend-remote \
--disable-documentation \
--prefix=${MINGW64_EXTRA_DIR}
make -j2
make install

11
appveyor/create_cache.sh Normal file
View File

@ -0,0 +1,11 @@
set -e
ARCHIVE_NAME="deps_windows_windows.zip"
SSH_KEY=C:\\projects\\kiwix-build\\appveyor\\nightlybot_id_key
7z a -tzip $ARCHIVE_NAME $MINGW64_EXTRA_DIR
scp -P 30022 -vrp -i ${SSH_KEY} -o StrictHostKeyChecking=no \
${ARCHIVE_NAME} \
ci@tmp.kiwix.org:/data/tmp/ci

21
appveyor/deploy.sh Executable file
View File

@ -0,0 +1,21 @@
set -e
NIGHTLY_DATE=$(date +%Y-%m-%d)
NIGHTLY_KIWIX_ARCHIVES_DIR=/c/projects/NIGHTLY_KIWIX_ARCHIVES/${NIGHTLY_DATE}
RELEASE_KIWIX_ARCHIVES_DIR=/c/projects/RELEASE_KIWIX_ARCHIVES
SSH_KEY=C:\\projects\\kiwix-build\\appveyor\\nightlybot_id_key
if [[ "$APPVEYOR_SCHEDULED_BUILD" = "True" ]]
then
scp -P 30022 -vrp -i ${SSH_KEY} -o StrictHostKeyChecking=no \
${NIGHTLY_KIWIX_ARCHIVES_DIR} \
ci@master.download.kiwix.org:/data/download/nightly
fi
if [[ "$APPVEYOR_REPO_TAG" = "true" ]]
then
RELEASE_ARCHIVES=$(find $RELEASE_KIWIX_ARCHIVES_DIR -type f)
scp -P 30022 -vrp -i ${SSH_KEY} -o StrictHostKeyChecking=no \
${RELEASE_ARCHIVES} \
ci@master.download.kiwix.org:/data/download/release/kiwix-desktop
fi

View File

@ -0,0 +1,29 @@
import re, sys, os
makefile = sys.argv[-1]
print("Patching '{}'".format(makefile))
with open(makefile, 'r') as f:
lines = f.readlines()
with open(makefile, 'w') as f:
for line in lines:
# if "/SUBSYSTEM:WINDOWS" in line:
# line = line.replace("/SUBSYSTEM:WINDOWS", "/SUBSYSTEM:CONSOLE")
# f.write(line)
# continue
if not line.startswith('LIBS '):
f.write(line)
continue
print("-- INPUT : {}".format(line))
for lib in ('kiwix', 'zim', 'pugixml', 'z', 'zstd'):
line = line.replace('{}.lib'.format(lib), 'lib{}.a'.format(lib))
for lib in ('lzma', 'curl'):
line = line.replace('lib{}.lib'.format(lib), 'lib{}.a'.format(lib))
line = line.strip()
# line += " pthreadVC2.lib"
# line += " icuin.lib icudt.lib icuuc.lib"
line += " Rpcrt4.lib Ws2_32.lib winmm.lib Shlwapi.lib"
line += os.linesep
print("++ OUTPUT : {}".format(line))
f.write(line)

View File

@ -0,0 +1,4 @@
REM ========================================================
REM Install aria2
curl -fsSL -O https://github.com/aria2/aria2/releases/download/release-1.36.0/aria2-1.36.0-win-64bit-build1.zip || exit /b 1
7z e aria2-1.36.0-win-64bit-build1.zip -o%EXTRA_DIR% aria2-1.36.0-win-64bit-build1\aria2c.exe || exit /b 1

View File

@ -0,0 +1,3 @@
REM ========================================================
REM Install dirent.h
curl -fsSL -o%EXTRA_DIR%\include\dirent.h https://raw.githubusercontent.com/tronkko/dirent/master/include/dirent.h || exit /b 1

View File

@ -0,0 +1,14 @@
REM Set VC 2017
"C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat"
mkdir C:\extra\bin
cd C:\projects
C:\Python36\Scripts\pip install meson || exit /b 1
REM Set ninja
curl -fsSL -o ninja-win.zip https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-win.zip || exit /b 1
7z e ninja-win.zip -o%EXTRA_DIR%\bin || exit /b 1
REM Set pkg-config-lit
curl --insecure -fsSL -o pkg-config-lite-0.28-1.zip https://netix.dl.sourceforge.net/project/pkgconfiglite/0.28-1/pkg-config-lite-0.28-1_bin-win32.zip || exit /b 1
7z e pkg-config-lite-0.28-1.zip -o%EXTRA_DIR%\bin pkg-config-lite-0.28-1/bin/pkg-config.exe || exit /b 1
cd kiwix-build

7
appveyor/install_icu.cmd Normal file
View File

@ -0,0 +1,7 @@
REM ========================================================
REM Install icu
curl -SL -O http://public.kymeria.fr/KIWIX/windows/icu4c-62_1-Win64-MSVC2017.zip || exit /b 1
7z x icu4c-62_1-Win64-MSVC2017.zip -o%EXTRA_DIR% -r include || exit /b 1
7z e icu4c-62_1-Win64-MSVC2017.zip -o%EXTRA_DIR%\lib lib64\* || exit /b 1
7z e icu4c-62_1-Win64-MSVC2017.zip -o%EXTRA_DIR%\bin bin64\*.dll || exit /b 1
curl -fsSL -o%PKG_CONFIG_PATH%\icu-i18n.pc http://public.kymeria.fr/KIWIX/windows/icu-i18n.pc || exit /b 1

View File

@ -0,0 +1,22 @@
REM ========================================================
REM Install kiwix-desktop
git clone https://github.com/kiwix/kiwix-desktop || exit /b 1
cd kiwix-desktop
echo "Running qmake"
SET _WITH_CONSOLE=1
IF %KIWIX_DESKTOP_RELEASE% EQU 1 (
IF %APPVEYOR_REPO_TAG% == true (
SET _WITH_CONSOLE=0
)
)
IF %_WITH_CONSOLE% EQU 1 (
C:\Qt\5.15\msvc2019_64\bin\qmake.exe "CONFIG+=static console" || exit /b 1
) else (
C:\Qt\5.15\msvc2019_64\bin\qmake.exe "CONFIG+=static" || exit /b 1
)
echo "Running fix_desktop"
C:\Python36\Python ..\appveyor\fix_desktop_makefile.py Makefile.Release || exit /b 1
echo "Running nmake"
nmake || exit /b 1
cd ..

View File

@ -0,0 +1,14 @@
REM ========================================================
REM Install libcurl
curl -fsSL -O https://curl.haxx.se/download/curl-7.61.1.zip || exit /b 1
7z x curl-7.61.1.zip || exit /b 1
cd curl-7.61.1\winbuild
nmake /f Makefile.vc mode=static MACHINE=x64 DEBUG=no VC=15 ENABLE_IDN=no || exit /b 1
mkdir %EXTRA_DIR%\include\curl
copy ..\builds\libcurl-vc15-x64-release-static-ipv6-sspi-winssl\include\curl\*.h %EXTRA_DIR%\include\curl
copy ..\builds\libcurl-vc15-x64-release-static-ipv6-sspi-winssl\lib\libcurl_a.lib %EXTRA_DIR%\lib
move %EXTRA_DIR%\lib\libcurl_a.lib %EXTRA_DIR%\lib\libcurl.a
dir %EXTRA_DIR%\include\curl
dir %EXTRA_DIR%\lib
curl -fsSl -o%PKG_CONFIG_PATH%\libcurl.pc http://public.kymeria.fr/KIWIX/windows/libcurl.pc || exit /b 1
cd ..\..

View File

@ -0,0 +1,10 @@
REM ========================================================
REM Install libkiwix
git clone https://github.com/kiwix/libkiwix.git || exit /b 1
cd libkiwix
set CPPFLAGS="-I%EXTRA_DIR%/include"
meson . build --prefix %EXTRA_DIR% --default-library static --buildtype release || exit /b 1
cd build
ninja || exit /b 1
ninja install || exit /b 1
cd ..\..

View File

@ -0,0 +1,11 @@
REM ========================================================
REM Install libmicrohttpd
curl -fsSL -O https://ftpmirror.gnu.org/libmicrohttpd/libmicrohttpd-latest-w32-bin.zip || exit /b 1
7z e libmicrohttpd-latest-w32-bin.zip -o%EXTRA_DIR%/include libmicrohttpd-*-w32-bin/x86_64/VS2019/Release-static/microhttpd.h || exit /b 1
7z e libmicrohttpd-latest-w32-bin.zip -o%EXTRA_DIR%/lib libmicrohttpd-*-w32-bin/x86_64/VS2019/Release-static/libmicrohttpd.lib || exit /b 1
7z e libmicrohttpd-latest-w32-bin.zip -o%EXTRA_DIR%/lib/pkgconfig libmicrohttpd-*-w32-bin/x86_64/MinGW/static/mingw64/lib/pkgconfig/* || exit /b 1
rename %EXTRA_DIR%\lib\libmicrohttpd.lib microhttpd.lib
dir %EXTRA_DIR%
dir %EXTRA_DIR%\lib
dir %EXTRA_DIR%\lib\pkgconfig

View File

@ -0,0 +1,9 @@
REM ========================================================
REM Install libzim
git clone https://github.com/openzim/libzim.git || exit /b 1
cd libzim
meson . build --prefix %EXTRA_DIR% --default-library static --buildtype release || exit /b 1
cd build
ninja || exit /b 1
ninja install || exit /b 1
cd ..\..

View File

@ -0,0 +1,6 @@
REM ========================================================
REM Install lzma
curl -fsSL -O https://tukaani.org/xz/xz-5.2.4-windows.zip || exit /b 1
7z x xz-5.2.4-windows.zip -o%EXTRA_DIR% -r include || exit /b 1
7z e xz-5.2.4-windows.zip -o%EXTRA_DIR%\lib bin_x86-64\liblzma.a || exit /b 1
curl -fsSL -o%PKG_CONFIG_PATH%\liblzma.pc http://public.kymeria.fr/KIWIX/windows/liblzma.pc || exit /b 1

View File

@ -0,0 +1,3 @@
REM ========================================================
REM Install mustache.hpp
curl -fsSL -o%EXTRA_DIR%\include\mustache.hpp https://raw.githubusercontent.com/kainjow/mustache/v3.2.1/mustache.hpp || exit /b 1

View File

@ -0,0 +1,14 @@
REM ========================================================
REM Install pthread
curl -fsSL -O ftp://sourceware.org/pub/pthreads-win32/pthreads-w32-2-9-1-release.zip || exit /b 1
7z x pthreads-w32-2-9-1-release.zip -r pthreads.2 || exit /b 1
cd pthreads.2
REM Patch is pthread_timespec.patch
curl -fsSL -O http://public.kymeria.fr/KIWIX/windows/pthread.h || exit /b 1
nmake clean VC-inlined || exit /b 1
copy pthread.h %EXTRA_DIR%\include
copy sched.h %EXTRA_DIR%\include
copy pthreadVC2.lib %EXTRA_DIR%\lib
copy pthreadVC2.dll %EXTRA_DIR%\bin
curl -fsSL -o%PKG_CONFIG_PATH%\libpthreadVC2.pc http://public.kymeria.fr/KIWIX/windows/libpthreadVC2.pc || exit /b 1
cd ..

View File

@ -0,0 +1,10 @@
REM ========================================================
REM Install pugixml
curl -fsSL -O http://public.kymeria.fr/KIWIX/windows/pugixml-1.2-meson.zip || exit /b 1
7z x pugixml-1.2-meson.zip -o. || exit /b 1
cd pugixml-1.2-meson
meson . build --prefix %EXTRA_DIR% --default-library static --buildtype release || exit /b 1
cd build
ninja || exit /b 1
ninja install || exit /b 1
cd ..\..

View File

@ -0,0 +1,9 @@
REM ========================================================
REM Install xapian
curl -fsSL -O http://mirror.download.kiwix.org/dev/kiwix-build/xapian-core-1.4.18.zip || exit /b 1
7z x xapian-core-1.4.18.zip || exit /b 1
cd xapian-core-1.4.18
mkdir build
cd build
%MINGW64_RUN% "cd /c/Projects/kiwix-build/xapian-core-1.4.18/build && /c/Projects/kiwix-build/appveyor/build_xapian.sh" > build_xapian.log || exit /b 1
cd ..\..

10
appveyor/install_zlib.cmd Normal file
View File

@ -0,0 +1,10 @@
REM ========================================================
REM Install zlib
curl -fsSL -O http://mirror.download.kiwix.org/dev/kiwix-build/zlib-1.2.12.meson.zip || exit /b 1
7z x zlib-1.2.12.meson.zip || exit /b 1
cd zlib-1.2.12
meson . build --prefix %EXTRA_DIR% --default-library static --buildtype release || exit /b 1
cd build
ninja || exit /b 1
ninja install || exit /b 1
cd ..\..

10
appveyor/install_zstd.cmd Normal file
View File

@ -0,0 +1,10 @@
REM ========================================================
REM Install zstd
curl -fsSL -o zstd-v1.5.2.zip https://github.com/facebook/zstd/archive/refs/tags/v1.5.2.zip || exit /b 1
7z x zstd-v1.5.2.zip || exit /b 1
cd zstd-1.5.2/build/meson
meson . builddir --prefix %EXTRA_DIR% --default-library static --buildtype release -Dbin_programs=false -Dbin_contrib=false || exit /b 1
cd builddir
ninja || exit /b 1
ninja install || exit /b 1
cd ..\..\..\..

BIN
appveyor/kiwix.pfx.enc Normal file

Binary file not shown.

Binary file not shown.

View File

@ -0,0 +1,35 @@
set -e
if [[ "$APPVEYOR_REPO_TAG" = "false" ]]
then
NIGHTLY_DATE=$(date +%Y-%m-%d)
KIWIX_ARCHIVES_DIR=/c/projects/NIGHTLY_KIWIX_ARCHIVES/${NIGHTLY_DATE}
KIWIX_DIR=kiwix-desktop_windows_x64_$NIGHTLY_DATE
else
if [ $KIWIX_DESKTOP_RELEASE -eq 1 ]
then
KIWIX_ARCHIVES_DIR=/c/projects/RELEASE_KIWIX_ARCHIVES
KIWIX_DIR=kiwix-desktop_windows_x64_${KIWIX_DESKTOP_VERSION}
fi
fi
if [[ "$KIWIX_DIR" ]]
then
KIWIX_ARCH_NAME=${KIWIX_DIR}.zip
mkdir $KIWIX_DIR
mkdir -p KIWIX_ARCHIVES_DIR
cp /c/projects/kiwix-build/kiwix-desktop/Release/kiwix-desktop.exe $KIWIX_DIR
/c/Qt/5.15/msvc2019_64/bin/windeployqt --compiler-runtime $KIWIX_DIR
cp $MINGW64_EXTRA_DIR/aria2c.exe $KIWIX_DIR
cp $MINGW64_EXTRA_DIR/bin/*.dll $KIWIX_DIR
cp /c/OpenSSL-v111-Win64/bin/libcrypto-1_1-x64.dll $KIWIX_DIR
cp /c/OpenSSL-v111-Win64/bin/libssl-1_1-x64.dll $KIWIX_DIR
signtool.exe sign -f appveyor/kiwix.pfx -p $win_certificate_password -t http://timestamp.digicert.com -d "Kiwix-desktop application" -fd SHA256 $KIWIX_DIR/kiwix-desktop.exe
7z a -tzip $KIWIX_ARCHIVES_DIR/$KIWIX_ARCH_NAME $KIWIX_DIR
fi

View File

@ -0,0 +1,4 @@
REM ========================================================
REM Setup from cache
curl -fsSL -O http://tmp.kiwix.org/ci/deps_windows_windows.zip || exit /b 1
7z x deps_windows_windows.zip -oc: -aoa || exit /b 1

View File

@ -0,0 +1,23 @@
FROM alpine:3.16
ENV LANG C.UTF-8
ENV OS_NAME alpine
RUN apk update -q \
&& apk add -q --no-cache \
# Base build tools
bash build-base git py3-pip \
# Packaged dependencies
xz-dev \
zstd-dev \
xapian-core-dev \
icu-dev icu-data-full \
gtest-dev
# Create user
RUN addgroup --gid 121 runner
RUN adduser -u 1001 -G runner -h /home/runner -D runner
USER runner
ENV PATH /home/runner/.local/bin:$PATH
RUN pip3 install meson ninja ; \
ln -s /usr/bin/python3 /home/runner/.local/bin/python

View File

@ -0,0 +1,26 @@
FROM ubuntu:bionic
ENV LANG C.UTF-8
ENV OS_NAME bionic
RUN apt update -q \
&& apt-get update \
&& apt install -q -y --no-install-recommends \
# Base build tools
build-essential automake libtool cmake ccache pkg-config autopoint patch \
python3-pip python3-setuptools python3-wheel git subversion wget unzip \
ninja-build openssh-client curl \
# Packaged dependencies
libbz2-dev uuid-dev zlib1g-dev \
libgtest-dev \
# Other tools (to remove)
# vim less grep \
&& apt-get clean -y \
&& rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/* \
&& pip3 install meson pytest gcovr requests distro
# Create user
RUN groupadd --gid 121 runner
RUN useradd --uid 1001 --gid 121 --create-home runner
USER runner
ENV PATH /home/runner/.local/bin:$PATH

View File

@ -0,0 +1,28 @@
FROM fedora:35
ENV LANG C.UTF-8
ENV OS_NAME f35
RUN dnf install -y --nodocs \
# Base build tools
make automake libtool cmake git-core subversion pkg-config gcc-c++ \
wget unzip ninja-build ccache which patch gcovr xz openssh-clients \
python3-pip \
# Cross win32 compiler
mingw32-gcc-c++ mingw32-bzip2-static mingw32-win-iconv-static \
mingw32-winpthreads-static mingw32-zlib-static mingw32-xz-libs-static \
mingw32-libmicrohttpd \
# python3
python3-pip python-unversioned-command \
# Other tools (to remove)
# vim less grep
&& dnf remove -y "*-doc" \
&& dnf autoremove -y \
&& dnf clean all \
&& pip3 install meson pytest requests distro
# Create user
RUN groupadd --gid 121 runner
RUN useradd --uid 1001 --gid 121 --create-home runner
USER runner
ENV PATH /home/runner/.local/bin:$PATH

View File

@ -0,0 +1,40 @@
FROM ubuntu:focal
ENV LANG C.UTF-8
ENV OS_NAME focal
ENV DEBIAN_FRONTEND noninteractive
RUN apt update -q \
&& apt install -q -y --no-install-recommends \
# Base build tools
build-essential automake libtool cmake ccache pkg-config autopoint patch \
python3-pip python3-setuptools python3-wheel git subversion wget unzip \
ninja-build openssh-client curl libgl-dev \
# Python (2) is needed to install android-ndk
python \
# Packaged dependencies
libbz2-dev libmagic-dev uuid-dev zlib1g-dev \
libmicrohttpd-dev aria2 libgtest-dev libgl-dev \
# Devel package to compile python modules
libxml2-dev libxslt-dev python3-dev \
# Qt packages
libqt5gui5 qtbase5-dev qtwebengine5-dev libqt5svg5-dev qt5-image-formats-plugins qt5-default \
# To create the appimage of kiwix-desktop
libfuse2 fuse patchelf \
# Flatpak tools
elfutils flatpak flatpak-builder \
# Cross win32 compiler
g++-mingw-w64-i686 gcc-mingw-w64-i686 gcc-mingw-w64-base mingw-w64-tools \
# Cross compile i586
libc6-dev-i386 lib32stdc++6 gcc-multilib g++-multilib \
# Other tools (to remove)
# vim less grep \
&& apt-get clean -y \
&& rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/debconf/* \
&& pip3 install meson pytest gcovr requests distro
# Create user
RUN groupadd --gid 121 runner
RUN useradd --uid 1001 --gid 121 --create-home runner
USER runner
ENV PATH /home/runner/.local/bin:$PATH

View File

@ -4,163 +4,81 @@ import os, sys
import argparse import argparse
from .dependencies import Dependency from .dependencies import Dependency
from .configs import ConfigInfo from .platforms import PlatformInfo
from .builder import Builder from .builder import Builder
from .flatpak_builder import FlatpakBuilder from .flatpak_builder import FlatpakBuilder
from . import _global from . import _global
def parse_args(): def parse_args():
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument( parser.add_argument('target', default='kiwix-tools', nargs='?', metavar='TARGET',
"target", choices=Dependency.all_deps.keys())
default="kiwix-tools", parser.add_argument('--working-dir', default=".",
nargs="?", help=("Directory where kiwix-build puts all its files "
metavar="TARGET", "(source, archive and build)\n"
choices=Dependency.all_deps.keys(), "working-dir can be absolute path or a relative (to cwd) one."))
) parser.add_argument('--build-dir', default=".",
parser.add_argument( help=("Directory where kiwix-build puts all build files.\n"
"--working-dir", "build-dir can be absolute path or a relative (to working-dir) one."))
default=".", parser.add_argument('--libprefix', default=None)
help=( parser.add_argument('--target-platform', choices=PlatformInfo.all_platforms)
"Directory where kiwix-build puts all its files " parser.add_argument('--verbose', '-v', action="store_true",
"(source, archive and build)\n" help=("Print all logs on stdout instead of in specific"
"working-dir can be absolute path or a relative (to cwd) one." " log files per commands"))
), parser.add_argument('--hide-progress', action='store_false', dest='show_progress',
) help="Hide intermediate progress information.")
parser.add_argument( parser.add_argument('--skip-source-prepare', action='store_true',
"--build-dir", help="Skip the source download part")
default=".", parser.add_argument('--build-deps-only', action='store_true',
help=( help="Build only the dependencies of the specified target.")
"Directory where kiwix-build puts all build files.\n" parser.add_argument('--build-nodeps', action='store_true',
"build-dir can be absolute path or a relative (to working-dir) one." help="Build only the target, not its dependencies.")
), parser.add_argument('--make-dist', action='store_true',
) help="Build distrubution (dist) source archive")
parser.add_argument("--libprefix", default=None) parser.add_argument('--make-release', action='store_true',
parser.add_argument( help="Build a release version")
"--config", choices=ConfigInfo.all_configs, default="native_dyn" subgroup = parser.add_argument_group('advanced')
) subgroup.add_argument('--no-cert-check', action='store_true',
parser.add_argument( help="Skip SSL certificate verification during download")
"--verbose", subgroup.add_argument('--clean-at-end', action='store_true',
"-v", help="Clean all intermediate files after the (successfull) build")
action="store_true", subgroup.add_argument('--dont-install-packages', action='store_true',
help=( help="Do not try to install packages before compiling")
"Print all logs on stdout instead of in specific" " log files per commands" subgroup.add_argument('--assume-packages-installed', action='store_true',
), help="Assume the package to install to be aleady installed")
) subgroup.add_argument('--android-arch', action='append',
parser.add_argument( help=("Specify the architecture to build for android application/libraries.\n"
"--hide-progress", "Can be specified several times to build for several architectures.\n"
action="store_false", "If not specified, all architectures will be build."))
dest="show_progress", subgroup.add_argument('--ios-arch', action='append',
help="Hide intermediate progress information.", help=("Specify the architecture to build for ios application/libraries.\n"
) "Can be specified several times to build for several architectures.\n"
parser.add_argument( "If not specified, all architectures will be build."))
"--skip-source-prepare", subgroup.add_argument('--fast-clone', action='store_true',
action="store_true", help=("Do not clone the whole repository.\n"
help="Skip the source download part", "This is useful for one shot build but it is not recommended if you want "
) "to develop with the cloned sources."))
parser.add_argument(
"--build-deps-only",
action="store_true",
help="Build only the dependencies of the specified target.",
)
parser.add_argument(
"--build-nodeps",
action="store_true",
help="Build only the target, not its dependencies.",
)
parser.add_argument(
"--make-dist",
action="store_true",
help="Build distrubution (dist) source archive",
)
parser.add_argument(
"--make-release", action="store_true", help="Build a release version"
)
subgroup = parser.add_argument_group("advanced")
subgroup.add_argument(
"--no-cert-check",
action="store_true",
help="Skip SSL certificate verification during download",
)
subgroup.add_argument(
"--clean-at-end",
action="store_true",
help="Clean all intermediate files after the (successfull) build",
)
subgroup.add_argument(
"--dont-install-packages",
action="store_true",
help="Do not try to install packages before compiling",
)
subgroup.add_argument(
"--assume-packages-installed",
action="store_true",
help="Assume the package to install to be aleady installed",
)
subgroup.add_argument(
"--android-arch",
action="append",
help=(
"Specify the architecture to build for android application/libraries.\n"
"Can be specified several times to build for several architectures.\n"
"If not specified, all architectures will be build."
),
)
subgroup.add_argument(
"--ios-arch",
action="append",
help=(
"Specify the architecture to build for ios application/libraries.\n"
"Can be specified several times to build for several architectures.\n"
"If not specified, all architectures will be build."
),
)
subgroup.add_argument(
"--fast-clone",
action="store_true",
help=(
"Do not clone the whole repository.\n"
"This is useful for one shot build but it is not recommended if you want "
"to develop with the cloned sources."
),
)
subgroup.add_argument(
"--use-target-arch-name",
action="store_true",
help=(
"Name the build directory using the arch name instead of the config name.\n"
"Different configs may create binary for the same arch so this option is "
"not recommended when working with several config on the same computer.\n"
"However, when generating dependencies for other it is better to have a "
"directory named using the target instead of the used config.\n"
"Intended to be used in CI only."
),
)
subgroup.add_argument(
"--get-build-dir", action="store_true", help="Print the output directory."
)
options = parser.parse_args() options = parser.parse_args()
if not options.android_arch: if not options.android_arch:
options.android_arch = ["arm", "arm64", "x86", "x86_64"] options.android_arch = ['arm', 'arm64', 'x86', 'x86_64']
if not options.ios_arch: if not options.ios_arch:
options.ios_arch = ["arm64", "x86_64"] options.ios_arch = ['arm64', 'x86_64']
if not options.target_platform:
options.target_platform = 'native_dyn'
return options return options
def main(): def main():
options = parse_args() options = parse_args()
options.working_dir = os.path.abspath(options.working_dir) options.working_dir = os.path.abspath(options.working_dir)
_global.set_options(options) _global.set_options(options)
neutralEnv = buildenv.NeutralEnv(options.get_build_dir) neutralEnv = buildenv.PlatformNeutralEnv()
_global.set_neutralEnv(neutralEnv) _global.set_neutralEnv(neutralEnv)
if options.config == "flatpak": if options.target_platform == 'flatpak':
builder = FlatpakBuilder() builder = FlatpakBuilder()
else: else:
builder = Builder() builder = Builder()
if options.get_build_dir: builder.run()
print(ConfigInfo.get_config(options.config).buildEnv.build_dir)
else:
builder.run()

View File

@ -5,29 +5,23 @@ _neutralEnv = None
_options = None _options = None
_target_steps = _OrderedDict() _target_steps = _OrderedDict()
def set_neutralEnv(env): def set_neutralEnv(env):
global _neutralEnv global _neutralEnv
_neutralEnv = env _neutralEnv = env
def neutralEnv(what): def neutralEnv(what):
return getattr(_neutralEnv, what) return getattr(_neutralEnv, what)
def set_options(options): def set_options(options):
global _options global _options
_options = options _options = options
def option(what): def option(what):
return getattr(_options, what) return getattr(_options, what)
def add_target_step(key, what): def add_target_step(key, what):
_target_steps[key] = what _target_steps[key] = what
def get_target_step(key, default_context=None): def get_target_step(key, default_context=None):
if isinstance(key, tuple): if isinstance(key, tuple):
context, target = key context, target = key
@ -35,6 +29,24 @@ def get_target_step(key, default_context=None):
context, target = default_context, key context, target = default_context, key
return _target_steps[(context, target)] return _target_steps[(context, target)]
def target_steps(): def target_steps():
return _target_steps return _target_steps
def backend():
global _backend
if _backend is not None:
return _backend
_platform = platform.system()
if _platform == 'Windows':
print('ERROR: kiwix-build is not intented to run on Windows platform.\n'
'There is no backend for Windows, so we can\'t launch any commands.')
sys.exit(0)
if _platform == 'Linux':
_platform, _, _ = platform.linux_distribution()
_platform = _platform.lower()
_backend = backends.Linux()
return _backend

View File

@ -1,96 +1,93 @@
import os, sys, shutil import os, sys, shutil
import subprocess import subprocess
import platform import platform
import distro import distro
from .utils import pj, download_remote, escape_path from .utils import pj, download_remote, Defaultdict
from ._global import neutralEnv, option from ._global import neutralEnv, option
class NeutralEnv: class PlatformNeutralEnv:
def __init__(self, dummy_run): def __init__(self):
self.working_dir = option("working_dir") self.working_dir = option('working_dir')
self.source_dir = pj(self.working_dir, "SOURCE") self.source_dir = pj(self.working_dir, "SOURCE")
self.archive_dir = pj(self.working_dir, "ARCHIVE") self.archive_dir = pj(self.working_dir, "ARCHIVE")
self.toolchain_dir = pj(self.working_dir, "TOOLCHAINS") self.toolchain_dir = pj(self.working_dir, "TOOLCHAINS")
self.log_dir = pj(self.working_dir, "LOGS") self.log_dir = pj(self.working_dir, 'LOGS')
for d in (self.source_dir, self.archive_dir, self.toolchain_dir, self.log_dir): for d in (self.source_dir,
self.archive_dir,
self.toolchain_dir,
self.log_dir):
os.makedirs(d, exist_ok=True) os.makedirs(d, exist_ok=True)
self.detect_platform() self.detect_platform()
if dummy_run: self.ninja_command = self._detect_ninja()
# If this is for a dummy run, we will not run anything. if not self.ninja_command:
# To check for command (and so, don't enforce their presence) sys.exit("ERROR: ninja command not found.")
return self.meson_command = self._detect_meson()
self.ninja_command = self._detect_command( if not self.meson_command:
"ninja", default=[["ninja"], ["ninja-build"]] sys.exit("ERROR: meson command not found.")
) self.qmake_command = self._detect_qmake()
self.meson_command = self._detect_command( if not self.qmake_command:
"meson", default=[["meson.py"], ["meson"]] print("WARNING: qmake command not found.", file=sys.stderr)
) self.mesontest_command = "{} test".format(self.meson_command)
self.mesontest_command = [*self.meson_command, "test"]
self.patch_command = self._detect_command("patch")
self.git_command = self._detect_command("git")
if platform.system() == "Windows":
self.make_command = self._detect_command("nmake", options=["/?", "/NOLOGO"])
else:
self.make_command = self._detect_command("make")
self.cmake_command = self._detect_command("cmake")
self.qmake_command = self._detect_command(
"qmake", required=False, default=[["qmake"], ["qmake-qt5"]]
)
def detect_platform(self): def detect_platform(self):
_platform = platform.system() _platform = platform.system()
self.distname = _platform self.distname = _platform
if _platform == "Linux": if _platform == 'Windows':
print('ERROR: kiwix-build is not intented to run on Windows platform.\n'
'It should probably not work, but well, you still can have a try.')
cont = input('Do you want to continue ? [y/N]')
if cont.lower() != 'y':
sys.exit(0)
if _platform == 'Linux':
self.distname = distro.id() self.distname = distro.id()
if self.distname == "ubuntu": if self.distname == 'ubuntu':
self.distname = "debian" self.distname = 'debian'
def download(self, what, where=None): def download(self, what, where=None):
where = where or self.archive_dir where = where or self.archive_dir
download_remote(what, where) download_remote(what, where)
def _detect_command(self, name, default=None, options=["--version"], required=True): def _detect_binary(self, *bin_variants):
if default is None: for n in bin_variants:
default = [[name]]
env_key = "KBUILD_{}_COMMAND".format(name.upper())
if env_key in os.environ:
default = [os.environ[env_key].split()] + default
for command in default:
try: try:
retcode = subprocess.check_call( retcode = subprocess.check_call([n, '--version'],
command + options, stdout=subprocess.DEVNULL stdout=subprocess.DEVNULL)
) except (FileNotFoundError, PermissionError):
except (FileNotFoundError, PermissionError, OSError):
# Doesn't exist in PATH or isn't executable # Doesn't exist in PATH or isn't executable
continue continue
if retcode == 0: if retcode == 0:
return command return n
else:
if required:
sys.exit("ERROR: {} command not found".format(name)) def _detect_ninja(self):
else: return self._detect_binary('ninja', 'ninja-build')
print("WARNING: {} command not found".format(name), file=sys.stderr)
return ["{}_NOT_FOUND".format(name.upper())] def _detect_meson(self):
return self._detect_binary('meson.py', 'meson')
def _detect_qmake(self):
return self._detect_binary('qmake-qt5', 'qmake')
class BuildEnv: class BuildEnv:
def __init__(self, configInfo): def __init__(self, platformInfo):
self.configInfo = configInfo build_dir = "BUILD_{}".format(platformInfo.name)
self.base_build_dir = pj(option("working_dir"), option("build_dir")) self.platformInfo = platformInfo
build_dir = ( self.base_build_dir = pj(option('working_dir'), option('build_dir'))
configInfo.arch_name if option("use_target_arch_name") else configInfo.name
)
build_dir = f"BUILD_{build_dir}"
self.build_dir = pj(self.base_build_dir, build_dir) self.build_dir = pj(self.base_build_dir, build_dir)
self.install_dir = pj(self.build_dir, "INSTALL") self.install_dir = pj(self.build_dir, "INSTALL")
self.toolchain_dir = pj(self.build_dir, "TOOLCHAINS") self.toolchain_dir = pj(self.build_dir, "TOOLCHAINS")
self.log_dir = pj(self.build_dir, "LOGS") self.log_dir = pj(self.build_dir, 'LOGS')
for d in (self.build_dir, self.install_dir, self.toolchain_dir, self.log_dir): for d in (self.build_dir,
self.install_dir,
self.toolchain_dir,
self.log_dir):
os.makedirs(d, exist_ok=True) os.makedirs(d, exist_ok=True)
self.libprefix = option("libprefix") or self._detect_libdir() self.libprefix = option('libprefix') or self._detect_libdir()
def clean_intermediate_directories(self): def clean_intermediate_directories(self):
for subdir in os.listdir(self.build_dir): for subdir in os.listdir(self.build_dir):
@ -103,81 +100,69 @@ class BuildEnv:
os.remove(subpath) os.remove(subpath)
def _is_debianlike(self): def _is_debianlike(self):
return os.path.isfile("/etc/debian_version") return os.path.isfile('/etc/debian_version')
def _detect_libdir(self): def _detect_libdir(self):
if self.configInfo.libdir is not None: if self.platformInfo.libdir is not None:
return self.configInfo.libdir return self.platformInfo.libdir
if self._is_debianlike(): if self._is_debianlike():
try: try:
pc = subprocess.Popen( pc = subprocess.Popen(['dpkg-architecture', '-qDEB_HOST_MULTIARCH'],
["dpkg-architecture", "-qDEB_HOST_MULTIARCH"], stdout=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
stderr=subprocess.DEVNULL,
)
(stdo, _) = pc.communicate() (stdo, _) = pc.communicate()
if pc.returncode == 0: if pc.returncode == 0:
archpath = stdo.decode().strip() archpath = stdo.decode().strip()
return "lib/" + archpath return 'lib/' + archpath
except Exception: except Exception:
pass pass
if os.path.isdir("/usr/lib64") and not os.path.islink("/usr/lib64"): if os.path.isdir('/usr/lib64') and not os.path.islink('/usr/lib64'):
return "lib64" return 'lib64'
return "lib" return 'lib'
def get_env(self, *, cross_comp_flags, cross_compilers, cross_path): def get_env(self, *, cross_comp_flags, cross_compilers, cross_path):
env = self.configInfo.get_env() env = self.platformInfo.get_env()
pkgconfig_path = pj(self.install_dir, self.libprefix, "pkgconfig") pkgconfig_path = pj(self.install_dir, self.libprefix, 'pkgconfig')
env["PKG_CONFIG_PATH"].append(pkgconfig_path) env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path])
env["PATH"].insert(0, pj(self.install_dir, "bin")) env['PATH'] = ':'.join([pj(self.install_dir, 'bin'), env['PATH']])
env["LD_LIBRARY_PATH"].extend( env['LD_LIBRARY_PATH'] = ':'.join([env['LD_LIBRARY_PATH'],
[ pj(self.install_dir, 'lib'),
pj(self.install_dir, "lib"), pj(self.install_dir, self.libprefix)
pj(self.install_dir, self.libprefix), ])
]
)
env["QMAKE_CXXFLAGS"] = " ".join( env['QMAKE_CXXFLAGS'] = " ".join(['-I'+pj(self.install_dir, 'include'), env['QMAKE_CXXFLAGS']])
[escape_path("-I" + pj(self.install_dir, "include")), env["QMAKE_CXXFLAGS"]] env['CPPFLAGS'] = " ".join(['-I'+pj(self.install_dir, 'include'), env['CPPFLAGS']])
) env['QMAKE_LFLAGS'] = " ".join(['-L'+pj(self.install_dir, 'lib'),
env["CPPFLAGS"] = " ".join( '-L'+pj(self.install_dir, self.libprefix),
[escape_path("-I" + pj(self.install_dir, "include")), env["CPPFLAGS"]] env['QMAKE_LFLAGS']])
) env['LDFLAGS'] = " ".join(['-L'+pj(self.install_dir, 'lib'),
env["QMAKE_LFLAGS"] = " ".join( '-L'+pj(self.install_dir, self.libprefix),
[ env['LDFLAGS']])
escape_path("-L" + pj(self.install_dir, "lib")),
escape_path("-L" + pj(self.install_dir, self.libprefix)),
env["QMAKE_LFLAGS"],
]
)
env["LDFLAGS"] = " ".join(
[
escape_path("-L" + pj(self.install_dir, "lib")),
escape_path("-L" + pj(self.install_dir, self.libprefix)),
env["LDFLAGS"],
]
)
if cross_comp_flags: if cross_comp_flags:
self.configInfo.set_comp_flags(env) self.platformInfo.set_comp_flags(env)
if cross_compilers: if cross_compilers:
self.configInfo.set_compiler(env) self.platformInfo.set_compiler(env)
if cross_path: if cross_path:
env["PATH"][0:0] = self.configInfo.get_bin_dir() env['PATH'] = ':'.join(self.platformInfo.get_bin_dir() + [env['PATH']])
return env return env
@property @property
def configure_wrapper(self): def configure_wrapper(self):
try: wrapper = getattr(self.platformInfo, "configure_wrapper", "")
yield self.configInfo.configure_wrapper if wrapper:
except AttributeError: return "{} ".format(wrapper)
pass else:
return ""
@property @property
def make_wrapper(self): def make_wrapper(self):
try: wrapper = getattr(self.platformInfo, "make_wrapper", "")
yield self.configInfo.make_wrapper if wrapper:
except AttributeError: return "{} ".format(wrapper)
pass else:
return ""

View File

@ -1,36 +1,30 @@
import sys import sys
from collections import OrderedDict from collections import OrderedDict
from .buildenv import * from .buildenv import *
from .configs import ConfigInfo from .platforms import PlatformInfo
from .utils import remove_duplicates, StopBuild, colorize from .utils import remove_duplicates, StopBuild, colorize
from .dependencies import Dependency from .dependencies import Dependency
from .packages import PACKAGE_NAME_MAPPERS from .packages import PACKAGE_NAME_MAPPERS
from ._global import ( from ._global import (
neutralEnv, neutralEnv, option,
option, add_target_step, get_target_step, target_steps,
add_target_step, backend)
get_target_step,
target_steps,
)
from . import _global from . import _global
class Builder: class Builder:
def __init__(self): def __init__(self):
self._targets = {} self._targets = {}
ConfigInfo.get_config("neutral", self._targets) PlatformInfo.get_platform('neutral', self._targets)
config_name = option("config") target_platform = option('target_platform')
config = ConfigInfo.get_config(config_name, self._targets) platform = PlatformInfo.get_platform(target_platform, self._targets)
if neutralEnv("distname") not in config.compatible_hosts: if neutralEnv('distname') not in platform.compatible_hosts:
print( print((colorize('ERROR')+': The target platform {} cannot be build on host {}.\n'
( 'Select another target platform or change your host system.'
colorize("ERROR") + ": The config {} cannot be build on host {}.\n" ).format(platform.name, neutralEnv('distname')))
"Select another config or change your host system." self.targetDefs = platform.add_targets(option('target'), self._targets)
).format(config.name, neutralEnv("distname"))
)
self.targetDefs = config.add_targets(option("target"), self._targets)
def finalize_target_steps(self): def finalize_target_steps(self):
steps = [] steps = []
@ -38,36 +32,36 @@ class Builder:
steps += self.order_steps(targetDef) steps += self.order_steps(targetDef)
steps = list(remove_duplicates(steps)) steps = list(remove_duplicates(steps))
if option("build_nodeps"): if option('build_nodeps'):
# add all config steps # add all platform steps
for dep in steps: for dep in steps:
stepClass = Dependency.all_deps[dep[1]] stepClass = Dependency.all_deps[dep[1]]
if stepClass.dont_skip: if stepClass.dont_skip:
add_target_step(dep, self._targets[dep]) add_target_step(dep, self._targets[dep])
src_targetDef = ("source", targetDef[1]) src_targetDef = ('source', targetDef[1])
add_target_step(src_targetDef, self._targets[src_targetDef]) add_target_step(src_targetDef, self._targets[src_targetDef])
add_target_step(targetDef, self._targets[targetDef]) add_target_step(targetDef, self._targets[targetDef])
else: else:
for dep in steps: for dep in steps:
if option("build_deps_only") and dep[1] == targetDef[1]: if option('build_deps_only') and dep[1] == targetDef[1]:
continue continue
add_target_step(dep, self._targets[dep]) add_target_step(dep, self._targets[dep])
self.instanciate_steps() self.instanciate_steps()
def order_steps(self, targetDef): def order_steps(self, targetDef):
for cfgName in ConfigInfo.all_running_configs: for pltName in PlatformInfo.all_running_platforms:
cfg = ConfigInfo.all_configs[cfgName] plt = PlatformInfo.all_platforms[pltName]
for tlcName in cfg.toolchain_names: for tlcName in plt.toolchain_names:
tlc = Dependency.all_deps[tlcName] tlc = Dependency.all_deps[tlcName]
yield ("source", tlcName) yield('source', tlcName)
yield ("neutral" if tlc.neutral else cfgName, tlcName) yield('neutral' if tlc.neutral else pltName, tlcName)
_targets = dict(self._targets) _targets =dict(self._targets)
yield from self.order_dependencies(targetDef, _targets) yield from self.order_dependencies(targetDef, _targets)
def order_dependencies(self, targetDef, targets): def order_dependencies(self, targetDef, targets):
targetConfigName, targetName = targetDef targetPlatformName, targetName = targetDef
if targetConfigName == "source": if targetPlatformName == 'source':
# Do not try to order sources, they will be added as dep by the # Do not try to order sources, they will be added as dep by the
# build step two lines later. # build step two lines later.
return return
@ -76,45 +70,43 @@ class Builder:
except KeyError: except KeyError:
return return
targetConfig = ConfigInfo.get_config(targetConfigName) targetPlatform = PlatformInfo.get_platform(targetPlatformName)
for dep in target.get_dependencies(targetConfig, True): for dep in target.get_dependencies(targetPlatform, True):
depConfig, depName = targetConfig.get_fully_qualified_dep(dep) depPlatform, depName = targetPlatform.get_fully_qualified_dep(dep)
if (depConfig, depName) in targets: if (depPlatform, depName) in targets:
yield from self.order_dependencies((depConfig, depName), targets) yield from self.order_dependencies((depPlatform, depName), targets)
yield ("source", targetName) yield ('source', targetName)
yield targetDef yield targetDef
def instanciate_steps(self): def instanciate_steps(self):
for stepDef in list(target_steps()): for stepDef in list(target_steps()):
stepConfig, stepName = stepDef stepPlatform, stepName = stepDef
stepClass = Dependency.all_deps[stepName] stepClass = Dependency.all_deps[stepName]
if stepConfig == "source": if stepPlatform == 'source':
source = get_target_step(stepDef)(stepClass) source = get_target_step(stepDef)(stepClass)
add_target_step(stepDef, source) add_target_step(stepDef, source)
else: else:
source = get_target_step(stepName, "source") source = get_target_step(stepName, 'source')
env = ConfigInfo.get_config(stepConfig).buildEnv env = PlatformInfo.get_platform(stepPlatform).buildEnv
builder = get_target_step(stepDef)(stepClass, source, env) builder = get_target_step(stepDef)(stepClass, source, env)
add_target_step(stepDef, builder) add_target_step(stepDef, builder)
def prepare_sources(self): def prepare_sources(self):
if option("skip_source_prepare"): if option('skip_source_prepare'):
print(colorize("SKIP")) print(colorize("SKIP"))
return return
sourceDefs = remove_duplicates( sourceDefs = remove_duplicates(tDef for tDef in target_steps() if tDef[0]=='source')
tDef for tDef in target_steps() if tDef[0] == "source"
)
for sourceDef in sourceDefs: for sourceDef in sourceDefs:
print("prepare sources {} :".format(sourceDef[1])) print("prepare sources {} :".format(sourceDef[1]))
source = get_target_step(sourceDef) source = get_target_step(sourceDef)
source.prepare() source.prepare()
def build(self): def build(self):
builderDefs = (tDef for tDef in target_steps() if tDef[0] != "source") builderDefs = (tDef for tDef in target_steps() if tDef[0] != 'source')
for builderDef in builderDefs: for builderDef in builderDefs:
builder = get_target_step(builderDef) builder = get_target_step(builderDef)
if option("make_dist") and builderDef[1] == option("target"): if option('make_dist') and builderDef[1] == option('target'):
print("make dist {} ({}):".format(builder.name, builderDef[0])) print("make dist {} ({}):".format(builder.name, builderDef[0]))
builder.make_dist() builder.make_dist()
continue continue
@ -124,19 +116,19 @@ class Builder:
def _get_packages(self): def _get_packages(self):
packages_list = [] packages_list = []
for config in ConfigInfo.all_running_configs.values(): for platform in PlatformInfo.all_running_platforms.values():
mapper_name = "{host}_{config}".format( mapper_name = "{host}_{target}".format(
host=neutralEnv("distname"), config=config host=neutralEnv('distname'),
) target=platform)
package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {}) package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {})
packages_list += package_name_mapper.get("COMMON", []) packages_list += package_name_mapper.get('COMMON', [])
to_drop = [] to_drop = []
for builderDef in self._targets: for builderDef in self._targets:
configName, builderName = builderDef platformName, builderName = builderDef
mapper_name = "{host}_{config}".format( mapper_name = "{host}_{target}".format(
host=neutralEnv("distname"), config=configName host=neutralEnv('distname'),
) target=platformName)
package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {}) package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {})
packages = package_name_mapper.get(builderName) packages = package_name_mapper.get(builderName)
if packages: if packages:
@ -152,20 +144,20 @@ class Builder:
packages_to_have = self._get_packages() packages_to_have = self._get_packages()
packages_to_have = remove_duplicates(packages_to_have) packages_to_have = remove_duplicates(packages_to_have)
if option("assume_packages_installed"): if option('assume_packages_installed'):
print(colorize("SKIP") + ", Assume package installed") print(colorize("SKIP") + ", Assume package installed")
return return
distname = neutralEnv("distname") distname = neutralEnv('distname')
if distname in ("fedora", "redhat", "centos"): if distname in ('fedora', 'redhat', 'centos'):
package_installer = "sudo dnf install {}" package_installer = 'sudo dnf install {}'
package_checker = "rpm -q --quiet {}" package_checker = 'rpm -q --quiet {}'
elif distname in ("debian", "Ubuntu"): elif distname in ('debian', 'Ubuntu'):
package_installer = "sudo apt-get install {}" package_installer = 'sudo apt-get install {}'
package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1' package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1'
elif distname == "Darwin": elif distname == 'Darwin':
package_installer = "brew install {}" package_installer = 'brew install {}'
package_checker = "brew ls --version {} > /dev/null" package_checker = 'brew ls --version {} > /dev/null'
packages_to_install = [] packages_to_install = []
for package in packages_to_have: for package in packages_to_have:
@ -184,30 +176,31 @@ class Builder:
print(command) print(command)
subprocess.check_call(command, shell=True) subprocess.check_call(command, shell=True)
else: else:
print(colorize("SKIP") + ", No package to install.") print(colorize("SKIP")+ ", No package to install.")
def run(self): def run(self):
try: try:
print("[INSTALL PACKAGES]") print("[INSTALL PACKAGES]")
if option("dont_install_packages"): if option('dont_install_packages'):
print(colorize("SKIP")) print(colorize("SKIP"))
else: else:
self.install_packages() self.install_packages()
self.finalize_target_steps() self.finalize_target_steps()
print("[SETUP TOOLCHAINS]") print("[SETUP PLATFORMS]")
for config in ConfigInfo.all_running_configs.values(): for platform in PlatformInfo.all_running_platforms.values():
config.finalize_setup() platform.finalize_setup()
print("[PREPARE]") print("[PREPARE]")
self.prepare_sources() self.prepare_sources()
print("[BUILD]") print("[BUILD]")
self.build() self.build()
# No error, clean intermediate file at end of build if needed. # No error, clean intermediate file at end of build if needed.
print("[CLEAN]") print("[CLEAN]")
if option("clean_at_end"): if option('clean_at_end'):
for config in ConfigInfo.all_running_configs.values(): for platform in PlatformInfo.all_running_platforms.values():
config.clean_intermediate_directories() platform.clean_intermediate_directories()
else: else:
print(colorize("SKIP")) print(colorize("SKIP"))
except StopBuild as e: except StopBuild as e:
print(e) print(e)
sys.exit("Stopping build due to errors") sys.exit("Stopping build due to errors")

View File

@ -1,3 +0,0 @@
from .base import *
from . import android, armhf, musl, flatpak, i586, ios, native, neutral, wasm

View File

@ -1,169 +0,0 @@
from .base import ConfigInfo, MetaConfigInfo
from kiwixbuild.utils import pj
from kiwixbuild._global import get_target_step, option
class AndroidConfigInfo(ConfigInfo):
build = "android"
static = True
toolchain_names = ["android-ndk"]
compatible_hosts = ["fedora", "debian"]
def __str__(self):
return "android"
@property
def libdir(self):
return "lib/{}".format(self.arch_full)
@property
def binaries_name(self):
arch_full = self.arch_full
return {
"CC": "{}-{}".format(arch_full, "clang"),
"CXX": "{}-{}".format(arch_full, "clang++"),
"AR": "{}-{}".format(arch_full, "ar"),
"STRIP": "{}-{}".format(arch_full, "strip"),
"RANLIB": "{}-{}".format(arch_full, "ranlib"),
"LD": "{}-{}".format(arch_full, "ld"),
}
def binaries(self):
install_path = self.install_path
binaries = {
k: pj(install_path, "bin", v) for k, v in self.binaries_name.items()
}
binaries["PKGCONFIG"] = "pkg-config"
return binaries
@property
def ndk_builder(self):
return get_target_step("android-ndk", self.name)
@property
def install_path(self):
return self.ndk_builder.install_path
def get_cross_config(self):
extra_libs = [
"-llog",
"-Wl,--exclude-libs,libgcc.a",
"-Wl,--exclude-libs,libunwind.a",
]
extra_cflags = [
"-I{}".format(include_dir) for include_dir in self.get_include_dirs()
]
if hasattr(self, "march"):
extra_libs.append("-march={}".format(self.march))
extra_cflags.append("-march={}".format(self.march))
return {
"exe_wrapper_def": "",
"install_path": self.install_path,
"binaries": self.binaries(),
"root_path": pj(self.install_path, "sysroot"),
"extra_libs": extra_libs,
"extra_cflags": extra_cflags,
"host_machine": {
"system": "Android",
"lsystem": "android",
"cpu_family": self.arch,
"cpu": self.cpu,
"endian": "little",
"abi": self.abi,
},
}
def get_env(self):
env = super().get_env()
root_path = pj(self.install_path, "sysroot")
env["PKG_CONFIG_LIBDIR"] = pj(root_path, "lib", "pkgconfig")
env["NDK_DEBUG"] = "0"
return env
def get_bin_dir(self):
return [pj(self.install_path, "bin")]
def set_comp_flags(self, env):
super().set_comp_flags(env)
root_path = pj(self.install_path, "sysroot")
march = "-march={}".format(self.march) if hasattr(self, "march") else ""
env["CFLAGS"] = (
"-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} {} ".format(
root_path, march
)
+ env["CFLAGS"]
)
env["CXXFLAGS"] = (
"-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} {} ".format(
root_path, march
)
+ env["CXXFLAGS"]
)
env["LDFLAGS"] = "--sysroot={} {} ".format(root_path, march) + env["LDFLAGS"]
def set_compiler(self, env):
binaries = self.binaries()
for k, v in binaries.items():
env[k] = v
@property
def configure_options(self):
yield "--host={}".format(self.arch_full)
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile(
"cmake_android_cross_file.txt", "cmake_cross_file.txt"
)
self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
class AndroidArm(AndroidConfigInfo):
name = "android_arm"
arch = cpu = "arm"
arch_full = "arm-linux-androideabi"
abi = "armeabi-v7a"
march = "armv7-a"
class AndroidArm64(AndroidConfigInfo):
name = "android_arm64"
arch = "arm64"
arch_full = "aarch64-linux-android"
cpu = "aarch64"
abi = "arm64-v8a"
class AndroidX86(AndroidConfigInfo):
name = "android_x86"
arch = abi = "x86"
arch_full = "i686-linux-android"
cpu = "i686"
class AndroidX8664(AndroidConfigInfo):
name = "android_x86_64"
arch = cpu = abi = "x86_64"
arch_full = "x86_64-linux-android"
class Android(MetaConfigInfo):
name = "android"
compatible_hosts = ["fedora", "debian"]
@property
def arch_name(self):
return "multi-linux-android"
@property
def subConfigNames(self):
return ["android_{}".format(arch) for arch in option("android_arch")]
def add_targets(self, targetName, targets):
return super().add_targets(targetName, targets)
def __str__(self):
return self.name
def set_comp_flags(self, env):
pass

View File

@ -1,183 +0,0 @@
from .base import ConfigInfo, MixedMixin
from kiwixbuild.utils import pj
from kiwixbuild._global import get_target_step
# Base config for arm
class ArmConfigInfo(ConfigInfo):
compatible_hosts = ["fedora", "debian", "almalinux"]
def get_cross_config(self):
return {
"binaries": self.binaries,
"exe_wrapper_def": "",
"root_path": self.root_path,
"extra_libs": [],
"extra_cflags": [
"-I{}".format(include_dir) for include_dir in self.get_include_dirs()
],
"host_machine": {
"system": "linux",
"lsystem": "linux",
"cpu_family": self.cpu_family,
"cpu": self.cpu,
"endian": "little",
"abi": "",
},
}
@property
def libdir(self):
return "lib/{}".format(self.arch_full)
@property
def toolchain(self):
return get_target_step(self.build, "neutral")
@property
def root_path(self):
return self.toolchain.build_path
@property
def binaries(self):
binaries = (
(k, "{}-{}".format(self.arch_full, v))
for k, v in (
("CC", "gcc"),
("CXX", "g++"),
("AR", "ar"),
("STRIP", "strip"),
("WINDRES", "windres"),
("RANLIB", "ranlib"),
("LD", "ld"),
("LDSHARED", "g++ -shared"),
)
)
binaries = {k: pj(self.root_path, "bin", v) for k, v in binaries}
binaries["PKGCONFIG"] = "pkg-config"
return binaries
@property
def exe_wrapper_def(self):
try:
which("qemu-arm")
except subprocess.CalledProcessError:
return ""
else:
return "exe_wrapper = 'qemu-arm'"
@property
def configure_options(self):
yield "--host={}".format(self.arch_full)
def get_bin_dir(self):
return [pj(self.root_path, "bin")]
def get_env(self):
env = super().get_env()
env["LD_LIBRARY_PATH"][0:0] = [
pj(self.root_path, self.arch_full, "lib64"),
pj(self.root_path, "lib"),
]
env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig")
env["QEMU_LD_PREFIX"] = pj(self.root_path, self.arch_full, "libc")
env["QEMU_SET_ENV"] = "LD_LIBRARY_PATH={}".format(
":".join(
[pj(self.root_path, self.arch_full, "lib"), str(env["LD_LIBRARY_PATH"])]
)
)
return env
def set_comp_flags(self, env):
super().set_comp_flags(env)
env["CFLAGS"] = (
" -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
+ env["CFLAGS"]
)
env["CXXFLAGS"] = (
" -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
+ env["CXXFLAGS"]
)
def set_compiler(self, env):
for k, v in self.binaries.items():
env[k] = v
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt")
self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
class Armv6(ArmConfigInfo):
build = "armv6"
arch_full = "armv6-rpi-linux-gnueabihf"
toolchain_names = ["armv6"]
cpu_family = "arm"
cpu = "armv6"
class Armv6Dyn(Armv6):
name = "armv6_dyn"
static = False
class Armv6Static(Armv6):
name = "armv6_static"
static = True
class Armv6Mixed(MixedMixin("armv6_static"), Armv6):
name = "armv6_mixed"
static = False
class Armv8(ArmConfigInfo):
build = "armv8"
arch_full = "armv8-rpi3-linux-gnueabihf"
toolchain_names = ["armv8"]
cpu_family = "arm"
cpu = "armv8"
class Armv8Dyn(Armv8):
name = "armv8_dyn"
static = False
class Armv8Static(Armv8):
name = "armv8_static"
static = True
class Armv8Mixed(MixedMixin("armv8_static"), Armv8):
name = "armv8_mixed"
static = False
class Aarch64(ArmConfigInfo):
build = "aarch64"
arch_full = "aarch64-linux-gnu"
toolchain_names = ["aarch64"]
cpu_family = "aarch64"
cpu = "aarch64"
@property
def root_path(self):
return self.toolchain.build_path
class Aarch64Dyn(Aarch64):
name = "aarch64_dyn"
static = False
class Aarch64Static(Aarch64):
name = "aarch64_static"
static = True
class Aarch64Mixed(MixedMixin("aarch64_static"), Aarch64):
name = "aarch64_mixed"
static = False

View File

@ -1,19 +0,0 @@
from .base import ConfigInfo
from kiwixbuild._global import option, neutralEnv
class FlatpakConfigInfo(ConfigInfo):
name = "flatpak"
arch_name = "flatpak"
build = "flatpak"
static = ""
toolchain_names = ["org.kde", "io.qt.qtwebengine"]
compatible_hosts = ["debian", "fedora"]
def __str__(self):
return "flatpak"
def get_env(self):
env = super().get_env()
env["FLATPAK_USER_DIR"] = self.buildEnv.build_dir
return env

View File

@ -1,79 +0,0 @@
import os
from .base import ConfigInfo
from kiwixbuild.utils import which, pj
class I586ConfigInfo(ConfigInfo):
build = "i586"
arch_full = "i586-linux-gnu"
compatible_hosts = ["fedora", "debian"]
def get_cross_config(self):
return {
"binaries": self.binaries,
"exe_wrapper_def": "",
"extra_libs": ["-m32", "-march=i586", "-mno-sse"],
"extra_cflags": [
"-m32",
"-march=i586",
"-mno-sse",
*(
"-I{}".format(include_dir)
for include_dir in self.get_include_dirs()
),
],
"host_machine": {
"system": "linux",
"lsystem": "linux",
"cpu_family": "x86",
"cpu": "i586",
"endian": "little",
"abi": "",
},
}
@property
def configure_options(self):
yield f"--host={self.arch_full}"
@property
def binaries(self):
return {
k: which(v)
for k, v in (
("CC", os.environ.get("CC", "gcc")),
("CXX", os.environ.get("CXX", "g++")),
("AR", "ar"),
("STRIP", "strip"),
("RANLIB", "ranlib"),
("LD", "ld"),
("PKGCONFIG", "pkg-config"),
)
}
def set_comp_flags(self, env):
super().set_comp_flags(env)
env["CFLAGS"] = "-m32 -march=i586 -mno-sse " + env["CFLAGS"]
env["CXXFLAGS"] = "-m32 -march=i586 -mno-sse " + env["CXXFLAGS"]
env["LDFLAGS"] = "-m32 -march=i586 -mno-sse " + env["LDFLAGS"]
def get_bin_dir(self):
return []
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile(
"cmake_i586_cross_file.txt", "cmake_cross_file.txt"
)
self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
class I586Dyn(I586ConfigInfo):
name = "i586_dyn"
static = False
class I586Static(I586ConfigInfo):
name = "i586_static"
static = True

View File

@ -1,249 +0,0 @@
import subprocess
from kiwixbuild._global import option
from kiwixbuild.utils import pj, xrun_find
from .base import ConfigInfo, MetaConfigInfo, MixedMixin
from kiwixbuild.dependencies.apple_xcframework import AppleXCFramework
MIN_MACOS_VERSION = "12.0"
class AppleConfigInfo(ConfigInfo):
build = "iOS"
static = True
compatible_hosts = ["Darwin"]
arch = None
host = None
target = None
sdk_name = None
min_iphoneos_version = None
min_macos_version = None
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._root_path = None
@property
def arch_name(self):
return self.target
@property
def root_path(self):
if self._root_path is None:
command = "xcrun --sdk {} --show-sdk-path".format(self.sdk_name)
self._root_path = subprocess.check_output(command, shell=True)[:-1].decode()
return self._root_path
def __str__(self):
return "iOS"
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile(
"cmake_ios_cross_file.txt", "cmake_cross_file.txt"
)
self.buildEnv.meson_crossfile = self._gen_crossfile(
"meson_ios_cross_file.txt", "meson_cross_file.txt"
)
def get_cross_config(self):
config = {
"root_path": self.root_path,
"binaries": self.binaries,
"exe_wrapper_def": "",
"extra_libs": [
"-isysroot",
self.root_path,
"-arch",
self.arch,
"-target",
self.target,
],
"extra_cflags": [
"-isysroot",
self.root_path,
"-arch",
self.arch,
"-target",
self.target,
*(
"-I{}".format(include_dir)
for include_dir in self.get_include_dirs()
),
],
"host_machine": {
"system": "Darwin",
"lsystem": "darwin",
"cpu_family": self.arch,
"cpu": self.cpu,
"endian": "",
"abi": "",
},
}
if self.min_iphoneos_version:
config["extra_libs"].append(
"-miphoneos-version-min={}".format(self.min_iphoneos_version)
)
config["extra_cflags"].append(
"-miphoneos-version-min={}".format(self.min_iphoneos_version)
)
if self.min_macos_version:
config["extra_libs"].append(
"-mmacosx-version-min={}".format(self.min_macos_version)
)
config["extra_cflags"].append(
"-mmacosx-version-min={}".format(self.min_macos_version)
)
return config
def get_env(self):
env = super().get_env()
cflags = [env["CFLAGS"]]
if self.min_iphoneos_version:
cflags.append("-miphoneos-version-min={}".format(self.min_iphoneos_version))
if self.min_macos_version:
cflags.append("-mmacosx-version-min={}".format(self.min_macos_version))
env["CFLAGS"] = " ".join(cflags)
return env
def set_comp_flags(self, env):
super().set_comp_flags(env)
cflags = [
"-isysroot {}".format(self.root_path),
"-arch {}".format(self.arch),
"-target {}".format(self.target),
env["CFLAGS"],
]
if self.min_iphoneos_version:
cflags.append("-miphoneos-version-min={}".format(self.min_iphoneos_version))
env["CFLAGS"] = " ".join(cflags)
env["CXXFLAGS"] = " ".join(
[
env["CFLAGS"],
"-std=c++11",
env["CXXFLAGS"],
]
)
env["LDFLAGS"] = " ".join(
[
" -arch {}".format(self.arch),
"-isysroot {}".format(self.root_path),
]
)
def get_bin_dir(self):
return [pj(self.root_path, "bin")]
@property
def binaries(self):
return {
"CC": xrun_find("clang"),
"CXX": xrun_find("clang++"),
"AR": xrun_find("ar"),
"STRIP": xrun_find("strip"),
"RANLIB": xrun_find("ranlib"),
"LD": xrun_find("ld"),
"PKGCONFIG": "pkg-config",
}
@property
def configure_options(self):
yield f"--host={self.host}"
class iOSArm64(AppleConfigInfo):
name = "iOS_arm64"
arch = cpu = "arm64"
host = "arm-apple-darwin"
target = "aarch64-apple-ios"
sdk_name = "iphoneos"
min_iphoneos_version = "15.0"
class iOSx64Simulator(AppleConfigInfo):
name = "iOSSimulator_x86_64"
arch = cpu = "x86_64"
host = "x86_64-apple-darwin"
target = "x86-apple-ios-simulator"
sdk_name = "iphonesimulator"
min_iphoneos_version = "15.0"
class iOSArm64Simulator(AppleConfigInfo):
name = "iOSSimulator_arm64"
arch = cpu = "arm64"
host = "arm-apple-darwin"
target = "aarch64-apple-ios-simulator"
sdk_name = "iphonesimulator"
min_iphoneos_version = "15.0"
class macOSArm64(AppleConfigInfo):
name = "macOS_arm64_static"
arch = cpu = "arm64"
host = "aarch64-apple-darwin"
target = "arm64-apple-macos"
sdk_name = "macosx"
min_iphoneos_version = None
min_macos_version = MIN_MACOS_VERSION
class macOSArm64Mixed(MixedMixin("macOS_arm64_static"), AppleConfigInfo):
name = "macOS_arm64_mixed"
arch = cpu = "arm64"
host = "aarch64-apple-darwin"
target = "arm64-apple-macos"
sdk_name = "macosx"
min_iphoneos_version = None
min_macos_version = MIN_MACOS_VERSION
class macOSx64(AppleConfigInfo):
name = "macOS_x86_64"
arch = cpu = "x86_64"
host = "x86_64-apple-darwin"
target = "x86_64-apple-macos"
sdk_name = "macosx"
min_iphoneos_version = None
min_macos_version = MIN_MACOS_VERSION
class IOS(MetaConfigInfo):
name = "iOS_multi"
compatible_hosts = ["Darwin"]
@property
def arch_name(self):
return self.name
@property
def subConfigNames(self):
return ["iOS_{}".format(arch) for arch in option("ios_arch")]
def add_targets(self, targetName, targets):
super().add_targets(targetName, targets)
return ConfigInfo.add_targets(self, "_ios_fat_lib", targets)
def __str__(self):
return self.name
class AppleStaticAll(MetaConfigInfo):
name = "apple_all_static"
compatible_hosts = ["Darwin"]
@property
def arch_name(self):
return self.name
@property
def subConfigNames(self):
return AppleXCFramework.subConfigNames
def add_targets(self, targetName, targets):
super().add_targets(targetName, targets)
return ConfigInfo.add_targets(self, "apple_xcframework", targets)
def __str__(self):
return self.name

View File

@ -1,157 +0,0 @@
from .base import ConfigInfo, MixedMixin
from kiwixbuild.utils import pj
from kiwixbuild._global import get_target_step
class MuslConfigInfo(ConfigInfo):
compatible_hosts = ["fedora", "debian"]
def get_cross_config(self):
return {
"binaries": self.binaries,
"exe_wrapper_def": "",
"root_path": self.root_path,
"extra_libs": [],
"extra_cflags": [
"-I{}".format(include_dir) for include_dir in self.get_include_dirs()
],
"host_machine": {
"system": "linux",
"lsystem": "linux",
"cpu_family": self.cpu_family,
"cpu": self.cpu,
"endian": "little",
"abi": "",
},
}
@property
def toolchain(self):
return get_target_step(self.build, "neutral")
@property
def root_path(self):
return self.toolchain.build_path
@property
def binaries(self):
binaries = (
(k, "{}-{}".format(self.arch_full, v))
for k, v in (
("CC", "gcc"),
("CXX", "g++"),
("AR", "ar"),
("STRIP", "strip"),
("WINDRES", "windres"),
("RANLIB", "ranlib"),
("LD", "ld"),
("LDSHARED", "g++ -shared"),
)
)
binaries = {k: pj(self.root_path, "bin", v) for k, v in binaries}
binaries["PKGCONFIG"] = "pkg-config"
return binaries
@property
def exe_wrapper_def(self):
try:
which(self.qemu)
except subprocess.CalledProcessError:
return ""
except AttributeError:
return ""
else:
return f"exe_wrapper = '{self.qemu}'"
@property
def configure_options(self):
return [f"--host={self.arch_full}"]
def get_bin_dir(self):
return [pj(self.root_path, "bin")]
def get_env(self):
env = super().get_env()
env["LD_LIBRARY_PATH"][0:0] = [
pj(self.root_path, self.arch_full, "lib64"),
pj(self.root_path, "lib"),
]
env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig")
env["QEMU_LD_PREFIX"] = pj(self.root_path, self.arch_full, "libc")
env["QEMU_SET_ENV"] = "LD_LIBRARY_PATH={}".format(
":".join(
[pj(self.root_path, self.arch_full, "lib"), str(env["LD_LIBRARY_PATH"])]
)
)
return env
def set_comp_flags(self, env):
super().set_comp_flags(env)
env["LD_LIBRARY_PATH"].insert(0, pj(self.root_path, self.arch_full, "lib"))
env["CFLAGS"] = (
" -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
+ env["CFLAGS"]
)
env["CXXFLAGS"] = (
" -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
+ env["CXXFLAGS"]
)
def set_compiler(self, env):
for k, v in self.binaries.items():
env[k] = v
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt")
self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
class Aarch64MuslConfigInfo(MuslConfigInfo):
build = "aarch64_musl"
arch_full = "aarch64-linux-musl"
toolchain_names = ["aarch64_musl"]
libdir = "lib/aarch64-linux-musl"
cpu_family = "arm"
cpu = "armhf"
qemu = "qemu-arm"
class Aarch64MuslDyn(Aarch64MuslConfigInfo):
name = "aarch64_musl_dyn"
static = False
class Aarch64MuslStatic(Aarch64MuslConfigInfo):
name = "aarch64_musl_static"
static = True
class Aarch64MuslMixed(MixedMixin("aarch64_musl_static"), Aarch64MuslConfigInfo):
name = "aarch64_musl_mixed"
static = False
class X86_64MuslConfigInfo(MuslConfigInfo):
build = "x86-64_musl"
arch_full = "x86_64-linux-musl"
toolchain_names = ["x86-64_musl"]
libdir = "lib/x86_64-linux-musl"
cpu_family = "x86_64"
cpu = "x86_64"
class X86_64MuslDyn(X86_64MuslConfigInfo):
name = "x86-64_musl_dyn"
static = False
class X86_64MuslStatic(X86_64MuslConfigInfo):
name = "x86-64_musl_static"
static = True
class x86_64MuslMixed(MixedMixin("x86-64_musl_static"), X86_64MuslConfigInfo):
name = "x86-64_musl_mixed"
static = False

View File

@ -1,44 +0,0 @@
from .base import ConfigInfo, MixedMixin
from kiwixbuild.utils import pj
from kiwixbuild._global import option, neutralEnv
from kiwixbuild.configs.ios import MIN_MACOS_VERSION
import sysconfig
import platform
import sys
class NativeConfigInfo(ConfigInfo):
build = "native"
def get_env(self):
env = super().get_env()
if neutralEnv("distname") == "fedora":
env["QT_SELECT"] = "5-64"
if neutralEnv("distname") == "Darwin":
env["CFLAGS"] += f"-mmacosx-version-min={MIN_MACOS_VERSION}"
return env
@property
def arch_name(self):
if sys.platform == "darwin":
return f"{platform.machine()}-apple-darwin"
return sysconfig.get_platform()
class NativeDyn(NativeConfigInfo):
name = "native_dyn"
static = False
compatible_hosts = ["fedora", "debian", "Darwin", "almalinux", "Windows"]
class NativeStatic(NativeConfigInfo):
name = "native_static"
static = True
compatible_hosts = ["fedora", "debian", "Darwin", "almalinux", "Windows"]
class NativeMixed(MixedMixin("native_static"), NativeConfigInfo):
name = "native_mixed"
static = False
compatible_hosts = ["fedora", "debian", "Darwin", "almalinux", "Windows"]

View File

@ -1,11 +0,0 @@
from .base import ConfigInfo
class NeutralConfigInfo(ConfigInfo):
name = "neutral"
arch_name = "neutral"
static = ""
compatible_hosts = ["fedora", "debian", "Darwin"]
def __str__(self):
return "neutral"

View File

@ -1,113 +0,0 @@
from .base import ConfigInfo
from kiwixbuild.utils import pj
from kiwixbuild._global import get_target_step
class WasmConfigInfo(ConfigInfo):
name = "wasm"
static = True
build = "wasm"
arch_full = "wasm64-emscripten"
libdir = "lib"
# arch_full = 'wasm64-linux'
toolchain_names = ["emsdk"]
compatible_hosts = ["fedora", "debian"]
exe_wrapper_def = ""
def get_cross_config(self):
return {
"binaries": self.binaries,
"exe_wrapper_def": "",
"root_path": self.root_path,
"extra_libs": [],
"extra_cflags": [],
"host_machine": {
"system": "emscripten",
"lsystem": "emscripten",
"cpu_family": "wasm64",
"cpu": "wasm64",
"endian": "little",
"abi": "",
},
}
@property
def wasm_sdk(self):
return get_target_step("emsdk", self.name)
@property
def install_path(self):
return self.wasm_sdk.install_path
@property
def root_path(self):
return self.install_path
@property
def binaries(self):
binaries = (
("CC", "emcc"),
("CXX", "em++"),
("AR", "emar"),
("STRIP", "emstrip"),
("WINDRES", "windres"),
("RANLIB", "emranlib"),
("LD", "wasm-ld"),
)
binaries = {
k: pj(self.install_path, "upstream", "emscripten", v) for k, v in binaries
}
binaries["PKGCONFIG"] = "pkg-config"
return binaries
@property
def configure_options(self):
# return ""
return [f"--host={self.arch_full}"]
@property
def configure_wrapper(self):
return "emconfigure"
@property
def make_wrapper(self):
return "emmake"
def get_bin_dir(self):
return [pj(self.install_path, "bin")]
def get_env(self):
env = super().get_env()
env["PATH"].extend(
[
self.install_path,
pj(self.install_path, "upstream", "emscripten"),
pj(self.install_path, "node", "14.18.2_64bit", "bin"),
]
)
env["EMSDK"] = self.install_path
env["EMSDK_NODE"] = pj(
self.install_path, "node", "14.18.2_64bit", "bin", "node"
)
return env
def set_comp_flags(self, env):
super().set_comp_flags(env)
env["CFLAGS"] = (
" -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
+ env["CFLAGS"]
)
env["CXXFLAGS"] = (
" -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
+ env["CXXFLAGS"]
)
def set_compiler(self, env):
for k, v in self.binaries.items():
env[k] = v
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt")
self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")

View File

@ -1,103 +0,0 @@
import subprocess
from .base import ConfigInfo
from kiwixbuild.utils import which, pj
from kiwixbuild._global import neutralEnv
class Win64ConfigInfo(ConfigInfo):
extra_libs = [
"-lmingw32",
"-lwinmm",
"-lws2_32",
"-lshlwapi",
"-lrpcrt4",
"-lmsvcr100",
"-liphlpapi",
"-lshell32",
"-lkernel32",
]
build = "win64"
compatible_hosts = ["fedora", "debian"]
arch_full = "x86_64-w64-mingw32"
def get_cross_config(self):
return {
"exe_wrapper_def": self.exe_wrapper_def,
"binaries": self.binaries,
"root_path": self.root_path,
"extra_libs": self.extra_libs,
"extra_cflags": ["-DWIN32"],
"host_machine": {
"system": "Windows",
"lsystem": "windows",
"cpu_family": "x86_64",
"cpu": "x86_64",
"endian": "little",
"abi": "",
},
}
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt")
self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
@property
def root_path(self):
root_paths = {
"fedora": "/usr/x86_64-w64-mingw32/sys-root/mingw",
"debian": "/usr/x86_64-w64-mingw32",
}
return root_paths[neutralEnv("distname")]
@property
def binaries(self):
return {
k: which("{}-{}".format(self.arch_full, v))
for k, v in (
("CC", "gcc"),
("CXX", "g++"),
("AR", "ar"),
("STRIP", "strip"),
("WINDRES", "windres"),
("RANLIB", "ranlib"),
("PKGCONFIG", "pkg-config"),
)
}
@property
def exe_wrapper_def(self):
try:
which("wine")
except subprocess.CalledProcessError:
return ""
else:
return "exe_wrapper = 'wine'"
@property
def configure_options(self):
return [f"--host={self.arch_full}"]
def set_compiler(self, env):
for k, v in self.binaries.items():
env[k] = v
def get_bin_dir(self):
return [pj(self.root_path, "bin")]
def get_env(self):
env = super().get_env()
env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig")
env["LIBS"] = " ".join(self.extra_libs) + " " + env["LIBS"]
return env
class Win64Dyn(Win64ConfigInfo):
name = "win64_dyn"
static = False
class Win64Static(Win64ConfigInfo):
name = "win64_static"
static = True

View File

@ -1,14 +1,14 @@
from .base import * from .base import *
from . import ( from . import (
all_dependencies, all_dependencies,
boostregex, android_ndk,
tc_android_ndk,
aria2, aria2,
tc_armhf, armhf,
tc_musl, arm_musl,
docoptcpp, docoptcpp,
tc_emsdk, emsdk,
tc_flatpak, flatpak,
gumbo, gumbo,
icu4c, icu4c,
ios_fat_lib, ios_fat_lib,
@ -28,5 +28,5 @@ from . import (
zim_tools, zim_tools,
zim_testing_suite, zim_testing_suite,
zlib, zlib,
zstd, zstd
) )

View File

@ -1,61 +1,30 @@
from os import environ from .base import (
Dependency,
from .base import Dependency, NoopSource, NoopBuilder NoopSource,
NoopBuilder)
from kiwixbuild._global import neutralEnv from kiwixbuild._global import neutralEnv
class AllBaseDependencies(Dependency): class AllBaseDependencies(Dependency):
name = "alldependencies" name = "alldependencies"
Source = NoopSource Source = NoopSource
class Builder(NoopBuilder): class Builder(NoopBuilder):
@classmethod @classmethod
def get_dependencies(cls, configInfo, allDeps): def get_dependencies(cls, platformInfo, allDeps):
if configInfo.build == "wasm" or environ.get("OS_NAME") == "manylinux": if platformInfo.build == "wasm":
return ["zlib", "lzma", "zstd", "icu4c", "xapian-core"] return ['zlib', 'lzma', 'zstd', 'icu4c', 'xapian-core']
if neutralEnv("distname") == "Windows": base_deps = ['zlib', 'lzma', 'zstd', 'xapian-core', 'pugixml', 'libcurl', 'icu4c', 'mustache', 'libmicrohttpd', 'zim-testing-suite']
base_deps = [ # Add specific dependencies depending of the platform
"zlib", if platformInfo.build not in ('android', 'iOS'):
"zstd", # For zimtools
"xapian-core", base_deps += ['docoptcpp']
"zim-testing-suite", if platformInfo.build != 'win32':
"icu4c", # zimwriterfs
"boostregex", base_deps += ['libmagic', 'gumbo']
"docoptcpp", if platformInfo.build == 'native' and neutralEnv('distname') != 'Darwin':
] # We compile kiwix-desktop only on native and not on `Darwin`
# So we need aria2 only there
if not configInfo.name.endswith("_dyn"): base_deps += ['aria2']
base_deps += [
"pugixml",
"libcurl",
"mustache",
"libmicrohttpd",
]
else:
base_deps = [
"zlib",
"lzma",
"zstd",
"xapian-core",
"pugixml",
"libcurl",
"icu4c",
"mustache",
"libmicrohttpd",
"zim-testing-suite",
]
# Add specific dependencies depending of the config
if configInfo.build not in ("android", "iOS"):
# For zimtools
base_deps += ["docoptcpp", "libmagic", "gumbo"]
if (
configInfo.build == "native"
and neutralEnv("distname") != "Darwin"
):
# We compile kiwix-desktop only on native and not on `Darwin`
# So we need aria2 only there
base_deps += ["aria2"]
return base_deps return base_deps

View File

@ -0,0 +1,79 @@
import os
from .base import Dependency, ReleaseDownload, Builder
from kiwixbuild.utils import Remotefile, add_execution_right, run_command
pj = os.path.join
class android_ndk(Dependency):
dont_skip = True
neutral = False
name = 'android-ndk'
gccver = '4.9.x'
api = '24'
class Source(ReleaseDownload):
archive = Remotefile('android-ndk-r21e-linux-x86_64.zip',
'ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e',
'https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip')
@property
def source_dir(self):
return self.target.full_name()
class Builder(Builder):
@property
def install_path(self):
return self.build_path
@property
def api(self):
return self.target.api
@property
def platform(self):
return 'android-'+self.api
@property
def arch(self):
return self.buildEnv.platformInfo.arch
@property
def arch_full(self):
return self.buildEnv.platformInfo.arch_full
def _build_platform(self, context):
context.try_skip(self.build_path)
script = pj(self.source_path, 'build/tools/make_standalone_toolchain.py')
add_execution_right(script)
command = '{script} --arch={arch} --api={api} --install-dir={install_dir} --force'
command = command.format(
script=script,
arch=self.arch,
api=self.api,
install_dir=self.install_path
)
env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False)
run_command(command, self.build_path, context, env=env)
def _fix_permission_right(self, context):
context.try_skip(self.build_path)
bin_dirs = [pj(self.install_path, 'bin'),
pj(self.install_path, self.arch_full, 'bin'),
pj(self.install_path, 'libexec', 'gcc', self.arch_full, self.target.gccver)
]
for root, dirs, files in os.walk(self.install_path):
if not root in bin_dirs:
continue
for file_ in files:
file_path = pj(root, file_)
if os.path.islink(file_path):
continue
add_execution_right(file_path)
def build(self):
self.command('build_platform', self._build_platform)
self.command('fix_permission_right', self._fix_permission_right)

View File

@ -1,126 +0,0 @@
import os
import shutil
from pathlib import Path
from kiwixbuild.configs import ConfigInfo
from kiwixbuild.utils import pj, run_command
from .base import Dependency, NoopSource, Builder as BaseBuilder
class AppleXCFramework(Dependency):
name = "apple_xcframework"
subConfigNames = [
"macOS_x86_64",
"macOS_arm64_static",
"iOS_arm64",
"iOSSimulator_x86_64",
"iOSSimulator_arm64",
]
Source = NoopSource
class Builder(BaseBuilder):
@property
def all_subconfigs(self):
return self.buildEnv.configInfo.subConfigNames
@property
def macos_subconfigs(self):
return [
target for target in self.all_subconfigs if target.startswith("macOS")
]
@property
def iossimulator_subconfigs(self):
return [
target
for target in self.all_subconfigs
if target.startswith("iOSSimulator")
]
@property
def ios_subconfigs(self):
return [
target for target in self.all_subconfigs if target.startswith("iOS_")
]
@classmethod
def get_dependencies(cls, configInfo, alldeps):
return [(target, "libkiwix") for target in AppleXCFramework.subConfigNames]
@property
def final_path(self):
return pj(self.buildEnv.install_dir, "lib", "CoreKiwix.xcframework")
def _remove_if_exists(self, context):
if not os.path.exists(self.final_path):
return
shutil.rmtree(self.final_path)
def _merge_libs(self, context):
"""create merged.a in all targets to bundle all static archives"""
xcf_libs = []
for target in self.all_subconfigs:
static_ars = []
cfg = ConfigInfo.get_config(target)
lib_dir = pj(cfg.buildEnv.install_dir, "lib")
static_ars = [str(f) for f in Path(lib_dir).glob("*.a")]
# create merged.a from all *.a in install_dir/lib
command = ["libtool", "-static", "-o", "merged.a", *static_ars]
run_command(command, lib_dir, context)
# will be included in xcframework
if target in self.ios_subconfigs:
xcf_libs.append(pj(lib_dir, "merged.a"))
return xcf_libs
def make_fat_with(self, configs, folder_name, context):
"""create fat merged.a in {folder_name} install/lib with {configs}"""
libs = []
for target in configs:
cfg = ConfigInfo.get_config(target)
libs.append(pj(cfg.buildEnv.install_dir, "lib", "merged.a"))
fat_dir = pj(self.buildEnv.build_dir, folder_name)
os.makedirs(fat_dir, exist_ok=True)
output_merged = pj(fat_dir, "merged.a")
command = ["lipo", "-create", "-output", output_merged, *libs]
run_command(command, self.buildEnv.build_dir, context)
return [output_merged]
def _build_xcframework(self, xcf_libs, context):
# create xcframework
ref_conf = ConfigInfo.get_config(self.macos_subconfigs[0])
command = ["xcodebuild", "-create-xcframework"]
for lib in xcf_libs:
command += [
"-library",
lib,
"-headers",
pj(ref_conf.buildEnv.install_dir, "include"),
]
command += ["-output", self.final_path]
run_command(command, self.buildEnv.build_dir, context)
def build(self):
xcf_libs = []
self.command("remove_if_exists", self._remove_if_exists)
xcf_libs += self.command("merge_libs", self._merge_libs)
xcf_libs += self.command(
"make_macos_fat",
self.make_fat_with,
self.macos_subconfigs,
"macOS_fat",
)
xcf_libs += self.command(
"make_simulator_fat",
self.make_fat_with,
self.iossimulator_subconfigs,
"iOS-simulator_fat",
)
self.command("build_xcframework", self._build_xcframework, xcf_libs)

View File

@ -1,53 +1,24 @@
from .base import Dependency, ReleaseDownload, MakeBuilder, NoopBuilder from .base import (
Dependency,
ReleaseDownload,
MakeBuilder
)
from kiwixbuild.utils import Remotefile, run_command, pj from kiwixbuild.utils import Remotefile, run_command
import platform
from shutil import copy2
# Important: in case of aria2c update,
# 'scripts/create_kiwix-desktop_appImage.sh' should not be forgotten!
class Aria2(Dependency): class Aria2(Dependency):
name = "aria2" name = "aria2"
if platform.system() == "Windows": class Source(ReleaseDownload):
archive = Remotefile('aria2-1.36.0.tar.xz',
'58d1e7608c12404f0229a3d9a4953d0d00c18040504498b483305bcb3de907a5',
'https://github.com/aria2/aria2/releases/download/release-1.36.0/aria2-1.36.0.tar.xz')
class Source(ReleaseDownload): def _post_prepare_script(self, context):
archive = Remotefile( context.try_skip(self.extract_path)
"aria2-1.37.0-win-64bit-build1.zip", command = "autoreconf -i"
"67d015301eef0b612191212d564c5bb0a14b5b9c4796b76454276a4d28d9b288", run_command(command, self.extract_path, context)
"https://dev.kiwix.org/kiwix-desktop/aria2-1.37.0-win-64bit-build1.zip",
)
class Builder(NoopBuilder): class Builder(MakeBuilder):
def build(self): dependencies = ['zlib']
self.command("copy_binary", self._copy_binary) configure_option = "--disable-libaria2 --disable-websocket --without-sqlite3"
def _copy_binary(self, context):
context.try_skip(self.build_path)
copy2(
pj(self.source_path, "aria2c.exe"),
pj(self.buildEnv.install_dir, "bin"),
)
else:
class Source(ReleaseDownload):
archive = Remotefile(
"aria2-1.37.0.tar.xz",
"60a420ad7085eb616cb6e2bdf0a7206d68ff3d37fb5a956dc44242eb2f79b66b",
"https://dev.kiwix.org/kiwix-desktop/aria2-1.37.0.tar.xz",
)
def _post_prepare_script(self, context):
context.try_skip(self.extract_path)
command = ["autoreconf", "-i"]
run_command(command, self.extract_path, context)
class Builder(MakeBuilder):
dependencies = ["zlib"]
configure_options = [
"--disable-libaria2",
"--disable-websocket",
"--without-sqlite3",
]

View File

@ -0,0 +1,14 @@
from .base import Dependency, ReleaseDownload, NoopBuilder
from kiwixbuild.utils import Remotefile
class aarch64_musl_toolchain(Dependency):
dont_skip = True
neutral = True
name = "aarch64_musl"
class Source(ReleaseDownload):
archive = Remotefile('aarch64-linux-musl-cross.tgz',
'c909817856d6ceda86aa510894fa3527eac7989f0ef6e87b5721c58737a06c38',
'https://musl.cc/aarch64-linux-musl-cross.tgz')
Builder = NoopBuilder

View File

@ -0,0 +1,47 @@
from .base import Dependency, ReleaseDownload, NoopBuilder
from kiwixbuild.utils import Remotefile
# The arm toolchains
# This is based on toolchains published here : https://github.com/tttapa/docker-arm-cross-toolchain
base_url = "https://github.com/tttapa/docker-arm-cross-toolchain/releases/download/0.1.0/"
aarch_base_url = 'https://master.dl.sourceforge.net/project/raspberry-pi-cross-compilers/Bonus%20Raspberry%20Pi%20GCC%2064-Bit%20Toolchains/Raspberry%20Pi%20GCC%2064-Bit%20Cross-Compiler%20Toolchains/Stretch/GCC%206.3.0/'
class armv6_toolchain(Dependency):
dont_skip = True
neutral = True
name = 'armv6'
class Source(ReleaseDownload):
archive = Remotefile('x-tools-armv6-rpi-linux-gnueabihf.tar.xz',
'4c371c4c5b55ebd1f3d7dd26b14703632d9ba47423f901bcd9303d83ad444434',
base_url + 'x-tools-armv6-rpi-linux-gnueabihf.tar.xz')
Builder = NoopBuilder
class armv8_toolchain(Dependency):
dont_skip = True
neutral = True
name = 'armv8'
class Source(ReleaseDownload):
archive = Remotefile('x-tools-armv8-rpi-linux-gnueabihf.tar.xz',
'cc28f5c3f6a3e7d9985f98779c4e72224b4eb5a7e4dc2bcdefd90cb241fb94a5',
base_url + 'x-tools-armv8-rpi3-linux-gnueabihf.tar.xz')
Builder = NoopBuilder
class aarch64_toolchain(Dependency):
dont_skip = True
neutral = True
name = "aarch64"
class Source(ReleaseDownload):
archive = Remotefile('cross-gcc-6.3.0-pi_64.tar.gz',
'1b048bb8886ad63d21797cd9129fc37b9ea0dfaac7e3c36f888aa16fbec1d320',
aarch_base_url + 'cross-gcc-6.3.0-pi_64.tar.gz')
Builder = NoopBuilder

View File

@ -2,19 +2,8 @@ import subprocess
import os import os
import shutil import shutil
import time import time
import platform
from kiwixbuild.utils import ( from kiwixbuild.utils import pj, Context, SkipCommand, WarningMessage, extract_archive, Defaultdict, StopBuild, run_command, colorize
pj,
Context,
SkipCommand,
WarningMessage,
extract_archive,
StopBuild,
run_command,
colorize,
copy_tree,
)
from kiwixbuild.versions import main_project_versions, base_deps_versions from kiwixbuild.versions import main_project_versions, base_deps_versions
from kiwixbuild._global import neutralEnv, option, get_target_step from kiwixbuild._global import neutralEnv, option, get_target_step
@ -24,8 +13,8 @@ SCRIPT_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
class _MetaDependency(type): class _MetaDependency(type):
def __new__(cls, name, bases, dct): def __new__(cls, name, bases, dct):
_class = type.__new__(cls, name, bases, dct) _class = type.__new__(cls, name, bases, dct)
if name != "Dependency": if name != 'Dependency':
dep_name = dct["name"] dep_name = dct['name']
Dependency.all_deps[dep_name] = _class Dependency.all_deps[dep_name] = _class
return _class return _class
@ -40,7 +29,7 @@ class Dependency(metaclass=_MetaDependency):
def version(cls): def version(cls):
if cls.name in base_deps_versions: if cls.name in base_deps_versions:
return base_deps_versions[cls.name] return base_deps_versions[cls.name]
elif option("make_release"): elif option('make_release'):
return main_project_versions.get(cls.name, None) return main_project_versions.get(cls.name, None)
return None return None
@ -53,9 +42,8 @@ class Dependency(metaclass=_MetaDependency):
class Source: class Source:
"""Base Class to the real preparator """Base Class to the real preparator
A source preparator must install source in the self.source_dir attribute A source preparator must install source in the self.source_dir attribute
inside the neutralEnv.source_dir.""" inside the neutralEnv.source_dir."""
def __init__(self, target): def __init__(self, target):
self.target = target self.target = target
@ -73,22 +61,22 @@ class Source:
@property @property
def source_path(self): def source_path(self):
return pj(neutralEnv("source_dir"), self.source_dir) return pj(neutralEnv('source_dir'), self.source_dir)
@property @property
def _log_dir(self): def _log_dir(self):
return neutralEnv("log_dir") return neutralEnv('log_dir')
def _patch(self, context): def _patch(self, context):
context.try_skip(self.source_path) context.try_skip(self.source_path)
for p in self.patches: for p in self.patches:
patch_file_path = pj(SCRIPT_DIR, "patches", p) patch_file_path = pj(SCRIPT_DIR, 'patches', p)
patch_command = [*neutralEnv("patch_command"), "-p1", "-i", patch_file_path] patch_command = "patch -p1 -i {patch}".format(patch=patch_file_path)
run_command(patch_command, self.source_path, context) run_command(patch_command, self.source_path, context)
def command(self, name, function, *args): def command(self, name, function, *args):
print(" {} {} : ".format(name, self.name), end="", flush=True) print(" {} {} : ".format(name, self.name), end="", flush=True)
log = pj(self._log_dir, "cmd_{}_{}.log".format(name, self.name)) log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
context = Context(name, log, True) context = Context(name, log, True)
try: try:
start_time = time.time() start_time = time.time()
@ -104,7 +92,7 @@ class Source:
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
print(colorize("ERROR")) print(colorize("ERROR"))
try: try:
with open(log, "r") as f: with open(log, 'r') as f:
print(f.read()) print(f.read())
except: except:
pass pass
@ -122,47 +110,30 @@ class NoopSource(Source):
class ReleaseDownload(Source): class ReleaseDownload(Source):
archive_top_dir = None archive_top_dir = None
@property
def archives(self):
return (self.archive,)
@property @property
def extract_path(self): def extract_path(self):
return pj(neutralEnv("source_dir"), self.source_dir) return pj(neutralEnv('source_dir'), self.source_dir)
def _download(self, context): def _download(self, context):
context.try_skip(neutralEnv("archive_dir"), self.full_name) context.try_skip(neutralEnv('archive_dir'), self.full_name)
archive_iter = iter(self.archives) neutralEnv('download')(self.archive)
archive = next(archive_iter, None)
while archive:
try:
neutralEnv("download")(archive)
except SkipCommand as e:
archive = next(archive_iter, None)
if not archive:
raise e
continue
archive = next(archive_iter, None)
def _extract(self, context): def _extract(self, context):
context.try_skip(self.extract_path) context.try_skip(self.extract_path)
if os.path.exists(self.extract_path): if os.path.exists(self.extract_path):
shutil.rmtree(self.extract_path) shutil.rmtree(self.extract_path)
for archive in self.archives: extract_archive(pj(neutralEnv('archive_dir'), self.archive.name),
extract_archive( neutralEnv('source_dir'),
pj(neutralEnv("archive_dir"), archive.name), topdir=self.archive_top_dir,
neutralEnv("source_dir"), name=self.source_dir)
topdir=self.archive_top_dir,
name=self.source_dir,
)
def prepare(self): def prepare(self):
self.command("download", self._download) self.command('download', self._download)
self.command("extract", self._extract) self.command('extract', self._extract)
if hasattr(self, "patches"): if hasattr(self, 'patches'):
self.command("patch", self._patch) self.command('patch', self._patch)
if hasattr(self, "_post_prepare_script"): if hasattr(self, '_post_prepare_script'):
self.command("post_prepare_script", self._post_prepare_script) self.command('post_prepare_script', self._post_prepare_script)
class GitClone(Source): class GitClone(Source):
@ -175,66 +146,70 @@ class GitClone(Source):
@property @property
def source_dir(self): def source_dir(self):
if option("make_release"): if option('make_release'):
return "{}_release".format(self.git_dir) return "{}_release".format(self.git_dir)
else: else:
return self.git_dir return self.git_dir
@property @property
def git_path(self): def git_path(self):
return pj(neutralEnv("source_dir"), self.source_dir) return pj(neutralEnv('source_dir'), self.source_dir)
@property @property
def git_ref(self): def git_ref(self):
if option("make_release"): if option('make_release'):
return self.release_git_ref return self.release_git_ref
else: else:
return self.base_git_ref return self.base_git_ref
def _git_init(self, context): def _git_init(self, context):
if option("fast_clone") and self.force_full_clone == False: if option('fast_clone') and self.force_full_clone == False:
command = [ command = "git clone --depth=1 --branch {} {} {}".format(
*neutralEnv("git_command"), self.git_ref, self.git_remote, self.source_dir)
"clone", run_command(command, neutralEnv('source_dir'), context)
"--depth=1",
"--branch",
self.git_ref,
self.git_remote,
self.source_dir,
]
run_command(command, neutralEnv("source_dir"), context)
else: else:
command = [ command = "git clone {} {}".format(self.git_remote, self.source_dir)
*neutralEnv("git_command"), run_command(command, neutralEnv('source_dir'), context)
"clone", command = "git checkout {}".format(self.git_ref)
self.git_remote,
self.source_dir,
]
run_command(command, neutralEnv("source_dir"), context)
command = [*neutralEnv("git_command"), "checkout", self.git_ref]
run_command(command, self.git_path, context) run_command(command, self.git_path, context)
def _git_update(self, context): def _git_update(self, context):
command = [*neutralEnv("git_command"), "fetch", "origin", self.git_ref] command = "git fetch origin {}".format(self.git_ref)
run_command(command, self.git_path, context) run_command(command, self.git_path, context)
try: try:
command = [ command = "git merge --ff-only origin/{}".format(self.git_ref)
*neutralEnv("git_command"),
"merge",
"--ff-only",
f"origin/{self.git_ref}",
]
run_command(command, self.git_path, context) run_command(command, self.git_path, context)
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
raise WarningMessage("Cannot update, please check log for information") raise WarningMessage("Cannot update, please check log for information")
def prepare(self): def prepare(self):
if not os.path.exists(self.git_path): if not os.path.exists(self.git_path):
self.command("gitinit", self._git_init) self.command('gitinit', self._git_init)
else: else:
self.command("gitupdate", self._git_update) self.command('gitupdate', self._git_update)
if hasattr(self, "_post_prepare_script"): if hasattr(self, '_post_prepare_script'):
self.command("post_prepare_script", self._post_prepare_script) self.command('post_prepare_script', self._post_prepare_script)
class SvnClone(Source):
@property
def source_dir(self):
return self.svn_dir
@property
def svn_path(self):
return pj(neutralEnv('source_dir'), self.svn_dir)
def _svn_export(self, context):
if os.path.exists(self.svn_path):
raise SkipCommand()
command = "svn export {} {}".format(self.svn_remote, self.svn_dir)
run_command(command, neutralEnv('source_dir'), context)
def prepare(self):
self.command('svnexport', self._svn_export)
if hasattr(self, 'patches'):
self.command('patch', self._patch)
class Builder: class Builder:
@ -247,7 +222,7 @@ class Builder:
self.buildEnv = buildEnv self.buildEnv = buildEnv
@classmethod @classmethod
def get_dependencies(cls, configInfo, allDeps): def get_dependencies(cls, platformInfo, allDeps):
return cls.dependencies return cls.dependencies
@property @property
@ -271,7 +246,7 @@ class Builder:
def command(self, name, function, *args): def command(self, name, function, *args):
print(" {} {} : ".format(name, self.name), end="", flush=True) print(" {} {} : ".format(name, self.name), end="", flush=True)
log = pj(self._log_dir, "cmd_{}_{}.log".format(name, self.name)) log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
context = Context(name, log, self.target.force_native_build) context = Context(name, log, self.target.force_native_build)
if self.target.force_build: if self.target.force_build:
context.no_skip = True context.no_skip = True
@ -289,7 +264,7 @@ class Builder:
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
print(colorize("ERROR")) print(colorize("ERROR"))
try: try:
with open(log, "r") as f: with open(log, 'r') as f:
print(f.read()) print(f.read())
except: except:
pass pass
@ -299,43 +274,37 @@ class Builder:
raise raise
def build(self): def build(self):
if hasattr(self, "_pre_build_script"): if hasattr(self, '_pre_build_script'):
self.command("pre_build_script", self._pre_build_script) self.command('pre_build_script', self._pre_build_script)
self.command("configure", self._configure) self.command('configure', self._configure)
if hasattr(self, "_post_configure_script"): self.command('compile', self._compile)
self.command("post_configure_script", self._post_configure_script) if hasattr(self, '_test'):
self.command("compile", self._compile) self.command('test', self._test)
if hasattr(self, "_test"): self.command('install', self._install)
self.command("test", self._test) if hasattr(self, '_post_build_script'):
self.command("install", self._install) self.command('post_build_script', self._post_build_script)
if hasattr(self, "_post_build_script"):
self.command("post_build_script", self._post_build_script)
def make_dist(self): def make_dist(self):
if hasattr(self, "_pre_build_script"): if hasattr(self, '_pre_build_script'):
self.command("pre_build_script", self._pre_build_script) self.command('pre_build_script', self._pre_build_script)
self.command("configure", self._configure) self.command('configure', self._configure)
self.command("make_dist", self._make_dist) self.command('make_dist', self._make_dist)
def set_flatpak_buildsystem(self, module): def set_flatpak_buildsystem(self, module):
if getattr(self, "flatpak_buildsystem", None): if getattr(self, 'flatpak_buildsystem', None):
module["buildsystem"] = self.flatpak_buildsystem module['buildsystem'] = self.flatpak_buildsystem
if getattr(self, "subsource_dir", None): if getattr(self, 'subsource_dir', None):
module["subdir"] = self.subsource_dir module['subdir'] = self.subsource_dir
if getattr(self, "flatpack_build_options", None): if getattr(self, 'flatpack_build_options', None):
module["build-options"] = self.flatpack_build_options module['build-options'] = self.flatpack_build_options
if getattr(self, "configure_options", ""): if getattr(self, 'configure_option', ''):
module["config-opts"] = list(self.configure_options) module['config-opts'] = self.configure_option.split(' ')
def get_env(self, *, cross_comp_flags, cross_compilers, cross_path): def get_env(self, *, cross_comp_flags, cross_compilers, cross_path):
env = self.buildEnv.get_env( env = self.buildEnv.get_env(cross_comp_flags=cross_comp_flags, cross_compilers=cross_compilers, cross_path=cross_path)
cross_comp_flags=cross_comp_flags, for dep in self.get_dependencies(self.buildEnv.platformInfo, False):
cross_compilers=cross_compilers,
cross_path=cross_path,
)
for dep in self.get_dependencies(self.buildEnv.configInfo, False):
try: try:
builder = get_target_step(dep, self.buildEnv.configInfo.name) builder = get_target_step(dep, self.buildEnv.platformInfo.name)
builder.set_env(env) builder.set_env(env)
except KeyError: except KeyError:
# Some target may be missing (installed by a package, ...) # Some target may be missing (installed by a package, ...)
@ -354,262 +323,234 @@ class NoopBuilder(Builder):
pass pass
class TcCopyBuilder(Builder):
src_subdir = None
@property
def build_path(self):
return pj(self.buildEnv.toolchain_dir, self.target.full_name())
def build(self):
self.command("copy", self._copy)
def _copy(self, context):
context.try_skip(self.build_path)
if self.src_subdir:
source_path = pj(self.source_path, self.src_subdir)
else:
source_path = self.source_path
copy_tree(source_path, self.build_path)
def make_dist(self):
pass
class MakeBuilder(Builder): class MakeBuilder(Builder):
configure_options = [] configure_option_template = "{dep_options} {static_option} {env_option} --prefix {install_dir} --libdir {libdir}"
dynamic_configure_options = ["--enable-shared", "--disable-static"] configure_option = ""
static_configure_options = ["--enable-static", "--disable-shared"] dynamic_configure_option = "--enable-shared --disable-static"
make_options = ["-j4"] static_configure_option = "--enable-static --disable-shared"
install_options = [] make_option = ""
install_option = ""
configure_script = "configure" configure_script = "configure"
configure_env = { configure_env = {
"_format_CFLAGS": "{env[CFLAGS]} -O3", '_format_CFLAGS' : '{env[CFLAGS]} -O3',
"_format_CXXFLAGS": "{env[CXXFLAGS]} -O3", '_format_CXXFLAGS': '{env[CXXFLAGS]} -O3'
} }
make_targets = [] make_target = ""
flatpak_buildsystem = None flatpak_buildsystem = None
@property @property
def make_install_targets(self): def make_install_target(self):
if self.buildEnv.configInfo.build in ("iOS", "wasm"): if self.buildEnv.platformInfo.build in ('iOS', "wasm"):
yield "install" return 'install'
else: return 'install-strip'
yield "install-strip"
@property @property
def all_configure_options(self): def all_configure_option(self):
yield from self.configure_options option = self.configure_option_template.format(
if self.buildEnv.configInfo.static: dep_options=self.configure_option,
yield from self.static_configure_options static_option=self.static_configure_option if self.buildEnv.platformInfo.static else self.dynamic_configure_option,
else: env_option=self.buildEnv.platformInfo.configure_option if not self.target.force_native_build else "",
yield from self.dynamic_configure_options install_dir=self.buildEnv.install_dir,
if not self.target.force_native_build: libdir=pj(self.buildEnv.install_dir, self.buildEnv.libprefix)
yield from self.buildEnv.configInfo.configure_options )
yield from ("--prefix", self.buildEnv.install_dir) return option
yield from ("--libdir", pj(self.buildEnv.install_dir, self.buildEnv.libprefix))
def set_configure_env(self, env): def set_configure_env(self, env):
dep_conf_env = self.configure_env dep_conf_env = self.configure_env
if not dep_conf_env: if not dep_conf_env:
return return
for k, v in dep_conf_env.items(): for k, v in dep_conf_env.items():
if k.startswith("_format_"): if k.startswith('_format_'):
v = v.format(buildEnv=self.buildEnv, env=env) v = v.format(buildEnv=self.buildEnv, env=env)
env[k[8:]] = v env[k[8:]] = v
else: else:
env[k] = v env[k] = v
def _configure(self, context): def _configure(self, context):
context.try_skip(self.build_path) context.try_skip(self.build_path)
command = [ command = "{configure_wrapper}{configure_script} {configure_option}"
*self.buildEnv.configure_wrapper, command = command.format(
pj(self.source_path, self.configure_script), configure_wrapper=self.buildEnv.configure_wrapper,
*self.all_configure_options, configure_script=pj(self.source_path, self.configure_script),
] configure_option=self.all_configure_option
)
env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
self.set_configure_env(env) self.set_configure_env(env)
run_command(command, self.build_path, context, env=env) run_command(command, self.build_path, context, env=env)
def _compile(self, context): def _compile(self, context):
context.try_skip(self.build_path) context.try_skip(self.build_path)
command = [ command = "{make_wrapper}make -j4 {make_target} {make_option}".format(
*self.buildEnv.make_wrapper, make_wrapper=self.buildEnv.make_wrapper,
*neutralEnv("make_command"), make_target=self.make_target,
*self.make_targets, make_option=self.make_option
*self.make_options, )
]
env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
run_command(command, self.build_path, context, env=env) run_command(command, self.build_path, context, env=env)
def _install(self, context): def _install(self, context):
context.try_skip(self.build_path) context.try_skip(self.build_path)
command = [ command = "{make_wrapper}make {make_install_target} {make_option}".format(
*self.buildEnv.make_wrapper, make_wrapper=self.buildEnv.make_wrapper,
*neutralEnv("make_command"), make_install_target=self.make_install_target,
*self.make_install_targets, make_option=self.make_option
*self.make_options, )
]
env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
run_command(command, self.build_path, context, env=env) run_command(command, self.build_path, context, env=env)
def _make_dist(self, context): def _make_dist(self, context):
context.try_skip(self.build_path) context.try_skip(self.build_path)
command = [*self.buildEnv.make_wrapper, *neutralEnv("make_command"), "dist"] command = "{make_wrapper}make dist".format(
make_wrapper=self.buildEnv.make_wrapper
)
env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
run_command(command, self.build_path, context, env=env) run_command(command, self.build_path, context, env=env)
class CMakeBuilder(MakeBuilder): class CMakeBuilder(MakeBuilder):
flatpak_buildsystem = "cmake" flatpak_buildsystem = 'cmake'
def _configure(self, context): def _configure(self, context):
context.try_skip(self.build_path) context.try_skip(self.build_path)
cross_options = [] cross_option = ""
if not self.target.force_native_build and self.buildEnv.cmake_crossfile: if not self.target.force_native_build and self.buildEnv.cmake_crossfile:
cross_options += [f"-DCMAKE_TOOLCHAIN_FILE={self.buildEnv.cmake_crossfile}"] cross_option = "-DCMAKE_TOOLCHAIN_FILE={}".format(self.buildEnv.cmake_crossfile)
command = [ command = ("cmake {configure_option}"
*neutralEnv("cmake_command"), " -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON"
*self.configure_options, " -DCMAKE_INSTALL_PREFIX={install_dir}"
"-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON", " -DCMAKE_INSTALL_LIBDIR={libdir}"
f"-DCMAKE_INSTALL_PREFIX={self.buildEnv.install_dir}", " {source_path}"
f"-DCMAKE_INSTALL_LIBDIR={self.buildEnv.libprefix}", " {cross_option}")
self.source_path, command = command.format(
*cross_options, configure_option=self.configure_option,
] install_dir=self.buildEnv.install_dir,
env = self.get_env( libdir=self.buildEnv.libprefix,
cross_comp_flags=True, cross_compilers=False, cross_path=True source_path=self.source_path,
cross_option=cross_option
) )
env = self.get_env(cross_comp_flags=True, cross_compilers=False, cross_path=True)
self.set_configure_env(env) self.set_configure_env(env)
run_command(command, self.build_path, context, env=env) run_command(command, self.build_path, context, env=env)
def set_flatpak_buildsystem(self, module): def set_flatpak_buildsystem(self, module):
super().set_flatpak_buildsystem(module) super().set_flatpak_buildsystem( module)
module["buildir"] = True module['buildir'] = True
class QMakeBuilder(MakeBuilder): class QMakeBuilder(MakeBuilder):
qmake_targets = [] qmake_target = ""
flatpak_buildsystem = "qmake" flatpak_buildsystem = 'qmake'
@property @property
def make_options(self): def env_option(self):
if platform.system() == "Windows": options = ""
return [] if 'QMAKE_CC' in os.environ:
return super().make_options options += 'QMAKE_CC={} '.format(os.environ['QMAKE_CC'])
if 'QMAKE_CXX' in os.environ:
@property options += 'QMAKE_CXX={} '.format(os.environ['QMAKE_CXX'])
def env_options(self): return options
if "QMAKE_CC" in os.environ:
yield f"QMAKE_CC={os.environ['QMAKE_CC']}"
if "QMAKE_CXX" in os.environ:
yield f"QMAKE_CXX={os.environ['QMAKE_CXX']}"
def _configure(self, context): def _configure(self, context):
context.try_skip(self.build_path) context.try_skip(self.build_path)
command = [ cross_option = ""
*neutralEnv("qmake_command"), command = ("{command} {configure_option}"
*self.configure_options, " {env_option}"
*self.env_options, " {source_path}"
self.source_path, " {cross_option}")
] command = command.format(
env = self.get_env( command = neutralEnv('qmake_command'),
cross_comp_flags=True, cross_compilers=False, cross_path=True configure_option=self.configure_option,
env_option=self.env_option,
source_path=self.source_path,
cross_option=cross_option
) )
env = self.get_env(cross_comp_flags=True, cross_compilers=False, cross_path=True)
self.set_configure_env(env) self.set_configure_env(env)
run_command(command, self.build_path, context, env=env) run_command(command, self.build_path, context, env=env)
def _make_dist(self, context): def _make_dist(self, context):
command = [ command = "git archive -o {build_dir}/{name}.tar.gz --prefix={name}/ HEAD"
*neutralEnv("git_command"), command = command.format(
"archive", build_dir = self.build_path,
"-o", name = self.target.full_name()
f"{self.build_path}/{self.target.full_name()}.tar.gz", )
f"--prefix={self.target.full_name()}/",
"HEAD",
]
run_command(command, self.source_path, context) run_command(command, self.source_path, context)
class MesonBuilder(Builder): class MesonBuilder(Builder):
configure_options = [] configure_option = ""
test_options = [] test_option = ""
flatpak_buildsystem = "meson" flatpak_buildsystem = 'meson'
@property @property
def build_type(self): def build_type(self):
if platform.system() == "Windows": return 'release' if option('make_release') else 'debug'
return "release"
return "release" if option("make_release") else "debug"
@property @property
def strip_options(self): def strip_option(self):
if option("make_release"): return '--strip' if option('make_release') else ''
yield "--strip"
@property @property
def library_type(self): def library_type(self):
return "static" if self.buildEnv.configInfo.static else "shared" return 'static' if self.buildEnv.platformInfo.static else 'shared'
def _configure(self, context): def _configure(self, context):
context.no_skip = False
context.try_skip(self.build_path) context.try_skip(self.build_path)
if os.path.exists(self.build_path): if os.path.exists(self.build_path):
shutil.rmtree(self.build_path) shutil.rmtree(self.build_path)
os.makedirs(self.build_path) os.makedirs(self.build_path)
cross_options = [] configure_option = self.configure_option.format(buildEnv=self.buildEnv)
cross_option = ""
if not self.target.force_native_build and self.buildEnv.meson_crossfile: if not self.target.force_native_build and self.buildEnv.meson_crossfile:
cross_options += ["--cross-file", self.buildEnv.meson_crossfile] cross_option = "--cross-file {}".format(
command = [ self.buildEnv.meson_crossfile)
*neutralEnv("meson_command"), command = ("{command} . {build_path}"
"setup", " --buildtype={build_type} {strip_option}"
".", " --default-library={library_type}"
self.build_path, " {configure_option}"
f"--buildtype={self.build_type}", " --prefix={buildEnv.install_dir}"
*self.strip_options, " --libdir={buildEnv.libprefix}"
f"--default-library={self.library_type}", " {cross_option}")
*self.configure_options, command = command.format(
f"--prefix={self.buildEnv.install_dir}", command=neutralEnv('meson_command'),
f"--libdir={self.buildEnv.libprefix}", build_type=self.build_type,
*cross_options, strip_option=self.strip_option,
] library_type=self.library_type,
env = self.get_env( configure_option=configure_option,
cross_comp_flags=False, cross_compilers=False, cross_path=True build_path=self.build_path,
buildEnv=self.buildEnv,
cross_option=cross_option
) )
env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True)
run_command(command, self.source_path, context, env=env) run_command(command, self.source_path, context, env=env)
def _compile(self, context): def _compile(self, context):
context.try_skip(self.build_path) context.try_skip(self.build_path)
command = [*neutralEnv("ninja_command"), "-v"] command = "{} -v".format(neutralEnv('ninja_command'))
env = self.get_env( env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True)
cross_comp_flags=False, cross_compilers=False, cross_path=True
)
run_command(command, self.build_path, context, env=env) run_command(command, self.build_path, context, env=env)
def _test(self, context): def _test(self, context):
context.try_skip(self.build_path) context.try_skip(self.build_path)
if self.buildEnv.configInfo.build == "android" or ( if ( self.buildEnv.platformInfo.build == 'android'
self.buildEnv.configInfo.build != "native" or (self.buildEnv.platformInfo.build != 'native'
and not self.buildEnv.configInfo.static and not self.buildEnv.platformInfo.static)
): ):
raise SkipCommand() raise SkipCommand()
command = [*neutralEnv("mesontest_command"), "--verbose", *self.test_options] command = "{} --verbose {}".format(neutralEnv('mesontest_command'), self.test_option)
env = self.get_env( env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True)
cross_comp_flags=False, cross_compilers=False, cross_path=True
)
run_command(command, self.build_path, context, env=env) run_command(command, self.build_path, context, env=env)
def _install(self, context): def _install(self, context):
context.try_skip(self.build_path) context.try_skip(self.build_path)
command = [*neutralEnv("ninja_command"), "-v", "install"] command = "{} -v install".format(neutralEnv('ninja_command'))
env = self.get_env( env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True)
cross_comp_flags=False, cross_compilers=False, cross_path=True
)
run_command(command, self.build_path, context, env=env) run_command(command, self.build_path, context, env=env)
def _make_dist(self, context): def _make_dist(self, context):
command = [*neutralEnv("ninja_command"), "-v", "dist"] command = "{} -v dist".format(neutralEnv('ninja_command'))
env = self.get_env( env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True)
cross_comp_flags=False, cross_compilers=False, cross_path=True
)
run_command(command, self.build_path, context, env=env) run_command(command, self.build_path, context, env=env)

View File

@ -1,27 +0,0 @@
from .base import Dependency, ReleaseDownload, Builder as BaseBuilder
from kiwixbuild.utils import Remotefile, pj
from shutil import copytree
class BoostRegex(Dependency):
name = "boostregex"
class Source(ReleaseDownload):
archive = Remotefile(
"regex-boost-1.86.0.zip",
"",
"https://codeload.github.com/boostorg/regex/zip/refs/tags/boost-1.86.0",
)
class Builder(BaseBuilder):
def build(self):
self.command("copy_headers", self._copy_headers)
def _copy_headers(self, context):
context.try_skip(self.build_path)
copytree(
pj(self.source_path, "include", "boost"),
pj(self.buildEnv.install_dir, "include", "boost"),
dirs_exist_ok=True,
)

View File

@ -1,34 +1,21 @@
from .base import Dependency, ReleaseDownload, MesonBuilder from .base import (
Dependency,
GitClone,
CMakeBuilder)
from kiwixbuild.utils import Remotefile from kiwixbuild.utils import Remotefile
from kiwixbuild._global import neutralEnv
class docoptcpp(Dependency): class docoptcpp(Dependency):
name = "docoptcpp" name = 'docoptcpp'
class Source(ReleaseDownload): class Source(GitClone):
name = "docoptcpp" git_remote = "https://github.com/docopt/docopt.cpp.git"
src_archive = Remotefile( git_dir = "docopt.cpp"
"v0.6.3.tar.gz", force_full_clone = True
"28af5a0c482c6d508d22b14d588a3b0bd9ff97135f99c2814a5aa3cbff1d6632", git_ref = "3dd23e3280f213bacefdf5fcb04857bf52e90917"
"https://github.com/docopt/docopt.cpp/archive/v0.6.3.tar.gz",
)
meson_archive = Remotefile( class Builder(CMakeBuilder):
"docopt_0.6.3-3_patch.zip", make_install_target = 'install'
"1f641187f9d3f35b0a5ebd2011876ef8e9b04b69b7b163095dd7dfa16219ad01",
"https://wrapdb.mesonbuild.com/v2/docopt_0.6.3-3/get_patch",
)
archives = [src_archive, meson_archive]
patches = [
"docopt_meson_install_pkgconfig.patch",
"docopt_meson_use_boostregex.patch",
]
class Builder(MesonBuilder):
@classmethod
def get_dependencies(cls, configInfo, allDeps):
if neutralEnv("distname") == "Windows":
return ["boostregex"]
return []

View File

@ -0,0 +1,47 @@
import os
from .base import Dependency, ReleaseDownload, Builder
from kiwixbuild.utils import Remotefile, run_command, copy_tree
pj = os.path.join
class emsdk(Dependency):
dont_skip = True
neutral = False
name = 'emsdk'
class Source(ReleaseDownload):
archive = Remotefile('emsdk-3.1.24.tar.gz',
'1aa5365ccb2147701cc9d1e59a5a49577c1d6aea55da7c450df2d5ffa48b8a58',
'https://codeload.github.com/emscripten-core/emsdk/tar.gz/refs/tags/3.1.24')
@property
def source_dir(self):
return self.target.full_name()
class Builder(Builder):
@property
def install_path(self):
return self.build_path
def _copy_source(self, context):
context.try_skip(self.build_path)
copy_tree(self.source_path, self.build_path)
def _install(self, context):
context.try_skip(self.build_path)
command = "./emsdk install 3.1.24"
run_command(command, self.build_path, context)
def _activate(self, context):
context.try_skip(self.build_path)
command = "./emsdk activate 3.1.24"
run_command(command, self.build_path, context)
def build(self):
self.command('copy_source', self._copy_source)
self.command('install', self._install)
self.command('activate', self._activate)

View File

@ -0,0 +1,68 @@
import os
from .base import Dependency, NoopSource, Builder
from kiwixbuild.utils import Remotefile, add_execution_right, run_command
pj = os.path.join
class org_kde(Dependency):
neutral = False
name = 'org.kde'
Source = NoopSource
class Builder(Builder):
def _setup_remote(self, context):
command = "flatpak --user remote-add --if-not-exists {remote_name} {remote_url}"
command = command.format(
remote_name = 'flathub',
remote_url = 'https://flathub.org/repo/flathub.flatpakrepo'
)
env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False)
run_command(command, self.buildEnv.build_dir, context, env=env)
def _install_sdk(self, context):
command = "flatpak --user install --noninteractive --verbose -y {remote_name} {name}.Sdk//{version} {name}.Platform//{version}"
command = command.format(
remote_name = 'flathub',
name = self.target.name,
version = self.target.version()
)
env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False)
run_command(command, self.buildEnv.build_dir, context, env=env)
def build(self):
self.command('setup_remote', self._setup_remote)
self.command('install_sdk', self._install_sdk)
class io_qt_qtwebengine(Dependency):
neutral = False
name = 'io.qt.qtwebengine'
Source = NoopSource
class Builder(Builder):
def _setup_remote(self, context):
command = "flatpak --user remote-add --if-not-exists {remote_name} {remote_url}"
command = command.format(
remote_name = 'flathub',
remote_url = 'https://flathub.org/repo/flathub.flatpakrepo'
)
env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False)
run_command(command, self.buildEnv.build_dir, context, env=env)
def _install_sdk(self, context):
command = "flatpak --user install -y {remote_name} {name}.BaseApp//{version}"
command = command.format(
remote_name = 'flathub',
name = self.target.name,
version = self.target.version()
)
env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False)
run_command(command, self.buildEnv.build_dir, context, env=env)
def build(self):
self.command('setup_remote', self._setup_remote)
self.command('install_sdk', self._install_sdk)

View File

@ -1,4 +1,8 @@
from .base import Dependency, ReleaseDownload, MakeBuilder from .base import (
Dependency,
ReleaseDownload,
MakeBuilder
)
from kiwixbuild.utils import Remotefile, run_command from kiwixbuild.utils import Remotefile, run_command
@ -7,15 +11,13 @@ class Gumbo(Dependency):
name = "gumbo" name = "gumbo"
class Source(ReleaseDownload): class Source(ReleaseDownload):
archive = Remotefile( archive = Remotefile('gumbo-0.10.1.tar.gz',
"gumbo-parser-0.12.1.tar.gz", '28463053d44a5dfbc4b77bcf49c8cee119338ffa636cc17fc3378421d714efad',
"c0bb5354e46539680724d638dbea07296b797229a7e965b13305c930ddc10d82", 'https://github.com/google/gumbo-parser/archive/v0.10.1.tar.gz')
"https://dev.kiwix.org/kiwix-build/gumbo-parser-0.12.1.tar.gz",
)
def _post_prepare_script(self, context): def _post_prepare_script(self, context):
context.try_skip(self.extract_path) context.try_skip(self.extract_path)
command = ["./autogen.sh"] command = "./autogen.sh"
run_command(command, self.extract_path, context) run_command(command, self.extract_path, context)
Builder = MakeBuilder Builder = MakeBuilder

View File

@ -1,191 +1,49 @@
from .base import ( from .base import (
Dependency, Dependency,
ReleaseDownload, ReleaseDownload,
MakeBuilder, MakeBuilder
Builder as BaseBuilder,
) )
from kiwixbuild.utils import pj, SkipCommand, Remotefile, extract_archive from kiwixbuild.utils import SkipCommand, Remotefile
from kiwixbuild._global import get_target_step, neutralEnv from kiwixbuild._global import get_target_step
import os, shutil
import fileinput
import platform
if platform.system() == "Windows": class Icu(Dependency):
name = "icu4c"
class Icu(Dependency): class Source(ReleaseDownload):
name = "icu4c" archive = Remotefile('icu4c_svn_58-2.tar.gz',
'fd8fcc1f1b8b2b71b879e88844480ebec107189c21076c81573f71dca5686a0d')
patches = ["icu4c_fix_static_lib_name_mingw.patch",
"icu4c_android_elf64_st_info.patch",
"icu4c_custom_data.patch",
"icu4c_noxlocale.patch",
"icu4c_rpath.patch",
"icu4c_build_config.patch",
"icu4c_wasm.patch"
]
class Source(ReleaseDownload):
archive = Remotefile(
"icu4c-74_1-Win64-MSVC2022.zip",
"",
"https://github.com/unicode-org/icu/releases/download/release-74-1/icu4c-74_1-Win64-MSVC2022.zip",
)
class Builder(BaseBuilder): class Builder(MakeBuilder):
def build(self): subsource_dir = "source"
self.command("copy_headers", self._copy_headers) make_install_target = "install"
self.command("copy_bins", self._copy_bin)
self.command("generate_pkg_config", self._generate_pkg_config)
def _copy_headers(self, context): @classmethod
context.try_skip(self.build_path) def get_dependencies(cls, platformInfo, allDeps):
shutil.copytree( plt = 'native_static' if platformInfo.static else 'native_dyn'
pj(self.source_path, "include", "unicode"), return [(plt, 'icu4c')]
pj(self.buildEnv.install_dir, "include", "unicode"),
)
def _copy_bin(self, context): @property
context.try_skip(self.build_path) def configure_option(self):
shutil.copytree( options = ("--disable-samples --disable-tests --disable-extras "
pj(self.source_path, "lib64"), "--disable-dyload --enable-rpath "
pj(self.buildEnv.install_dir, "lib"), "--disable-icuio --disable-layoutex")
dirs_exist_ok=True, platformInfo = self.buildEnv.platformInfo
) if platformInfo.build != 'native':
icu_native_builder = get_target_step(
def ignore_non_dll(path, names): 'icu4c',
return [n for n in names if not n.endswith(".dll")] 'native_static' if platformInfo.static else 'native_dyn')
options += " --with-cross-build={} --disable-tools".format(
shutil.copytree( icu_native_builder.build_path)
pj(self.source_path, "bin64"), if platformInfo.build in ('android', 'wasm'):
pj(self.buildEnv.install_dir, "bin"), options += " --with-data-packaging=archive"
ignore=ignore_non_dll, return options
dirs_exist_ok=True,
)
def _generate_pkg_config(self, context):
context.try_skip(self.build_path)
pkg_config_template = """ Copyright (C) 2016 and later: Unicode, Inc. and others.
# License & terms of use: http://www.unicode.org/copyright.html
# Copyright (C) 2010-2013, International Business Machines Corporation. All Rights Reserved.
# CFLAGS contains only anything end users should set
CFLAGS =
# CXXFLAGS contains only anything end users should set
CXXFLAGS = -std=c++11
# DEFS only contains those UCONFIG_CPPFLAGS which are not auto-set by platform.h
DEFS =
prefix = {prefix}
exec_prefix = ${{prefix}}
libdir = ${{exec_prefix}}/lib
includedir = ${{prefix}}/include
baselibs = -lpthread -lm
UNICODE_VERSION=15.0
ICUPREFIX=icu
ICULIBSUFFIX=
LIBICU=lib${{ICUPREFIX}}
pkglibdir=${{libdir}}/icu${{ICULIBSUFFIX}}/73.1
ICUDATA_NAME = icudt73l
ICUDESC=International Components for Unicode
Version: 73.1
Cflags: -I${{includedir}}
Description: International Components for Unicode: Internationalization library
Name: icu-i18n
Libs: -L${{libdir}} -licuin -licuuc -licudt"""
pkg_config_content = pkg_config_template.format(
prefix=self.buildEnv.install_dir
)
os.makedirs(
pj(self.buildEnv.install_dir, "lib", "pkgconfig"), exist_ok=True
)
with open(
pj(self.buildEnv.install_dir, "lib", "pkgconfig", "icu-i18n.pc"),
mode="w",
) as f:
f.write(pkg_config_content)
else:
class Icu(Dependency):
name = "icu4c"
class Source(ReleaseDownload):
archive_src = Remotefile(
"icu4c-73_2-src.tgz",
"818a80712ed3caacd9b652305e01afc7fa167e6f2e94996da44b90c2ab604ce1",
"https://github.com/unicode-org/icu/releases/download/release-73-2/icu4c-73_2-src.tgz",
)
archive_data = Remotefile(
"icu4c-73_2-data.zip",
"ca1ee076163b438461e484421a7679fc33a64cd0a54f9d4b401893fa1eb42701",
"https://github.com/unicode-org/icu/releases/download/release-73-2/icu4c-73_2-data.zip",
)
archives = [archive_src, archive_data]
def _extract(self, context):
context.try_skip(self.extract_path)
if os.path.exists(self.extract_path):
shutil.rmtree(self.extract_path)
extract_archive(
pj(neutralEnv("archive_dir"), self.archive_src.name),
neutralEnv("source_dir"),
topdir=None,
name=self.source_dir,
)
shutil.rmtree(
pj(neutralEnv("source_dir"), self.source_dir, "source", "data")
)
extract_archive(
pj(neutralEnv("archive_dir"), self.archive_data.name),
pj(neutralEnv("source_dir"), self.source_dir, "source"),
topdir="data",
name="data",
)
patches = [
"icu4c_fix_static_lib_name_mingw.patch",
"icu4c_rpath.patch",
"icu4c_wasm.patch",
]
class Builder(MakeBuilder):
subsource_dir = "source"
make_install_targets = ["install"]
@classmethod
def get_dependencies(cls, configInfo, allDeps):
plt = "native_static" if configInfo.static else "native_dyn"
return [(plt, "icu4c")]
@property
def configure_options(self):
yield "--disable-samples"
yield "--disable-tests"
yield "--disable-extras"
yield "--disable-dyload"
yield "--enable-rpath"
yield "--disable-icuio"
yield "--disable-layoutex"
configInfo = self.buildEnv.configInfo
if configInfo.build != "native":
icu_native_builder = get_target_step(
"icu4c", "native_static" if configInfo.static else "native_dyn"
)
yield f"--with-cross-build={icu_native_builder.build_path}"
yield "--disable-tools"
if configInfo.build in ("android", "wasm"):
yield "--with-data-packaging=archive"
def set_env(self, env):
env["ICU_DATA_FILTER_FILE"] = pj(
os.path.dirname(os.path.realpath(__file__)),
"icu4c_data_filter.json",
)
def _post_configure_script(self, context):
if self.buildEnv.configInfo.build != "wasm":
context.skip()
context.try_skip(self.build_path)
for line in fileinput.input(
pj(self.build_path, "Makefile"), inplace=True
):
if line == "#DATASUBDIR = data\n":
print("DATASUBDIR = data")
else:
print(line, end="")

View File

@ -1,14 +0,0 @@
{
"strategy": "additive",
"featureFilters": {
"lang_tree": "include",
"locales_tree": "include",
"translit": "include",
"misc": {
"includelist": [
"likelySubtags",
"metadata"
]
}
}
}

View File

@ -1,9 +1,13 @@
import os import os
from kiwixbuild.configs import ConfigInfo from kiwixbuild.platforms import PlatformInfo
from kiwixbuild.utils import pj, copy_tree, run_command from kiwixbuild.utils import pj, copy_tree, run_command
from kiwixbuild._global import option from kiwixbuild._global import option
from .base import Dependency, NoopSource, Builder as BaseBuilder from .base import (
Dependency,
NoopSource,
Builder as BaseBuilder)
class IOSFatLib(Dependency): class IOSFatLib(Dependency):
@ -12,39 +16,37 @@ class IOSFatLib(Dependency):
Source = NoopSource Source = NoopSource
class Builder(BaseBuilder): class Builder(BaseBuilder):
@classmethod @classmethod
def get_dependencies(self, platfomInfo, alldeps): def get_dependencies(self, platfomInfo, alldeps):
base_target = option("target") base_target = option('target')
return [("iOS_{}".format(arch), base_target) for arch in option("ios_arch")] return [('iOS_{}'.format(arch), base_target) for arch in option('ios_arch')]
def _copy_headers(self, context): def _copy_headers(self, context):
plt = ConfigInfo.get_config("iOS_{}".format(option("ios_arch")[0])) plt = PlatformInfo.get_platform('iOS_{}'.format(option('ios_arch')[0]))
include_src = pj(plt.buildEnv.install_dir, "include") include_src = pj(plt.buildEnv.install_dir, 'include')
include_dst = pj(self.buildEnv.install_dir, "include") include_dst = pj(self.buildEnv.install_dir, 'include')
copy_tree(include_src, include_dst) copy_tree(include_src, include_dst)
def _merge_libs(self, context): def _merge_libs(self, context):
lib_dirs = [] lib_dirs = []
for arch in option("ios_arch"): for arch in option('ios_arch'):
plt = ConfigInfo.get_config("iOS_{}".format(arch)) plt = PlatformInfo.get_platform('iOS_{}'.format(arch))
lib_dirs.append(pj(plt.buildEnv.install_dir, "lib")) lib_dirs.append(pj(plt.buildEnv.install_dir, 'lib'))
libs = [] libs = []
for f in os.listdir(lib_dirs[0]): for f in os.listdir(lib_dirs[0]):
if os.path.islink(pj(lib_dirs[0], f)): if os.path.islink(pj(lib_dirs[0], f)):
continue continue
if f.endswith(".a") or f.endswith(".dylib"): if f.endswith('.a') or f.endswith('.dylib'):
libs.append(f) libs.append(f)
os.makedirs(pj(self.buildEnv.install_dir, "lib"), exist_ok=True) os.makedirs(pj(self.buildEnv.install_dir, 'lib'), exist_ok=True)
command_tmp = "lipo -create {input} -output {output}"
for l in libs: for l in libs:
command = [ command = command_tmp.format(
"lipo", input=" ".join(pj(d, l) for d in lib_dirs),
"-create", output=pj(self.buildEnv.install_dir, 'lib', l))
*[pj(d, l) for d in lib_dirs],
"-output",
pj(self.buildEnv.install_dir, "lib", l),
]
run_command(command, self.buildEnv.install_dir, context) run_command(command, self.buildEnv.install_dir, context)
def build(self): def build(self):
self.command("copy_headers", self._copy_headers) self.command('copy_headers', self._copy_headers)
self.command("merge_libs", self._merge_libs) self.command('merge_libs', self._merge_libs)

View File

@ -1,7 +1,7 @@
from kiwixbuild._global import option from .base import (
from .base import Dependency, GitClone, QMakeBuilder Dependency,
import platform GitClone,
QMakeBuilder)
class KiwixDesktop(Dependency): class KiwixDesktop(Dependency):
name = "kiwix-desktop" name = "kiwix-desktop"
@ -13,27 +13,25 @@ class KiwixDesktop(Dependency):
class Builder(QMakeBuilder): class Builder(QMakeBuilder):
dependencies = ["qt", "qtwebengine", "libkiwix", "aria2"] dependencies = ["qt", "qtwebengine", "libkiwix", "aria2"]
make_install_target = 'install'
configure_env = None configure_env = None
flatpack_build_options = {"env": {"QMAKEPATH": "/app"}} flatpack_build_options = {
"env": [
"QMAKEPATH=/app/lib"
]
}
@property @property
def make_targets(self): def configure_option(self):
if platform.system() == "Windows": if self.buildEnv.platformInfo.name == 'flatpak':
yield "release-all" options = [
'QMAKE_INCDIR+=/app/include/QtWebEngine',
'QMAKE_INCDIR+=/app/include/QtWebEngineCore',
'QMAKE_INCDIR+=/app/include/QtWebEngineWidgets'
]
else: else:
yield from super().make_targets options = ["PREFIX={}".format(self.buildEnv.install_dir)]
if self.buildEnv.platformInfo.static:
@property options.append('"CONFIG+=static"')
def make_install_targets(self): return " ".join(options)
if platform.system() == "Windows":
yield "release-install"
else:
yield "install"
@property
def configure_options(self):
if self.buildEnv.configInfo.name != "flatpak":
yield f"PREFIX={self.buildEnv.install_dir}"
if self.buildEnv.configInfo.static:
yield "CONFIG+=static"

View File

@ -1,5 +1,7 @@
from .base import Dependency, GitClone, MesonBuilder from .base import (
Dependency,
GitClone,
MesonBuilder)
class KiwixTools(Dependency): class KiwixTools(Dependency):
name = "kiwix-tools" name = "kiwix-tools"
@ -10,9 +12,10 @@ class KiwixTools(Dependency):
git_dir = "kiwix-tools" git_dir = "kiwix-tools"
class Builder(MesonBuilder): class Builder(MesonBuilder):
dependencies = ["libkiwix", "docoptcpp"] dependencies = ["libkiwix"]
@property @property
def configure_options(self): def configure_option(self):
if self.buildEnv.configInfo.static: if self.buildEnv.platformInfo.static:
yield "-Dstatic-linkage=true" return "-Dstatic-linkage=true"
return ""

View File

@ -3,59 +3,28 @@ import os
from .base import ( from .base import (
Dependency, Dependency,
ReleaseDownload, ReleaseDownload,
MesonBuilder, MakeBuilder,
) )
from kiwixbuild.utils import Remotefile, pj, Defaultdict, SkipCommand, run_command from kiwixbuild.utils import Remotefile, pj, Defaultdict, SkipCommand, run_command
from kiwixbuild._global import get_target_step from kiwixbuild._global import get_target_step
class LibCurl(Dependency): class LibCurl(Dependency):
name = "libcurl" name = "libcurl"
class Source(ReleaseDownload): class Source(ReleaseDownload):
name = "libcurl" name = "libcurl"
src_archive = Remotefile( archive = Remotefile('curl-7.67.0.tar.xz',
"curl-8.4.0.tar.xz", 'f5d2e7320379338c3952dcc7566a140abb49edb575f9f99272455785c40e536c',
"16c62a9c4af0f703d28bda6d7bbf37ba47055ad3414d70dec63e2e6336f2a82d", 'https://curl.haxx.se/download/curl-7.67.0.tar.xz')
"https://curl.se/download/curl-8.4.0.tar.xz",
)
meson_archive = Remotefile(
"curl_8.4.0-2_patch.zip",
"bbb6ae75225c36ef9bb336cface729794c7c070c623a003fff40bd416042ff6e",
"https://dev.kiwix.org/libkiwix/curl_8.4.0-2_patch.zip",
)
archives = [src_archive, meson_archive]
class Builder(MesonBuilder): class Builder(MakeBuilder):
dependencies = ["zlib"] dependencies = ['zlib']
configure_options = [ configure_option = " ".join(
f"-D{p}=disabled" ["--without-{}".format(p)
for p in ( for p in ('libssh2', 'ssl', 'libmetalink', 'librtmp',
"psl", 'nghttp2', 'libidn2', 'brotli')] +
"kerberos-auth", ["--disable-{}".format(p)
"gss-api", for p in ('ftp', 'file', 'ldap', 'ldaps', 'rtsp', 'dict',
"ssh", 'telnet', 'tftp', 'pop3', 'imap', 'smb', 'smtp',
"rtmp", 'gopher', 'manual')])
"http2",
"idn",
"brotli",
"ftp",
"file",
"ldap",
"ldaps",
"rtsp",
"dict",
"telnet",
"tftp",
"pop3",
"imap",
"smb",
"smtp",
"gopher",
"tool",
)
]
def _test(self, context):
context.skip("No Test")

View File

@ -1,10 +1,12 @@
import shutil, os import shutil, os
from .base import Dependency, GitClone, MesonBuilder from .base import (
Dependency,
GitClone,
MesonBuilder)
from kiwixbuild.utils import pj, copy_tree from kiwixbuild.utils import pj, copy_tree
from kiwixbuild._global import option, get_target_step, neutralEnv from kiwixbuild._global import option, get_target_step, neutralEnv
class Libkiwix(Dependency): class Libkiwix(Dependency):
name = "libkiwix" name = "libkiwix"
force_build = True force_build = True
@ -14,39 +16,24 @@ class Libkiwix(Dependency):
git_dir = "libkiwix" git_dir = "libkiwix"
class Builder(MesonBuilder): class Builder(MesonBuilder):
dependencies = [ dependencies = ["pugixml", "libzim", "zlib", "lzma", "libcurl", "libmicrohttpd", "icu4c", "mustache", "xapian-core"]
"pugixml", strip_option = ''
"libzim",
"zlib",
"libcurl",
"libmicrohttpd",
"icu4c",
"mustache",
"xapian-core",
]
strip_options = []
@property @property
def build_type(self): def configure_option(self):
if self.buildEnv.configInfo.build == "android": platformInfo = self.buildEnv.platformInfo
return "debug" if platformInfo.build == 'android':
return super().build_type return '-Dstatic-linkage=true -Dwerror=false'
if platformInfo.build == 'iOS':
@property return '-Db_bitcode=true'
def configure_options(self): if platformInfo.name == 'flatpak':
configInfo = self.buildEnv.configInfo return '--wrap-mode=nodownload'
if configInfo.build == "android": if platformInfo.mixed and option('target') == 'libkiwix':
yield "-Dstatic-linkage=true" return "-Dstatic-linkage=true"
yield "-Dwerror=false" return ''
if configInfo.build == "iOS":
yield "-Db_bitcode=true"
if configInfo.name == "flatpak":
yield "--wrap-mode=nodownload"
if configInfo.mixed and option("target") == "libkiwix":
yield "-Dstatic-linkage=true"
@property @property
def library_type(self): def library_type(self):
if self.buildEnv.configInfo.build == "android": if self.buildEnv.platformInfo.build == 'android':
return "shared" return 'shared'
return super().library_type return super().library_type

View File

@ -6,45 +6,44 @@ from .base import (
MakeBuilder, MakeBuilder,
) )
from kiwixbuild.utils import Remotefile, pj, SkipCommand, run_command from kiwixbuild.utils import Remotefile, pj, Defaultdict, SkipCommand, run_command
from kiwixbuild._global import get_target_step from kiwixbuild._global import get_target_step
class LibMagic(Dependency): class LibMagic(Dependency):
name = "libmagic" name = "libmagic"
class Source(ReleaseDownload): class Source(ReleaseDownload):
name = "libmagic" name = "libmagic"
source_dir = "libmagic" source_dir = "libmagic"
archive_top_dir = "file-5.44" archive_top_dir = 'file-5.44'
archive = Remotefile( archive = Remotefile('file-5.44.tar.gz',
"file-5.44.tar.gz", '3751c7fba8dbc831cb8d7cc8aff21035459b8ce5155ef8b0880a27d028475f3b')
"3751c7fba8dbc831cb8d7cc8aff21035459b8ce5155ef8b0880a27d028475f3b",
)
class Builder(MakeBuilder): class Builder(MakeBuilder):
@property @property
def configure_options(self): def configure_option(self):
yield "--disable-bzlib" return ("--disable-bzlib "
yield "--disable-xzlib" "--disable-xzlib "
yield "--disable-zstdlib" "--disable-zstdlib "
yield "--disable-lzlib" "--disable-lzlib "
)
@classmethod @classmethod
def get_dependencies(cls, configInfo, allDeps): def get_dependencies(cls, platformInfo, allDeps):
if configInfo.build != "native": if platformInfo.build != 'native':
return [("native_static", "libmagic")] return [('native_static', 'libmagic')]
return [] return []
def _compile(self, context): def _compile(self, context):
configInfo = self.buildEnv.configInfo platformInfo = self.buildEnv.platformInfo
if configInfo.build == "native": if platformInfo.build == 'native':
return super()._compile(context) return super()._compile(context)
context.try_skip(self.build_path) context.try_skip(self.build_path)
command = ["make", "-j4", *self.make_targets, *self.make_options] command = "make -j4 {make_target} {make_option}".format(
env = self.buildEnv.get_env( make_target=self.make_target,
cross_comp_flags=True, cross_compilers=True, cross_path=True make_option=self.make_option
) )
libmagic_native_builder = get_target_step("libmagic", "native_static") env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
env["PATH"].insert(0, pj(libmagic_native_builder.build_path, "src")) libmagic_native_builder = get_target_step('libmagic', 'native_static')
env['PATH'] = ':'.join([pj(libmagic_native_builder.build_path, 'src'), env['PATH']])
run_command(command, self.build_path, context, env=env) run_command(command, self.build_path, context, env=env)

View File

@ -1,33 +1,17 @@
from .base import Dependency, ReleaseDownload, MesonBuilder from .base import (
Dependency,
ReleaseDownload,
MakeBuilder)
from kiwixbuild.utils import Remotefile from kiwixbuild.utils import Remotefile
class MicroHttpd(Dependency): class MicroHttpd(Dependency):
name = "libmicrohttpd" name = "libmicrohttpd"
class Source(ReleaseDownload): class Source(ReleaseDownload):
src_archive = Remotefile( archive = Remotefile('libmicrohttpd-0.9.76.tar.gz',
"libmicrohttpd-0.9.76.tar.gz", 'f0b1547b5a42a6c0f724e8e1c1cb5ce9c4c35fb495e7d780b9930d35011ceb4c',
"f0b1547b5a42a6c0f724e8e1c1cb5ce9c4c35fb495e7d780b9930d35011ceb4c", 'https://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.76.tar.gz')
"https://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.76.tar.gz",
)
meson_archive = Remotefile(
"libmicrohttpd_0.9.16-3_patch.zip",
"0954c094a0d4cfe0dd799d8df8a04face6669f7b4d51a7386a9c3e2d37b9c3b3",
"https://wrapdb.mesonbuild.com/v2/libmicrohttpd_0.9.76-3/get_patch",
)
archives = [src_archive, meson_archive]
patches = [
"libmicrohttpd_meson_pkgconfig.patch",
"libmicrohttpd_meson_timeval_tvsec_size.patch",
"libmicrohttpd_meson_winet6.patch",
]
class Builder(MesonBuilder): class Builder(MakeBuilder):
configure_options = [ configure_option = "--disable-https --without-libgcrypt --without-libcurl --disable-doc --disable-examples"
"-Dgnutls=disabled",
"-Dgcrypt=disabled",
"-Dcurl=disabled",
"-Dexpat=disabled",
]

View File

@ -1,6 +1,8 @@
from .base import Dependency, GitClone, MesonBuilder from .base import (
from kiwixbuild._global import option, get_target_step, neutralEnv Dependency,
GitClone,
MesonBuilder)
from kiwixbuild._global import option, get_target_step
class Libzim(Dependency): class Libzim(Dependency):
name = "libzim" name = "libzim"
@ -11,47 +13,40 @@ class Libzim(Dependency):
git_dir = "libzim" git_dir = "libzim"
class Builder(MesonBuilder): class Builder(MesonBuilder):
test_options = ["-t", "8"] test_option = "-t 8"
strip_options = [] strip_option = ''
@property
def build_type(self):
if self.buildEnv.configInfo.build == "android":
return "debug"
return super().build_type
@classmethod @classmethod
def get_dependencies(cls, configInfo, allDeps): def get_dependencies(cls, platformInfo, allDeps):
if neutralEnv("distname") == "Windows": deps = ['lzma', 'zstd', 'xapian-core', 'icu4c']
return ["zstd", "xapian-core", "icu4c", "zim-testing-suite"] if platformInfo.name not in ('flatpak', 'wasm'):
deps = ["lzma", "zstd", "xapian-core", "icu4c"] deps.append('zim-testing-suite')
if configInfo.name not in ("flatpak", "wasm"):
deps.append("zim-testing-suite")
return deps return deps
@property @property
def configure_options(self): def configure_option(self):
configInfo = self.buildEnv.configInfo platformInfo = self.buildEnv.platformInfo
if neutralEnv("distname") == "Windows": config_options = []
yield "-Dwith_xapian_fuller=false" if platformInfo.build == 'android':
yield "-Dwerror=false" config_options.append("-DUSE_BUFFER_HEADER=false")
if configInfo.build == "android": config_options.append("-Dstatic-linkage=true")
yield "-DUSE_BUFFER_HEADER=false" if platformInfo.build == 'iOS':
yield "-Dstatic-linkage=true" config_options.append("-Db_bitcode=true")
if configInfo.mixed and option("target") == "libzim": if platformInfo.mixed and option('target') == 'libzim':
yield "-Dstatic-linkage=true" config_options.append("-Dstatic-linkage=true")
if configInfo.name == "flatpak": if platformInfo.name == "flatpak":
yield "--wrap-mode=nodownload" config_options.append("--wrap-mode=nodownload")
yield "-Dtest_data_dir=none" config_options.append("-Dtest_data_dir=none")
if configInfo.name == "wasm": if platformInfo.name == "wasm":
yield "-Dexamples=false" config_options.append("-Dexamples=false")
yield "-DUSE_MMAP=false" config_options.append("-DUSE_MMAP=false")
if configInfo.name not in ("flatpak", "wasm"): if platformInfo.name not in ("flatpak", "wasm"):
zim_testing_suite = get_target_step("zim-testing-suite", "source") zim_testing_suite = get_target_step('zim-testing-suite', 'source')
yield "-Dtest_data_dir={}".format(zim_testing_suite.source_path) config_options.append('-Dtest_data_dir={}'.format(zim_testing_suite.source_path))
return " ".join(config_options)
@property @property
def library_type(self): def library_type(self):
if self.buildEnv.configInfo.build == "android": if self.buildEnv.platformInfo.build == 'android':
return "shared" return 'shared'
return super().library_type return super().library_type

View File

@ -1,26 +1,28 @@
from .base import ( from .base import (
Dependency, Dependency,
ReleaseDownload, ReleaseDownload,
MesonBuilder) MakeBuilder)
from kiwixbuild.utils import Remotefile from kiwixbuild.utils import Remotefile
class lzma(Dependency): class lzma(Dependency):
name = 'lzma' name = 'lzma'
class Source(ReleaseDownload): class Source(ReleaseDownload):
src_archive = Remotefile( archive = Remotefile('xz-5.2.6.tar.gz',
"xz-5.2.6.tar.gz", 'a2105abee17bcd2ebd15ced31b4f5eda6e17efd6b10f921a01cda4a44c91b3a0',
"a2105abee17bcd2ebd15ced31b4f5eda6e17efd6b10f921a01cda4a44c91b3a0", 'https://altushost-swe.dl.sourceforge.net/project/lzmautils/xz-5.2.6.tar.gz'
"https://altushost-swe.dl.sourceforge.net/project/lzmautils/xz-5.2.6.tar.gz", )
)
meson_patch = Remotefile(
"liblzma_5.2.6-3_patch.zip",
"1c71536d364e1a3ce6bea61266576f89cc5cce4d3b9e11f3494417dafa29780b",
"https://wrapdb.mesonbuild.com/v2/liblzma_5.2.6-3/get_patch",
)
archives = [src_archive, meson_patch]
patches = ['lzma_meson_install.patch']
Builder = MesonBuilder class Builder(MakeBuilder):
@property
def configure_option(self):
return ("--disable-xz "
"--disable-xzdec "
"--disable-lzmadec "
"--disable-lzmainfo "
"--disable-lzma-links "
"--disable-scripts "
"--disable-doc "
# "--disable-symbol-versions "
)

View File

@ -1,30 +1,28 @@
from .base import Dependency, ReleaseDownload, Builder as BaseBuilder from .base import (
Dependency,
ReleaseDownload,
Builder as BaseBuilder)
from kiwixbuild.utils import Remotefile, pj from kiwixbuild.utils import Remotefile, pj
from shutil import copy2 from shutil import copy2
class Mustache(Dependency): class Mustache(Dependency):
name = "mustache" name = "mustache"
class Source(ReleaseDownload): class Source(ReleaseDownload):
archive = Remotefile( archive = Remotefile('Mustache-4.1.tar.gz',
"Mustache-4.1.tar.gz", 'acd66359feb4318b421f9574cfc5a511133a77d916d0b13c7caa3783c0bfe167',
"acd66359feb4318b421f9574cfc5a511133a77d916d0b13c7caa3783c0bfe167", 'https://github.com/kainjow/Mustache/archive/v4.1.tar.gz')
"https://dev.kiwix.org/kiwix-build/mustache-4.1.tar.gz",
)
class Builder(BaseBuilder): class Builder(BaseBuilder):
def build(self): def build(self):
self.command("copy_header", self._copy_header) self.command('copy_header', self._copy_header)
def _copy_header(self, context): def _copy_header(self, context):
context.try_skip(self.build_path) context.try_skip(self.build_path)
copy2( copy2(pj(self.source_path, 'mustache.hpp'),
pj(self.source_path, "mustache.hpp"), pj(self.buildEnv.install_dir, 'include'))
pj(self.buildEnv.install_dir, "include"),
)
def set_flatpak_buildsystem(self, module): def set_flatpak_buildsystem(self, module):
module["buildsystem"] = "simple" module['buildsystem'] = 'simple'
module["build-commands"] = ["cp mustache.hpp /app/include"] module['build-commands'] = ['cp mustache.hpp /app/include']

View File

@ -1,18 +1,19 @@
from .base import Dependency, ReleaseDownload, MesonBuilder from .base import (
Dependency,
ReleaseDownload,
MesonBuilder)
from kiwixbuild.utils import Remotefile from kiwixbuild.utils import Remotefile
class Pugixml(Dependency): class Pugixml(Dependency):
name = "pugixml" name = "pugixml"
class Source(ReleaseDownload): class Source(ReleaseDownload):
archive = Remotefile( archive = Remotefile('pugixml-1.2.tar.gz',
"pugixml-1.2.tar.gz", '0f422dad86da0a2e56a37fb2a88376aae6e931f22cc8b956978460c9db06136b')
"0f422dad86da0a2e56a37fb2a88376aae6e931f22cc8b956978460c9db06136b",
)
patches = ["pugixml_meson.patch"] patches = ["pugixml_meson.patch"]
flatpak_dest = "src" flatpak_dest = "src"
class Builder(MesonBuilder): class Builder(MesonBuilder):
strip_options = [] build_type = 'release'
strip_option = ''

View File

@ -1,25 +1,70 @@
import shutil import shutil
from .base import Dependency, NoopBuilder, NoopSource from .base import (
Dependency,
ReleaseDownload,
MakeBuilder,
QMakeBuilder)
from kiwixbuild.utils import SkipCommand, colorize from kiwixbuild.utils import Remotefile, pj, SkipCommand
class Qt(Dependency): class Qt(Dependency):
name = "qt" name = 'qt'
Source = NoopSource class Source(ReleaseDownload):
name = "qt"
source_dir = "qt-5.10.1"
archive = Remotefile('qt-everywhere-src-5.10.1.tar.xz',
'',
'http://ftp.oregonstate.edu/.1/blfs/conglomeration/qt5/qt-everywhere-src-5.10.1.tar.xz')
class Builder(NoopBuilder): class Builder(MakeBuilder):
def build(self): dependencies = ['icu4c', 'zlib']
error_msg = f"""WARNING: kiwix-build cannot build {self.name} for you. configure_option_template = "{dep_options} {static_option} {env_option} -prefix {install_dir} -libdir {libdir}"
You must install it yourself using official Qt installer or your distribution system.""" dynamic_configure_option = "-shared"
print(colorize(error_msg, "WARNING")) static_configure_option = "-static"
@property
def configure_option(self):
skip_modules = [
'qt3d',
'qtcanvas3d',
'qtcharts',
'qtconnectivity',
'qtdatavis3d',
# 'qtdeclarative',
'qtdoc',
'qtgamepad',
'qtgraphicaleffects',
'qtlocation',
'qtmultimedia',
'qtnetworkauth',
'qtpurchasing',
# 'qtquickcontrols',
'qtquickcontrols2',
'qtremoteobjects',
'qtscript',
'qtscxml',
'qtsensors',
'qtserialbus',
'qtserialport',
'qtspeech',
'qtvirtualkeyboard',
'qtwayland',
'qtwebglplugin',
'qtwebsockets',
# 'qtwebview',
]
skip_modules = " ".join("-skip {}".format(m) for m in skip_modules)
options = "-recheck -opensource -confirm-license -ccache -make libs {}".format(skip_modules)
return options
class QtWebEngine(Dependency): class QtWebEngine(Dependency):
name = "qtwebengine" name = "qtwebengine"
Source = NoopSource Source = Qt.Source
Builder = Qt.Builder class Builder(QMakeBuilder):
dependencies = ['qt']
subsource_dir = "qtwebengine"

View File

@ -1,93 +0,0 @@
import os
from .base import Dependency, ReleaseDownload, Builder
from kiwixbuild.utils import Remotefile, add_execution_right, run_command
pj = os.path.join
class android_ndk(Dependency):
dont_skip = True
neutral = False
name = "android-ndk"
gccver = "4.9.x"
api = "24"
class Source(ReleaseDownload):
archive = Remotefile(
"android-ndk-r21e-linux-x86_64.zip",
"ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e",
"https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip",
)
@property
def source_dir(self):
return self.target.full_name()
patches = [
"android-ndk-r21e-linux-x86_64-python3.12+.patch",
]
class Builder(Builder):
@property
def install_path(self):
return pj(self.buildEnv.toolchain_dir, self.target.full_name())
@property
def api(self):
return self.target.api
@property
def config(self):
return "android-" + self.api
@property
def arch(self):
return self.buildEnv.configInfo.arch
@property
def arch_full(self):
return self.buildEnv.configInfo.arch_full
def _build_toolchain(self, context):
context.try_skip(self.build_path)
script = pj(self.source_path, "build/tools/make_standalone_toolchain.py")
add_execution_right(script)
command = [
script,
f"--arch={self.arch}",
f"--api={self.api}",
f"--install-dir={self.install_path}",
"--force",
]
env = self.buildEnv.get_env(
cross_comp_flags=False, cross_compilers=False, cross_path=False
)
run_command(command, self.build_path, context, env=env)
def _fix_permission_right(self, context):
context.try_skip(self.build_path)
bin_dirs = [
pj(self.install_path, "bin"),
pj(self.install_path, self.arch_full, "bin"),
pj(
self.install_path,
"libexec",
"gcc",
self.arch_full,
self.target.gccver,
),
]
for root, dirs, files in os.walk(self.install_path):
if not root in bin_dirs:
continue
for file_ in files:
file_path = pj(root, file_)
if os.path.islink(file_path):
continue
add_execution_right(file_path)
def build(self):
self.command("build_toolchain", self._build_toolchain)
self.command("fix_permission_right", self._fix_permission_right)

View File

@ -1,55 +0,0 @@
from .base import Dependency, ReleaseDownload, TcCopyBuilder
from kiwixbuild.utils import Remotefile
# The arm toolchains
# This is based on toolchains published here : https://github.com/tttapa/docker-arm-cross-toolchain
base_url = (
"https://github.com/tttapa/docker-arm-cross-toolchain/releases/download/0.1.0/"
)
class armv6_toolchain(Dependency):
dont_skip = True
neutral = True
name = "armv6"
class Source(ReleaseDownload):
archive = Remotefile(
"x-tools-armv6-rpi-linux-gnueabihf.tar.xz",
"4c371c4c5b55ebd1f3d7dd26b14703632d9ba47423f901bcd9303d83ad444434",
base_url + "x-tools-armv6-rpi-linux-gnueabihf.tar.xz",
)
class Builder(TcCopyBuilder):
src_subdir = "armv6-rpi-linux-gnueabihf"
class armv8_toolchain(Dependency):
dont_skip = True
neutral = True
name = "armv8"
class Source(ReleaseDownload):
archive = Remotefile(
"x-tools-armv8-rpi-linux-gnueabihf.tar.xz",
"cc28f5c3f6a3e7d9985f98779c4e72224b4eb5a7e4dc2bcdefd90cb241fb94a5",
base_url + "x-tools-armv8-rpi3-linux-gnueabihf.tar.xz",
)
class Builder(TcCopyBuilder):
src_subdir = "armv8-rpi3-linux-gnueabihf"
class aarch64_toolchain(Dependency):
dont_skip = True
neutral = True
name = "aarch64"
class Source(ReleaseDownload):
archive = Remotefile(
"cross-gcc-6.3.0-pi_64.tar.gz",
"1b048bb8886ad63d21797cd9129fc37b9ea0dfaac7e3c36f888aa16fbec1d320",
)
Builder = TcCopyBuilder

View File

@ -1,47 +0,0 @@
import os
from .base import Dependency, ReleaseDownload, Builder
from kiwixbuild.utils import Remotefile, run_command, copy_tree
pj = os.path.join
class emsdk(Dependency):
dont_skip = True
neutral = False
name = "emsdk"
class Source(ReleaseDownload):
archive = Remotefile(
"emsdk-3.1.41.tar.gz",
"147a2d72df34227bdb4ffedc587a8cb674a42269c40458f3f69ae37e8966cdc6",
"https://codeload.github.com/emscripten-core/emsdk/tar.gz/refs/tags/3.1.41",
)
@property
def source_dir(self):
return self.target.full_name()
class Builder(Builder):
@property
def install_path(self):
return pj(self.buildEnv.toolchain_dir, self.target.full_name())
def _copy_source(self, context):
context.try_skip(self.build_path)
copy_tree(self.source_path, self.install_path)
def _install(self, context):
context.try_skip(self.build_path)
command = ["./emsdk", "install", "3.1.24"]
run_command(command, self.install_path, context)
def _activate(self, context):
context.try_skip(self.build_path)
command = ["./emsdk", "activate", "3.1.24"]
run_command(command, self.install_path, context)
def build(self):
self.command("copy_source", self._copy_source)
self.command("install", self._install)
self.command("activate", self._activate)

View File

@ -1,89 +0,0 @@
import os
from .base import Dependency, NoopSource, Builder
from kiwixbuild.utils import Remotefile, add_execution_right, run_command
pj = os.path.join
class org_kde(Dependency):
neutral = False
name = "org.kde"
Source = NoopSource
class Builder(Builder):
def _setup_remote(self, context):
command = [
"flatpak",
"--user",
"remote-add",
"--if-not-exists",
"flathub",
"https://flathub.org/repo/flathub.flatpakrepo",
]
env = self.buildEnv.get_env(
cross_comp_flags=False, cross_compilers=False, cross_path=False
)
run_command(command, self.buildEnv.build_dir, context, env=env)
def _install_sdk(self, context):
command = [
"flatpak",
"--user",
"install",
"--noninteractive",
"--verbose",
"-y",
"flathub",
f"{self.target.name}.Sdk//{self.target.version()}",
f"{self.target.name}.Platform//{self.target.version()}",
]
env = self.buildEnv.get_env(
cross_comp_flags=False, cross_compilers=False, cross_path=False
)
run_command(command, self.buildEnv.build_dir, context, env=env)
def build(self):
self.command("setup_remote", self._setup_remote)
self.command("install_sdk", self._install_sdk)
class io_qt_qtwebengine(Dependency):
neutral = False
name = "io.qt.qtwebengine"
Source = NoopSource
class Builder(Builder):
def _setup_remote(self, context):
command = [
"flatpak",
"--user",
"remote-add",
"--if-not-exists",
"flathub",
"https://flathub.org/repo/flathub.flatpakrepo",
]
env = self.buildEnv.get_env(
cross_comp_flags=False, cross_compilers=False, cross_path=False
)
run_command(command, self.buildEnv.build_dir, context, env=env)
def _install_sdk(self, context):
command = [
"flatpak",
"--user",
"install",
"-y",
"flathub",
f"{self.target.name}.BaseApp//{self.target.version()}",
]
env = self.buildEnv.get_env(
cross_comp_flags=False, cross_compilers=False, cross_path=False
)
run_command(command, self.buildEnv.build_dir, context, env=env)
def build(self):
self.command("setup_remote", self._setup_remote)
self.command("install_sdk", self._install_sdk)

View File

@ -1,32 +0,0 @@
from .base import Dependency, ReleaseDownload, TcCopyBuilder
from kiwixbuild.utils import Remotefile
class aarch64_musl_toolchain(Dependency):
dont_skip = True
neutral = True
name = "aarch64_musl"
class Source(ReleaseDownload):
archive = Remotefile(
"aarch64-linux-musl-cross.tgz",
"0f18a885b161815520bbb5757a4b4ab40d0898c29bebee58d0cddd6112e59cc6",
"https://more.musl.cc/10/x86_64-linux-musl/aarch64-linux-musl-cross.tgz",
)
Builder = TcCopyBuilder
class x86_64_musl_toolchain(Dependency):
dont_skip = True
neutral = True
name = "x86-64_musl"
class Source(ReleaseDownload):
archive = Remotefile(
"x86_64-linux-musl-cross.tgz",
"a3d55de8105739fcfb8b10eaa72cdb5d779319726bacff24149388d7608d1ed8",
"https://more.musl.cc/10/x86_64-linux-musl/x86_64-linux-musl-cross.tgz",
)
Builder = TcCopyBuilder

View File

@ -1,30 +1,25 @@
from .base import Dependency, ReleaseDownload, MakeBuilder from .base import (
Dependency,
ReleaseDownload,
MakeBuilder
)
from kiwixbuild.utils import Remotefile from kiwixbuild.utils import Remotefile
class UUID(Dependency): class UUID(Dependency):
name = "uuid" name = 'uuid'
class Source(ReleaseDownload): class Source(ReleaseDownload):
archive = Remotefile( archive = Remotefile('e2fsprogs-libs-1.43.4.tar.gz',
"e2fsprogs-libs-1.43.4.tar.gz", 'eed4516325768255c9745e7b82c9d7d0393abce302520a5b2cde693204b0e419',
"eed4516325768255c9745e7b82c9d7d0393abce302520a5b2cde693204b0e419", 'https://www.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v1.43.4/e2fsprogs-libs-1.43.4.tar.gz')
"https://www.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v1.43.4/e2fsprogs-libs-1.43.4.tar.gz", extract_dir = 'e2fsprogs-libs-1.43.4'
)
extract_dir = "e2fsprogs-libs-1.43.4"
class Builder(MakeBuilder): class Builder(MakeBuilder):
configure_options = [ configure_option = ("--enable-libuuid --disable-debugfs --disable-imager --disable-resizer --disable-defrag --enable-fsck"
"--enable-libuuid", " --disable-uuidd")
"--disable-debugfs", configure_env = {'_format_CFLAGS': "{env.CFLAGS} -O3 -fPIC"}
"--disable-imager", static_configure_option = dynamic_configure_option = ""
"--disable-resizer", make_target = 'libs'
"--disable-defrag", make_install_target = 'install-libs'
"--enable-fsck",
"--disable-uuidd",
]
configure_env = {"_format_CFLAGS": "{env.CFLAGS} -O3 -fPIC"}
static_configure_options = dynamic_configure_options = []
make_targets = ["libs"]
make_install_targets = ["install-libs"]

Some files were not shown because too many files have changed in this diff Show More