diff --git a/.github/scripts/build_definition.py b/.github/scripts/build_definition.py index 01fb906..e3ecd80 100644 --- a/.github/scripts/build_definition.py +++ b/.github/scripts/build_definition.py @@ -3,7 +3,7 @@ import csv, io, re # Definition of what to build. # Array is read line by line. -# Empty cells under (OS_NAME, PLATFORM_TARGET) mean "always match" (catch all, or `.*` regex) +# Empty cells under (OS_NAME, COMPILE_CONFIG) mean "always match" (catch all, or `.*` regex) # Once a cell doesn't match, skip to the next line. # Once a line matches, other lines are not read, so put more specific combination first. # Lines composed of `-` , or `=`, or starting by `#` are ignored. @@ -15,7 +15,7 @@ import csv, io, re # 'D' letter means we trigger the docker forkflow to build the docker image. # If a cell contains several letters, all are done. BUILD_DEF = """ - | OS_NAME | PLATFORM_TARGET | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | platform_name | + | OS_NAME | COMPILE_CONFIG | libzim | libkiwix | zim-tools | kiwix-tools | kiwix-desktop | platform_name | ===================================================================================================================== # Bionic is a special case as we need to compile libzim on old arch for python | bionic | native_mixed | BP | | | | | linux-x86_64-bionic | @@ -92,10 +92,10 @@ def selector_match(selector, value): class Context(NamedTuple): OS_NAME: str - PLATFORM_TARGET: str + COMPILE_CONFIG: str def match(self, row): - for key in ["OS_NAME", "PLATFORM_TARGET"]: + for key in ["OS_NAME", "COMPILE_CONFIG"]: context_value = getattr(self, key) selector = row[key] if not selector_match(selector, context_value): @@ -109,10 +109,11 @@ SOURCE_PUBLISH = "S" DEPS = "d" DOCKER = "D" -def select_build_targets(criteria): - from common import PLATFORM_TARGET, OS_NAME - context = Context(PLATFORM_TARGET=PLATFORM_TARGET, OS_NAME=OS_NAME) +def select_build_targets(criteria): + from common import COMPILE_CONFIG, OS_NAME + + context = Context(COMPILE_CONFIG=COMPILE_CONFIG, OS_NAME=OS_NAME) reader = csv.DictReader(strip_array(BUILD_DEF), dialect=TableDialect()) for row in reader: @@ -133,10 +134,11 @@ def select_build_targets(criteria): raise ValueError("No definition match with current context.") -def get_platform_name(): - from common import PLATFORM_TARGET, OS_NAME - context = Context(PLATFORM_TARGET=PLATFORM_TARGET, OS_NAME=OS_NAME) +def get_platform_name(): + from common import COMPILE_CONFIG, OS_NAME + + context = Context(COMPILE_CONFIG=COMPILE_CONFIG, OS_NAME=OS_NAME) reader = csv.DictReader(strip_array(BUILD_DEF), dialect=TableDialect()) for row in reader: diff --git a/.github/scripts/build_projects.py b/.github/scripts/build_projects.py index b267b64..68cdce9 100755 --- a/.github/scripts/build_projects.py +++ b/.github/scripts/build_projects.py @@ -8,16 +8,16 @@ from common import ( fix_macos_rpath, upload_archive, OS_NAME, - PLATFORM_TARGET, + COMPILE_CONFIG, DEV_BRANCH, ) for target in select_build_targets(BUILD): - run_kiwix_build(target, platform=PLATFORM_TARGET) + run_kiwix_build(target, config=COMPILE_CONFIG) if target == "kiwix-desktop": archive = create_desktop_image(make_release=False) else: - if PLATFORM_TARGET == "native_mixed" and OS_NAME == "macos": + if COMPILE_CONFIG == "native_mixed" and OS_NAME == "macos": fix_macos_rpath(target) archive = make_archive(target, make_release=False) if archive and DEV_BRANCH: diff --git a/.github/scripts/build_release_nightly.py b/.github/scripts/build_release_nightly.py index e97cda6..5ddf3c9 100755 --- a/.github/scripts/build_release_nightly.py +++ b/.github/scripts/build_release_nightly.py @@ -13,7 +13,7 @@ from common import ( fix_macos_rpath, BASE_DIR, OS_NAME, - PLATFORM_TARGET, + COMPILE_CONFIG, MAKE_RELEASE, notarize_macos_build, ) @@ -21,23 +21,23 @@ from common import ( from build_definition import select_build_targets, BUILD, PUBLISH, SOURCE_PUBLISH - - # Filter what to build if we are doing a release. if MAKE_RELEASE: TARGETS = select_build_targets(PUBLISH) + def release_filter(project): return release_versions.get(project) is not None + TARGETS = tuple(filter(release_filter, TARGETS)) else: TARGETS = select_build_targets(BUILD) for target in TARGETS: - run_kiwix_build(target, platform=PLATFORM_TARGET, make_release=MAKE_RELEASE) + run_kiwix_build(target, config=COMPILE_CONFIG, make_release=MAKE_RELEASE) if target == "kiwix-desktop": archive = create_desktop_image(make_release=MAKE_RELEASE) else: - if OS_NAME == "macos" and PLATFORM_TARGET.endswith("_mixed"): + if OS_NAME == "macos" and COMPILE_CONFIG.endswith("_mixed"): fix_macos_rpath(target) notarize_macos_build(target) archive = make_archive(target, make_release=MAKE_RELEASE) @@ -57,13 +57,11 @@ if MAKE_RELEASE: if target not in source_published_targets: continue run_kiwix_build( - target, platform=PLATFORM_TARGET, make_release=MAKE_RELEASE, make_dist=True + target, config=COMPILE_CONFIG, make_release=MAKE_RELEASE, make_dist=True ) full_target_name = "{}-{}".format(target, main_project_versions[target]) if target == "kiwix-desktop": - archive = ( - BASE_DIR / full_target_name / "{}.tar.gz".format(full_target_name) - ) + archive = BASE_DIR / full_target_name / "{}.tar.gz".format(full_target_name) else: archive = ( BASE_DIR @@ -74,5 +72,5 @@ if MAKE_RELEASE: upload_archive(archive, target, make_release=MAKE_RELEASE) # Publish flathub - if PLATFORM_TARGET == "flatpak" and "kiwix-desktop" in TARGETS: + if COMPILE_CONFIG == "flatpak" and "kiwix-desktop" in TARGETS: update_flathub_git() diff --git a/.github/scripts/common.py b/.github/scripts/common.py index bf635cf..360f193 100644 --- a/.github/scripts/common.py +++ b/.github/scripts/common.py @@ -20,12 +20,11 @@ from kiwixbuild.versions import ( ) - -PLATFORM_TARGET = _environ["PLATFORM_TARGET"] +COMPILE_CONFIG = _environ["COMPILE_CONFIG"] OS_NAME = _environ["OS_NAME"] HOME = Path(os.path.expanduser("~")) -BASE_DIR = HOME / "BUILD_{}".format(PLATFORM_TARGET) +BASE_DIR = HOME / "BUILD_{}".format(COMPILE_CONFIG) SOURCE_DIR = HOME / "SOURCE" ARCHIVE_DIR = HOME / "ARCHIVE" TOOLCHAIN_DIR = BASE_DIR / "TOOLCHAINS" @@ -35,7 +34,7 @@ KBUILD_SOURCE_DIR = HOME / "kiwix-build" _ref = _environ.get("GITHUB_REF", "").split("/")[-1] MAKE_RELEASE = re.fullmatch(r"r_[0-9]+", _ref) is not None -MAKE_RELEASE = MAKE_RELEASE and (_environ.get('GITHUB_EVENT_NAME') != 'schedule') +MAKE_RELEASE = MAKE_RELEASE and (_environ.get("GITHUB_EVENT_NAME") != "schedule") if not MAKE_RELEASE and _ref != "main": DEV_BRANCH = _ref @@ -45,19 +44,18 @@ else: FLATPAK_HTTP_GIT_REMOTE = "https://github.com/flathub/org.kiwix.desktop.git" FLATPAK_GIT_REMOTE = "git@github.com:flathub/org.kiwix.desktop.git" -BIN_EXT = ".exe" if PLATFORM_TARGET.startswith("win32_") else "" +BIN_EXT = ".exe" if COMPILE_CONFIG.startswith("win32_") else "" + def major_version(version: str) -> str: return version.split(".")[0] + # We have build everything. Now create archives for public deployement. EXPORT_FILES = { "kiwix-tools": ( INSTALL_DIR / "bin", - [ - f + BIN_EXT - for f in ("kiwix-manage", "kiwix-search", "kiwix-serve") - ], + [f + BIN_EXT for f in ("kiwix-manage", "kiwix-search", "kiwix-serve")], ), "zim-tools": ( INSTALL_DIR / "bin", @@ -72,7 +70,7 @@ EXPORT_FILES = { "zimpatch", "zimsplit", "zimwriterfs", - "zimrecreate" + "zimrecreate", ) ], ), @@ -80,11 +78,9 @@ EXPORT_FILES = { INSTALL_DIR, ( # We need to package all dependencies (`*.a`) on wasm - "lib/*/libzim.a" if PLATFORM_TARGET != "wasm" else "lib/*.a", + "lib/*/libzim.a" if COMPILE_CONFIG != "wasm" else "lib/*.a", "lib/*/libzim.so", - "lib/*/libzim.so.{version}".format( - version=main_project_versions["libzim"] - ), + "lib/*/libzim.so.{version}".format(version=main_project_versions["libzim"]), "lib/*/libzim.so.{version}".format( version=major_version(main_project_versions["libzim"]) ), @@ -95,9 +91,8 @@ EXPORT_FILES = { "lib/*/libzim.pc", "include/zim/**/*.h", "share/icu/{}/icudt{}l.dat".format( - base_deps_versions["icu4c"], - major_version(base_deps_versions["icu4c"]) - ) + base_deps_versions["icu4c"], major_version(base_deps_versions["icu4c"]) + ), ), ), "libkiwix": ( @@ -118,15 +113,15 @@ EXPORT_FILES = { "lib/*/libkiwix.pc", "include/kiwix/**/*.h", "share/icu/{}/icudt{}l.dat".format( - base_deps_versions["icu4c"], - major_version(base_deps_versions["icu4c"]) - ) + base_deps_versions["icu4c"], major_version(base_deps_versions["icu4c"]) + ), ), ), } DATE = date.today().isoformat() + def print_message(message, *args, **kwargs): message = message.format(*args, **kwargs) message = "{0} {1} {0}".format("-" * 3, message) @@ -136,23 +131,26 @@ def print_message(message, *args, **kwargs): MANIFEST_TEMPLATE = """{archive_name} *************************** -Dependencies archive for {target} on platform {platform} +Dependencies archive for {target} using config {config} Generated at {date} """ -def write_manifest(manifest_file, archive_name, target, platform): +def write_manifest(manifest_file, archive_name, target, config): with manifest_file.open(mode="w") as f: f.write( MANIFEST_TEMPLATE.format( - archive_name=archive_name, target=target, platform=platform, date=DATE, + archive_name=archive_name, + target=target, + config=config, + date=DATE, ) ) def run_kiwix_build( target, - platform, + config, build_deps_only=False, target_only=False, make_release=False, @@ -163,7 +161,7 @@ def run_kiwix_build( command.append("--hide-progress") command.append("--fast-clone") command.append("--assume-packages-installed") - command.extend(["--target-platform", platform]) + command.extend(["--config", config]) if build_deps_only: command.append("--build-deps-only") if target_only: @@ -240,7 +238,7 @@ def upload_archive(archive, project, make_release, dev_branch=None): print_message("No archive {} to upload!", archive) return - if project.startswith("kiwix-") or project in ['libkiwix']: + if project.startswith("kiwix-") or project in ["libkiwix"]: host = "ci@master.download.kiwix.org:30022" dest_path = "/data/download/" else: @@ -264,36 +262,39 @@ def upload_archive(archive, project, make_release, dev_branch=None): # This remove "share/doc" and "share/man" from the thing to copy in the deps archive def filter_install_dir(path): - for dir in path.glob('*'): - if dir.name not in ['share']: + for dir in path.glob("*"): + if dir.name not in ["share"]: yield dir else: - for sub_dir in dir.glob('*'): - if sub_dir.name not in ['doc', 'man']: + for sub_dir in dir.glob("*"): + if sub_dir.name not in ["doc", "man"]: yield sub_dir -# Full: True if we are creating a full archive to be used as cache by kiwix-build (base_deps2_{os}_{platform}_{base_deps_version}.tar.xz) -# Full: False if we are creating a archive to be used as pre-cached dependencies for project's CI (deps2_{os}_{platform}_{target}.tar.xz) + +# Full: True if we are creating a full archive to be used as cache by kiwix-build (base_deps2_{os}_{config}_{base_deps_version}.tar.xz) +# Full: False if we are creating a archive to be used as pre-cached dependencies for project's CI (deps2_{os}_{config}_{target}.tar.xz) def make_deps_archive(target=None, name=None, full=False): archive_name = name or "deps2_{}_{}_{}.tar.xz".format( - OS_NAME, PLATFORM_TARGET, target + OS_NAME, COMPILE_CONFIG, target ) print_message("Create archive {}.", archive_name) files_to_archive = list(filter_install_dir(INSTALL_DIR)) files_to_archive += HOME.glob("BUILD_*/LOGS") - if PLATFORM_TARGET == "apple_all_static": - for subplatform in AppleXCFramework.subPlatformNames: - base_dir = HOME / "BUILD_{}".format(subplatform) + if COMPILE_CONFIG == "apple_all_static": + for subconfig in AppleXCFramework.subConfigNames: + base_dir = HOME / "BUILD_{}".format(subconfig) files_to_archive += filter_install_dir(base_dir / "INSTALL") if (base_dir / "meson_cross_file.txt").exists(): files_to_archive.append(base_dir / "meson_cross_file.txt") - if PLATFORM_TARGET.endswith("_mixed"): - static_platform = PLATFORM_TARGET.replace("_mixed", "_static") - files_to_archive += filter_install_dir(HOME / ("BUILD_" + static_platform) / "INSTALL") - if PLATFORM_TARGET.startswith("android_"): + if COMPILE_CONFIG.endswith("_mixed"): + static_config = COMPILE_CONFIG.replace("_mixed", "_static") + files_to_archive += filter_install_dir( + HOME / ("BUILD_" + static_config) / "INSTALL" + ) + if COMPILE_CONFIG.startswith("android_"): files_to_archive += filter_install_dir(HOME / "BUILD_neutral" / "INSTALL") - base_dir = HOME / "BUILD_{}".format(PLATFORM_TARGET) + base_dir = HOME / "BUILD_{}".format(COMPILE_CONFIG) if (base_dir / "meson_cross_file.txt").exists(): files_to_archive.append(base_dir / "meson_cross_file.txt") # Copy any toolchain @@ -303,7 +304,7 @@ def make_deps_archive(target=None, name=None, full=False): files_to_archive.append(BASE_DIR / "meson_cross_file.txt") manifest_file = BASE_DIR / "manifest.txt" - write_manifest(manifest_file, archive_name, target, PLATFORM_TARGET) + write_manifest(manifest_file, archive_name, target, COMPILE_CONFIG) files_to_archive.append(manifest_file) relative_path = HOME @@ -311,9 +312,9 @@ def make_deps_archive(target=None, name=None, full=False): files_to_archive += ARCHIVE_DIR.glob(".*_ok") files_to_archive += BASE_DIR.glob("*/.*_ok") # Add also static build for mixed target - if PLATFORM_TARGET.endswith("_mixed"): - static_platform = PLATFORM_TARGET.replace("_mixed", "_static") - files_to_archive += (HOME / ("BUILD_" + static_platform)).glob("*/.*_ok") + if COMPILE_CONFIG.endswith("_mixed"): + static_config = COMPILE_CONFIG.replace("_mixed", "_static") + files_to_archive += (HOME / ("BUILD_" + static_config)).glob("*/.*_ok") # Native dyn and static is needed for potential cross compilation that use native tools (icu) files_to_archive += (HOME / "BUILD_native_dyn").glob("*/.*_ok") files_to_archive += (HOME / "BUILD_native_static").glob("*/.*_ok") @@ -391,7 +392,7 @@ def create_desktop_image(make_release): postfix = DATE src_dir = SOURCE_DIR / "kiwix-desktop" - if PLATFORM_TARGET == "flatpak": + if COMPILE_CONFIG == "flatpak": build_path = BASE_DIR / "org.kiwix.desktop.flatpak" app_name = "org.kiwix.desktop.{}.flatpak".format(postfix) print_message("archive is {}", build_path) @@ -459,7 +460,6 @@ def update_flathub_git(): def fix_macos_rpath(project): - base_dir, export_files = EXPORT_FILES[project] for file in filter(lambda f: f.endswith(".dylib"), export_files): lib = base_dir / file @@ -478,22 +478,35 @@ def trigger_workflow(repo, workflow="docker.yml", ref="main", inputs=None): ref: branch or tag name inputs: dict of inputs to pass to the workflow""" print_message( - "triggering workflow `{workflow}` on {repo}@{ref} " - "with inputs={inputs}", workflow=workflow, repo=repo, ref=ref, inputs=inputs) + "triggering workflow `{workflow}` on {repo}@{ref} " "with inputs={inputs}", + workflow=workflow, + repo=repo, + ref=ref, + inputs=inputs, + ) url = "{base_url}/repos/{repo}/actions/workflows/{workflow}/dispatches".format( base_url=os.getenv("GITHUB_API_URL", "https://api.github.com"), - repo=repo, workflow=workflow) + repo=repo, + workflow=workflow, + ) - resp = requests.post(url, headers={ + resp = requests.post( + url, + headers={ "Content-Type": "application/json", - "Authorization": "token {token}".format( - token=os.getenv('GITHUB_PAT', '')), + "Authorization": "token {token}".format(token=os.getenv("GITHUB_PAT", "")), "Accept": "application/vnd.github.v3+json", - }, json={"ref": ref, "inputs": inputs}, timeout=5) + }, + json={"ref": ref, "inputs": inputs}, + timeout=5, + ) if resp.status_code != 204: - raise ValueError("Unexpected HTTP {code}: {reason}".format( - code=resp.status_code, reason=resp.reason)) + raise ValueError( + "Unexpected HTTP {code}: {reason}".format( + code=resp.status_code, reason=resp.reason + ) + ) def trigger_docker_publish(target): @@ -501,13 +514,14 @@ def trigger_docker_publish(target): return version = get_postfix(target) - repo = { - "zim-tools": "openzim/zim-tools", - "kiwix-tools": "kiwix/kiwix-tools"}.get(target) + repo = {"zim-tools": "openzim/zim-tools", "kiwix-tools": "kiwix/kiwix-tools"}.get( + target + ) try: - trigger_workflow(repo, workflow="docker.yml", ref="main", - inputs={"version": version}) + trigger_workflow( + repo, workflow="docker.yml", ref="main", inputs={"version": version} + ) print_message("triggered docker workflow on {repo}", repo=repo) except Exception as exc: print_message("Error triggering workflow: {exc}", exc=exc) @@ -515,39 +529,52 @@ def trigger_docker_publish(target): def notarize_macos_build(project): - """ sign and notarize files for macOS + """sign and notarize files for macOS - Expects the following environment: - - `SIGNING_IDENTITY` environ with Certificate name/identity - - `KEYCHAIN` environ with path to the keychain storing credentials - - `KEYCHAIN_PROFILE` environ with name of the profile in that keychain - - `KEYCHAIN_PASSWORD` environ with password to unlock the keychain - """ + Expects the following environment: + - `SIGNING_IDENTITY` environ with Certificate name/identity + - `KEYCHAIN` environ with path to the keychain storing credentials + - `KEYCHAIN_PROFILE` environ with name of the profile in that keychain + - `KEYCHAIN_PASSWORD` environ with password to unlock the keychain + """ if project != "libzim": return # currently only supports libzim use case: sign every dylib base_dir, export_files = EXPORT_FILES[project] - filepaths = [base_dir.joinpath(file) - for file in filter(lambda f: f.endswith(".dylib"), export_files) - if not base_dir.joinpath(file).is_symlink()] + filepaths = [ + base_dir.joinpath(file) + for file in filter(lambda f: f.endswith(".dylib"), export_files) + if not base_dir.joinpath(file).is_symlink() + ] if not filepaths: return for filepath in filepaths: - subprocess.check_call(["/usr/bin/codesign", "--force", "--sign", - os.getenv("SIGNING_IDENTITY", "no-signing-ident"), - "--keychain", - os.getenv("KEYCHAIN", "no-keychain-path"), - str(filepath), "--deep", "--timestamp"], env=os.environ) + subprocess.check_call( + [ + "/usr/bin/codesign", + "--force", + "--sign", + os.getenv("SIGNING_IDENTITY", "no-signing-ident"), + "--keychain", + os.getenv("KEYCHAIN", "no-keychain-path"), + str(filepath), + "--deep", + "--timestamp", + ], + env=os.environ, + ) # create a zip of the dylibs and upload for notarization zip_name = "{}.zip".format(project) subprocess.check_call( ["/usr/bin/ditto", "-c", "-k", "--keepParent"] - + [str(f) for f in filepaths] + [zip_name], - env=os.environ) + + [str(f) for f in filepaths] + + [zip_name], + env=os.environ, + ) # make sure keychain is unlocked subprocess.check_call( diff --git a/.github/scripts/compile_all_deps.py b/.github/scripts/compile_all_deps.py index 0912763..8a62c73 100755 --- a/.github/scripts/compile_all_deps.py +++ b/.github/scripts/compile_all_deps.py @@ -6,13 +6,13 @@ from common import ( run_kiwix_build, make_deps_archive, upload, - PLATFORM_TARGET, + COMPILE_CONFIG, DEV_BRANCH, ) from build_definition import select_build_targets, DEPS for target in select_build_targets(DEPS): - run_kiwix_build(target, platform=PLATFORM_TARGET, build_deps_only=True) + run_kiwix_build(target, config=COMPILE_CONFIG, build_deps_only=True) archive_file = make_deps_archive(target=target) if DEV_BRANCH: destination = "/data/tmp/ci/dev_preview/" + DEV_BRANCH diff --git a/.github/scripts/ensure_base_deps.py b/.github/scripts/ensure_base_deps.py index ffa012e..0995f9a 100755 --- a/.github/scripts/ensure_base_deps.py +++ b/.github/scripts/ensure_base_deps.py @@ -13,10 +13,11 @@ from common import ( upload, make_deps_archive, HOME, - PLATFORM_TARGET, + COMPILE_CONFIG, OS_NAME, ) + def download_base_archive(base_name): url = "http://tmp.kiwix.org/ci/{}".format(base_name) file_path = str(HOME / base_name) @@ -30,14 +31,15 @@ def download_base_archive(base_name): file.write(batch) return file_path -ARCHIVE_NAME_TEMPLATE = "base_deps2_{os}_{platform}_{version}.tar.xz" -if PLATFORM_TARGET == 'flatpak': +ARCHIVE_NAME_TEMPLATE = "base_deps2_{os}_{config}_{version}.tar.xz" + +if COMPILE_CONFIG == "flatpak": base_dep_archive_name = "base_deps2_flatpak.tar.xz" else: base_dep_archive_name = ARCHIVE_NAME_TEMPLATE.format( os=OS_NAME, - platform=PLATFORM_TARGET, + config=COMPILE_CONFIG, version=base_deps_meta_version, ) @@ -48,11 +50,11 @@ try: f.extractall(str(HOME)) os.remove(str(local_filename)) except URLError: - if PLATFORM_TARGET == "flatpak": + if COMPILE_CONFIG == "flatpak": print_message("Cannot get archive. Move on") else: print_message("Cannot get archive. Build dependencies") - run_kiwix_build("alldependencies", platform=PLATFORM_TARGET) + run_kiwix_build("alldependencies", config=COMPILE_CONFIG) archive_file = make_deps_archive(name=base_dep_archive_name, full=True) upload(archive_file, "ci@tmp.kiwix.org:30022", "/data/tmp/ci") os.remove(str(archive_file)) diff --git a/.github/scripts/upload_failure_logs.sh b/.github/scripts/upload_failure_logs.sh index a284afb..0714d37 100755 --- a/.github/scripts/upload_failure_logs.sh +++ b/.github/scripts/upload_failure_logs.sh @@ -4,7 +4,7 @@ set -e cd $HOME -ARCHIVE_NAME=fail_log_${OS_NAME}_${PLATFORM_TARGET}.tar.gz +ARCHIVE_NAME=fail_log_${OS_NAME}_${COMPILE_CONFIG}.tar.gz tar -czf ${ARCHIVE_NAME} $HOME/BUILD_* $HOME/SOURCE $HOME/LOGS $HOME/TOOLCHAINS echo "Uploading archive $ARCHIVE_NAME" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da9d298..65da67b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,7 @@ jobs: strategy: fail-fast: false matrix: - target: + config: - native_static - native_dyn - native_mixed @@ -37,13 +37,13 @@ jobs: - android_x86_64 image_variant: ['focal'] include: - - target: native_mixed + - config: native_mixed image_variant: bionic - - target: aarch64_mixed + - config: aarch64_mixed image_variant: bionic - - target: win32_static + - config: win32_static image_variant: f35 - - target: win32_dyn + - config: win32_dyn image_variant: f35 env: HOME: /home/runner @@ -74,26 +74,26 @@ jobs: cd $HOME kiwix-build/.github/scripts/ensure_base_deps.py env: - PLATFORM_TARGET: ${{matrix.target}} + COMPILE_CONFIG: ${{matrix.config}} - name: Compile all deps shell: bash run: | cd $HOME kiwix-build/.github/scripts/compile_all_deps.py env: - PLATFORM_TARGET: ${{matrix.target}} + COMPILE_CONFIG: ${{matrix.config}} - name: Build projects shell: bash run: | cd $HOME kiwix-build/.github/scripts/build_projects.py env: - PLATFORM_TARGET: ${{matrix.target}} + COMPILE_CONFIG: ${{matrix.config}} - name: Upload failure logs if: failure() run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh env: - PLATFORM_TARGET: ${{matrix.target}} + COMPILE_CONFIG: ${{matrix.config}} Flatpak: strategy: @@ -101,7 +101,7 @@ jobs: env: HOME: /home/runner SSH_KEY: /tmp/id_rsa - PLATFORM_TARGET: flatpak + COMPILE_CONFIG: flatpak OS_NAME: focal runs-on: ubuntu-22.04 steps: @@ -142,7 +142,7 @@ jobs: strategy: fail-fast: false matrix: - target: + config: - native_dyn - native_static - native_mixed @@ -185,23 +185,23 @@ jobs: cd $HOME $GITHUB_WORKSPACE/.github/scripts/ensure_base_deps.py env: - PLATFORM_TARGET: ${{matrix.target}} + COMPILE_CONFIG: ${{matrix.config}} - name: Compile all deps shell: bash run: | cd $HOME $GITHUB_WORKSPACE/.github/scripts/compile_all_deps.py env: - PLATFORM_TARGET: ${{matrix.target}} + COMPILE_CONFIG: ${{matrix.config}} - name: Build projects shell: bash run: | cd $HOME $GITHUB_WORKSPACE/.github/scripts/build_projects.py env: - PLATFORM_TARGET: ${{matrix.target}} + COMPILE_CONFIG: ${{matrix.config}} - name: Upload failure logs if: failure() run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.sh env: - PLATFORM_TARGET: ${{matrix.target}} + COMPILE_CONFIG: ${{matrix.config}} diff --git a/.github/workflows/releaseNigthly.yml b/.github/workflows/releaseNigthly.yml index 32c48d2..7d786ff 100644 --- a/.github/workflows/releaseNigthly.yml +++ b/.github/workflows/releaseNigthly.yml @@ -12,7 +12,7 @@ jobs: strategy: fail-fast: false matrix: - target: + config: - native_static - native_mixed - native_dyn @@ -34,11 +34,11 @@ jobs: - android_x86_64 image_variant: ['focal'] include: - - target: native_mixed + - config: native_mixed image_variant: bionic - - target: aarch64_mixed + - config: aarch64_mixed image_variant: bionic - - target: win32_static + - config: win32_static image_variant: f35 env: HOME: /home/runner @@ -69,21 +69,21 @@ jobs: cd $HOME kiwix-build/.github/scripts/ensure_base_deps.py env: - PLATFORM_TARGET: ${{matrix.target}} + COMPILE_CONFIG: ${{matrix.config}} - name: Build release shell: bash run: | cd $HOME kiwix-build/.github/scripts/build_release_nightly.py env: - PLATFORM_TARGET: ${{matrix.target}} + COMPILE_CONFIG: ${{matrix.config}} BINTRAY_USER: kiwix BINTRAY_PASS: ${{secrets.bintray_pass}} - name: Upload failure logs if: failure() run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh env: - PLATFORM_TARGET: ${{matrix.target}} + COMPILE_CONFIG: ${{matrix.config}} Flatpak: strategy: @@ -91,7 +91,7 @@ jobs: env: HOME: /home/runner SSH_KEY: /tmp/id_rsa - PLATFORM_TARGET: flatpak + COMPILE_CONFIG: flatpak OS_NAME: focal runs-on: ubuntu-22.04 steps: @@ -132,7 +132,7 @@ jobs: strategy: fail-fast: false matrix: - target: + config: - native_dyn - native_static - native_mixed @@ -195,26 +195,26 @@ jobs: cd $HOME $GITHUB_WORKSPACE/.github/scripts/ensure_base_deps.py env: - PLATFORM_TARGET: ${{matrix.target}} + COMPILE_CONFIG: ${{matrix.config}} - name: Build release shell: bash run: | cd $HOME $GITHUB_WORKSPACE/.github/scripts/build_release_nightly.py env: - PLATFORM_TARGET: ${{matrix.target}} + COMPILE_CONFIG: ${{matrix.config}} - name: Upload failure logs if: failure() run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.sh env: - PLATFORM_TARGET: ${{matrix.target}} + COMPILE_CONFIG: ${{matrix.config}} Trigger_Docker: needs: [Linux] runs-on: ubuntu-22.04 env: - PLATFORM_TARGET: native_static + COMPILE_CONFIG: native_static OS_NAME: linux steps: - name: Checkout code diff --git a/README.md b/README.md index 4381815..1ba27ed 100644 --- a/README.md +++ b/README.md @@ -78,12 +78,12 @@ invalid choice: 'not-existing-target' (choose from 'alldependencies', 'android-n ... ``` -#### Target platform +#### Config -If no target platform is specified, the default will be `native_dyn`. +If no config is specified, the default will be `native_dyn`. -You can select another target platform using the option -`--target-platform`. For now, there is ten different supported +You can select another config using the option +`--config`. For now, there is ten different supported platforms: - native_dyn @@ -98,9 +98,12 @@ platforms: - android_x86_64 - flatpak -So, if you want to compile `kiwix-tools` for win32 using static linkage: +All `native_*` config means using the native compiler without any cross-compilation option. +Other may simply use cross-compilation or may download a specific toolchain to use. + +If you want to compile `kiwix-tools` for win32 using static linkage: ```bash -kiwix-build --target-platform win32_dyn +kiwix-build --config win32_dyn ``` Android @@ -108,32 +111,23 @@ Android `kiwix-android` (https://github.com/kiwix/kiwix-android) depends of the `libkiwix` project. -It uses a special `.aar` file that represent (and embed) the libkiwix for -all supported android arch. This is a kind of fat archive we have for MacOs. - -The `.aar` file is build using the `libkiwix-app` project. -`libkiwix-app` itself is architecture independent (it is just a packaging of -other archives) but it use `libkiwix` who is architecture dependent. When building `libkiwix`, you should directly use the target-platform `android_`: ```bash -kiwix-build libkiwix --target-platform android_arm +kiwix-build libkiwix --config android_arm ``` -But, `libkiwix-app` is mainly multi arch. -To compile `libkiwix-app`, you must use the `android` platform: +You may directly use the special config `android` which will build different android architectures ```bash -kiwix-build --target-platform android libkiwix-app -kiwix-build libkiwix-app # because `android` platform is the default for `libkiwix-app` +kiwix-build --config android libkiwix ``` -By default, when using platform `android`, `libkiwix` will be build for -all architectures. This can be changed by using the option `--android-arch`: +By default, it will build for all android architecture, +you can limit this with option `--android-arch`: ```bash -kiwix-build libkiwix-app # aar with all architectures -kiwix-build libkiwix-app --android-arch arm # aar with arm architecture -kiwix-build libkiwix-app --android-arch arm --android-arch arm64 # aan with arm and arm64 architectures +kiwix-build libkiwix --config android --android-arch arm # aar with arm architecture +kiwix-build libkiwix --config android --android-arch arm --android-arch arm64 # aan with arm and arm64 architectures ``` To build `kiwix-android` itself, you should see the documentation of `kiwix-android`. @@ -148,13 +142,13 @@ To do so, you should directly use the target-platfrom `ios_multi`. As for `android`, `kiwix-build` will build the library several times (once for each platform) and then create the fat library. ```bash -kiwix-build --target-platform iOS_multi libkiwix +kiwix-build --config iOS_multi libkiwix ``` You can specify the supported architectures with the option `--ios-arch`: ```bash -kiwix-build --target-platform iOS_multi libkiwix # all architetures -kiwix-build --target-platform iOS_multi --ios-arch arm --ios-arch arm64 # arm and arm64 arch only +kiwix-build --config iOS_multi libkiwix # all architetures +kiwix-build --config iOS_multi --ios-arch arm --ios-arch arm64 # arm and arm64 arch only ``` Outputs @@ -163,9 +157,9 @@ Outputs Kiwix-build.py will create several directories: - `ARCHIVES`: All the downloaded archives go there. - `SOURCES`: All the sources (extracted from archives and patched) go there. -- `BUILD_`: All the build files go there. -- `BUILD_/INSTALL`: The installed files go there. -- `BUILD_/LOGS`: The logs files of the build. +- `BUILD_`: All the build files go there. +- `BUILD_/INSTALL`: The installed files go there. +- `BUILD_/LOGS`: The logs files of the build. If you want to install all those directories elsewhere, you can pass the `--working-dir` option to `kiwix-build`: diff --git a/kiwixbuild/__init__.py b/kiwixbuild/__init__.py index cdbd6b9..33f2926 100644 --- a/kiwixbuild/__init__.py +++ b/kiwixbuild/__init__.py @@ -4,81 +4,145 @@ import os, sys import argparse from .dependencies import Dependency -from .platforms import PlatformInfo +from .configs import ConfigInfo from .builder import Builder from .flatpak_builder import FlatpakBuilder from . import _global + def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument('target', default='kiwix-tools', nargs='?', metavar='TARGET', - choices=Dependency.all_deps.keys()) - parser.add_argument('--working-dir', default=".", - help=("Directory where kiwix-build puts all its files " - "(source, archive and build)\n" - "working-dir can be absolute path or a relative (to cwd) one.")) - parser.add_argument('--build-dir', default=".", - help=("Directory where kiwix-build puts all build files.\n" - "build-dir can be absolute path or a relative (to working-dir) one.")) - parser.add_argument('--libprefix', default=None) - parser.add_argument('--target-platform', choices=PlatformInfo.all_platforms) - parser.add_argument('--verbose', '-v', action="store_true", - help=("Print all logs on stdout instead of in specific" - " log files per commands")) - parser.add_argument('--hide-progress', action='store_false', dest='show_progress', - help="Hide intermediate progress information.") - parser.add_argument('--skip-source-prepare', action='store_true', - help="Skip the source download part") - parser.add_argument('--build-deps-only', action='store_true', - help="Build only the dependencies of the specified target.") - parser.add_argument('--build-nodeps', action='store_true', - help="Build only the target, not its dependencies.") - parser.add_argument('--make-dist', action='store_true', - help="Build distrubution (dist) source archive") - parser.add_argument('--make-release', action='store_true', - help="Build a release version") - subgroup = parser.add_argument_group('advanced') - subgroup.add_argument('--no-cert-check', action='store_true', - help="Skip SSL certificate verification during download") - subgroup.add_argument('--clean-at-end', action='store_true', - help="Clean all intermediate files after the (successfull) build") - subgroup.add_argument('--dont-install-packages', action='store_true', - help="Do not try to install packages before compiling") - subgroup.add_argument('--assume-packages-installed', action='store_true', - help="Assume the package to install to be aleady installed") - subgroup.add_argument('--android-arch', action='append', - help=("Specify the architecture to build for android application/libraries.\n" - "Can be specified several times to build for several architectures.\n" - "If not specified, all architectures will be build.")) - subgroup.add_argument('--ios-arch', action='append', - help=("Specify the architecture to build for ios application/libraries.\n" - "Can be specified several times to build for several architectures.\n" - "If not specified, all architectures will be build.")) - subgroup.add_argument('--fast-clone', action='store_true', - help=("Do not clone the whole repository.\n" - "This is useful for one shot build but it is not recommended if you want " - "to develop with the cloned sources.")) + parser.add_argument( + "target", + default="kiwix-tools", + nargs="?", + metavar="TARGET", + choices=Dependency.all_deps.keys(), + ) + parser.add_argument( + "--working-dir", + default=".", + help=( + "Directory where kiwix-build puts all its files " + "(source, archive and build)\n" + "working-dir can be absolute path or a relative (to cwd) one." + ), + ) + parser.add_argument( + "--build-dir", + default=".", + help=( + "Directory where kiwix-build puts all build files.\n" + "build-dir can be absolute path or a relative (to working-dir) one." + ), + ) + parser.add_argument("--libprefix", default=None) + parser.add_argument( + "--config", choices=ConfigInfo.all_configs, default="native_dyn" + ) + parser.add_argument( + "--verbose", + "-v", + action="store_true", + help=( + "Print all logs on stdout instead of in specific" " log files per commands" + ), + ) + parser.add_argument( + "--hide-progress", + action="store_false", + dest="show_progress", + help="Hide intermediate progress information.", + ) + parser.add_argument( + "--skip-source-prepare", + action="store_true", + help="Skip the source download part", + ) + parser.add_argument( + "--build-deps-only", + action="store_true", + help="Build only the dependencies of the specified target.", + ) + parser.add_argument( + "--build-nodeps", + action="store_true", + help="Build only the target, not its dependencies.", + ) + parser.add_argument( + "--make-dist", + action="store_true", + help="Build distrubution (dist) source archive", + ) + parser.add_argument( + "--make-release", action="store_true", help="Build a release version" + ) + subgroup = parser.add_argument_group("advanced") + subgroup.add_argument( + "--no-cert-check", + action="store_true", + help="Skip SSL certificate verification during download", + ) + subgroup.add_argument( + "--clean-at-end", + action="store_true", + help="Clean all intermediate files after the (successfull) build", + ) + subgroup.add_argument( + "--dont-install-packages", + action="store_true", + help="Do not try to install packages before compiling", + ) + subgroup.add_argument( + "--assume-packages-installed", + action="store_true", + help="Assume the package to install to be aleady installed", + ) + subgroup.add_argument( + "--android-arch", + action="append", + help=( + "Specify the architecture to build for android application/libraries.\n" + "Can be specified several times to build for several architectures.\n" + "If not specified, all architectures will be build." + ), + ) + subgroup.add_argument( + "--ios-arch", + action="append", + help=( + "Specify the architecture to build for ios application/libraries.\n" + "Can be specified several times to build for several architectures.\n" + "If not specified, all architectures will be build." + ), + ) + subgroup.add_argument( + "--fast-clone", + action="store_true", + help=( + "Do not clone the whole repository.\n" + "This is useful for one shot build but it is not recommended if you want " + "to develop with the cloned sources." + ), + ) options = parser.parse_args() if not options.android_arch: - options.android_arch = ['arm', 'arm64', 'x86', 'x86_64'] + options.android_arch = ["arm", "arm64", "x86", "x86_64"] if not options.ios_arch: - options.ios_arch = ['arm64', 'x86_64'] - - if not options.target_platform: - options.target_platform = 'native_dyn' + options.ios_arch = ["arm64", "x86_64"] return options + def main(): options = parse_args() options.working_dir = os.path.abspath(options.working_dir) _global.set_options(options) - neutralEnv = buildenv.PlatformNeutralEnv() + neutralEnv = buildenv.NeutralEnv() _global.set_neutralEnv(neutralEnv) - if options.target_platform == 'flatpak': + if options.config == "flatpak": builder = FlatpakBuilder() else: builder = Builder() builder.run() - diff --git a/kiwixbuild/_global.py b/kiwixbuild/_global.py index c476c09..971555c 100644 --- a/kiwixbuild/_global.py +++ b/kiwixbuild/_global.py @@ -5,23 +5,29 @@ _neutralEnv = None _options = None _target_steps = _OrderedDict() + def set_neutralEnv(env): global _neutralEnv _neutralEnv = env + def neutralEnv(what): return getattr(_neutralEnv, what) + def set_options(options): global _options _options = options + def option(what): return getattr(_options, what) + def add_target_step(key, what): _target_steps[key] = what + def get_target_step(key, default_context=None): if isinstance(key, tuple): context, target = key @@ -29,24 +35,6 @@ def get_target_step(key, default_context=None): context, target = default_context, key return _target_steps[(context, target)] + def target_steps(): return _target_steps - - -def backend(): - global _backend - if _backend is not None: - return _backend - - _platform = platform.system() - if _platform == 'Windows': - print('ERROR: kiwix-build is not intented to run on Windows platform.\n' - 'There is no backend for Windows, so we can\'t launch any commands.') - sys.exit(0) - if _platform == 'Linux': - _platform, _, _ = platform.linux_distribution() - _platform = _platform.lower() - _backend = backends.Linux() - - return _backend - diff --git a/kiwixbuild/buildenv.py b/kiwixbuild/buildenv.py index 458542f..37fe7fc 100644 --- a/kiwixbuild/buildenv.py +++ b/kiwixbuild/buildenv.py @@ -1,4 +1,3 @@ - import os, sys, shutil import subprocess import platform @@ -8,62 +7,60 @@ from .utils import pj, download_remote, escape_path from ._global import neutralEnv, option -class PlatformNeutralEnv: +class NeutralEnv: def __init__(self): - self.working_dir = option('working_dir') + self.working_dir = option("working_dir") self.source_dir = pj(self.working_dir, "SOURCE") self.archive_dir = pj(self.working_dir, "ARCHIVE") self.toolchain_dir = pj(self.working_dir, "TOOLCHAINS") - self.log_dir = pj(self.working_dir, 'LOGS') - for d in (self.source_dir, - self.archive_dir, - self.toolchain_dir, - self.log_dir): + self.log_dir = pj(self.working_dir, "LOGS") + for d in (self.source_dir, self.archive_dir, self.toolchain_dir, self.log_dir): os.makedirs(d, exist_ok=True) self.detect_platform() self.ninja_command = self._detect_command( - 'ninja', - default=[['ninja'], ['ninja-build']]) + "ninja", default=[["ninja"], ["ninja-build"]] + ) self.meson_command = self._detect_command( - 'meson', - default= [['meson.py'], ['meson']]) + "meson", default=[["meson.py"], ["meson"]] + ) self.mesontest_command = [*self.meson_command, "test"] - self.patch_command = self._detect_command('patch') - self.git_command = self._detect_command('git') - self.make_command = self._detect_command('make') - self.cmake_command = self._detect_command('cmake') - self.qmake_command = self._detect_command('qmake', required=False) - + self.patch_command = self._detect_command("patch") + self.git_command = self._detect_command("git") + self.make_command = self._detect_command("make") + self.cmake_command = self._detect_command("cmake") + self.qmake_command = self._detect_command("qmake", required=False) def detect_platform(self): _platform = platform.system() self.distname = _platform - if _platform == 'Windows': - print('ERROR: kiwix-build is not intented to run on Windows platform.\n' - 'It should probably not work, but well, you still can have a try.') - cont = input('Do you want to continue ? [y/N]') - if cont.lower() != 'y': + if _platform == "Windows": + print( + "ERROR: kiwix-build is not intented to run on Windows platform.\n" + "It should probably not work, but well, you still can have a try." + ) + cont = input("Do you want to continue ? [y/N]") + if cont.lower() != "y": sys.exit(0) - if _platform == 'Linux': + if _platform == "Linux": self.distname = distro.id() - if self.distname == 'ubuntu': - self.distname = 'debian' + if self.distname == "ubuntu": + self.distname = "debian" def download(self, what, where=None): where = where or self.archive_dir download_remote(what, where) - - def _detect_command(self, name, default=None, options=['--version'], required=True): + def _detect_command(self, name, default=None, options=["--version"], required=True): if default is None: default = [[name]] - env_key = 'KBUILD_{}_COMMAND'.format(name.upper()) + env_key = "KBUILD_{}_COMMAND".format(name.upper()) if env_key in os.environ: default = [os.environ[env_key].split()] + default for command in default: try: - retcode = subprocess.check_call(command + options, - stdout=subprocess.DEVNULL) + retcode = subprocess.check_call( + command + options, stdout=subprocess.DEVNULL + ) except (FileNotFoundError, PermissionError, OSError): # Doesn't exist in PATH or isn't executable continue @@ -76,22 +73,20 @@ class PlatformNeutralEnv: print("WARNING: {} command not found".format(name)) return ["{}_NOT_FOUND".format(name.upper())] + class BuildEnv: - def __init__(self, platformInfo): - build_dir = "BUILD_{}".format(platformInfo.name) - self.platformInfo = platformInfo - self.base_build_dir = pj(option('working_dir'), option('build_dir')) + def __init__(self, configInfo): + build_dir = "BUILD_{}".format(configInfo.name) + self.configInfo = configInfo + self.base_build_dir = pj(option("working_dir"), option("build_dir")) self.build_dir = pj(self.base_build_dir, build_dir) self.install_dir = pj(self.build_dir, "INSTALL") self.toolchain_dir = pj(self.build_dir, "TOOLCHAINS") - self.log_dir = pj(self.build_dir, 'LOGS') - for d in (self.build_dir, - self.install_dir, - self.toolchain_dir, - self.log_dir): + self.log_dir = pj(self.build_dir, "LOGS") + for d in (self.build_dir, self.install_dir, self.toolchain_dir, self.log_dir): os.makedirs(d, exist_ok=True) - self.libprefix = option('libprefix') or self._detect_libdir() + self.libprefix = option("libprefix") or self._detect_libdir() def clean_intermediate_directories(self): for subdir in os.listdir(self.build_dir): @@ -104,79 +99,82 @@ class BuildEnv: os.remove(subpath) def _is_debianlike(self): - return os.path.isfile('/etc/debian_version') + return os.path.isfile("/etc/debian_version") def _detect_libdir(self): - if self.platformInfo.libdir is not None: - return self.platformInfo.libdir + if self.configInfo.libdir is not None: + return self.configInfo.libdir if self._is_debianlike(): try: - pc = subprocess.Popen(['dpkg-architecture', '-qDEB_HOST_MULTIARCH'], - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL) + pc = subprocess.Popen( + ["dpkg-architecture", "-qDEB_HOST_MULTIARCH"], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) (stdo, _) = pc.communicate() if pc.returncode == 0: archpath = stdo.decode().strip() - return 'lib/' + archpath + return "lib/" + archpath except Exception: pass - if os.path.isdir('/usr/lib64') and not os.path.islink('/usr/lib64'): - return 'lib64' - return 'lib' + if os.path.isdir("/usr/lib64") and not os.path.islink("/usr/lib64"): + return "lib64" + return "lib" def get_env(self, *, cross_comp_flags, cross_compilers, cross_path): - env = self.platformInfo.get_env() - pkgconfig_path = pj(self.install_dir, self.libprefix, 'pkgconfig') - env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path]) + env = self.configInfo.get_env() + pkgconfig_path = pj(self.install_dir, self.libprefix, "pkgconfig") + env["PKG_CONFIG_PATH"] = ":".join([env["PKG_CONFIG_PATH"], pkgconfig_path]) - env['PATH'] = ':'.join([ - escape_path(pj(self.install_dir, 'bin')), - env['PATH'] - ]) + env["PATH"] = ":".join([escape_path(pj(self.install_dir, "bin")), env["PATH"]]) - env['LD_LIBRARY_PATH'] = ':'.join([env['LD_LIBRARY_PATH'], - pj(self.install_dir, 'lib'), - pj(self.install_dir, self.libprefix) - ]) + env["LD_LIBRARY_PATH"] = ":".join( + [ + env["LD_LIBRARY_PATH"], + pj(self.install_dir, "lib"), + pj(self.install_dir, self.libprefix), + ] + ) - env['QMAKE_CXXFLAGS'] = " ".join([ - escape_path('-I'+pj(self.install_dir, 'include')), - env['QMAKE_CXXFLAGS'] - ]) - env['CPPFLAGS'] = " ".join([ - escape_path('-I'+pj(self.install_dir, 'include')), - env['CPPFLAGS'] - ]) - env['QMAKE_LFLAGS'] = " ".join([ - escape_path('-L'+pj(self.install_dir, 'lib')), - escape_path('-L'+pj(self.install_dir, self.libprefix)), - env['QMAKE_LFLAGS'] - ]) - env['LDFLAGS'] = " ".join([ - escape_path('-L'+pj(self.install_dir, 'lib')), - escape_path('-L'+pj(self.install_dir, self.libprefix)), - env['LDFLAGS'] - ]) + env["QMAKE_CXXFLAGS"] = " ".join( + [escape_path("-I" + pj(self.install_dir, "include")), env["QMAKE_CXXFLAGS"]] + ) + env["CPPFLAGS"] = " ".join( + [escape_path("-I" + pj(self.install_dir, "include")), env["CPPFLAGS"]] + ) + env["QMAKE_LFLAGS"] = " ".join( + [ + escape_path("-L" + pj(self.install_dir, "lib")), + escape_path("-L" + pj(self.install_dir, self.libprefix)), + env["QMAKE_LFLAGS"], + ] + ) + env["LDFLAGS"] = " ".join( + [ + escape_path("-L" + pj(self.install_dir, "lib")), + escape_path("-L" + pj(self.install_dir, self.libprefix)), + env["LDFLAGS"], + ] + ) if cross_comp_flags: - self.platformInfo.set_comp_flags(env) + self.configInfo.set_comp_flags(env) if cross_compilers: - self.platformInfo.set_compiler(env) + self.configInfo.set_compiler(env) if cross_path: - env['PATH'] = ':'.join(self.platformInfo.get_bin_dir() + [env['PATH']]) + env["PATH"] = ":".join(self.configInfo.get_bin_dir() + [env["PATH"]]) return env - @property def configure_wrapper(self): try: - yield self.platformInfo.configure_wrapper + yield self.configInfo.configure_wrapper except AttributeError: pass @property def make_wrapper(self): try: - yield self.platformInfo.make_wrapper + yield self.configInfo.make_wrapper except AttributeError: pass diff --git a/kiwixbuild/builder.py b/kiwixbuild/builder.py index 78f8ca3..587a3a7 100644 --- a/kiwixbuild/builder.py +++ b/kiwixbuild/builder.py @@ -1,30 +1,36 @@ - import sys from collections import OrderedDict from .buildenv import * -from .platforms import PlatformInfo +from .configs import ConfigInfo from .utils import remove_duplicates, StopBuild, colorize from .dependencies import Dependency from .packages import PACKAGE_NAME_MAPPERS from ._global import ( - neutralEnv, option, - add_target_step, get_target_step, target_steps, - backend) + neutralEnv, + option, + add_target_step, + get_target_step, + target_steps, +) from . import _global + class Builder: def __init__(self): self._targets = {} - PlatformInfo.get_platform('neutral', self._targets) + ConfigInfo.get_config("neutral", self._targets) - target_platform = option('target_platform') - platform = PlatformInfo.get_platform(target_platform, self._targets) - if neutralEnv('distname') not in platform.compatible_hosts: - print((colorize('ERROR')+': The target platform {} cannot be build on host {}.\n' - 'Select another target platform or change your host system.' - ).format(platform.name, neutralEnv('distname'))) - self.targetDefs = platform.add_targets(option('target'), self._targets) + config_name = option("config") + config = ConfigInfo.get_config(config_name, self._targets) + if neutralEnv("distname") not in config.compatible_hosts: + print( + ( + colorize("ERROR") + ": The config {} cannot be build on host {}.\n" + "Select another config or change your host system." + ).format(config.name, neutralEnv("distname")) + ) + self.targetDefs = config.add_targets(option("target"), self._targets) def finalize_target_steps(self): steps = [] @@ -32,36 +38,36 @@ class Builder: steps += self.order_steps(targetDef) steps = list(remove_duplicates(steps)) - if option('build_nodeps'): - # add all platform steps + if option("build_nodeps"): + # add all config steps for dep in steps: stepClass = Dependency.all_deps[dep[1]] if stepClass.dont_skip: add_target_step(dep, self._targets[dep]) - src_targetDef = ('source', targetDef[1]) + src_targetDef = ("source", targetDef[1]) add_target_step(src_targetDef, self._targets[src_targetDef]) add_target_step(targetDef, self._targets[targetDef]) else: for dep in steps: - if option('build_deps_only') and dep[1] == targetDef[1]: + if option("build_deps_only") and dep[1] == targetDef[1]: continue add_target_step(dep, self._targets[dep]) self.instanciate_steps() def order_steps(self, targetDef): - for pltName in PlatformInfo.all_running_platforms: - plt = PlatformInfo.all_platforms[pltName] - for tlcName in plt.toolchain_names: + for cfgName in ConfigInfo.all_running_configs: + cfg = ConfigInfo.all_configs[cfgName] + for tlcName in cfg.toolchain_names: tlc = Dependency.all_deps[tlcName] - yield('source', tlcName) - yield('neutral' if tlc.neutral else pltName, tlcName) - _targets =dict(self._targets) + yield ("source", tlcName) + yield ("neutral" if tlc.neutral else cfgName, tlcName) + _targets = dict(self._targets) yield from self.order_dependencies(targetDef, _targets) def order_dependencies(self, targetDef, targets): - targetPlatformName, targetName = targetDef - if targetPlatformName == 'source': + targetConfigName, targetName = targetDef + if targetConfigName == "source": # Do not try to order sources, they will be added as dep by the # build step two lines later. return @@ -70,43 +76,45 @@ class Builder: except KeyError: return - targetPlatform = PlatformInfo.get_platform(targetPlatformName) - for dep in target.get_dependencies(targetPlatform, True): - depPlatform, depName = targetPlatform.get_fully_qualified_dep(dep) - if (depPlatform, depName) in targets: - yield from self.order_dependencies((depPlatform, depName), targets) - yield ('source', targetName) + targetConfig = ConfigInfo.get_config(targetConfigName) + for dep in target.get_dependencies(targetConfig, True): + depConfig, depName = targetConfig.get_fully_qualified_dep(dep) + if (depConfig, depName) in targets: + yield from self.order_dependencies((depConfig, depName), targets) + yield ("source", targetName) yield targetDef def instanciate_steps(self): for stepDef in list(target_steps()): - stepPlatform, stepName = stepDef + stepConfig, stepName = stepDef stepClass = Dependency.all_deps[stepName] - if stepPlatform == 'source': + if stepConfig == "source": source = get_target_step(stepDef)(stepClass) add_target_step(stepDef, source) else: - source = get_target_step(stepName, 'source') - env = PlatformInfo.get_platform(stepPlatform).buildEnv + source = get_target_step(stepName, "source") + env = ConfigInfo.get_config(stepConfig).buildEnv builder = get_target_step(stepDef)(stepClass, source, env) add_target_step(stepDef, builder) def prepare_sources(self): - if option('skip_source_prepare'): + if option("skip_source_prepare"): print(colorize("SKIP")) return - sourceDefs = remove_duplicates(tDef for tDef in target_steps() if tDef[0]=='source') + sourceDefs = remove_duplicates( + tDef for tDef in target_steps() if tDef[0] == "source" + ) for sourceDef in sourceDefs: print("prepare sources {} :".format(sourceDef[1])) source = get_target_step(sourceDef) source.prepare() def build(self): - builderDefs = (tDef for tDef in target_steps() if tDef[0] != 'source') + builderDefs = (tDef for tDef in target_steps() if tDef[0] != "source") for builderDef in builderDefs: builder = get_target_step(builderDef) - if option('make_dist') and builderDef[1] == option('target'): + if option("make_dist") and builderDef[1] == option("target"): print("make dist {} ({}):".format(builder.name, builderDef[0])) builder.make_dist() continue @@ -116,19 +124,19 @@ class Builder: def _get_packages(self): packages_list = [] - for platform in PlatformInfo.all_running_platforms.values(): - mapper_name = "{host}_{target}".format( - host=neutralEnv('distname'), - target=platform) + for config in ConfigInfo.all_running_configs.values(): + mapper_name = "{host}_{config}".format( + host=neutralEnv("distname"), config=config + ) package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {}) - packages_list += package_name_mapper.get('COMMON', []) + packages_list += package_name_mapper.get("COMMON", []) to_drop = [] for builderDef in self._targets: - platformName, builderName = builderDef - mapper_name = "{host}_{target}".format( - host=neutralEnv('distname'), - target=platformName) + configName, builderName = builderDef + mapper_name = "{host}_{config}".format( + host=neutralEnv("distname"), config=configName + ) package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {}) packages = package_name_mapper.get(builderName) if packages: @@ -144,20 +152,20 @@ class Builder: packages_to_have = self._get_packages() packages_to_have = remove_duplicates(packages_to_have) - if option('assume_packages_installed'): + if option("assume_packages_installed"): print(colorize("SKIP") + ", Assume package installed") return - distname = neutralEnv('distname') - if distname in ('fedora', 'redhat', 'centos'): - package_installer = 'sudo dnf install {}' - package_checker = 'rpm -q --quiet {}' - elif distname in ('debian', 'Ubuntu'): - package_installer = 'sudo apt-get install {}' + distname = neutralEnv("distname") + if distname in ("fedora", "redhat", "centos"): + package_installer = "sudo dnf install {}" + package_checker = "rpm -q --quiet {}" + elif distname in ("debian", "Ubuntu"): + package_installer = "sudo apt-get install {}" package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1' - elif distname == 'Darwin': - package_installer = 'brew install {}' - package_checker = 'brew ls --version {} > /dev/null' + elif distname == "Darwin": + package_installer = "brew install {}" + package_checker = "brew ls --version {} > /dev/null" packages_to_install = [] for package in packages_to_have: @@ -176,31 +184,30 @@ class Builder: print(command) subprocess.check_call(command, shell=True) else: - print(colorize("SKIP")+ ", No package to install.") + print(colorize("SKIP") + ", No package to install.") def run(self): try: print("[INSTALL PACKAGES]") - if option('dont_install_packages'): + if option("dont_install_packages"): print(colorize("SKIP")) else: self.install_packages() self.finalize_target_steps() - print("[SETUP PLATFORMS]") - for platform in PlatformInfo.all_running_platforms.values(): - platform.finalize_setup() + print("[SETUP TOOLCHAINS]") + for config in ConfigInfo.all_running_configs.values(): + config.finalize_setup() print("[PREPARE]") self.prepare_sources() print("[BUILD]") self.build() # No error, clean intermediate file at end of build if needed. print("[CLEAN]") - if option('clean_at_end'): - for platform in PlatformInfo.all_running_platforms.values(): - platform.clean_intermediate_directories() + if option("clean_at_end"): + for config in ConfigInfo.all_running_configs.values(): + config.clean_intermediate_directories() else: print(colorize("SKIP")) except StopBuild as e: print(e) sys.exit("Stopping build due to errors") - diff --git a/kiwixbuild/configs/__init__.py b/kiwixbuild/configs/__init__.py new file mode 100644 index 0000000..e82de4d --- /dev/null +++ b/kiwixbuild/configs/__init__.py @@ -0,0 +1,3 @@ +from .base import * + +from . import android, armhf, musl, flatpak, i586, ios, native, neutral, win32, wasm diff --git a/kiwixbuild/configs/android.py b/kiwixbuild/configs/android.py new file mode 100644 index 0000000..51d6d91 --- /dev/null +++ b/kiwixbuild/configs/android.py @@ -0,0 +1,161 @@ +from .base import ConfigInfo, MetaConfigInfo +from kiwixbuild.utils import pj +from kiwixbuild._global import get_target_step, option + + +class AndroidConfigInfo(ConfigInfo): + build = "android" + static = True + toolchain_names = ["android-ndk"] + compatible_hosts = ["fedora", "debian"] + + def __str__(self): + return "android" + + @property + def libdir(self): + return "lib/{}".format(self.arch_full) + + @property + def binaries_name(self): + arch_full = self.arch_full + return { + "CC": "{}-{}".format(arch_full, "clang"), + "CXX": "{}-{}".format(arch_full, "clang++"), + "AR": "{}-{}".format(arch_full, "ar"), + "STRIP": "{}-{}".format(arch_full, "strip"), + "RANLIB": "{}-{}".format(arch_full, "ranlib"), + "LD": "{}-{}".format(arch_full, "ld"), + } + + def binaries(self): + install_path = self.install_path + binaries = { + k: pj(install_path, "bin", v) for k, v in self.binaries_name.items() + } + binaries["PKGCONFIG"] = "pkg-config" + return binaries + + @property + def ndk_builder(self): + return get_target_step("android-ndk", self.name) + + @property + def install_path(self): + return self.ndk_builder.install_path + + def get_cross_config(self): + extra_libs = ["-llog"] + extra_cflags = [ + "-I{}".format(include_dir) for include_dir in self.get_include_dirs() + ] + if hasattr(self, "march"): + extra_libs.append("-march={}".format(self.march)) + extra_cflags.append("-march={}".format(self.march)) + return { + "exe_wrapper_def": "", + "install_path": self.install_path, + "binaries": self.binaries(), + "root_path": pj(self.install_path, "sysroot"), + "extra_libs": extra_libs, + "extra_cflags": extra_cflags, + "host_machine": { + "system": "Android", + "lsystem": "android", + "cpu_family": self.arch, + "cpu": self.cpu, + "endian": "little", + "abi": self.abi, + }, + } + + def get_env(self): + env = super().get_env() + root_path = pj(self.install_path, "sysroot") + env["PKG_CONFIG_LIBDIR"] = pj(root_path, "lib", "pkgconfig") + env["NDK_DEBUG"] = "0" + return env + + def get_bin_dir(self): + return [pj(self.install_path, "bin")] + + def set_comp_flags(self, env): + super().set_comp_flags(env) + root_path = pj(self.install_path, "sysroot") + march = "-march={}".format(self.march) if hasattr(self, "march") else "" + env["CFLAGS"] = ( + "-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} {} ".format( + root_path, march + ) + + env["CFLAGS"] + ) + env["CXXFLAGS"] = ( + "-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} {} ".format( + root_path, march + ) + + env["CXXFLAGS"] + ) + env["LDFLAGS"] = "--sysroot={} {} ".format(root_path, march) + env["LDFLAGS"] + + def set_compiler(self, env): + binaries = self.binaries() + for k, v in binaries.items(): + env[k] = v + + @property + def configure_options(self): + yield "--host={}".format(self.arch_full) + + def finalize_setup(self): + super().finalize_setup() + self.buildEnv.cmake_crossfile = self._gen_crossfile( + "cmake_android_cross_file.txt", "cmake_cross_file.txt" + ) + self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt") + + +class AndroidArm(AndroidConfigInfo): + name = "android_arm" + arch = cpu = "arm" + arch_full = "arm-linux-androideabi" + abi = "armeabi-v7a" + march = "armv7-a" + + +class AndroidArm64(AndroidConfigInfo): + name = "android_arm64" + arch = "arm64" + arch_full = "aarch64-linux-android" + cpu = "aarch64" + abi = "arm64-v8a" + + +class AndroidX86(AndroidConfigInfo): + name = "android_x86" + arch = abi = "x86" + arch_full = "i686-linux-android" + cpu = "i686" + + +class AndroidX8664(AndroidConfigInfo): + name = "android_x86_64" + arch = cpu = abi = "x86_64" + arch_full = "x86_64-linux-android" + + +class Android(MetaConfigInfo): + name = "android" + compatible_hosts = ["fedora", "debian"] + + @property + def subConfigNames(self): + return ["android_{}".format(arch) for arch in option("android_arch")] + + def add_targets(self, targetName, targets): + return super().add_targets(targetName, targets) + + def __str__(self): + return self.name + + def set_comp_flags(self, env): + pass diff --git a/kiwixbuild/configs/armhf.py b/kiwixbuild/configs/armhf.py new file mode 100644 index 0000000..de666a3 --- /dev/null +++ b/kiwixbuild/configs/armhf.py @@ -0,0 +1,186 @@ +from .base import ConfigInfo, MixedMixin + +from kiwixbuild.utils import pj +from kiwixbuild._global import get_target_step + + +# Base config for arm +class ArmConfigInfo(ConfigInfo): + compatible_hosts = ["fedora", "debian"] + + def get_cross_config(self): + return { + "binaries": self.binaries, + "exe_wrapper_def": "", + "root_path": self.root_path, + "extra_libs": [], + "extra_cflags": [ + "-I{}".format(include_dir) for include_dir in self.get_include_dirs() + ], + "host_machine": { + "system": "linux", + "lsystem": "linux", + "cpu_family": self.cpu_family, + "cpu": self.cpu, + "endian": "little", + "abi": "", + }, + } + + @property + def libdir(self): + return "lib/{}".format(self.arch_full) + + @property + def toolchain(self): + return get_target_step(self.build, "neutral") + + @property + def root_path(self): + return self.toolchain.build_path + + @property + def binaries(self): + binaries = ( + (k, "{}-{}".format(self.arch_full, v)) + for k, v in ( + ("CC", "gcc"), + ("CXX", "g++"), + ("AR", "ar"), + ("STRIP", "strip"), + ("WINDRES", "windres"), + ("RANLIB", "ranlib"), + ("LD", "ld"), + ("LDSHARED", "g++ -shared"), + ) + ) + binaries = {k: pj(self.root_path, "bin", v) for k, v in binaries} + binaries["PKGCONFIG"] = "pkg-config" + return binaries + + @property + def exe_wrapper_def(self): + try: + which("qemu-arm") + except subprocess.CalledProcessError: + return "" + else: + return "exe_wrapper = 'qemu-arm'" + + @property + def configure_options(self): + yield "--host={}".format(self.arch_full) + + def get_bin_dir(self): + return [pj(self.root_path, "bin")] + + def get_env(self): + env = super().get_env() + env["LD_LIBRARY_PATH"] = ":".join( + [ + pj(self.root_path, self.arch_full, "lib64"), + pj(self.root_path, "lib"), + env["LD_LIBRARY_PATH"], + ] + ) + env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig") + env["QEMU_LD_PREFIX"] = pj(self.root_path, self.arch_full, "libc") + env["QEMU_SET_ENV"] = "LD_LIBRARY_PATH={}".format( + ":".join( + [pj(self.root_path, self.arch_full, "lib"), env["LD_LIBRARY_PATH"]] + ) + ) + return env + + def set_comp_flags(self, env): + super().set_comp_flags(env) + env["CFLAGS"] = ( + " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 " + + env["CFLAGS"] + ) + env["CXXFLAGS"] = ( + " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 " + + env["CXXFLAGS"] + ) + + def set_compiler(self, env): + for k, v in self.binaries.items(): + env[k] = v + + def finalize_setup(self): + super().finalize_setup() + self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt") + self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt") + + +class Armv6(ArmConfigInfo): + build = "armv6" + arch_full = "armv6-rpi-linux-gnueabihf" + toolchain_names = ["armv6"] + cpu_family = "arm" + cpu = "armv6" + + +class Armv6Dyn(Armv6): + name = "armv6_dyn" + static = False + + +class Armv6Static(Armv6): + name = "armv6_static" + static = True + + +class Armv6Mixed(MixedMixin("armv6_static"), Armv6): + name = "armv6_mixed" + static = False + + +class Armv8(ArmConfigInfo): + build = "armv8" + arch_full = "armv8-rpi3-linux-gnueabihf" + toolchain_names = ["armv8"] + cpu_family = "arm" + cpu = "armv8" + + +class Armv8Dyn(Armv8): + name = "armv8_dyn" + static = False + + +class Armv8Static(Armv8): + name = "armv8_static" + static = True + + +class Armv8Mixed(MixedMixin("armv8_static"), Armv8): + name = "armv8_mixed" + static = False + + +class Aarch64(ArmConfigInfo): + build = "aarch64" + arch_full = "aarch64-linux-gnu" + toolchain_names = ["aarch64"] + cpu_family = "aarch64" + cpu = "aarch64" + + @property + def root_path(self): + return self.toolchain.build_path + + +class Aarch64Dyn(Aarch64): + name = "aarch64_dyn" + static = False + + +class Aarch64Static(Aarch64): + name = "aarch64_static" + static = True + + +class Aarch64Mixed(MixedMixin("aarch64_static"), Aarch64): + name = "aarch64_mixed" + static = False diff --git a/kiwixbuild/platforms/base.py b/kiwixbuild/configs/base.py similarity index 52% rename from kiwixbuild/platforms/base.py rename to kiwixbuild/configs/base.py index 41cbee2..43798f2 100644 --- a/kiwixbuild/platforms/base.py +++ b/kiwixbuild/configs/base.py @@ -1,4 +1,3 @@ - import os, sys import subprocess @@ -8,107 +7,101 @@ from kiwixbuild.buildenv import BuildEnv from kiwixbuild._global import neutralEnv, option, target_steps _SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -TEMPLATES_DIR = pj(os.path.dirname(_SCRIPT_DIR), 'templates') +TEMPLATES_DIR = pj(os.path.dirname(_SCRIPT_DIR), "templates") -class _MetaPlatform(type): + +class _MetaConfig(type): def __new__(cls, name, bases, dct): _class = type.__new__(cls, name, bases, dct) - if name not in ('PlatformInfo', 'MetaPlatformInfo') and 'name' in dct: - dep_name = dct['name'] - PlatformInfo.all_platforms[dep_name] = _class + if name not in ("ConfigInfo", "MetaConfigInfo") and "name" in dct: + dep_name = dct["name"] + ConfigInfo.all_configs[dep_name] = _class return _class -class PlatformInfo(metaclass=_MetaPlatform): - all_platforms = {} - all_running_platforms = {} +class ConfigInfo(metaclass=_MetaConfig): + all_configs = {} + all_running_configs = {} toolchain_names = [] configure_options = [] mixed = False libdir = None @classmethod - def get_platform(cls, name, targets=None): - if name not in cls.all_running_platforms: + def get_config(cls, name, targets=None): + if name not in cls.all_running_configs: if targets is None: print("Should not got there.") - print(cls.all_running_platforms) + print(cls.all_running_configs) raise KeyError(name) - cls.all_running_platforms[name] = cls.all_platforms[name](targets) - return cls.all_running_platforms[name] + cls.all_running_configs[name] = cls.all_configs[name](targets) + return cls.all_running_configs[name] def __init__(self, targets): - self.all_running_platforms[self.name] = self + self.all_running_configs[self.name] = self self.buildEnv = BuildEnv(self) self.setup_toolchains(targets) def __str__(self): - return "{}_{}".format(self.build, 'static' if self.static else 'dyn') + return "{}_{}".format(self.build, "static" if self.static else "dyn") def setup_toolchains(self, targets): for tlc_name in self.toolchain_names: ToolchainClass = Dependency.all_deps[tlc_name] - targets[('source', tlc_name)] = ToolchainClass.Source - plt_name = 'neutral' if ToolchainClass.neutral else self.name - targets[(plt_name, tlc_name)] = ToolchainClass.Builder + targets[("source", tlc_name)] = ToolchainClass.Source + cfg_name = "neutral" if ToolchainClass.neutral else self.name + targets[(cfg_name, tlc_name)] = ToolchainClass.Builder def add_targets(self, targetName, targets): if (self.name, targetName) in targets: return [] targetClass = Dependency.all_deps[targetName] - targets[('source', targetName)] = targetClass.Source + targets[("source", targetName)] = targetClass.Source targets[(self.name, targetName)] = targetClass.Builder for dep in targetClass.Builder.get_dependencies(self, False): if isinstance(dep, tuple): - depPlatformName, depName = dep + depConfigName, depName = dep else: - depPlatformName, depName = self.name, dep - depPlatform = self.get_platform(depPlatformName, targets) - depPlatform.add_targets(depName, targets) + depConfigName, depName = self.name, dep + depConfig = self.get_config(depConfigName, targets) + depConfig.add_targets(depName, targets) return [(self.name, targetName)] def get_fully_qualified_dep(self, dep): - if isinstance(dep, tuple): + if isinstance(dep, tuple): return dep - else: + else: return self.name, dep - def get_cross_config(self): return {} def get_include_dirs(self): - return [pj(self.buildEnv.install_dir, 'include')] + return [pj(self.buildEnv.install_dir, "include")] def get_env(self): return DefaultEnv() - def get_bin_dir(self): return [] - def set_compiler(self, env): pass - def set_comp_flags(self, env): if self.static: - env['CFLAGS'] = env['CFLAGS'] + ' -fPIC' - env['CXXFLAGS'] = env['CXXFLAGS'] + ' -fPIC' - + env["CFLAGS"] = env["CFLAGS"] + " -fPIC" + env["CXXFLAGS"] = env["CXXFLAGS"] + " -fPIC" def _gen_crossfile(self, name, outname=None): if outname is None: outname = name crossfile = pj(self.buildEnv.build_dir, outname) template_file = pj(TEMPLATES_DIR, name) - with open(template_file, 'r') as f: + with open(template_file, "r") as f: template = f.read() - content = template.format( - **self.get_cross_config() - ) - with open(crossfile, 'w') as outfile: + content = template.format(**self.get_cross_config()) + with open(crossfile, "w") as outfile: outfile.write(content) return crossfile @@ -121,19 +114,17 @@ class PlatformInfo(metaclass=_MetaPlatform): self.buildEnv.clean_intermediate_directories() - -class MetaPlatformInfo(PlatformInfo): - subPlatformNames = [] +class MetaConfigInfo(ConfigInfo): + subConfigNames = [] def add_targets(self, targetName, targets): targetDefs = [] - for platformName in self.subPlatformNames: - platform = self.get_platform(platformName, targets) - targetDefs += platform.add_targets(targetName, targets) + for configName in self.subConfigNames: + config = self.get_config(configName, targets) + targetDefs += config.add_targets(targetName, targets) return targetDefs - def MixedMixin(static_name): class MixedMixinClass: mixed = True @@ -141,36 +132,47 @@ def MixedMixin(static_name): def add_targets(self, targetName, targets): print(targetName) - if option('target') == targetName: + if option("target") == targetName: return super().add_targets(targetName, targets) else: - static_platform = self.get_platform(static_name, targets) - return static_platform.add_targets(targetName, targets) + static_config = self.get_config(static_name, targets) + return static_config.add_targets(targetName, targets) def get_fully_qualified_dep(self, dep): if isinstance(dep, tuple): return dep - if option('target') == dep: + if option("target") == dep: return self.name, dep return static_name, dep @property def static_buildEnv(self): - static_platform = self.get_platform(static_name) - return static_platform.buildEnv + static_config = self.get_config(static_name) + return static_config.buildEnv def get_include_dirs(self): return [ - pj(self.buildEnv.install_dir, 'include'), - pj(self.static_buildEnv.install_dir, 'include') + pj(self.buildEnv.install_dir, "include"), + pj(self.static_buildEnv.install_dir, "include"), ] def get_env(self): env = super().get_env() - env['PATH'] = ':'.join([pj(self.static_buildEnv.install_dir, 'bin')] + [env['PATH']]) - pkgconfig_path = pj(self.static_buildEnv.install_dir, self.static_buildEnv.libprefix, 'pkgconfig') - env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path]) - env['CPPFLAGS'] = " ".join(['-I'+pj(self.static_buildEnv.install_dir, 'include'), env['CPPFLAGS']]) + env["PATH"] = ":".join( + [pj(self.static_buildEnv.install_dir, "bin")] + [env["PATH"]] + ) + pkgconfig_path = pj( + self.static_buildEnv.install_dir, + self.static_buildEnv.libprefix, + "pkgconfig", + ) + env["PKG_CONFIG_PATH"] = ":".join([env["PKG_CONFIG_PATH"], pkgconfig_path]) + env["CPPFLAGS"] = " ".join( + [ + "-I" + pj(self.static_buildEnv.install_dir, "include"), + env["CPPFLAGS"], + ] + ) return env return MixedMixinClass diff --git a/kiwixbuild/configs/flatpak.py b/kiwixbuild/configs/flatpak.py new file mode 100644 index 0000000..a49acdd --- /dev/null +++ b/kiwixbuild/configs/flatpak.py @@ -0,0 +1,18 @@ +from .base import ConfigInfo +from kiwixbuild._global import option, neutralEnv + + +class FlatpakConfigInfo(ConfigInfo): + name = "flatpak" + build = "flatpak" + static = "" + toolchain_names = ["org.kde", "io.qt.qtwebengine"] + compatible_hosts = ["debian", "fedora"] + + def __str__(self): + return "flatpak" + + def get_env(self): + env = super().get_env() + env["FLATPAK_USER_DIR"] = self.buildEnv.build_dir + return env diff --git a/kiwixbuild/configs/i586.py b/kiwixbuild/configs/i586.py new file mode 100644 index 0000000..8c402a3 --- /dev/null +++ b/kiwixbuild/configs/i586.py @@ -0,0 +1,79 @@ +import os + +from .base import ConfigInfo +from kiwixbuild.utils import which, pj + + +class I586ConfigInfo(ConfigInfo): + build = "i586" + arch_full = "i586-linux-gnu" + compatible_hosts = ["fedora", "debian"] + + def get_cross_config(self): + return { + "binaries": self.binaries, + "exe_wrapper_def": "", + "extra_libs": ["-m32", "-march=i586", "-mno-sse"], + "extra_cflags": [ + "-m32", + "-march=i586", + "-mno-sse", + *( + "-I{}".format(include_dir) + for include_dir in self.get_include_dirs() + ), + ], + "host_machine": { + "system": "linux", + "lsystem": "linux", + "cpu_family": "x86", + "cpu": "i586", + "endian": "little", + "abi": "", + }, + } + + @property + def configure_options(self): + yield f"--host={self.arch_full}" + + @property + def binaries(self): + return { + k: which(v) + for k, v in ( + ("CC", os.environ.get("CC", "gcc")), + ("CXX", os.environ.get("CXX", "g++")), + ("AR", "ar"), + ("STRIP", "strip"), + ("RANLIB", "ranlib"), + ("LD", "ld"), + ("PKGCONFIG", "pkg-config"), + ) + } + + def set_comp_flags(self, env): + super().set_comp_flags(env) + env["CFLAGS"] = "-m32 -march=i586 -mno-sse " + env["CFLAGS"] + env["CXXFLAGS"] = "-m32 -march=i586 -mno-sse " + env["CXXFLAGS"] + env["LDFLAGS"] = "-m32 -march=i586 -mno-sse " + env["LDFLAGS"] + + def get_bin_dir(self): + return [] + + def finalize_setup(self): + super().finalize_setup() + self.buildEnv.cmake_crossfile = self._gen_crossfile( + "cmake_i586_cross_file.txt", "cmake_cross_file.txt" + ) + self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt") + + +class I586Dyn(I586ConfigInfo): + name = "i586_dyn" + static = False + + +class I586Static(I586ConfigInfo): + name = "i586_static" + static = True diff --git a/kiwixbuild/configs/ios.py b/kiwixbuild/configs/ios.py new file mode 100644 index 0000000..bce93a3 --- /dev/null +++ b/kiwixbuild/configs/ios.py @@ -0,0 +1,237 @@ +import subprocess + +from kiwixbuild._global import option +from kiwixbuild.utils import pj, xrun_find +from .base import ConfigInfo, MetaConfigInfo, MixedMixin +from kiwixbuild.dependencies.apple_xcframework import AppleXCFramework + + +MIN_MACOS_VERSION = "12.0" + + +class AppleConfigInfo(ConfigInfo): + build = "iOS" + static = True + compatible_hosts = ["Darwin"] + arch = None + host = None + target = None + sdk_name = None + min_iphoneos_version = None + min_macos_version = None + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._root_path = None + + @property + def root_path(self): + if self._root_path is None: + command = "xcrun --sdk {} --show-sdk-path".format(self.sdk_name) + self._root_path = subprocess.check_output(command, shell=True)[:-1].decode() + return self._root_path + + def __str__(self): + return "iOS" + + def finalize_setup(self): + super().finalize_setup() + self.buildEnv.cmake_crossfile = self._gen_crossfile( + "cmake_ios_cross_file.txt", "cmake_cross_file.txt" + ) + self.buildEnv.meson_crossfile = self._gen_crossfile( + "meson_ios_cross_file.txt", "meson_cross_file.txt" + ) + + def get_cross_config(self): + config = { + "root_path": self.root_path, + "binaries": self.binaries, + "exe_wrapper_def": "", + "extra_libs": [ + "-isysroot", + self.root_path, + "-arch", + self.arch, + "-target", + self.target, + ], + "extra_cflags": [ + "-isysroot", + self.root_path, + "-arch", + self.arch, + "-target", + self.target, + *( + "-I{}".format(include_dir) + for include_dir in self.get_include_dirs() + ), + ], + "host_machine": { + "system": "Darwin", + "lsystem": "darwin", + "cpu_family": self.arch, + "cpu": self.cpu, + "endian": "", + "abi": "", + }, + } + if self.min_iphoneos_version: + config["extra_libs"].append( + "-miphoneos-version-min={}".format(self.min_iphoneos_version) + ) + config["extra_cflags"].append( + "-miphoneos-version-min={}".format(self.min_iphoneos_version) + ) + if self.min_macos_version: + config["extra_libs"].append( + "-mmacosx-version-min={}".format(self.min_macos_version) + ) + config["extra_cflags"].append( + "-mmacosx-version-min={}".format(self.min_macos_version) + ) + return config + + def get_env(self): + env = super().get_env() + cflags = [env["CFLAGS"]] + if self.min_iphoneos_version: + cflags.append("-miphoneos-version-min={}".format(self.min_iphoneos_version)) + if self.min_macos_version: + cflags.append("-mmacosx-version-min={}".format(self.min_macos_version)) + env["CFLAGS"] = " ".join(cflags) + return env + + def set_comp_flags(self, env): + super().set_comp_flags(env) + cflags = [ + "-isysroot {}".format(self.root_path), + "-arch {}".format(self.arch), + "-target {}".format(self.target), + env["CFLAGS"], + ] + if self.min_iphoneos_version: + cflags.append("-miphoneos-version-min={}".format(self.min_iphoneos_version)) + env["CFLAGS"] = " ".join(cflags) + env["CXXFLAGS"] = " ".join( + [ + env["CFLAGS"], + "-std=c++11", + env["CXXFLAGS"], + ] + ) + env["LDFLAGS"] = " ".join( + [ + " -arch {}".format(self.arch), + "-isysroot {}".format(self.root_path), + ] + ) + + def get_bin_dir(self): + return [pj(self.root_path, "bin")] + + @property + def binaries(self): + return { + "CC": xrun_find("clang"), + "CXX": xrun_find("clang++"), + "AR": xrun_find("ar"), + "STRIP": xrun_find("strip"), + "RANLIB": xrun_find("ranlib"), + "LD": xrun_find("ld"), + "PKGCONFIG": "pkg-config", + } + + @property + def configure_options(self): + yield f"--host={self.host}" + + +class iOSArm64(AppleConfigInfo): + name = "iOS_arm64" + arch = cpu = "arm64" + host = "arm-apple-darwin" + target = "aarch64-apple-ios" + sdk_name = "iphoneos" + min_iphoneos_version = "15.0" + + +class iOSx64Simulator(AppleConfigInfo): + name = "iOSSimulator_x86_64" + arch = cpu = "x86_64" + host = "x86_64-apple-darwin" + target = "x86-apple-ios-simulator" + sdk_name = "iphonesimulator" + min_iphoneos_version = "15.0" + + +class iOSArm64Simulator(AppleConfigInfo): + name = "iOSSimulator_arm64" + arch = cpu = "arm64" + host = "arm-apple-darwin" + target = "aarch64-apple-ios-simulator" + sdk_name = "iphonesimulator" + min_iphoneos_version = "15.0" + + +class macOSArm64(AppleConfigInfo): + name = "macOS_arm64_static" + arch = cpu = "arm64" + host = "aarch64-apple-darwin" + target = "arm64-apple-macos" + sdk_name = "macosx" + min_iphoneos_version = None + min_macos_version = MIN_MACOS_VERSION + + +class macOSArm64Mixed(MixedMixin("macOS_arm64_static"), AppleConfigInfo): + name = "macOS_arm64_mixed" + arch = cpu = "arm64" + host = "aarch64-apple-darwin" + target = "arm64-apple-macos" + sdk_name = "macosx" + min_iphoneos_version = None + min_macos_version = MIN_MACOS_VERSION + + +class macOSx64(AppleConfigInfo): + name = "macOS_x86_64" + arch = cpu = "x86_64" + host = "x86_64-apple-darwin" + target = "x86_64-apple-macos" + sdk_name = "macosx" + min_iphoneos_version = None + min_macos_version = MIN_MACOS_VERSION + + +class IOS(MetaConfigInfo): + name = "iOS_multi" + compatible_hosts = ["Darwin"] + + @property + def subConfigNames(self): + return ["iOS_{}".format(arch) for arch in option("ios_arch")] + + def add_targets(self, targetName, targets): + super().add_targets(targetName, targets) + return ConfigInfo.add_targets(self, "_ios_fat_lib", targets) + + def __str__(self): + return self.name + + +class AppleStaticAll(MetaConfigInfo): + name = "apple_all_static" + compatible_hosts = ["Darwin"] + + @property + def subConfigNames(self): + return AppleXCFramework.subConfigNames + + def add_targets(self, targetName, targets): + super().add_targets(targetName, targets) + return ConfigInfo.add_targets(self, "apple_xcframework", targets) + + def __str__(self): + return self.name diff --git a/kiwixbuild/configs/musl.py b/kiwixbuild/configs/musl.py new file mode 100644 index 0000000..c1eebd8 --- /dev/null +++ b/kiwixbuild/configs/musl.py @@ -0,0 +1,162 @@ +from .base import ConfigInfo, MixedMixin + +from kiwixbuild.utils import pj +from kiwixbuild._global import get_target_step + + +class MuslConfigInfo(ConfigInfo): + compatible_hosts = ["fedora", "debian"] + + def get_cross_config(self): + return { + "binaries": self.binaries, + "exe_wrapper_def": "", + "root_path": self.root_path, + "extra_libs": [], + "extra_cflags": [ + "-I{}".format(include_dir) for include_dir in self.get_include_dirs() + ], + "host_machine": { + "system": "linux", + "lsystem": "linux", + "cpu_family": self.cpu_family, + "cpu": self.cpu, + "endian": "little", + "abi": "", + }, + } + + @property + def toolchain(self): + return get_target_step(self.build, "neutral") + + @property + def root_path(self): + return self.toolchain.build_path + + @property + def binaries(self): + binaries = ( + (k, "{}-{}".format(self.arch_full, v)) + for k, v in ( + ("CC", "gcc"), + ("CXX", "g++"), + ("AR", "ar"), + ("STRIP", "strip"), + ("WINDRES", "windres"), + ("RANLIB", "ranlib"), + ("LD", "ld"), + ("LDSHARED", "g++ -shared"), + ) + ) + binaries = {k: pj(self.root_path, "bin", v) for k, v in binaries} + binaries["PKGCONFIG"] = "pkg-config" + return binaries + + @property + def exe_wrapper_def(self): + try: + which(self.qemu) + except subprocess.CalledProcessError: + return "" + except AttributeError: + return "" + else: + return f"exe_wrapper = '{self.qemu}'" + + @property + def configure_options(self): + return [f"--host={self.arch_full}"] + + def get_bin_dir(self): + return [pj(self.root_path, "bin")] + + def get_env(self): + env = super().get_env() + env["LD_LIBRARY_PATH"] = ":".join( + [ + pj(self.root_path, self.arch_full, "lib64"), + pj(self.root_path, "lib"), + env["LD_LIBRARY_PATH"], + ] + ) + env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig") + env["QEMU_LD_PREFIX"] = pj(self.root_path, self.arch_full, "libc") + env["QEMU_SET_ENV"] = "LD_LIBRARY_PATH={}".format( + ":".join( + [pj(self.root_path, self.arch_full, "lib"), env["LD_LIBRARY_PATH"]] + ) + ) + return env + + def set_comp_flags(self, env): + super().set_comp_flags(env) + env["LD_LIBRARY_PATH"] = ":".join( + [pj(self.root_path, self.arch_full, "lib"), env["LD_LIBRARY_PATH"]] + ) + env["CFLAGS"] = ( + " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 " + + env["CFLAGS"] + ) + env["CXXFLAGS"] = ( + " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 " + + env["CXXFLAGS"] + ) + + def set_compiler(self, env): + for k, v in self.binaries.items(): + env[k] = v + + def finalize_setup(self): + super().finalize_setup() + self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt") + self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt") + + +class Aarch64MuslConfigInfo(MuslConfigInfo): + build = "aarch64_musl" + arch_full = "aarch64-linux-musl" + toolchain_names = ["aarch64_musl"] + libdir = "lib/aarch64-linux-musl" + cpu_family = "arm" + cpu = "armhf" + qemu = "qemu-arm" + + +class Aarch64MuslDyn(Aarch64MuslConfigInfo): + name = "aarch64_musl_dyn" + static = False + + +class Aarch64MuslStatic(Aarch64MuslConfigInfo): + name = "aarch64_musl_static" + static = True + + +class Aarch64MuslMixed(MixedMixin("aarch64_musl_static"), Aarch64MuslConfigInfo): + name = "aarch64_musl_mixed" + static = False + + +class X86_64MuslConfigInfo(MuslConfigInfo): + build = "x86-64_musl" + arch_full = "x86_64-linux-musl" + toolchain_names = ["x86-64_musl"] + libdir = "lib/x86_64-linux-musl" + cpu_family = "x86_64" + cpu = "x86_64" + + +class X86_64MuslDyn(X86_64MuslConfigInfo): + name = "x86-64_musl_dyn" + static = False + + +class X86_64MuslStatic(X86_64MuslConfigInfo): + name = "x86-64_musl_static" + static = True + + +class x86_64MuslMixed(MixedMixin("x86-64_musl_static"), X86_64MuslConfigInfo): + name = "x86-64_musl_mixed" + static = False diff --git a/kiwixbuild/configs/native.py b/kiwixbuild/configs/native.py new file mode 100644 index 0000000..6bf3e36 --- /dev/null +++ b/kiwixbuild/configs/native.py @@ -0,0 +1,35 @@ +from .base import ConfigInfo, MixedMixin + +from kiwixbuild.utils import pj +from kiwixbuild._global import option, neutralEnv +from kiwixbuild.configs.ios import MIN_MACOS_VERSION + + +class NativeConfigInfo(ConfigInfo): + build = "native" + + def get_env(self): + env = super().get_env() + if neutralEnv("distname") == "fedora": + env["QT_SELECT"] = "5-64" + if neutralEnv("distname") == "Darwin": + env["CFLAGS"] += f"-mmacosx-version-min={MIN_MACOS_VERSION}" + return env + + +class NativeDyn(NativeConfigInfo): + name = "native_dyn" + static = False + compatible_hosts = ["fedora", "debian", "Darwin"] + + +class NativeStatic(NativeConfigInfo): + name = "native_static" + static = True + compatible_hosts = ["fedora", "debian"] + + +class NativeMixed(MixedMixin("native_static"), NativeConfigInfo): + name = "native_mixed" + static = False + compatible_hosts = ["fedora", "debian", "Darwin"] diff --git a/kiwixbuild/configs/neutral.py b/kiwixbuild/configs/neutral.py new file mode 100644 index 0000000..f5ae53e --- /dev/null +++ b/kiwixbuild/configs/neutral.py @@ -0,0 +1,10 @@ +from .base import ConfigInfo + + +class NeutralConfigInfo(ConfigInfo): + name = "neutral" + static = "" + compatible_hosts = ["fedora", "debian", "Darwin"] + + def __str__(self): + return "neutral" diff --git a/kiwixbuild/configs/wasm.py b/kiwixbuild/configs/wasm.py new file mode 100644 index 0000000..ff0503e --- /dev/null +++ b/kiwixbuild/configs/wasm.py @@ -0,0 +1,114 @@ +from .base import ConfigInfo + +from kiwixbuild.utils import pj +from kiwixbuild._global import get_target_step + + +class WasmConfigInfo(ConfigInfo): + name = "wasm" + static = True + build = "wasm" + arch_full = "wasm64-emscripten" + libdir = "lib" + # arch_full = 'wasm64-linux' + toolchain_names = ["emsdk"] + compatible_hosts = ["fedora", "debian"] + exe_wrapper_def = "" + + def get_cross_config(self): + return { + "binaries": self.binaries, + "exe_wrapper_def": "", + "root_path": self.root_path, + "extra_libs": [], + "extra_cflags": [], + "host_machine": { + "system": "emscripten", + "lsystem": "emscripten", + "cpu_family": "wasm64", + "cpu": "wasm64", + "endian": "little", + "abi": "", + }, + } + + @property + def wasm_sdk(self): + return get_target_step("emsdk", self.name) + + @property + def install_path(self): + return self.wasm_sdk.install_path + + @property + def root_path(self): + return self.install_path + + @property + def binaries(self): + binaries = ( + ("CC", "emcc"), + ("CXX", "em++"), + ("AR", "emar"), + ("STRIP", "emstrip"), + ("WINDRES", "windres"), + ("RANLIB", "emranlib"), + ("LD", "wasm-ld"), + ) + binaries = { + k: pj(self.install_path, "upstream", "emscripten", v) for k, v in binaries + } + binaries["PKGCONFIG"] = "pkg-config" + return binaries + + @property + def configure_options(self): + # return "" + return [f"--host={self.arch_full}"] + + @property + def configure_wrapper(self): + return "emconfigure" + + @property + def make_wrapper(self): + return "emmake" + + def get_bin_dir(self): + return [pj(self.install_path, "bin")] + + def get_env(self): + env = super().get_env() + env["PATH"] = ":".join( + [ + env["PATH"], + self.install_path, + pj(self.install_path, "upstream", "emscripten"), + pj(self.install_path, "node", "14.18.2_64bit", "bin"), + ] + ) + env["EMSDK"] = self.install_path + env["EMSDK_NODE"] = pj( + self.install_path, "node", "14.18.2_64bit", "bin", "node" + ) + return env + + def set_comp_flags(self, env): + super().set_comp_flags(env) + env["CFLAGS"] = ( + " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 " + + env["CFLAGS"] + ) + env["CXXFLAGS"] = ( + " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 " + + env["CXXFLAGS"] + ) + + def set_compiler(self, env): + for k, v in self.binaries.items(): + env[k] = v + + def finalize_setup(self): + super().finalize_setup() + self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt") + self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt") diff --git a/kiwixbuild/configs/win32.py b/kiwixbuild/configs/win32.py new file mode 100644 index 0000000..6b665e7 --- /dev/null +++ b/kiwixbuild/configs/win32.py @@ -0,0 +1,99 @@ +import subprocess + +from .base import ConfigInfo +from kiwixbuild.utils import which, pj +from kiwixbuild._global import neutralEnv + + +class Win32ConfigInfo(ConfigInfo): + build = "win32" + compatible_hosts = ["fedora", "debian"] + arch_full = "i686-w64-mingw32" + extra_libs = ["-lwinmm", "-lshlwapi", "-lws2_32", "-lssp"] + + def get_cross_config(self): + return { + "exe_wrapper_def": self.exe_wrapper_def, + "binaries": self.binaries, + "root_path": self.root_path, + "extra_libs": self.extra_libs, + "extra_cflags": [ + "-DWIN32", + *( + "-I{}".format(include_dir) + for include_dir in self.get_include_dirs() + ), + ], + "host_machine": { + "system": "Windows", + "lsystem": "windows", + "cpu_family": "x86", + "cpu": "i686", + "endian": "little", + "abi": "", + }, + } + + def finalize_setup(self): + super().finalize_setup() + self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt") + self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt") + + @property + def root_path(self): + root_paths = { + "fedora": "/usr/i686-w64-mingw32/sys-root/mingw", + "debian": "/usr/i686-w64-mingw32", + } + return root_paths[neutralEnv("distname")] + + @property + def binaries(self): + return { + k: which("{}-{}".format(self.arch_full, v)) + for k, v in ( + ("CC", "gcc"), + ("CXX", "g++"), + ("AR", "ar"), + ("STRIP", "strip"), + ("WINDRES", "windres"), + ("RANLIB", "ranlib"), + ("PKGCONFIG", "pkg-config"), + ) + } + + @property + def exe_wrapper_def(self): + try: + which("wine") + except subprocess.CalledProcessError: + return "" + else: + return "exe_wrapper = 'wine'" + + @property + def configure_options(self): + yield f"--host={self.arch_full}" + + def set_compiler(self, env): + for k, v in self.binaries.items(): + env[k] = v + + def get_bin_dir(self): + return [pj(self.root_path, "bin")] + + def get_env(self): + env = super().get_env() + env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig") + env["LIBS"] = " ".join(self.extra_libs) + " " + env["LIBS"] + return env + + +class Win32Dyn(Win32ConfigInfo): + name = "win32_dyn" + static = False + + +class Win32Static(Win32ConfigInfo): + name = "win32_static" + static = True diff --git a/kiwixbuild/configs/win64.py b/kiwixbuild/configs/win64.py new file mode 100644 index 0000000..10f215a --- /dev/null +++ b/kiwixbuild/configs/win64.py @@ -0,0 +1,103 @@ +import subprocess + +from .base import ConfigInfo +from kiwixbuild.utils import which, pj +from kiwixbuild._global import neutralEnv + + +class Win64ConfigInfo(ConfigInfo): + extra_libs = [ + "-lmingw32", + "-lwinmm", + "-lws2_32", + "-lshlwapi", + "-lrpcrt4", + "-lmsvcr100", + "-liphlpapi", + "-lshell32", + "-lkernel32", + ] + build = "win64" + compatible_hosts = ["fedora", "debian"] + arch_full = "x86_64-w64-mingw32" + + def get_cross_config(self): + return { + "exe_wrapper_def": self.exe_wrapper_def, + "binaries": self.binaries, + "root_path": self.root_path, + "extra_libs": self.extra_libs, + "extra_cflags": ["-DWIN32"], + "host_machine": { + "system": "Windows", + "lsystem": "windows", + "cpu_family": "x86_64", + "cpu": "x86_64", + "endian": "little", + "abi": "", + }, + } + + def finalize_setup(self): + super().finalize_setup() + self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt") + self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt") + + @property + def root_path(self): + root_paths = { + "fedora": "/usr/x86_64-w64-mingw32/sys-root/mingw", + "debian": "/usr/x86_64-w64-mingw32", + } + return root_paths[neutralEnv("distname")] + + @property + def binaries(self): + return { + k: which("{}-{}".format(self.arch_full, v)) + for k, v in ( + ("CC", "gcc"), + ("CXX", "g++"), + ("AR", "ar"), + ("STRIP", "strip"), + ("WINDRES", "windres"), + ("RANLIB", "ranlib"), + ("PKGCONFIG", "pkg-config"), + ) + } + + @property + def exe_wrapper_def(self): + try: + which("wine") + except subprocess.CalledProcessError: + return "" + else: + return "exe_wrapper = 'wine'" + + @property + def configure_options(self): + return [f"--host={self.arch_full}"] + + def set_compiler(self, env): + for k, v in self.binaries.items(): + env[k] = v + + def get_bin_dir(self): + return [pj(self.root_path, "bin")] + + def get_env(self): + env = super().get_env() + env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig") + env["LIBS"] = " ".join(self.extra_libs) + " " + env["LIBS"] + return env + + +class Win64Dyn(Win64ConfigInfo): + name = "win64_dyn" + static = False + + +class Win64Static(Win64ConfigInfo): + name = "win64_static" + static = True diff --git a/kiwixbuild/dependencies/__init__.py b/kiwixbuild/dependencies/__init__.py index bdede5b..7a45018 100644 --- a/kiwixbuild/dependencies/__init__.py +++ b/kiwixbuild/dependencies/__init__.py @@ -1,4 +1,3 @@ - from .base import * from . import ( all_dependencies, @@ -28,5 +27,5 @@ from . import ( zim_tools, zim_testing_suite, zlib, - zstd + zstd, ) diff --git a/kiwixbuild/dependencies/all_dependencies.py b/kiwixbuild/dependencies/all_dependencies.py index 09f29b3..18cbeb7 100644 --- a/kiwixbuild/dependencies/all_dependencies.py +++ b/kiwixbuild/dependencies/all_dependencies.py @@ -1,32 +1,42 @@ from os import environ -from .base import ( - Dependency, - NoopSource, - NoopBuilder) +from .base import Dependency, NoopSource, NoopBuilder from kiwixbuild._global import neutralEnv + class AllBaseDependencies(Dependency): name = "alldependencies" Source = NoopSource + class Builder(NoopBuilder): @classmethod - def get_dependencies(cls, platformInfo, allDeps): - if platformInfo.build == "wasm" or environ.get('OS_NAME') == 'bionic': - return ['zlib', 'lzma', 'zstd', 'icu4c', 'xapian-core'] + def get_dependencies(cls, configInfo, allDeps): + if configInfo.build == "wasm" or environ.get("OS_NAME") == "bionic": + return ["zlib", "lzma", "zstd", "icu4c", "xapian-core"] - base_deps = ['zlib', 'lzma', 'zstd', 'xapian-core', 'pugixml', 'libcurl', 'icu4c', 'mustache', 'libmicrohttpd', 'zim-testing-suite'] - # Add specific dependencies depending of the platform - if platformInfo.build not in ('android', 'iOS'): + base_deps = [ + "zlib", + "lzma", + "zstd", + "xapian-core", + "pugixml", + "libcurl", + "icu4c", + "mustache", + "libmicrohttpd", + "zim-testing-suite", + ] + # Add specific dependencies depending of the config + if configInfo.build not in ("android", "iOS"): # For zimtools - base_deps += ['docoptcpp'] - if platformInfo.build != 'win32': + base_deps += ["docoptcpp"] + if configInfo.build != "win32": # zimwriterfs - base_deps += ['libmagic', 'gumbo'] - if platformInfo.build == 'native' and neutralEnv('distname') != 'Darwin': + base_deps += ["libmagic", "gumbo"] + if configInfo.build == "native" and neutralEnv("distname") != "Darwin": # We compile kiwix-desktop only on native and not on `Darwin` # So we need aria2 only there - base_deps += ['aria2'] + base_deps += ["aria2"] return base_deps diff --git a/kiwixbuild/dependencies/apple_xcframework.py b/kiwixbuild/dependencies/apple_xcframework.py index 33c179c..7f8b7f7 100644 --- a/kiwixbuild/dependencies/apple_xcframework.py +++ b/kiwixbuild/dependencies/apple_xcframework.py @@ -2,14 +2,14 @@ import os import shutil from pathlib import Path -from kiwixbuild.platforms import PlatformInfo +from kiwixbuild.configs import ConfigInfo from kiwixbuild.utils import pj, run_command from .base import Dependency, NoopSource, Builder as BaseBuilder class AppleXCFramework(Dependency): name = "apple_xcframework" - subPlatformNames = [ + subConfigNames = [ "macOS_x86_64", "macOS_arm64_static", "iOS_arm64", @@ -20,34 +20,32 @@ class AppleXCFramework(Dependency): class Builder(BaseBuilder): @property - def all_subplatforms(self): - return self.buildEnv.platformInfo.subPlatformNames + def all_subconfigs(self): + return self.buildEnv.configInfo.subConfigNames @property - def macos_subplatforms(self): + def macos_subconfigs(self): return [ - target for target in self.all_subplatforms if target.startswith("macOS") + target for target in self.all_subconfigs if target.startswith("macOS") ] @property - def iossimulator_subplatforms(self): + def iossimulator_subconfigs(self): return [ target - for target in self.all_subplatforms + for target in self.all_subconfigs if target.startswith("iOSSimulator") ] @property - def ios_subplatforms(self): + def ios_subconfigs(self): return [ - target for target in self.all_subplatforms if target.startswith("iOS_") + target for target in self.all_subconfigs if target.startswith("iOS_") ] @classmethod - def get_dependencies(cls, platfomInfo, alldeps): - return [ - (target, "libkiwix") for target in AppleXCFramework.subPlatformNames - ] + def get_dependencies(cls, configInfo, alldeps): + return [(target, "libkiwix") for target in AppleXCFramework.subConfigNames] @property def final_path(self): @@ -62,11 +60,11 @@ class AppleXCFramework(Dependency): def _merge_libs(self, context): """create merged.a in all targets to bundle all static archives""" xcf_libs = [] - for target in self.all_subplatforms: + for target in self.all_subconfigs: static_ars = [] - plt = PlatformInfo.get_platform(target) - lib_dir = pj(plt.buildEnv.install_dir, "lib") + cfg = ConfigInfo.get_config(target) + lib_dir = pj(cfg.buildEnv.install_dir, "lib") static_ars = [str(f) for f in Path(lib_dir).glob("*.a")] # create merged.a from all *.a in install_dir/lib @@ -74,17 +72,17 @@ class AppleXCFramework(Dependency): run_command(command, lib_dir, context) # will be included in xcframework - if target in self.ios_subplatforms: + if target in self.ios_subconfigs: xcf_libs.append(pj(lib_dir, "merged.a")) return xcf_libs - def make_fat_with(self, platforms, folder_name, context): - """create fat merged.a in {folder_name} install/lib with {platforms} archs""" + def make_fat_with(self, configs, folder_name, context): + """create fat merged.a in {folder_name} install/lib with {configs}""" libs = [] - for target in platforms: - plt = PlatformInfo.get_platform(target) - libs.append(pj(plt.buildEnv.install_dir, "lib", "merged.a")) + for target in configs: + cfg = ConfigInfo.get_config(target) + libs.append(pj(cfg.buildEnv.install_dir, "lib", "merged.a")) fat_dir = pj(self.buildEnv.build_dir, folder_name) os.makedirs(fat_dir, exist_ok=True) @@ -97,14 +95,14 @@ class AppleXCFramework(Dependency): def _build_xcframework(self, xcf_libs, context): # create xcframework - ref_plat = PlatformInfo.get_platform(self.macos_subplatforms[0]) + ref_conf = ConfigInfo.get_config(self.macos_subconfigs[0]) command = ["xcodebuild", "-create-xcframework"] for lib in xcf_libs: command += [ "-library", lib, "-headers", - pj(ref_plat.buildEnv.install_dir, "include"), + pj(ref_conf.buildEnv.install_dir, "include"), ] command += ["-output", self.final_path] run_command(command, self.buildEnv.build_dir, context) @@ -116,13 +114,13 @@ class AppleXCFramework(Dependency): xcf_libs += self.command( "make_macos_fat", self.make_fat_with, - self.macos_subplatforms, + self.macos_subconfigs, "macOS_fat", ) xcf_libs += self.command( "make_simulator_fat", self.make_fat_with, - self.iossimulator_subplatforms, + self.iossimulator_subconfigs, "iOS-simulator_fat", ) self.command("build_xcframework", self._build_xcframework, xcf_libs) diff --git a/kiwixbuild/dependencies/aria2.py b/kiwixbuild/dependencies/aria2.py index d543be4..b2b8ee7 100644 --- a/kiwixbuild/dependencies/aria2.py +++ b/kiwixbuild/dependencies/aria2.py @@ -1,18 +1,17 @@ -from .base import ( - Dependency, - ReleaseDownload, - MakeBuilder -) +from .base import Dependency, ReleaseDownload, MakeBuilder from kiwixbuild.utils import Remotefile, run_command + class Aria2(Dependency): name = "aria2" class Source(ReleaseDownload): - archive = Remotefile('aria2-1.37.0.tar.xz', - '60a420ad7085eb616cb6e2bdf0a7206d68ff3d37fb5a956dc44242eb2f79b66b', - 'https://github.com/aria2/aria2/releases/download/release-1.37.0/aria2-1.37.0.tar.xz') + archive = Remotefile( + "aria2-1.37.0.tar.xz", + "60a420ad7085eb616cb6e2bdf0a7206d68ff3d37fb5a956dc44242eb2f79b66b", + "https://github.com/aria2/aria2/releases/download/release-1.37.0/aria2-1.37.0.tar.xz", + ) def _post_prepare_script(self, context): context.try_skip(self.extract_path) @@ -20,5 +19,9 @@ class Aria2(Dependency): run_command(command, self.extract_path, context) class Builder(MakeBuilder): - dependencies = ['zlib'] - configure_options = ["--disable-libaria2", "--disable-websocket", "--without-sqlite3"] + dependencies = ["zlib"] + configure_options = [ + "--disable-libaria2", + "--disable-websocket", + "--without-sqlite3", + ] diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index a778c6a..dbe5ef2 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -3,7 +3,17 @@ import os import shutil import time -from kiwixbuild.utils import pj, Context, SkipCommand, WarningMessage, extract_archive, StopBuild, run_command, colorize, copy_tree +from kiwixbuild.utils import ( + pj, + Context, + SkipCommand, + WarningMessage, + extract_archive, + StopBuild, + run_command, + colorize, + copy_tree, +) from kiwixbuild.versions import main_project_versions, base_deps_versions from kiwixbuild._global import neutralEnv, option, get_target_step @@ -13,8 +23,8 @@ SCRIPT_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) class _MetaDependency(type): def __new__(cls, name, bases, dct): _class = type.__new__(cls, name, bases, dct) - if name != 'Dependency': - dep_name = dct['name'] + if name != "Dependency": + dep_name = dct["name"] Dependency.all_deps[dep_name] = _class return _class @@ -29,7 +39,7 @@ class Dependency(metaclass=_MetaDependency): def version(cls): if cls.name in base_deps_versions: return base_deps_versions[cls.name] - elif option('make_release'): + elif option("make_release"): return main_project_versions.get(cls.name, None) return None @@ -42,8 +52,9 @@ class Dependency(metaclass=_MetaDependency): class Source: """Base Class to the real preparator - A source preparator must install source in the self.source_dir attribute - inside the neutralEnv.source_dir.""" + A source preparator must install source in the self.source_dir attribute + inside the neutralEnv.source_dir.""" + def __init__(self, target): self.target = target @@ -61,22 +72,22 @@ class Source: @property def source_path(self): - return pj(neutralEnv('source_dir'), self.source_dir) + return pj(neutralEnv("source_dir"), self.source_dir) @property def _log_dir(self): - return neutralEnv('log_dir') + return neutralEnv("log_dir") def _patch(self, context): context.try_skip(self.source_path) for p in self.patches: - patch_file_path = pj(SCRIPT_DIR, 'patches', p) - patch_command = [*neutralEnv('patch_command'), "-p1", "-i", patch_file_path] + patch_file_path = pj(SCRIPT_DIR, "patches", p) + patch_command = [*neutralEnv("patch_command"), "-p1", "-i", patch_file_path] run_command(patch_command, self.source_path, context) def command(self, name, function, *args): print(" {} {} : ".format(name, self.name), end="", flush=True) - log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name)) + log = pj(self._log_dir, "cmd_{}_{}.log".format(name, self.name)) context = Context(name, log, True) try: start_time = time.time() @@ -92,7 +103,7 @@ class Source: except subprocess.CalledProcessError: print(colorize("ERROR")) try: - with open(log, 'r') as f: + with open(log, "r") as f: print(f.read()) except: pass @@ -112,34 +123,36 @@ class ReleaseDownload(Source): @property def archives(self): - return (self.archive, ) + return (self.archive,) @property def extract_path(self): - return pj(neutralEnv('source_dir'), self.source_dir) + return pj(neutralEnv("source_dir"), self.source_dir) def _download(self, context): - context.try_skip(neutralEnv('archive_dir'), self.full_name) + context.try_skip(neutralEnv("archive_dir"), self.full_name) for archive in self.archives: - neutralEnv('download')(archive) + neutralEnv("download")(archive) def _extract(self, context): context.try_skip(self.extract_path) if os.path.exists(self.extract_path): shutil.rmtree(self.extract_path) for archive in self.archives: - extract_archive(pj(neutralEnv('archive_dir'), archive.name), - neutralEnv('source_dir'), - topdir=self.archive_top_dir, - name=self.source_dir) + extract_archive( + pj(neutralEnv("archive_dir"), archive.name), + neutralEnv("source_dir"), + topdir=self.archive_top_dir, + name=self.source_dir, + ) def prepare(self): - self.command('download', self._download) - self.command('extract', self._extract) - if hasattr(self, 'patches'): - self.command('patch', self._patch) - if hasattr(self, '_post_prepare_script'): - self.command('post_prepare_script', self._post_prepare_script) + self.command("download", self._download) + self.command("extract", self._extract) + if hasattr(self, "patches"): + self.command("patch", self._patch) + if hasattr(self, "_post_prepare_script"): + self.command("post_prepare_script", self._post_prepare_script) class GitClone(Source): @@ -152,48 +165,66 @@ class GitClone(Source): @property def source_dir(self): - if option('make_release'): + if option("make_release"): return "{}_release".format(self.git_dir) else: return self.git_dir @property def git_path(self): - return pj(neutralEnv('source_dir'), self.source_dir) + return pj(neutralEnv("source_dir"), self.source_dir) @property def git_ref(self): - if option('make_release'): + if option("make_release"): return self.release_git_ref else: return self.base_git_ref def _git_init(self, context): - if option('fast_clone') and self.force_full_clone == False: - command = [*neutralEnv('git_command'), "clone" , "--depth=1", "--branch", self.git_ref, self.git_remote, self.source_dir] - run_command(command, neutralEnv('source_dir'), context) + if option("fast_clone") and self.force_full_clone == False: + command = [ + *neutralEnv("git_command"), + "clone", + "--depth=1", + "--branch", + self.git_ref, + self.git_remote, + self.source_dir, + ] + run_command(command, neutralEnv("source_dir"), context) else: - command = [*neutralEnv('git_command'), "clone", self.git_remote, self.source_dir] - run_command(command, neutralEnv('source_dir'), context) - command = [*neutralEnv('git_command'), "checkout", self.git_ref] + command = [ + *neutralEnv("git_command"), + "clone", + self.git_remote, + self.source_dir, + ] + run_command(command, neutralEnv("source_dir"), context) + command = [*neutralEnv("git_command"), "checkout", self.git_ref] run_command(command, self.git_path, context) def _git_update(self, context): - command = [*neutralEnv('git_command'), "fetch", "origin", self.git_ref] + command = [*neutralEnv("git_command"), "fetch", "origin", self.git_ref] run_command(command, self.git_path, context) try: - command = [*neutralEnv('git_command'), "merge", "--ff-only", f"origin/{self.git_ref}"] + command = [ + *neutralEnv("git_command"), + "merge", + "--ff-only", + f"origin/{self.git_ref}", + ] run_command(command, self.git_path, context) except subprocess.CalledProcessError: raise WarningMessage("Cannot update, please check log for information") def prepare(self): if not os.path.exists(self.git_path): - self.command('gitinit', self._git_init) + self.command("gitinit", self._git_init) else: - self.command('gitupdate', self._git_update) - if hasattr(self, '_post_prepare_script'): - self.command('post_prepare_script', self._post_prepare_script) + self.command("gitupdate", self._git_update) + if hasattr(self, "_post_prepare_script"): + self.command("post_prepare_script", self._post_prepare_script) class Builder: @@ -206,7 +237,7 @@ class Builder: self.buildEnv = buildEnv @classmethod - def get_dependencies(cls, platformInfo, allDeps): + def get_dependencies(cls, configInfo, allDeps): return cls.dependencies @property @@ -230,7 +261,7 @@ class Builder: def command(self, name, function, *args): print(" {} {} : ".format(name, self.name), end="", flush=True) - log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name)) + log = pj(self._log_dir, "cmd_{}_{}.log".format(name, self.name)) context = Context(name, log, self.target.force_native_build) if self.target.force_build: context.no_skip = True @@ -248,7 +279,7 @@ class Builder: except subprocess.CalledProcessError: print(colorize("ERROR")) try: - with open(log, 'r') as f: + with open(log, "r") as f: print(f.read()) except: pass @@ -258,39 +289,43 @@ class Builder: raise def build(self): - if hasattr(self, '_pre_build_script'): - self.command('pre_build_script', self._pre_build_script) - self.command('configure', self._configure) - if hasattr(self, '_post_configure_script'): - self.command('post_configure_script', self._post_configure_script) - self.command('compile', self._compile) - if hasattr(self, '_test'): - self.command('test', self._test) - self.command('install', self._install) - if hasattr(self, '_post_build_script'): - self.command('post_build_script', self._post_build_script) + if hasattr(self, "_pre_build_script"): + self.command("pre_build_script", self._pre_build_script) + self.command("configure", self._configure) + if hasattr(self, "_post_configure_script"): + self.command("post_configure_script", self._post_configure_script) + self.command("compile", self._compile) + if hasattr(self, "_test"): + self.command("test", self._test) + self.command("install", self._install) + if hasattr(self, "_post_build_script"): + self.command("post_build_script", self._post_build_script) def make_dist(self): - if hasattr(self, '_pre_build_script'): - self.command('pre_build_script', self._pre_build_script) - self.command('configure', self._configure) - self.command('make_dist', self._make_dist) + if hasattr(self, "_pre_build_script"): + self.command("pre_build_script", self._pre_build_script) + self.command("configure", self._configure) + self.command("make_dist", self._make_dist) def set_flatpak_buildsystem(self, module): - if getattr(self, 'flatpak_buildsystem', None): - module['buildsystem'] = self.flatpak_buildsystem - if getattr(self, 'subsource_dir', None): - module['subdir'] = self.subsource_dir - if getattr(self, 'flatpack_build_options', None): - module['build-options'] = self.flatpack_build_options - if getattr(self, 'configure_option', ''): - module['config-opts'] = self.configure_option.split(' ') + if getattr(self, "flatpak_buildsystem", None): + module["buildsystem"] = self.flatpak_buildsystem + if getattr(self, "subsource_dir", None): + module["subdir"] = self.subsource_dir + if getattr(self, "flatpack_build_options", None): + module["build-options"] = self.flatpack_build_options + if getattr(self, "configure_option", ""): + module["config-opts"] = self.configure_option.split(" ") def get_env(self, *, cross_comp_flags, cross_compilers, cross_path): - env = self.buildEnv.get_env(cross_comp_flags=cross_comp_flags, cross_compilers=cross_compilers, cross_path=cross_path) - for dep in self.get_dependencies(self.buildEnv.platformInfo, False): + env = self.buildEnv.get_env( + cross_comp_flags=cross_comp_flags, + cross_compilers=cross_compilers, + cross_path=cross_path, + ) + for dep in self.get_dependencies(self.buildEnv.configInfo, False): try: - builder = get_target_step(dep, self.buildEnv.platformInfo.name) + builder = get_target_step(dep, self.buildEnv.configInfo.name) builder.set_env(env) except KeyError: # Some target may be missing (installed by a package, ...) @@ -317,7 +352,7 @@ class TcCopyBuilder(Builder): return pj(self.buildEnv.toolchain_dir, self.target.full_name()) def build(self): - self.command('copy', self._copy) + self.command("copy", self._copy) def _copy(self, context): context.try_skip(self.build_path) @@ -339,49 +374,48 @@ class MakeBuilder(Builder): install_options = [] configure_script = "configure" configure_env = { - '_format_CFLAGS' : '{env[CFLAGS]} -O3', - '_format_CXXFLAGS': '{env[CXXFLAGS]} -O3' + "_format_CFLAGS": "{env[CFLAGS]} -O3", + "_format_CXXFLAGS": "{env[CXXFLAGS]} -O3", } make_targets = [] flatpak_buildsystem = None @property def make_install_targets(self): - if self.buildEnv.platformInfo.build in ('iOS', "wasm"): - yield 'install' + if self.buildEnv.configInfo.build in ("iOS", "wasm"): + yield "install" else: - yield 'install-strip' + yield "install-strip" @property def all_configure_options(self): yield from self.configure_options - if self.buildEnv.platformInfo.static: + if self.buildEnv.configInfo.static: yield from self.static_configure_options else: yield from self.dynamic_configure_options if not self.target.force_native_build: - yield from self.buildEnv.platformInfo.configure_options - yield from ('--prefix', self.buildEnv.install_dir) - yield from ('--libdir', pj(self.buildEnv.install_dir, self.buildEnv.libprefix)) + yield from self.buildEnv.configInfo.configure_options + yield from ("--prefix", self.buildEnv.install_dir) + yield from ("--libdir", pj(self.buildEnv.install_dir, self.buildEnv.libprefix)) def set_configure_env(self, env): dep_conf_env = self.configure_env if not dep_conf_env: return for k, v in dep_conf_env.items(): - if k.startswith('_format_'): + if k.startswith("_format_"): v = v.format(buildEnv=self.buildEnv, env=env) env[k[8:]] = v else: env[k] = v - def _configure(self, context): context.try_skip(self.build_path) command = [ *self.buildEnv.configure_wrapper, pj(self.source_path, self.configure_script), - *self.all_configure_options + *self.all_configure_options, ] env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) self.set_configure_env(env) @@ -391,10 +425,10 @@ class MakeBuilder(Builder): context.try_skip(self.build_path) command = [ *self.buildEnv.make_wrapper, - *neutralEnv('make_command'), + *neutralEnv("make_command"), "-j4", *self.make_targets, - *self.make_options + *self.make_options, ] env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) run_command(command, self.build_path, context, env=env) @@ -403,26 +437,22 @@ class MakeBuilder(Builder): context.try_skip(self.build_path) command = [ *self.buildEnv.make_wrapper, - *neutralEnv('make_command'), + *neutralEnv("make_command"), *self.make_install_targets, - *self.make_options + *self.make_options, ] env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) run_command(command, self.build_path, context, env=env) def _make_dist(self, context): context.try_skip(self.build_path) - command = [ - *self.buildEnv.make_wrapper, - *neutralEnv('make_command'), - "dist" - ] + command = [*self.buildEnv.make_wrapper, *neutralEnv("make_command"), "dist"] env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) run_command(command, self.build_path, context, env=env) class CMakeBuilder(MakeBuilder): - flatpak_buildsystem = 'cmake' + flatpak_buildsystem = "cmake" def _configure(self, context): context.try_skip(self.build_path) @@ -430,74 +460,79 @@ class CMakeBuilder(MakeBuilder): if not self.target.force_native_build and self.buildEnv.cmake_crossfile: cross_options += [f"-DCMAKE_TOOLCHAIN_FILE={self.buildEnv.cmake_crossfile}"] command = [ - *neutralEnv('cmake_command'), + *neutralEnv("cmake_command"), *self.configure_options, "-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON", f"-DCMAKE_INSTALL_PREFIX={self.buildEnv.install_dir}", f"-DCMAKE_INSTALL_LIBDIR={self.buildEnv.libprefix}", self.source_path, - *cross_options + *cross_options, ] - env = self.get_env(cross_comp_flags=True, cross_compilers=False, cross_path=True) + env = self.get_env( + cross_comp_flags=True, cross_compilers=False, cross_path=True + ) self.set_configure_env(env) run_command(command, self.build_path, context, env=env) def set_flatpak_buildsystem(self, module): - super().set_flatpak_buildsystem( module) - module['buildir'] = True + super().set_flatpak_buildsystem(module) + module["buildir"] = True class QMakeBuilder(MakeBuilder): qmake_targets = [] - flatpak_buildsystem = 'qmake' + flatpak_buildsystem = "qmake" @property def env_options(self): - if 'QMAKE_CC' in os.environ: + if "QMAKE_CC" in os.environ: yield f"QMAKE_CC={os.environ['QMAKE_CC']}" - if 'QMAKE_CXX' in os.environ: + if "QMAKE_CXX" in os.environ: yield f"QMAKE_CXX={os.environ['QMAKE_CXX']}" def _configure(self, context): context.try_skip(self.build_path) command = [ "qmake", - *self.configure_options, - *self.env_options, - self.source_path + *self.configure_options, + *self.env_options, + self.source_path, ] - env = self.get_env(cross_comp_flags=True, cross_compilers=False, cross_path=True) + env = self.get_env( + cross_comp_flags=True, cross_compilers=False, cross_path=True + ) self.set_configure_env(env) run_command(command, self.build_path, context, env=env) def _make_dist(self, context): command = [ - *neutralEnv('git_command'), "archive", - "-o", f"{self.build_path}/{self.target_full_name()}.tar.gz", + *neutralEnv("git_command"), + "archive", + "-o", + f"{self.build_path}/{self.target_full_name()}.tar.gz", f"--prefix={self.target_full_name()}/", - "HEAD" + "HEAD", ] run_command(command, self.source_path, context) - class MesonBuilder(Builder): configure_options = [] test_options = [] - flatpak_buildsystem = 'meson' + flatpak_buildsystem = "meson" @property def build_type(self): - return 'release' if option('make_release') else 'debug' + return "release" if option("make_release") else "debug" @property def strip_options(self): - if option('make_release'): - yield '--strip' + if option("make_release"): + yield "--strip" @property def library_type(self): - return 'static' if self.buildEnv.platformInfo.static else 'shared' + return "static" if self.buildEnv.configInfo.static else "shared" def _configure(self, context): context.try_skip(self.build_path) @@ -508,47 +543,54 @@ class MesonBuilder(Builder): if not self.target.force_native_build and self.buildEnv.meson_crossfile: cross_options += ["--cross-file", self.buildEnv.meson_crossfile] command = [ - *neutralEnv('meson_command'), - '.', self.build_path, - f'--buildtype={self.build_type}', + *neutralEnv("meson_command"), + ".", + self.build_path, + f"--buildtype={self.build_type}", *self.strip_options, - f'--default-library={self.library_type}', + f"--default-library={self.library_type}", *self.configure_options, - f'--prefix={self.buildEnv.install_dir}', - f'--libdir={self.buildEnv.libprefix}', - *cross_options + f"--prefix={self.buildEnv.install_dir}", + f"--libdir={self.buildEnv.libprefix}", + *cross_options, ] - env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) + env = self.get_env( + cross_comp_flags=False, cross_compilers=False, cross_path=True + ) run_command(command, self.source_path, context, env=env) def _compile(self, context): context.try_skip(self.build_path) - command = [*neutralEnv('ninja_command'), "-v"] - env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) + command = [*neutralEnv("ninja_command"), "-v"] + env = self.get_env( + cross_comp_flags=False, cross_compilers=False, cross_path=True + ) run_command(command, self.build_path, context, env=env) def _test(self, context): context.try_skip(self.build_path) - if ( self.buildEnv.platformInfo.build == 'android' - or (self.buildEnv.platformInfo.build != 'native' - and not self.buildEnv.platformInfo.static) - ): + if self.buildEnv.configInfo.build == "android" or ( + self.buildEnv.configInfo.build != "native" + and not self.buildEnv.configInfo.static + ): raise SkipCommand() - command = [ - *neutralEnv('mesontest_command'), - '--verbose', - *self.test_options - ] - env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) + command = [*neutralEnv("mesontest_command"), "--verbose", *self.test_options] + env = self.get_env( + cross_comp_flags=False, cross_compilers=False, cross_path=True + ) run_command(command, self.build_path, context, env=env) def _install(self, context): context.try_skip(self.build_path) - command = [*neutralEnv('ninja_command'), '-v', 'install'] - env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) + command = [*neutralEnv("ninja_command"), "-v", "install"] + env = self.get_env( + cross_comp_flags=False, cross_compilers=False, cross_path=True + ) run_command(command, self.build_path, context, env=env) def _make_dist(self, context): - command = [*neutralEnv('ninja_command'), "-v", "dist"] - env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True) + command = [*neutralEnv("ninja_command"), "-v", "dist"] + env = self.get_env( + cross_comp_flags=False, cross_compilers=False, cross_path=True + ) run_command(command, self.build_path, context, env=env) diff --git a/kiwixbuild/dependencies/docoptcpp.py b/kiwixbuild/dependencies/docoptcpp.py index 73c8f4c..b5bf4e1 100644 --- a/kiwixbuild/dependencies/docoptcpp.py +++ b/kiwixbuild/dependencies/docoptcpp.py @@ -1,14 +1,10 @@ -from .base import ( - Dependency, - GitClone, - CMakeBuilder) +from .base import Dependency, GitClone, CMakeBuilder from kiwixbuild.utils import Remotefile - class docoptcpp(Dependency): - name = 'docoptcpp' + name = "docoptcpp" class Source(GitClone): git_remote = "https://github.com/docopt/docopt.cpp.git" @@ -17,5 +13,4 @@ class docoptcpp(Dependency): git_ref = "3dd23e3280f213bacefdf5fcb04857bf52e90917" class Builder(CMakeBuilder): - make_install_targets = ['install'] - + make_install_targets = ["install"] diff --git a/kiwixbuild/dependencies/gumbo.py b/kiwixbuild/dependencies/gumbo.py index 23693c3..76412fa 100644 --- a/kiwixbuild/dependencies/gumbo.py +++ b/kiwixbuild/dependencies/gumbo.py @@ -1,8 +1,4 @@ -from .base import ( - Dependency, - ReleaseDownload, - MakeBuilder -) +from .base import Dependency, ReleaseDownload, MakeBuilder from kiwixbuild.utils import Remotefile, run_command @@ -11,9 +7,11 @@ class Gumbo(Dependency): name = "gumbo" class Source(ReleaseDownload): - archive = Remotefile('gumbo-0.10.1.tar.gz', - '28463053d44a5dfbc4b77bcf49c8cee119338ffa636cc17fc3378421d714efad', - 'https://github.com/google/gumbo-parser/archive/v0.10.1.tar.gz') + archive = Remotefile( + "gumbo-0.10.1.tar.gz", + "28463053d44a5dfbc4b77bcf49c8cee119338ffa636cc17fc3378421d714efad", + "https://github.com/google/gumbo-parser/archive/v0.10.1.tar.gz", + ) def _post_prepare_script(self, context): context.try_skip(self.extract_path) diff --git a/kiwixbuild/dependencies/icu4c.py b/kiwixbuild/dependencies/icu4c.py index 55492b6..20993f4 100644 --- a/kiwixbuild/dependencies/icu4c.py +++ b/kiwixbuild/dependencies/icu4c.py @@ -1,25 +1,25 @@ -from .base import ( - Dependency, - ReleaseDownload, - MakeBuilder -) +from .base import Dependency, ReleaseDownload, MakeBuilder from kiwixbuild.utils import pj, SkipCommand, Remotefile, extract_archive from kiwixbuild._global import get_target_step, neutralEnv import os, shutil import fileinput + class Icu(Dependency): name = "icu4c" class Source(ReleaseDownload): - archive_src = Remotefile('icu4c-73_2-src.tgz', - '818a80712ed3caacd9b652305e01afc7fa167e6f2e94996da44b90c2ab604ce1', - 'https://github.com/unicode-org/icu/releases/download/release-73-2/icu4c-73_2-src.tgz') - archive_data = Remotefile('icu4c-73_2-data.zip', - 'ca1ee076163b438461e484421a7679fc33a64cd0a54f9d4b401893fa1eb42701', - 'https://github.com/unicode-org/icu/releases/download/release-73-2/icu4c-73_2-data.zip') - + archive_src = Remotefile( + "icu4c-73_2-src.tgz", + "818a80712ed3caacd9b652305e01afc7fa167e6f2e94996da44b90c2ab604ce1", + "https://github.com/unicode-org/icu/releases/download/release-73-2/icu4c-73_2-src.tgz", + ) + archive_data = Remotefile( + "icu4c-73_2-data.zip", + "ca1ee076163b438461e484421a7679fc33a64cd0a54f9d4b401893fa1eb42701", + "https://github.com/unicode-org/icu/releases/download/release-73-2/icu4c-73_2-data.zip", + ) archives = [archive_src, archive_data] @@ -28,37 +28,39 @@ class Icu(Dependency): if os.path.exists(self.extract_path): shutil.rmtree(self.extract_path) extract_archive( - pj(neutralEnv('archive_dir'), self.archive_src.name), - neutralEnv('source_dir'), + pj(neutralEnv("archive_dir"), self.archive_src.name), + neutralEnv("source_dir"), topdir=None, - name=self.source_dir) - shutil.rmtree(pj(neutralEnv('source_dir'), self.source_dir, 'source', 'data')) + name=self.source_dir, + ) + shutil.rmtree( + pj(neutralEnv("source_dir"), self.source_dir, "source", "data") + ) extract_archive( - pj(neutralEnv('archive_dir'), self.archive_data.name), - pj(neutralEnv('source_dir'), self.source_dir, 'source'), - topdir='data', - name='data' + pj(neutralEnv("archive_dir"), self.archive_data.name), + pj(neutralEnv("source_dir"), self.source_dir, "source"), + topdir="data", + name="data", ) patches = [ - "icu4c_fix_static_lib_name_mingw.patch", - # "icu4c_android_elf64_st_info.patch", - # "icu4c_custom_data.patch", - # "icu4c_noxlocale.patch", - "icu4c_rpath.patch", - # "icu4c_build_config.patch", - "icu4c_wasm.patch" - ] - + "icu4c_fix_static_lib_name_mingw.patch", + # "icu4c_android_elf64_st_info.patch", + # "icu4c_custom_data.patch", + # "icu4c_noxlocale.patch", + "icu4c_rpath.patch", + # "icu4c_build_config.patch", + "icu4c_wasm.patch", + ] class Builder(MakeBuilder): subsource_dir = "source" make_install_targets = ["install"] @classmethod - def get_dependencies(cls, platformInfo, allDeps): - plt = 'native_static' if platformInfo.static else 'native_dyn' - return [(plt, 'icu4c')] + def get_dependencies(cls, configInfo, allDeps): + plt = "native_static" if configInfo.static else "native_dyn" + return [(plt, "icu4c")] @property def configure_options(self): @@ -69,24 +71,26 @@ class Icu(Dependency): yield "--enable-rpath" yield "--disable-icuio" yield "--disable-layoutex" - platformInfo = self.buildEnv.platformInfo - if platformInfo.build != 'native': + configInfo = self.buildEnv.configInfo + if configInfo.build != "native": icu_native_builder = get_target_step( - 'icu4c', - 'native_static' if platformInfo.static else 'native_dyn') + "icu4c", "native_static" if configInfo.static else "native_dyn" + ) yield f"--with-cross-build={icu_native_builder.build_path}" yield "--disable-tools" - if platformInfo.build in ('android', 'wasm'): + if configInfo.build in ("android", "wasm"): yield "--with-data-packaging=archive" def set_env(self, env): - env['ICU_DATA_FILTER_FILE'] = pj(os.path.dirname(os.path.realpath(__file__)), "icu4c_data_filter.json") + env["ICU_DATA_FILTER_FILE"] = pj( + os.path.dirname(os.path.realpath(__file__)), "icu4c_data_filter.json" + ) def _post_configure_script(self, context): - if self.buildEnv.platformInfo.build != "wasm": + if self.buildEnv.configInfo.build != "wasm": context.skip() context.try_skip(self.build_path) - for line in fileinput.input(pj(self.build_path, 'Makefile'), inplace=True): + for line in fileinput.input(pj(self.build_path, "Makefile"), inplace=True): if line == "#DATASUBDIR = data\n": print("DATASUBDIR = data") else: diff --git a/kiwixbuild/dependencies/ios_fat_lib.py b/kiwixbuild/dependencies/ios_fat_lib.py index 27e6cfe..a0b74f2 100644 --- a/kiwixbuild/dependencies/ios_fat_lib.py +++ b/kiwixbuild/dependencies/ios_fat_lib.py @@ -1,13 +1,9 @@ import os -from kiwixbuild.platforms import PlatformInfo +from kiwixbuild.configs import ConfigInfo from kiwixbuild.utils import pj, copy_tree, run_command from kiwixbuild._global import option -from .base import ( - Dependency, - NoopSource, - Builder as BaseBuilder) - +from .base import Dependency, NoopSource, Builder as BaseBuilder class IOSFatLib(Dependency): @@ -16,38 +12,39 @@ class IOSFatLib(Dependency): Source = NoopSource class Builder(BaseBuilder): - @classmethod def get_dependencies(self, platfomInfo, alldeps): - base_target = option('target') - return [('iOS_{}'.format(arch), base_target) for arch in option('ios_arch')] + base_target = option("target") + return [("iOS_{}".format(arch), base_target) for arch in option("ios_arch")] def _copy_headers(self, context): - plt = PlatformInfo.get_platform('iOS_{}'.format(option('ios_arch')[0])) - include_src = pj(plt.buildEnv.install_dir, 'include') - include_dst = pj(self.buildEnv.install_dir, 'include') + plt = ConfigInfo.get_config("iOS_{}".format(option("ios_arch")[0])) + include_src = pj(plt.buildEnv.install_dir, "include") + include_dst = pj(self.buildEnv.install_dir, "include") copy_tree(include_src, include_dst) def _merge_libs(self, context): lib_dirs = [] - for arch in option('ios_arch'): - plt = PlatformInfo.get_platform('iOS_{}'.format(arch)) - lib_dirs.append(pj(plt.buildEnv.install_dir, 'lib')) + for arch in option("ios_arch"): + plt = ConfigInfo.get_config("iOS_{}".format(arch)) + lib_dirs.append(pj(plt.buildEnv.install_dir, "lib")) libs = [] for f in os.listdir(lib_dirs[0]): if os.path.islink(pj(lib_dirs[0], f)): continue - if f.endswith('.a') or f.endswith('.dylib'): + if f.endswith(".a") or f.endswith(".dylib"): libs.append(f) - os.makedirs(pj(self.buildEnv.install_dir, 'lib'), exist_ok=True) + os.makedirs(pj(self.buildEnv.install_dir, "lib"), exist_ok=True) for l in libs: command = [ - 'lipo', - '-create', *[pj(d, l) for d in lib_dirs], - '-output', pj(self.buildEnv.install_dir, 'lib', l) + "lipo", + "-create", + *[pj(d, l) for d in lib_dirs], + "-output", + pj(self.buildEnv.install_dir, "lib", l), ] run_command(command, self.buildEnv.install_dir, context) def build(self): - self.command('copy_headers', self._copy_headers) - self.command('merge_libs', self._merge_libs) + self.command("copy_headers", self._copy_headers) + self.command("merge_libs", self._merge_libs) diff --git a/kiwixbuild/dependencies/kiwix_desktop.py b/kiwixbuild/dependencies/kiwix_desktop.py index 51371a1..c72ff59 100644 --- a/kiwixbuild/dependencies/kiwix_desktop.py +++ b/kiwixbuild/dependencies/kiwix_desktop.py @@ -1,7 +1,5 @@ -from .base import ( - Dependency, - GitClone, - QMakeBuilder) +from .base import Dependency, GitClone, QMakeBuilder + class KiwixDesktop(Dependency): name = "kiwix-desktop" @@ -13,22 +11,18 @@ class KiwixDesktop(Dependency): class Builder(QMakeBuilder): dependencies = ["qt", "qtwebengine", "libkiwix", "aria2"] - make_install_targets = ['install'] + make_install_targets = ["install"] configure_env = None - flatpack_build_options = { - "env": [ - "QMAKEPATH=/app/lib" - ] - } + flatpack_build_options = {"env": ["QMAKEPATH=/app/lib"]} @property def configure_options(self): - if self.buildEnv.platformInfo.name == 'flatpak': - yield 'QMAKE_INCDIR+=/app/include/QtWebEngine' - yield 'QMAKE_INCDIR+=/app/include/QtWebEngineCore' - yield 'QMAKE_INCDIR+=/app/include/QtWebEngineWidgets' + if self.buildEnv.configInfo.name == "flatpak": + yield "QMAKE_INCDIR+=/app/include/QtWebEngine" + yield "QMAKE_INCDIR+=/app/include/QtWebEngineCore" + yield "QMAKE_INCDIR+=/app/include/QtWebEngineWidgets" else: yield f"PREFIX={self.buildEnv.install_dir}" - if self.buildEnv.platformInfo.static: - yield 'CONFIG+=static' + if self.buildEnv.configInfo.static: + yield "CONFIG+=static" diff --git a/kiwixbuild/dependencies/kiwix_tools.py b/kiwixbuild/dependencies/kiwix_tools.py index 2e73488..615b749 100644 --- a/kiwixbuild/dependencies/kiwix_tools.py +++ b/kiwixbuild/dependencies/kiwix_tools.py @@ -1,7 +1,5 @@ -from .base import ( - Dependency, - GitClone, - MesonBuilder) +from .base import Dependency, GitClone, MesonBuilder + class KiwixTools(Dependency): name = "kiwix-tools" @@ -16,5 +14,5 @@ class KiwixTools(Dependency): @property def configure_options(self): - if self.buildEnv.platformInfo.static: + if self.buildEnv.configInfo.static: yield "-Dstatic-linkage=true" diff --git a/kiwixbuild/dependencies/libcurl.py b/kiwixbuild/dependencies/libcurl.py index 2599180..f63ef7b 100644 --- a/kiwixbuild/dependencies/libcurl.py +++ b/kiwixbuild/dependencies/libcurl.py @@ -9,22 +9,50 @@ from .base import ( from kiwixbuild.utils import Remotefile, pj, Defaultdict, SkipCommand, run_command from kiwixbuild._global import get_target_step + class LibCurl(Dependency): name = "libcurl" class Source(ReleaseDownload): name = "libcurl" - archive = Remotefile('curl-7.67.0.tar.xz', - 'f5d2e7320379338c3952dcc7566a140abb49edb575f9f99272455785c40e536c', - 'https://curl.haxx.se/download/curl-7.67.0.tar.xz') + archive = Remotefile( + "curl-7.67.0.tar.xz", + "f5d2e7320379338c3952dcc7566a140abb49edb575f9f99272455785c40e536c", + "https://curl.haxx.se/download/curl-7.67.0.tar.xz", + ) class Builder(MakeBuilder): - dependencies = ['zlib'] + dependencies = ["zlib"] configure_options = [ - *[f"--without-{p}" for p in - ('libssh2', 'ssl', 'libmetalink', 'librtmp', 'nghttp2', 'libidn2', 'brotli') - ], - *[f"--disable-{p}" for p in - ('ftp', 'file', 'ldap', 'ldaps', 'rtsp', 'dict', 'telnet', - 'tftp', 'pop3', 'imap', 'smb', 'smtp', 'gopher', 'manual')] + *[ + f"--without-{p}" + for p in ( + "libssh2", + "ssl", + "libmetalink", + "librtmp", + "nghttp2", + "libidn2", + "brotli", + ) + ], + *[ + f"--disable-{p}" + for p in ( + "ftp", + "file", + "ldap", + "ldaps", + "rtsp", + "dict", + "telnet", + "tftp", + "pop3", + "imap", + "smb", + "smtp", + "gopher", + "manual", + ) + ], ] diff --git a/kiwixbuild/dependencies/libkiwix.py b/kiwixbuild/dependencies/libkiwix.py index 6c52a85..573cb23 100644 --- a/kiwixbuild/dependencies/libkiwix.py +++ b/kiwixbuild/dependencies/libkiwix.py @@ -1,12 +1,10 @@ import shutil, os -from .base import ( - Dependency, - GitClone, - MesonBuilder) +from .base import Dependency, GitClone, MesonBuilder from kiwixbuild.utils import pj, copy_tree from kiwixbuild._global import option, get_target_step, neutralEnv + class Libkiwix(Dependency): name = "libkiwix" force_build = True @@ -16,30 +14,40 @@ class Libkiwix(Dependency): git_dir = "libkiwix" class Builder(MesonBuilder): - dependencies = ["pugixml", "libzim", "zlib", "lzma", "libcurl", "libmicrohttpd", "icu4c", "mustache", "xapian-core"] + dependencies = [ + "pugixml", + "libzim", + "zlib", + "lzma", + "libcurl", + "libmicrohttpd", + "icu4c", + "mustache", + "xapian-core", + ] strip_options = [] @property def build_type(self): - if self.buildEnv.platformInfo.build == "android": + if self.buildEnv.configInfo.build == "android": return "debug" return super().build_type @property def configure_options(self): - platformInfo = self.buildEnv.platformInfo - if platformInfo.build == 'android': - yield '-Dstatic-linkage=true' - yield '-Dwerror=false' - if platformInfo.build == 'iOS': - yield '-Db_bitcode=true' - if platformInfo.name == 'flatpak': - yield '--wrap-mode=nodownload' - if platformInfo.mixed and option('target') == 'libkiwix': + configInfo = self.buildEnv.configInfo + if configInfo.build == "android": + yield "-Dstatic-linkage=true" + yield "-Dwerror=false" + if configInfo.build == "iOS": + yield "-Db_bitcode=true" + if configInfo.name == "flatpak": + yield "--wrap-mode=nodownload" + if configInfo.mixed and option("target") == "libkiwix": yield "-Dstatic-linkage=true" @property def library_type(self): - if self.buildEnv.platformInfo.build == 'android': - return 'shared' + if self.buildEnv.configInfo.build == "android": + return "shared" return super().library_type diff --git a/kiwixbuild/dependencies/libmagic.py b/kiwixbuild/dependencies/libmagic.py index df66c2e..18d2f9d 100644 --- a/kiwixbuild/dependencies/libmagic.py +++ b/kiwixbuild/dependencies/libmagic.py @@ -9,15 +9,18 @@ from .base import ( from kiwixbuild.utils import Remotefile, pj, SkipCommand, run_command from kiwixbuild._global import get_target_step + class LibMagic(Dependency): name = "libmagic" class Source(ReleaseDownload): name = "libmagic" source_dir = "libmagic" - archive_top_dir = 'file-5.44' - archive = Remotefile('file-5.44.tar.gz', - '3751c7fba8dbc831cb8d7cc8aff21035459b8ce5155ef8b0880a27d028475f3b') + archive_top_dir = "file-5.44" + archive = Remotefile( + "file-5.44.tar.gz", + "3751c7fba8dbc831cb8d7cc8aff21035459b8ce5155ef8b0880a27d028475f3b", + ) class Builder(MakeBuilder): @property @@ -28,23 +31,22 @@ class LibMagic(Dependency): yield "--disable-lzlib" @classmethod - def get_dependencies(cls, platformInfo, allDeps): - if platformInfo.build != 'native': - return [('native_static', 'libmagic')] + def get_dependencies(cls, configInfo, allDeps): + if configInfo.build != "native": + return [("native_static", "libmagic")] return [] def _compile(self, context): - platformInfo = self.buildEnv.platformInfo - if platformInfo.build == 'native': + configInfo = self.buildEnv.configInfo + if configInfo.build == "native": return super()._compile(context) context.try_skip(self.build_path) - command = [ - "make", - "-j4", - *self.make_targets, - *self.make_options - ] - env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) - libmagic_native_builder = get_target_step('libmagic', 'native_static') - env['PATH'] = ':'.join([pj(libmagic_native_builder.build_path, 'src'), env['PATH']]) + command = ["make", "-j4", *self.make_targets, *self.make_options] + env = self.buildEnv.get_env( + cross_comp_flags=True, cross_compilers=True, cross_path=True + ) + libmagic_native_builder = get_target_step("libmagic", "native_static") + env["PATH"] = ":".join( + [pj(libmagic_native_builder.build_path, "src"), env["PATH"]] + ) run_command(command, self.build_path, context, env=env) diff --git a/kiwixbuild/dependencies/libmicrohttpd.py b/kiwixbuild/dependencies/libmicrohttpd.py index 3e161a2..ddc0178 100644 --- a/kiwixbuild/dependencies/libmicrohttpd.py +++ b/kiwixbuild/dependencies/libmicrohttpd.py @@ -1,17 +1,23 @@ -from .base import ( - Dependency, - ReleaseDownload, - MakeBuilder) +from .base import Dependency, ReleaseDownload, MakeBuilder from kiwixbuild.utils import Remotefile + class MicroHttpd(Dependency): name = "libmicrohttpd" class Source(ReleaseDownload): - archive = Remotefile('libmicrohttpd-0.9.76.tar.gz', - 'f0b1547b5a42a6c0f724e8e1c1cb5ce9c4c35fb495e7d780b9930d35011ceb4c', - 'https://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.76.tar.gz') + archive = Remotefile( + "libmicrohttpd-0.9.76.tar.gz", + "f0b1547b5a42a6c0f724e8e1c1cb5ce9c4c35fb495e7d780b9930d35011ceb4c", + "https://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.76.tar.gz", + ) class Builder(MakeBuilder): - configure_options = ["--disable-https", "--without-libgcrypt", "--without-libcurl", "--disable-doc", "--disable-examples"] + configure_options = [ + "--disable-https", + "--without-libgcrypt", + "--without-libcurl", + "--disable-doc", + "--disable-examples", + ] diff --git a/kiwixbuild/dependencies/libzim.py b/kiwixbuild/dependencies/libzim.py index 0f62ca0..804cd38 100644 --- a/kiwixbuild/dependencies/libzim.py +++ b/kiwixbuild/dependencies/libzim.py @@ -1,9 +1,7 @@ -from .base import ( - Dependency, - GitClone, - MesonBuilder) +from .base import Dependency, GitClone, MesonBuilder from kiwixbuild._global import option, get_target_step + class Libzim(Dependency): name = "libzim" force_build = True @@ -18,37 +16,37 @@ class Libzim(Dependency): @property def build_type(self): - if self.buildEnv.platformInfo.build == "android": + if self.buildEnv.configInfo.build == "android": return "debug" return super().build_type @classmethod - def get_dependencies(cls, platformInfo, allDeps): - deps = ['lzma', 'zstd', 'xapian-core', 'icu4c'] - if platformInfo.name not in ('flatpak', 'wasm'): - deps.append('zim-testing-suite') + def get_dependencies(cls, configInfo, allDeps): + deps = ["lzma", "zstd", "xapian-core", "icu4c"] + if configInfo.name not in ("flatpak", "wasm"): + deps.append("zim-testing-suite") return deps @property def configure_options(self): - platformInfo = self.buildEnv.platformInfo - if platformInfo.build == 'android': + configInfo = self.buildEnv.configInfo + if configInfo.build == "android": yield "-DUSE_BUFFER_HEADER=false" yield "-Dstatic-linkage=true" - if platformInfo.mixed and option('target') == 'libzim': + if configInfo.mixed and option("target") == "libzim": yield "-Dstatic-linkage=true" - if platformInfo.name == "flatpak": + if configInfo.name == "flatpak": yield "--wrap-mode=nodownload" yield "-Dtest_data_dir=none" - if platformInfo.name == "wasm": + if configInfo.name == "wasm": yield "-Dexamples=false" yield "-DUSE_MMAP=false" - if platformInfo.name not in ("flatpak", "wasm"): - zim_testing_suite = get_target_step('zim-testing-suite', 'source') - yield '-Dtest_data_dir={}'.format(zim_testing_suite.source_path) + if configInfo.name not in ("flatpak", "wasm"): + zim_testing_suite = get_target_step("zim-testing-suite", "source") + yield "-Dtest_data_dir={}".format(zim_testing_suite.source_path) @property def library_type(self): - if self.buildEnv.platformInfo.build == 'android': - return 'shared' + if self.buildEnv.configInfo.build == "android": + return "shared" return super().library_type diff --git a/kiwixbuild/dependencies/lzma.py b/kiwixbuild/dependencies/lzma.py index f2d3cfa..62947f0 100644 --- a/kiwixbuild/dependencies/lzma.py +++ b/kiwixbuild/dependencies/lzma.py @@ -1,28 +1,28 @@ -from .base import ( - Dependency, - ReleaseDownload, - MakeBuilder) +from .base import Dependency, ReleaseDownload, MakeBuilder from kiwixbuild.utils import Remotefile + class lzma(Dependency): - name = 'lzma' + name = "lzma" class Source(ReleaseDownload): - archive = Remotefile('xz-5.2.6.tar.gz', - 'a2105abee17bcd2ebd15ced31b4f5eda6e17efd6b10f921a01cda4a44c91b3a0', - 'https://altushost-swe.dl.sourceforge.net/project/lzmautils/xz-5.2.6.tar.gz' - ) + archive = Remotefile( + "xz-5.2.6.tar.gz", + "a2105abee17bcd2ebd15ced31b4f5eda6e17efd6b10f921a01cda4a44c91b3a0", + "https://altushost-swe.dl.sourceforge.net/project/lzmautils/xz-5.2.6.tar.gz", + ) class Builder(MakeBuilder): @property def configure_options(self): - return ["--disable-xz", - "--disable-xzdec", - "--disable-lzmadec", - "--disable-lzmainfo", - "--disable-lzma-links", - "--disable-scripts", - "--disable-doc", -# "--disable-symbol-versions" - ] + return [ + "--disable-xz", + "--disable-xzdec", + "--disable-lzmadec", + "--disable-lzmainfo", + "--disable-lzma-links", + "--disable-scripts", + "--disable-doc", + # "--disable-symbol-versions" + ] diff --git a/kiwixbuild/dependencies/mustache.py b/kiwixbuild/dependencies/mustache.py index 2c07cdf..08eb947 100644 --- a/kiwixbuild/dependencies/mustache.py +++ b/kiwixbuild/dependencies/mustache.py @@ -1,28 +1,30 @@ -from .base import ( - Dependency, - ReleaseDownload, - Builder as BaseBuilder) +from .base import Dependency, ReleaseDownload, Builder as BaseBuilder from kiwixbuild.utils import Remotefile, pj from shutil import copy2 + class Mustache(Dependency): name = "mustache" class Source(ReleaseDownload): - archive = Remotefile('Mustache-4.1.tar.gz', - 'acd66359feb4318b421f9574cfc5a511133a77d916d0b13c7caa3783c0bfe167', - 'https://github.com/kainjow/Mustache/archive/v4.1.tar.gz') + archive = Remotefile( + "Mustache-4.1.tar.gz", + "acd66359feb4318b421f9574cfc5a511133a77d916d0b13c7caa3783c0bfe167", + "https://github.com/kainjow/Mustache/archive/v4.1.tar.gz", + ) class Builder(BaseBuilder): def build(self): - self.command('copy_header', self._copy_header) + self.command("copy_header", self._copy_header) def _copy_header(self, context): context.try_skip(self.build_path) - copy2(pj(self.source_path, 'mustache.hpp'), - pj(self.buildEnv.install_dir, 'include')) + copy2( + pj(self.source_path, "mustache.hpp"), + pj(self.buildEnv.install_dir, "include"), + ) def set_flatpak_buildsystem(self, module): - module['buildsystem'] = 'simple' - module['build-commands'] = ['cp mustache.hpp /app/include'] + module["buildsystem"] = "simple" + module["build-commands"] = ["cp mustache.hpp /app/include"] diff --git a/kiwixbuild/dependencies/pugixml.py b/kiwixbuild/dependencies/pugixml.py index 3c4a531..30bac3d 100644 --- a/kiwixbuild/dependencies/pugixml.py +++ b/kiwixbuild/dependencies/pugixml.py @@ -1,19 +1,19 @@ -from .base import ( - Dependency, - ReleaseDownload, - MesonBuilder) +from .base import Dependency, ReleaseDownload, MesonBuilder from kiwixbuild.utils import Remotefile + class Pugixml(Dependency): name = "pugixml" class Source(ReleaseDownload): - archive = Remotefile('pugixml-1.2.tar.gz', - '0f422dad86da0a2e56a37fb2a88376aae6e931f22cc8b956978460c9db06136b') + archive = Remotefile( + "pugixml-1.2.tar.gz", + "0f422dad86da0a2e56a37fb2a88376aae6e931f22cc8b956978460c9db06136b", + ) patches = ["pugixml_meson.patch"] flatpak_dest = "src" class Builder(MesonBuilder): - build_type = 'release' + build_type = "release" strip_options = [] diff --git a/kiwixbuild/dependencies/qt.py b/kiwixbuild/dependencies/qt.py index efdfd85..1a55ae3 100644 --- a/kiwixbuild/dependencies/qt.py +++ b/kiwixbuild/dependencies/qt.py @@ -1,79 +1,80 @@ import shutil -from .base import ( - Dependency, - ReleaseDownload, - MakeBuilder, - QMakeBuilder) +from .base import Dependency, ReleaseDownload, MakeBuilder, QMakeBuilder from kiwixbuild.utils import Remotefile, pj, SkipCommand class Qt(Dependency): - name = 'qt' + name = "qt" class Source(ReleaseDownload): name = "qt" source_dir = "qt-5.10.1" - archive = Remotefile('qt-everywhere-src-5.10.1.tar.xz', - '', - 'http://ftp.oregonstate.edu/.1/blfs/conglomeration/qt5/qt-everywhere-src-5.10.1.tar.xz') + archive = Remotefile( + "qt-everywhere-src-5.10.1.tar.xz", + "", + "http://ftp.oregonstate.edu/.1/blfs/conglomeration/qt5/qt-everywhere-src-5.10.1.tar.xz", + ) class Builder(MakeBuilder): - dependencies = ['icu4c', 'zlib'] + dependencies = ["icu4c", "zlib"] dynamic_configure_options = ["-shared"] static_configure_options = ["-static"] @property def all_configure_options(self): yield from self.configure_options - if self.buildEnv.platformInfo.static: + if self.buildEnv.configInfo.static: yield from self.static_configure_options else: yield from self.dynamic_configure_options if not self.target.force_native_build: - yield from self.buildEnv.platformInfo.configure_options - yield from ('-prefix', self.buildEnv.install_dir) - yield from ('-libdir', pj(self.buildEnv.install_dir, self.buildEnv.libprefix)) + yield from self.buildEnv.configInfo.configure_options + yield from ("-prefix", self.buildEnv.install_dir) + yield from ( + "-libdir", + pj(self.buildEnv.install_dir, self.buildEnv.libprefix), + ) @property def configure_options(self): skip_modules = [ - 'qt3d', - 'qtcanvas3d', - 'qtcharts', - 'qtconnectivity', - 'qtdatavis3d', - # 'qtdeclarative', - 'qtdoc', - 'qtgamepad', - 'qtgraphicaleffects', - 'qtlocation', - 'qtmultimedia', - 'qtnetworkauth', - 'qtpurchasing', - # 'qtquickcontrols', - 'qtquickcontrols2', - 'qtremoteobjects', - 'qtscript', - 'qtscxml', - 'qtsensors', - 'qtserialbus', - 'qtserialport', - 'qtspeech', - 'qtvirtualkeyboard', - 'qtwayland', - 'qtwebglplugin', - 'qtwebsockets', -# 'qtwebview', + "qt3d", + "qtcanvas3d", + "qtcharts", + "qtconnectivity", + "qtdatavis3d", + # 'qtdeclarative', + "qtdoc", + "qtgamepad", + "qtgraphicaleffects", + "qtlocation", + "qtmultimedia", + "qtnetworkauth", + "qtpurchasing", + # 'qtquickcontrols', + "qtquickcontrols2", + "qtremoteobjects", + "qtscript", + "qtscxml", + "qtsensors", + "qtserialbus", + "qtserialport", + "qtspeech", + "qtvirtualkeyboard", + "qtwayland", + "qtwebglplugin", + "qtwebsockets", + # 'qtwebview', ] - yield '-recheck' - yield '-opensource' - yield '-confirm-license' - yield '-ccache' - yield from ('-make', 'libs') + yield "-recheck" + yield "-opensource" + yield "-confirm-license" + yield "-ccache" + yield from ("-make", "libs") for module in skip_modules: - yield from ('-skip', module) + yield from ("-skip", module) class QtWebEngine(Dependency): @@ -82,5 +83,5 @@ class QtWebEngine(Dependency): Source = Qt.Source class Builder(QMakeBuilder): - dependencies = ['qt'] + dependencies = ["qt"] subsource_dir = "qtwebengine" diff --git a/kiwixbuild/dependencies/tc_android_ndk.py b/kiwixbuild/dependencies/tc_android_ndk.py index 4b565d4..3774976 100644 --- a/kiwixbuild/dependencies/tc_android_ndk.py +++ b/kiwixbuild/dependencies/tc_android_ndk.py @@ -5,23 +5,25 @@ from kiwixbuild.utils import Remotefile, add_execution_right, run_command pj = os.path.join + class android_ndk(Dependency): dont_skip = True neutral = False - name = 'android-ndk' - gccver = '4.9.x' - api = '24' + name = "android-ndk" + gccver = "4.9.x" + api = "24" class Source(ReleaseDownload): - archive = Remotefile('android-ndk-r21e-linux-x86_64.zip', - 'ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e', - 'https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip') + archive = Remotefile( + "android-ndk-r21e-linux-x86_64.zip", + "ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e", + "https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip", + ) @property def source_dir(self): return self.target.full_name() - class Builder(Builder): @property def install_path(self): @@ -32,37 +34,46 @@ class android_ndk(Dependency): return self.target.api @property - def platform(self): - return 'android-'+self.api + def config(self): + return "android-" + self.api @property def arch(self): - return self.buildEnv.platformInfo.arch + return self.buildEnv.configInfo.arch @property def arch_full(self): - return self.buildEnv.platformInfo.arch_full + return self.buildEnv.configInfo.arch_full - def _build_platform(self, context): + def _build_toolchain(self, context): context.try_skip(self.build_path) - script = pj(self.source_path, 'build/tools/make_standalone_toolchain.py') + script = pj(self.source_path, "build/tools/make_standalone_toolchain.py") add_execution_right(script) command = [ script, - f'--arch={self.arch}', - f'--api={self.api}', - f'--install-dir={self.install_path}', - '--force' + f"--arch={self.arch}", + f"--api={self.api}", + f"--install-dir={self.install_path}", + "--force", ] - env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False) + env = self.buildEnv.get_env( + cross_comp_flags=False, cross_compilers=False, cross_path=False + ) run_command(command, self.build_path, context, env=env) def _fix_permission_right(self, context): context.try_skip(self.build_path) - bin_dirs = [pj(self.install_path, 'bin'), - pj(self.install_path, self.arch_full, 'bin'), - pj(self.install_path, 'libexec', 'gcc', self.arch_full, self.target.gccver) - ] + bin_dirs = [ + pj(self.install_path, "bin"), + pj(self.install_path, self.arch_full, "bin"), + pj( + self.install_path, + "libexec", + "gcc", + self.arch_full, + self.target.gccver, + ), + ] for root, dirs, files in os.walk(self.install_path): if not root in bin_dirs: continue @@ -74,6 +85,5 @@ class android_ndk(Dependency): add_execution_right(file_path) def build(self): - self.command('build_platform', self._build_platform) - self.command('fix_permission_right', self._fix_permission_right) - + self.command("build_toolchain", self._build_toolchain) + self.command("fix_permission_right", self._fix_permission_right) diff --git a/kiwixbuild/dependencies/tc_armhf.py b/kiwixbuild/dependencies/tc_armhf.py index 507aee1..a40b640 100644 --- a/kiwixbuild/dependencies/tc_armhf.py +++ b/kiwixbuild/dependencies/tc_armhf.py @@ -4,21 +4,25 @@ from kiwixbuild.utils import Remotefile # The arm toolchains # This is based on toolchains published here : https://github.com/tttapa/docker-arm-cross-toolchain -base_url = "https://github.com/tttapa/docker-arm-cross-toolchain/releases/download/0.1.0/" +base_url = ( + "https://github.com/tttapa/docker-arm-cross-toolchain/releases/download/0.1.0/" +) -aarch_base_url = 'https://master.dl.sourceforge.net/project/raspberry-pi-cross-compilers/Bonus%20Raspberry%20Pi%20GCC%2064-Bit%20Toolchains/Raspberry%20Pi%20GCC%2064-Bit%20Cross-Compiler%20Toolchains/Stretch/GCC%206.3.0/' +aarch_base_url = "https://master.dl.sourceforge.net/project/raspberry-pi-cross-compilers/Bonus%20Raspberry%20Pi%20GCC%2064-Bit%20Toolchains/Raspberry%20Pi%20GCC%2064-Bit%20Cross-Compiler%20Toolchains/Stretch/GCC%206.3.0/" + class armv6_toolchain(Dependency): dont_skip = True neutral = True - name = 'armv6' + name = "armv6" class Source(ReleaseDownload): - archive = Remotefile('x-tools-armv6-rpi-linux-gnueabihf.tar.xz', - '4c371c4c5b55ebd1f3d7dd26b14703632d9ba47423f901bcd9303d83ad444434', - base_url + 'x-tools-armv6-rpi-linux-gnueabihf.tar.xz') - + archive = Remotefile( + "x-tools-armv6-rpi-linux-gnueabihf.tar.xz", + "4c371c4c5b55ebd1f3d7dd26b14703632d9ba47423f901bcd9303d83ad444434", + base_url + "x-tools-armv6-rpi-linux-gnueabihf.tar.xz", + ) class Builder(TcCopyBuilder): src_subdir = "armv6-rpi-linux-gnueabihf" @@ -27,24 +31,29 @@ class armv6_toolchain(Dependency): class armv8_toolchain(Dependency): dont_skip = True neutral = True - name = 'armv8' + name = "armv8" class Source(ReleaseDownload): - archive = Remotefile('x-tools-armv8-rpi-linux-gnueabihf.tar.xz', - 'cc28f5c3f6a3e7d9985f98779c4e72224b4eb5a7e4dc2bcdefd90cb241fb94a5', - base_url + 'x-tools-armv8-rpi3-linux-gnueabihf.tar.xz') + archive = Remotefile( + "x-tools-armv8-rpi-linux-gnueabihf.tar.xz", + "cc28f5c3f6a3e7d9985f98779c4e72224b4eb5a7e4dc2bcdefd90cb241fb94a5", + base_url + "x-tools-armv8-rpi3-linux-gnueabihf.tar.xz", + ) class Builder(TcCopyBuilder): src_subdir = "armv8-rpi3-linux-gnueabihf" + class aarch64_toolchain(Dependency): dont_skip = True neutral = True name = "aarch64" class Source(ReleaseDownload): - archive = Remotefile('cross-gcc-6.3.0-pi_64.tar.gz', - '1b048bb8886ad63d21797cd9129fc37b9ea0dfaac7e3c36f888aa16fbec1d320', - aarch_base_url + 'cross-gcc-6.3.0-pi_64.tar.gz') + archive = Remotefile( + "cross-gcc-6.3.0-pi_64.tar.gz", + "1b048bb8886ad63d21797cd9129fc37b9ea0dfaac7e3c36f888aa16fbec1d320", + aarch_base_url + "cross-gcc-6.3.0-pi_64.tar.gz", + ) Builder = TcCopyBuilder diff --git a/kiwixbuild/dependencies/tc_emsdk.py b/kiwixbuild/dependencies/tc_emsdk.py index 8303bed..5b26c3b 100644 --- a/kiwixbuild/dependencies/tc_emsdk.py +++ b/kiwixbuild/dependencies/tc_emsdk.py @@ -5,21 +5,23 @@ from kiwixbuild.utils import Remotefile, run_command, copy_tree pj = os.path.join + class emsdk(Dependency): dont_skip = True neutral = False - name = 'emsdk' + name = "emsdk" class Source(ReleaseDownload): - archive = Remotefile('emsdk-3.1.41.tar.gz', - '147a2d72df34227bdb4ffedc587a8cb674a42269c40458f3f69ae37e8966cdc6', - 'https://codeload.github.com/emscripten-core/emsdk/tar.gz/refs/tags/3.1.41') + archive = Remotefile( + "emsdk-3.1.41.tar.gz", + "147a2d72df34227bdb4ffedc587a8cb674a42269c40458f3f69ae37e8966cdc6", + "https://codeload.github.com/emscripten-core/emsdk/tar.gz/refs/tags/3.1.41", + ) @property def source_dir(self): return self.target.full_name() - class Builder(Builder): @property def install_path(self): @@ -39,9 +41,7 @@ class emsdk(Dependency): command = ["./emsdk", "activate", "3.1.24"] run_command(command, self.install_path, context) - def build(self): - self.command('copy_source', self._copy_source) - self.command('install', self._install) - self.command('activate', self._activate) - + self.command("copy_source", self._copy_source) + self.command("install", self._install) + self.command("activate", self._activate) diff --git a/kiwixbuild/dependencies/tc_flatpak.py b/kiwixbuild/dependencies/tc_flatpak.py index 8fe54c7..21b0fd5 100644 --- a/kiwixbuild/dependencies/tc_flatpak.py +++ b/kiwixbuild/dependencies/tc_flatpak.py @@ -5,9 +5,10 @@ from kiwixbuild.utils import Remotefile, add_execution_right, run_command pj = os.path.join + class org_kde(Dependency): neutral = False - name = 'org.kde' + name = "org.kde" Source = NoopSource @@ -15,32 +16,42 @@ class org_kde(Dependency): def _setup_remote(self, context): command = [ "flatpak", - "--user", "remote-add", "--if-not-exists", + "--user", + "remote-add", + "--if-not-exists", "flathub", - "https://flathub.org/repo/flathub.flatpakrepo" + "https://flathub.org/repo/flathub.flatpakrepo", ] - env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False) + env = self.buildEnv.get_env( + cross_comp_flags=False, cross_compilers=False, cross_path=False + ) run_command(command, self.buildEnv.build_dir, context, env=env) def _install_sdk(self, context): command = [ "flatpak", - "--user", "install", "--noninteractive", "--verbose", "-y", + "--user", + "install", + "--noninteractive", + "--verbose", + "-y", "flathub", f"{self.target.name}.Sdk//{self.target.version()}", - f"{self.target.name}.Platform//{self.target.version()}" + f"{self.target.name}.Platform//{self.target.version()}", ] - env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False) + env = self.buildEnv.get_env( + cross_comp_flags=False, cross_compilers=False, cross_path=False + ) run_command(command, self.buildEnv.build_dir, context, env=env) def build(self): - self.command('setup_remote', self._setup_remote) - self.command('install_sdk', self._install_sdk) + self.command("setup_remote", self._setup_remote) + self.command("install_sdk", self._install_sdk) class io_qt_qtwebengine(Dependency): neutral = False - name = 'io.qt.qtwebengine' + name = "io.qt.qtwebengine" Source = NoopSource @@ -48,23 +59,31 @@ class io_qt_qtwebengine(Dependency): def _setup_remote(self, context): command = [ "flatpak", - "--user", "remote-add", "--if-not-exists", + "--user", + "remote-add", + "--if-not-exists", "flathub", - "https://flathub.org/repo/flathub.flatpakrepo" + "https://flathub.org/repo/flathub.flatpakrepo", ] - env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False) + env = self.buildEnv.get_env( + cross_comp_flags=False, cross_compilers=False, cross_path=False + ) run_command(command, self.buildEnv.build_dir, context, env=env) def _install_sdk(self, context): command = [ "flatpak", - "--user", "install", "-y", + "--user", + "install", + "-y", "flathub", - f"{self.target.name}.BaseApp//{self.target.version()}" + f"{self.target.name}.BaseApp//{self.target.version()}", ] - env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False) + env = self.buildEnv.get_env( + cross_comp_flags=False, cross_compilers=False, cross_path=False + ) run_command(command, self.buildEnv.build_dir, context, env=env) def build(self): - self.command('setup_remote', self._setup_remote) - self.command('install_sdk', self._install_sdk) + self.command("setup_remote", self._setup_remote) + self.command("install_sdk", self._install_sdk) diff --git a/kiwixbuild/dependencies/tc_musl.py b/kiwixbuild/dependencies/tc_musl.py index 4ca68f5..103c846 100644 --- a/kiwixbuild/dependencies/tc_musl.py +++ b/kiwixbuild/dependencies/tc_musl.py @@ -1,15 +1,18 @@ from .base import Dependency, ReleaseDownload, TcCopyBuilder from kiwixbuild.utils import Remotefile + class aarch64_musl_toolchain(Dependency): dont_skip = True neutral = True name = "aarch64_musl" class Source(ReleaseDownload): - archive = Remotefile('aarch64-linux-musl-cross.tgz', - '0f18a885b161815520bbb5757a4b4ab40d0898c29bebee58d0cddd6112e59cc6', - 'https://more.musl.cc/10/x86_64-linux-musl/aarch64-linux-musl-cross.tgz') + archive = Remotefile( + "aarch64-linux-musl-cross.tgz", + "0f18a885b161815520bbb5757a4b4ab40d0898c29bebee58d0cddd6112e59cc6", + "https://more.musl.cc/10/x86_64-linux-musl/aarch64-linux-musl-cross.tgz", + ) Builder = TcCopyBuilder @@ -20,8 +23,10 @@ class x86_64_musl_toolchain(Dependency): name = "x86-64_musl" class Source(ReleaseDownload): - archive = Remotefile('x86_64-linux-musl-cross.tgz', - 'a3d55de8105739fcfb8b10eaa72cdb5d779319726bacff24149388d7608d1ed8', - 'https://more.musl.cc/10/x86_64-linux-musl/x86_64-linux-musl-cross.tgz') + archive = Remotefile( + "x86_64-linux-musl-cross.tgz", + "a3d55de8105739fcfb8b10eaa72cdb5d779319726bacff24149388d7608d1ed8", + "https://more.musl.cc/10/x86_64-linux-musl/x86_64-linux-musl-cross.tgz", + ) Builder = TcCopyBuilder diff --git a/kiwixbuild/dependencies/uuid.py b/kiwixbuild/dependencies/uuid.py index 245d064..b0b8976 100644 --- a/kiwixbuild/dependencies/uuid.py +++ b/kiwixbuild/dependencies/uuid.py @@ -1,30 +1,30 @@ -from .base import ( - Dependency, - ReleaseDownload, - MakeBuilder -) +from .base import Dependency, ReleaseDownload, MakeBuilder from kiwixbuild.utils import Remotefile class UUID(Dependency): - name = 'uuid' + name = "uuid" class Source(ReleaseDownload): - archive = Remotefile('e2fsprogs-libs-1.43.4.tar.gz', - 'eed4516325768255c9745e7b82c9d7d0393abce302520a5b2cde693204b0e419', - 'https://www.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v1.43.4/e2fsprogs-libs-1.43.4.tar.gz') - extract_dir = 'e2fsprogs-libs-1.43.4' + archive = Remotefile( + "e2fsprogs-libs-1.43.4.tar.gz", + "eed4516325768255c9745e7b82c9d7d0393abce302520a5b2cde693204b0e419", + "https://www.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v1.43.4/e2fsprogs-libs-1.43.4.tar.gz", + ) + extract_dir = "e2fsprogs-libs-1.43.4" class Builder(MakeBuilder): - configure_options = ["--enable-libuuid", - "--disable-debugfs", - "--disable-imager", - "--disable-resizer", - "--disable-defrag", - "--enable-fsck", - "--disable-uuidd"] - configure_env = {'_format_CFLAGS': "{env.CFLAGS} -O3 -fPIC"} + configure_options = [ + "--enable-libuuid", + "--disable-debugfs", + "--disable-imager", + "--disable-resizer", + "--disable-defrag", + "--enable-fsck", + "--disable-uuidd", + ] + configure_env = {"_format_CFLAGS": "{env.CFLAGS} -O3 -fPIC"} static_configure_options = dynamic_configure_options = [] - make_targets = ['libs'] - make_install_targets = ['install-libs'] + make_targets = ["libs"] + make_install_targets = ["install-libs"] diff --git a/kiwixbuild/dependencies/xapian.py b/kiwixbuild/dependencies/xapian.py index 6857d10..aac4fd7 100644 --- a/kiwixbuild/dependencies/xapian.py +++ b/kiwixbuild/dependencies/xapian.py @@ -1,8 +1,4 @@ -from .base import ( - Dependency, - ReleaseDownload, - MakeBuilder -) +from .base import Dependency, ReleaseDownload, MakeBuilder from kiwixbuild.utils import Remotefile from kiwixbuild._global import neutralEnv @@ -12,23 +8,29 @@ class Xapian(Dependency): name = "xapian-core" class Source(ReleaseDownload): - archive = Remotefile('xapian-core-1.4.23.tar.xz', - '30d3518172084f310dab86d262b512718a7f9a13635aaa1a188e61dc26b2288c') + archive = Remotefile( + "xapian-core-1.4.23.tar.xz", + "30d3518172084f310dab86d262b512718a7f9a13635aaa1a188e61dc26b2288c", + ) class Builder(MakeBuilder): configure_options = [ "--disable-sse", "--disable-backend-chert", "--disable-backend-remote", - "--disable-documentation"] - configure_env = {'_format_LDFLAGS': "{env.LDFLAGS} -L{buildEnv.install_dir}/{buildEnv.libprefix}", - '_format_CXXFLAGS': "{env.CXXFLAGS} -O3 -I{buildEnv.install_dir}/include"} - + "--disable-documentation", + ] + configure_env = { + "_format_LDFLAGS": "{env.LDFLAGS} -L{buildEnv.install_dir}/{buildEnv.libprefix}", + "_format_CXXFLAGS": "{env.CXXFLAGS} -O3 -I{buildEnv.install_dir}/include", + } @classmethod - def get_dependencies(cls, platformInfo, allDeps): - deps = ['zlib', 'lzma'] - if (platformInfo.build in ('win32', 'wasm') - or neutralEnv('distname') == 'Darwin'): + def get_dependencies(cls, configInfo, allDeps): + deps = ["zlib", "lzma"] + if ( + configInfo.build in ("win32", "wasm") + or neutralEnv("distname") == "Darwin" + ): return deps - return deps + ['uuid'] + return deps + ["uuid"] diff --git a/kiwixbuild/dependencies/zim_testing_suite.py b/kiwixbuild/dependencies/zim_testing_suite.py index 9f0f536..dd9224d 100644 --- a/kiwixbuild/dependencies/zim_testing_suite.py +++ b/kiwixbuild/dependencies/zim_testing_suite.py @@ -1,8 +1,4 @@ -from .base import ( - Dependency, - ReleaseDownload, - NoopBuilder -) +from .base import Dependency, ReleaseDownload, NoopBuilder from kiwixbuild.utils import Remotefile @@ -12,8 +8,10 @@ class ZimTestingSuite(Dependency): dont_skip = True class Source(ReleaseDownload): - archive = Remotefile('zim-testing-suite-0.3.tar.gz', - 'cd7d1ccc48af3783af9156cb6bf3c18d9a3319a73fdeefe65f0b4cae402d3d66', - 'https://github.com/openzim/zim-testing-suite/releases/download/v0.3/zim-testing-suite-0.3.tar.gz') + archive = Remotefile( + "zim-testing-suite-0.3.tar.gz", + "cd7d1ccc48af3783af9156cb6bf3c18d9a3319a73fdeefe65f0b4cae402d3d66", + "https://github.com/openzim/zim-testing-suite/releases/download/v0.3/zim-testing-suite-0.3.tar.gz", + ) Builder = NoopBuilder diff --git a/kiwixbuild/dependencies/zim_tools.py b/kiwixbuild/dependencies/zim_tools.py index 8179fbc..9e677c8 100644 --- a/kiwixbuild/dependencies/zim_tools.py +++ b/kiwixbuild/dependencies/zim_tools.py @@ -1,7 +1,5 @@ -from .base import ( - Dependency, - GitClone, - MesonBuilder) +from .base import Dependency, GitClone, MesonBuilder + class ZimTools(Dependency): name = "zim-tools" @@ -13,16 +11,16 @@ class ZimTools(Dependency): class Builder(MesonBuilder): @classmethod - def get_dependencies(cls, platformInfo, allDeps): - base_deps = ['libzim', 'docoptcpp', 'mustache'] - if platformInfo.build != 'win32': - base_deps += ['libmagic', 'gumbo'] + def get_dependencies(cls, configInfo, allDeps): + base_deps = ["libzim", "docoptcpp", "mustache"] + if configInfo.build != "win32": + base_deps += ["libmagic", "gumbo"] return base_deps @property def configure_options(self): # We don't build zimwriterfs on win32, and so we don't have magic - if self.buildEnv.platformInfo.build != 'win32': + if self.buildEnv.configInfo.build != "win32": yield f"-Dmagic-install-prefix={self.buildEnv.install_dir}" - if self.buildEnv.platformInfo.static: + if self.buildEnv.configInfo.static: yield "-Dstatic-linkage=true" diff --git a/kiwixbuild/dependencies/zlib.py b/kiwixbuild/dependencies/zlib.py index 1ca6eda..b0d712f 100644 --- a/kiwixbuild/dependencies/zlib.py +++ b/kiwixbuild/dependencies/zlib.py @@ -1,59 +1,63 @@ import shutil -from .base import ( - Dependency, - ReleaseDownload, - MakeBuilder) +from .base import Dependency, ReleaseDownload, MakeBuilder from kiwixbuild.utils import Remotefile, pj, SkipCommand - class zlib(Dependency): - name = 'zlib' + name = "zlib" class Source(ReleaseDownload): - archive = Remotefile('zlib-1.2.12.tar.gz', - '91844808532e5ce316b3c010929493c0244f3d37593afd6de04f71821d5136d9') - patches = ['zlib_std_libname.patch'] + archive = Remotefile( + "zlib-1.2.12.tar.gz", + "91844808532e5ce316b3c010929493c0244f3d37593afd6de04f71821d5136d9", + ) + patches = ["zlib_std_libname.patch"] class Builder(MakeBuilder): dynamic_configure_options = ["--shared"] static_configure_options = ["--static"] - make_install_targets = ['install'] + make_install_targets = ["install"] def _pre_build_script(self, context): context.try_skip(self.build_path) shutil.copytree(self.source_path, self.build_path) - def _configure(self, context): - if self.buildEnv.platformInfo.build == 'win32': + if self.buildEnv.configInfo.build == "win32": raise SkipCommand() return super()._configure(context) @property def all_configure_options(self): yield from self.configure_options - yield from self.static_configure_options if self.buildEnv.platformInfo.static else self.dynamic_configure_options - yield from ('--prefix', self.buildEnv.install_dir) - yield from ('--libdir', pj(self.buildEnv.install_dir, self.buildEnv.libprefix)) + yield from self.static_configure_options if self.buildEnv.configInfo.static else self.dynamic_configure_options + yield from ("--prefix", self.buildEnv.install_dir) + yield from ( + "--libdir", + pj(self.buildEnv.install_dir, self.buildEnv.libprefix), + ) @property def make_options(self): - if self.buildEnv.platformInfo.build != 'win32': + if self.buildEnv.configInfo.build != "win32": return yield "--makefile" yield "win32/Makefile.gcc" yield "PREFIX=i686-w64-mingw32-", - yield "SHARED_MODE={}".format("0" if self.buildEnv.platformInfo.static else "1"), - yield "INCLUDE_PATH={}".format(pj(self.buildEnv.install_dir, 'include')), - yield "LIBRARY_PATH={}".format(pj(self.buildEnv.install_dir, self.buildEnv.libprefix)), - yield "BINARY_PATH={}".format(pj(self.buildEnv.install_dir, 'bin')) + yield "SHARED_MODE={}".format( + "0" if self.buildEnv.configInfo.static else "1" + ), + yield "INCLUDE_PATH={}".format(pj(self.buildEnv.install_dir, "include")), + yield "LIBRARY_PATH={}".format( + pj(self.buildEnv.install_dir, self.buildEnv.libprefix) + ), + yield "BINARY_PATH={}".format(pj(self.buildEnv.install_dir, "bin")) @property def make_targets(self): - if self.buildEnv.platformInfo.static: + if self.buildEnv.configInfo.static: return ["static"] else: return ["shared"] diff --git a/kiwixbuild/dependencies/zstd.py b/kiwixbuild/dependencies/zstd.py index 09c93b7..87daf75 100644 --- a/kiwixbuild/dependencies/zstd.py +++ b/kiwixbuild/dependencies/zstd.py @@ -1,22 +1,20 @@ -from .base import ( - Dependency, - ReleaseDownload, - MesonBuilder) +from .base import Dependency, ReleaseDownload, MesonBuilder from kiwixbuild.utils import Remotefile - class zstd(Dependency): - name = 'zstd' + name = "zstd" class Source(ReleaseDownload): - archive = Remotefile('zstd-1.5.2.tar.gz', - 'f7de13462f7a82c29ab865820149e778cbfe01087b3a55b5332707abf9db4a6e', - 'https://github.com/facebook/zstd/archive/refs/tags/v1.5.2.tar.gz') + archive = Remotefile( + "zstd-1.5.2.tar.gz", + "f7de13462f7a82c29ab865820149e778cbfe01087b3a55b5332707abf9db4a6e", + "https://github.com/facebook/zstd/archive/refs/tags/v1.5.2.tar.gz", + ) class Builder(MesonBuilder): - subsource_dir = 'build/meson' - build_type = 'release' + subsource_dir = "build/meson" + build_type = "release" strip_options = [] - configure_options = ['-Dbin_programs=false', '-Dbin_contrib=false'] + configure_options = ["-Dbin_programs=false", "-Dbin_contrib=false"] diff --git a/kiwixbuild/flatpak_builder.py b/kiwixbuild/flatpak_builder.py index ef61734..ea0d934 100644 --- a/kiwixbuild/flatpak_builder.py +++ b/kiwixbuild/flatpak_builder.py @@ -1,17 +1,19 @@ - import sys from collections import OrderedDict from .buildenv import * -from .platforms import PlatformInfo +from .configs import ConfigInfo from .utils import remove_duplicates, run_command, StopBuild, Context from .dependencies import Dependency from .packages import PACKAGE_NAME_MAPPERS from .versions import base_deps_versions from ._global import ( - neutralEnv, option, - add_target_step, get_target_step, target_steps, - backend) + neutralEnv, + option, + add_target_step, + get_target_step, + target_steps, +) from . import _global from .dependencies.base import ( Source, @@ -22,7 +24,8 @@ from .dependencies.base import ( CMakeBuilder, QMakeBuilder, MakeBuilder, - SCRIPT_DIR) + SCRIPT_DIR, +) import json from shutil import copyfile from urllib.parse import urlparse @@ -30,54 +33,60 @@ from urllib.request import urlopen import json MANIFEST = { - 'app-id': 'org.kiwix.desktop', - 'runtime': 'org.kde.Platform', - 'runtime-version': base_deps_versions['org.kde'], - 'base': 'io.qt.qtwebengine.BaseApp', - 'base-version': base_deps_versions['org.kde'], # keep BaseApp (qwebengine) in sync with org.kde - 'sdk': 'org.kde.Sdk', - 'command': 'kiwix-desktop', - 'rename-icon': 'kiwix-desktop', - 'finish-args': [ - '--socket=wayland', - '--socket=x11', - '--share=network', - '--share=ipc', - '--device=dri', - '--socket=pulseaudio', + "app-id": "org.kiwix.desktop", + "runtime": "org.kde.Platform", + "runtime-version": base_deps_versions["org.kde"], + "base": "io.qt.qtwebengine.BaseApp", + "base-version": base_deps_versions[ + "org.kde" + ], # keep BaseApp (qwebengine) in sync with org.kde + "sdk": "org.kde.Sdk", + "command": "kiwix-desktop", + "rename-icon": "kiwix-desktop", + "finish-args": [ + "--socket=wayland", + "--socket=x11", + "--share=network", + "--share=ipc", + "--device=dri", + "--socket=pulseaudio", + ], + "cleanup": [ + "/include", + "/lib/pkgconfig", + "/lib/cmake", + "/lib/*.la", + "/bin/curl", + "/bin/copydatabase", + "/bin/kiwix-compile-resources", + "/bin/kiwix-manage", + "/bin/kiwix-read", + "/bin/kiwix-search", + "/bin/quest", + "/bin/simple*", + "/bin/xapian-*", + "/share/aclocal", + "/share/doc", + "/share/man", ], - 'cleanup': [ - '/include', - '/lib/pkgconfig', - '/lib/cmake', - '/lib/*.la', - '/bin/curl', - '/bin/copydatabase', - '/bin/kiwix-compile-resources', - '/bin/kiwix-manage', - '/bin/kiwix-read', - '/bin/kiwix-search', - '/bin/quest', - '/bin/simple*', - '/bin/xapian-*', - '/share/aclocal', - '/share/doc', - '/share/man' - ] } -GET_REF_URL_API_TEMPLATE = 'https://api.github.com/repos{repo}/git/refs/tags/{ref}' +GET_REF_URL_API_TEMPLATE = "https://api.github.com/repos{repo}/git/refs/tags/{ref}" + class FlatpakBuilder: def __init__(self): self._targets = {} - PlatformInfo.get_platform('neutral', self._targets) - self.platform = PlatformInfo.get_platform('flatpak', self._targets) - if neutralEnv('distname') not in self.platform.compatible_hosts: - print(('ERROR: The target platform {} cannot be build on host {}.\n' - 'Select another target platform or change your host system.' - ).format(self.platform.name, neutralEnv('distname'))) - self.targetDefs = self.platform.add_targets(option('target'), self._targets) + ConfigInfo.get_config("neutral", self._targets) + self.config = ConfigInfo.get_config("flatpak", self._targets) + if neutralEnv("distname") not in self.config.compatible_hosts: + print( + ( + "ERROR: The config {} cannot be build on host {}.\n" + "Select another config or change your host system." + ).format(self.config.name, neutralEnv("distname")) + ) + self.targetDefs = self.config.add_targets(option("target"), self._targets) def finalize_target_steps(self): steps = [] @@ -85,14 +94,14 @@ class FlatpakBuilder: steps += self.order_steps(targetDef) steps = list(remove_duplicates(steps)) - for pltName in PlatformInfo.all_running_platforms: - plt = PlatformInfo.all_platforms[pltName] - for tlcName in plt.toolchain_names: + for cfgName in ConfigInfo.all_running_configs: + cfg = ConfigInfo.all_configs[cfgName] + for tlcName in cfg.toolchain_names: tlc = Dependency.all_deps[tlcName] - src_plt_step = ('source', tlcName) - add_target_step(src_plt_step, self._targets[src_plt_step]) - blt_plt_step = ('neutral' if tlc.neutral else pltName, tlcName) - add_target_step(blt_plt_step, self._targets[blt_plt_step]) + src_cfg_step = ("source", tlcName) + add_target_step(src_cfg_step, self._targets[src_cfg_step]) + blt_cfg_step = ("neutral" if tlc.neutral else cfgName, tlcName) + add_target_step(blt_cfg_step, self._targets[blt_cfg_step]) for dep in steps: add_target_step(dep, self._targets[dep]) @@ -103,8 +112,8 @@ class FlatpakBuilder: yield from self.order_dependencies(targetDef, _targets) def order_dependencies(self, targetDef, targets): - targetPlatformName, targetName = targetDef - if targetPlatformName == 'source': + targetConfigName, targetName = targetDef + if targetConfigName == "source": # Do not try to order sources, they will be added as dep by the # build step two lines later. return @@ -113,27 +122,27 @@ class FlatpakBuilder: except KeyError: return - targetPlatform = PlatformInfo.get_platform(targetPlatformName) - for dep in target.get_dependencies(targetPlatform, True): + targetConfig = ConfigInfo.get_config(targetConfigName) + for dep in target.get_dependencies(targetConfig, True): if isinstance(dep, tuple): - depPlatform, depName = dep + depConfig, depName = dep else: - depPlatform, depName = targetPlatformName, dep - if (depPlatform, depName) in targets: - yield from self.order_dependencies((depPlatform, depName), targets) - yield ('source', targetName) + depConfig, depName = targetConfigName, dep + if (depConfig, depName) in targets: + yield from self.order_dependencies((depConfig, depName), targets) + yield ("source", targetName) yield targetDef def instanciate_steps(self): for stepDef in list(target_steps()): - stepPlatform, stepName = stepDef + stepConfig, stepName = stepDef stepClass = Dependency.all_deps[stepName] - if stepPlatform == 'source': + if stepConfig == "source": source = get_target_step(stepDef)(stepClass) add_target_step(stepDef, source) else: - source = get_target_step(stepName, 'source') - env = PlatformInfo.get_platform(stepPlatform).buildEnv + source = get_target_step(stepName, "source") + env = ConfigInfo.get_config(stepConfig).buildEnv builder = get_target_step(stepDef)(stepClass, source, env) add_target_step(stepDef, builder) @@ -142,125 +151,130 @@ class FlatpakBuilder: modules = OrderedDict() for stepDef in steps: module = modules.setdefault(stepDef[1], {}) - module['name'] = stepDef[1] - if stepDef[0] == 'source': + module["name"] = stepDef[1] + if stepDef[0] == "source": source = get_target_step(stepDef) - if getattr(source, 'flatpak_no_autogen', False): - module['no-autogen'] = True - module_sources = module.setdefault('sources', []) + if getattr(source, "flatpak_no_autogen", False): + module["no-autogen"] = True + module_sources = module.setdefault("sources", []) if isinstance(source, ReleaseDownload): src = { - 'type': 'archive', - 'sha256': source.archive.sha256, - 'url': source.archive.url + "type": "archive", + "sha256": source.archive.sha256, + "url": source.archive.url, } - if hasattr(source, 'flatpak_dest'): - src['dest'] = source.flatpak_dest + if hasattr(source, "flatpak_dest"): + src["dest"] = source.flatpak_dest module_sources.append(src) elif isinstance(source, GitClone): src = { - 'type': 'git', - 'url': source.git_remote, - 'tag': source.git_ref + "type": "git", + "url": source.git_remote, + "tag": source.git_ref, } try: parsed = urlparse(source.git_remote) - if parsed.hostname == 'github.com': + if parsed.hostname == "github.com": repo = parsed.path[:-4] api_url = GET_REF_URL_API_TEMPLATE.format( - repo = repo, - ref = source.git_ref) + repo=repo, ref=source.git_ref + ) with urlopen(api_url) as r: ret = json.loads(r.read()) - src['commit'] = ret['object']['sha'] + src["commit"] = ret["object"]["sha"] except: pass module_sources.append(src) - for p in getattr(source, 'patches', []): - patch = { - 'type': 'patch', - 'path': 'patches/' + p - } + for p in getattr(source, "patches", []): + patch = {"type": "patch", "path": "patches/" + p} module_sources.append(patch) - if hasattr(source, 'flatpak_command'): - command = { - 'type': 'shell', - 'commands': [source.flatpak_command] - } + if hasattr(source, "flatpak_command"): + command = {"type": "shell", "commands": [source.flatpak_command]} module_sources.append(command) else: builder = get_target_step(stepDef) builder.set_flatpak_buildsystem(module) - print(module['name']) + print(module["name"]) manifest = MANIFEST.copy() - modules = [m for m in modules.values() if m.get('sources')] + modules = [m for m in modules.values() if m.get("sources")] for m in modules: - temp = m['sources'] - del m['sources'] - m['sources'] = temp - manifest['modules'] = modules - manifest_name = "{}.json".format(MANIFEST['app-id']) - manifest_path = pj(self.platform.buildEnv.build_dir, manifest_name) - with open(manifest_path, 'w') as f: + temp = m["sources"] + del m["sources"] + m["sources"] = temp + manifest["modules"] = modules + manifest_name = "{}.json".format(MANIFEST["app-id"]) + manifest_path = pj(self.config.buildEnv.build_dir, manifest_name) + with open(manifest_path, "w") as f: f.write(json.dumps(manifest, indent=4)) def copy_patches(self): - sourceDefs = (tDef for tDef in target_steps() if tDef[0] == 'source') + sourceDefs = (tDef for tDef in target_steps() if tDef[0] == "source") for sourceDef in sourceDefs: source = get_target_step(sourceDef) - if not hasattr(source, 'patches'): + if not hasattr(source, "patches"): continue for p in source.patches: - path = pj(SCRIPT_DIR, 'patches', p) - os.makedirs(pj(self.platform.buildEnv.build_dir, 'patches'), exist_ok=True) - dest = pj(self.platform.buildEnv.build_dir, 'patches', p) + path = pj(SCRIPT_DIR, "patches", p) + os.makedirs( + pj(self.config.buildEnv.build_dir, "patches"), exist_ok=True + ) + dest = pj(self.config.buildEnv.build_dir, "patches", p) copyfile(path, dest) - def build(self): - log = pj(self.platform.buildEnv.log_dir, 'cmd_build_flatpak.log') - context = Context('build', log, False) + log = pj(self.config.buildEnv.log_dir, "cmd_build_flatpak.log") + context = Context("build", log, False) command = [ "flatpak-builder", - "--user", "--ccache", "--force-clean", "--keep-build-dirs", - "--disable-rofiles-fuse", "--repo=repo", "builddir", - f"{MANIFEST['app-id']}.json" + "--user", + "--ccache", + "--force-clean", + "--keep-build-dirs", + "--disable-rofiles-fuse", + "--repo=repo", + "builddir", + f"{MANIFEST['app-id']}.json", ] try: - run_command(command, self.platform.buildEnv.build_dir, context, env=self.platform.get_env()) + run_command( + command, + self.config.buildEnv.build_dir, + context, + env=self.config.get_env(), + ) context._finalise() except subprocess.CalledProcessError: - with open(log, 'r') as f: + with open(log, "r") as f: print(f.read()) raise StopBuild() def bundle(self): - log = pj(self.platform.buildEnv.log_dir, 'cmd_bundle_flatpak.log') - context = Context('bundle', log, False) - app_id = MANIFEST['app-id'] - command = [ - "flatpak", "build-bundle", "repo", - f"{app_id}.flatpak", - app_id - ] + log = pj(self.config.buildEnv.log_dir, "cmd_bundle_flatpak.log") + context = Context("bundle", log, False) + app_id = MANIFEST["app-id"] + command = ["flatpak", "build-bundle", "repo", f"{app_id}.flatpak", app_id] try: - run_command(command, self.platform.buildEnv.build_dir, context, env=self.platform.get_env()) + run_command( + command, + self.config.buildEnv.build_dir, + context, + env=self.config.get_env(), + ) context._finalise() except subprocess.CalledProcessError: - with open(log, 'r') as f: + with open(log, "r") as f: print(f.read()) raise StopBuild() - def _get_packages(self): - package_name_mapper = PACKAGE_NAME_MAPPERS.get('flatpak', {}) + package_name_mapper = PACKAGE_NAME_MAPPERS.get("flatpak", {}) to_drop = [] for builderDef in self._targets: - platformName, builderName = builderDef + configName, builderName = builderDef packages = package_name_mapper.get(builderName) if packages: to_drop.append(builderDef) @@ -274,16 +288,16 @@ class FlatpakBuilder: # dependencies we already have in the sdk. self._get_packages() self.finalize_target_steps() - print("[SETUP PLATFORMS]") - for platform in PlatformInfo.all_running_platforms.values(): - platform.finalize_setup() - for pltName in PlatformInfo.all_running_platforms: - plt = PlatformInfo.all_platforms[pltName] - for tlcName in plt.toolchain_names: + print("[SETUP TOOLCHAINS]") + for config in ConfigInfo.all_running_configs.values(): + config.finalize_setup() + for cfgName in ConfigInfo.all_running_configs: + cfg = ConfigInfo.all_configs[cfgName] + for tlcName in cfg.toolchain_names: tlc = Dependency.all_deps[tlcName] - builderDef = (pltName, tlcName) + builderDef = (cfgName, tlcName) builder = get_target_step(builderDef) - print("build {} ({}):".format(builder.name, pltName[0])) + print("build {} ({}):".format(builder.name, cfgName[0])) add_target_step(builderDef, builder) builder.build() print("[GENERATE FLATPAK MANIFEST]") @@ -295,11 +309,10 @@ class FlatpakBuilder: self.bundle() # No error, clean intermediate file at end of build if needed. print("[CLEAN]") - if option('clean_at_end'): - for platform in PlatformInfo.all_running_platforms.values(): - platform.clean_intermediate_directories() + if option("clean_at_end"): + for config in ConfigInfo.all_running_configs.values(): + config.clean_intermediate_directories() else: print("SKIP") except StopBuild: sys.exit("Stopping build due to errors") - diff --git a/kiwixbuild/packages.py b/kiwixbuild/packages.py index 5a229f3..e7364ff 100644 --- a/kiwixbuild/packages.py +++ b/kiwixbuild/packages.py @@ -1,107 +1,147 @@ - - -_fedora_common = ['automake', 'libtool', 'cmake', 'git', 'subversion', 'ccache', 'pkgconf-pkg-config', 'gcc-c++', 'gettext-devel'] -_debian_common = ['automake', 'libtool', 'cmake', 'git', 'subversion', 'ccache', 'pkg-config', 'gcc', 'autopoint'] +_fedora_common = [ + "automake", + "libtool", + "cmake", + "git", + "subversion", + "ccache", + "pkgconf-pkg-config", + "gcc-c++", + "gettext-devel", +] +_debian_common = [ + "automake", + "libtool", + "cmake", + "git", + "subversion", + "ccache", + "pkg-config", + "gcc", + "autopoint", +] PACKAGE_NAME_MAPPERS = { - 'flatpak': { - 'zlib': True, - 'lzma': True, - 'icu4c': True, - 'qt': True, - 'qtwebengine': True, - 'uuid': True, - 'libxml2': True, - 'libssl': True, - 'libcurl': True, + "flatpak": { + "zlib": True, + "lzma": True, + "icu4c": True, + "qt": True, + "qtwebengine": True, + "uuid": True, + "libxml2": True, + "libssl": True, + "libcurl": True, }, - 'fedora_native_dyn': { - 'COMMON': _fedora_common, - 'uuid': ['libuuid-devel'], - 'xapian-core': None, # Not the right version on fedora 25 - 'pugixml': None, # ['pugixml-devel'] but package doesn't provide pkg-config file - 'libmicrohttpd': ['libmicrohttpd-devel'], - 'zlib': ['zlib-devel'], - 'lzma': ['xz-devel'], - 'icu4c': None, - 'zimlib': None, - 'file' : ['file-devel'], - 'gumbo' : ['gumbo-parser-devel'], - 'aria2': ['aria2'], - 'qt': ['qt5-qtbase-devel', 'qt5-qtsvg'], - 'qtwebengine': ['qt5-qtwebengine-devel'] + "fedora_native_dyn": { + "COMMON": _fedora_common, + "uuid": ["libuuid-devel"], + "xapian-core": None, # Not the right version on fedora 25 + "pugixml": None, # ['pugixml-devel'] but package doesn't provide pkg-config file + "libmicrohttpd": ["libmicrohttpd-devel"], + "zlib": ["zlib-devel"], + "lzma": ["xz-devel"], + "icu4c": None, + "zimlib": None, + "file": ["file-devel"], + "gumbo": ["gumbo-parser-devel"], + "aria2": ["aria2"], + "qt": ["qt5-qtbase-devel", "qt5-qtsvg"], + "qtwebengine": ["qt5-qtwebengine-devel"], }, - 'fedora_native_static': { - 'COMMON': _fedora_common + ['glibc-static', 'libstdc++-static'], - 'lzma': ['xz-devel', 'xz-static'] + "fedora_native_static": { + "COMMON": _fedora_common + ["glibc-static", "libstdc++-static"], + "lzma": ["xz-devel", "xz-static"] # Either there is no packages, or no static or too old }, - 'fedora_i586_dyn': { - 'COMMON': _fedora_common + ['glibc-devel.i686', 'libstdc++-devel.i686'], + "fedora_i586_dyn": { + "COMMON": _fedora_common + ["glibc-devel.i686", "libstdc++-devel.i686"], }, - 'fedora_i586_static': { - 'COMMON': _fedora_common + ['glibc-devel.i686'], + "fedora_i586_static": { + "COMMON": _fedora_common + ["glibc-devel.i686"], }, - 'fedora_win32_dyn': { - 'COMMON': _fedora_common + ['mingw32-gcc-c++', 'mingw32-bzip2', 'mingw32-win-iconv', 'mingw32-winpthreads', 'wine'], - 'zlib': ['mingw32-zlib'], - 'lzma': ['mingw32-xz-libs'], - 'libmicrohttpd': ['mingw32-libmicrohttpd'], + "fedora_win32_dyn": { + "COMMON": _fedora_common + + [ + "mingw32-gcc-c++", + "mingw32-bzip2", + "mingw32-win-iconv", + "mingw32-winpthreads", + "wine", + ], + "zlib": ["mingw32-zlib"], + "lzma": ["mingw32-xz-libs"], + "libmicrohttpd": ["mingw32-libmicrohttpd"], }, - 'fedora_win32_static': { - 'COMMON': _fedora_common + ['mingw32-gcc-c++', 'mingw32-bzip2-static', 'mingw32-win-iconv-static', 'mingw32-winpthreads-static', 'wine'], - 'zlib': ['mingw32-zlib-static'], - 'lzma': ['mingw32-xz-libs-static'], - 'libmicrohttpd': None, # ['mingw32-libmicrohttpd-static'] packaging dependecy seems buggy, and some static lib are name libfoo.dll.a and - # gcc cannot found them. + "fedora_win32_static": { + "COMMON": _fedora_common + + [ + "mingw32-gcc-c++", + "mingw32-bzip2-static", + "mingw32-win-iconv-static", + "mingw32-winpthreads-static", + "wine", + ], + "zlib": ["mingw32-zlib-static"], + "lzma": ["mingw32-xz-libs-static"], + "libmicrohttpd": None, # ['mingw32-libmicrohttpd-static'] packaging dependecy seems buggy, and some static lib are name libfoo.dll.a and + # gcc cannot found them. }, - 'fedora_armhf_static': { - 'COMMON': _fedora_common + "fedora_armhf_static": {"COMMON": _fedora_common}, + "fedora_armhf_dyn": {"COMMON": _fedora_common}, + "fedora_android": {"COMMON": _fedora_common}, + "debian_native_dyn": { + "COMMON": _debian_common + ["libbz2-dev", "libmagic-dev"], + "zlib": ["zlib1g-dev"], + "uuid": ["uuid-dev"], + "libmicrohttpd": ["libmicrohttpd-dev", "ccache"], + "qt": ["libqt5gui5", "qtbase5-dev", "qt5-default"], + "qtwebengine": ["qtwebengine5-dev"], + "aria2": ["aria2"], }, - 'fedora_armhf_dyn': { - 'COMMON': _fedora_common + "debian_native_static": { + "COMMON": _debian_common + ["libbz2-dev", "libmagic-dev"], }, - 'fedora_android': { - 'COMMON': _fedora_common + "debian_i586_dyn": { + "COMMON": _debian_common + + ["libc6-dev-i386", "lib32stdc++6", "gcc-multilib", "g++-multilib"], }, - 'debian_native_dyn': { - 'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'], - 'zlib': ['zlib1g-dev'], - 'uuid': ['uuid-dev'], - 'libmicrohttpd': ['libmicrohttpd-dev', 'ccache'], - 'qt' : ['libqt5gui5', 'qtbase5-dev', 'qt5-default'], - 'qtwebengine' : ['qtwebengine5-dev'], - 'aria2': ['aria2'], + "debian_i586_static": { + "COMMON": _debian_common + + ["libc6-dev-i386", "lib32stdc++6", "gcc-multilib", "g++-multilib"], }, - 'debian_native_static': { - 'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'], + "debian_win32_dyn": { + "COMMON": _debian_common + + [ + "g++-mingw-w64-i686", + "gcc-mingw-w64-i686", + "gcc-mingw-w64-base", + "mingw-w64-tools", + ], }, - 'debian_i586_dyn': { - 'COMMON': _debian_common + ['libc6-dev-i386', 'lib32stdc++6', 'gcc-multilib', 'g++-multilib'], + "debian_win32_static": { + "COMMON": _debian_common + + [ + "g++-mingw-w64-i686", + "gcc-mingw-w64-i686", + "gcc-mingw-w64-base", + "mingw-w64-tools", + ], }, - 'debian_i586_static': { - 'COMMON': _debian_common + ['libc6-dev-i386', 'lib32stdc++6', 'gcc-multilib', 'g++-multilib'], + "debian_armhf_static": { + "COMMON": _debian_common, }, - 'debian_win32_dyn': { - 'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'], + "debian_armhf_dyn": { + "COMMON": _debian_common, }, - 'debian_win32_static': { - 'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'], + "debian_android": { + "COMMON": _debian_common, }, - 'debian_armhf_static': { - 'COMMON': _debian_common, + "Darwin_native_dyn": { + "COMMON": ["autoconf", "automake", "libtool", "cmake", "pkg-config"], + "file": ["libmagic"], }, - 'debian_armhf_dyn': { - 'COMMON': _debian_common, - }, - 'debian_android': { - 'COMMON': _debian_common, - }, - 'Darwin_native_dyn': { - 'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'], - 'file': ['libmagic'] - }, - 'Darwin_iOS': { - 'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'], - 'file': ['libmagic'] + "Darwin_iOS": { + "COMMON": ["autoconf", "automake", "libtool", "cmake", "pkg-config"], + "file": ["libmagic"], }, } diff --git a/kiwixbuild/platforms/__init__.py b/kiwixbuild/platforms/__init__.py deleted file mode 100644 index fac9e94..0000000 --- a/kiwixbuild/platforms/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ - -from .base import * - -from . import ( - android, - armhf, - musl, - flatpak, - i586, - ios, - native, - neutral, - win32, - wasm -) diff --git a/kiwixbuild/platforms/android.py b/kiwixbuild/platforms/android.py deleted file mode 100644 index 2fb6d09..0000000 --- a/kiwixbuild/platforms/android.py +++ /dev/null @@ -1,146 +0,0 @@ -from .base import PlatformInfo, MetaPlatformInfo -from kiwixbuild.utils import pj -from kiwixbuild._global import get_target_step, option - - -class AndroidPlatformInfo(PlatformInfo): - build = 'android' - static = True - toolchain_names = ['android-ndk'] - compatible_hosts = ['fedora', 'debian'] - - def __str__(self): - return "android" - - @property - def libdir(self): - return 'lib/{}'.format(self.arch_full) - - @property - def binaries_name(self): - arch_full = self.arch_full - return { - 'CC': '{}-{}'.format(arch_full, 'clang'), - 'CXX': '{}-{}'.format(arch_full, 'clang++'), - 'AR': '{}-{}'.format(arch_full, 'ar'), - 'STRIP': '{}-{}'.format(arch_full, 'strip'), - 'RANLIB': '{}-{}'.format(arch_full, 'ranlib'), - 'LD': '{}-{}'.format(arch_full, 'ld') - } - - def binaries(self): - install_path = self.install_path - binaries = {k:pj(install_path, 'bin', v) - for k,v in self.binaries_name.items()} - binaries['PKGCONFIG'] = 'pkg-config' - return binaries - - @property - def ndk_builder(self): - return get_target_step('android-ndk', self.name) - - @property - def install_path(self): - return self.ndk_builder.install_path - - def get_cross_config(self): - extra_libs = ['-llog'] - extra_cflags = ['-I{}'.format(include_dir) for include_dir in self.get_include_dirs()] - if hasattr(self, 'march'): - extra_libs.append('-march={}'.format(self.march)) - extra_cflags.append('-march={}'.format(self.march)) - return { - 'exe_wrapper_def': '', - 'install_path': self.install_path, - 'binaries': self.binaries(), - 'root_path': pj(self.install_path, 'sysroot'), - 'extra_libs': extra_libs, - 'extra_cflags': extra_cflags, - 'host_machine': { - 'system': 'Android', - 'lsystem': 'android', - 'cpu_family': self.arch, - 'cpu': self.cpu, - 'endian': 'little', - 'abi': self.abi - } - } - - def get_env(self): - env = super().get_env() - root_path = pj(self.install_path, 'sysroot') - env['PKG_CONFIG_LIBDIR'] = pj(root_path, 'lib', 'pkgconfig') - env['NDK_DEBUG'] = '0' - return env - - def get_bin_dir(self): - return [pj(self.install_path, 'bin')] - - def set_comp_flags(self, env): - super().set_comp_flags(env) - root_path = pj(self.install_path, 'sysroot') - march = '-march={}'.format(self.march) if hasattr(self,'march') else '' - env['CFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} {} '.format(root_path, march) + env['CFLAGS'] - env['CXXFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} {} '.format(root_path, march) + env['CXXFLAGS'] - env['LDFLAGS'] = '--sysroot={} {} '.format(root_path, march) + env['LDFLAGS'] - - def set_compiler(self, env): - binaries = self.binaries() - for k,v in binaries.items(): - env[k] = v - - @property - def configure_options(self): - yield '--host={}'.format(self.arch_full) - - def finalize_setup(self): - super().finalize_setup() - self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_android_cross_file.txt', 'cmake_cross_file.txt') - self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') - - -class AndroidArm(AndroidPlatformInfo): - name = 'android_arm' - arch = cpu = 'arm' - arch_full = 'arm-linux-androideabi' - abi = 'armeabi-v7a' - march = 'armv7-a' - - -class AndroidArm64(AndroidPlatformInfo): - name = 'android_arm64' - arch = 'arm64' - arch_full = 'aarch64-linux-android' - cpu = 'aarch64' - abi = 'arm64-v8a' - - -class AndroidX86(AndroidPlatformInfo): - name = 'android_x86' - arch = abi = 'x86' - arch_full = 'i686-linux-android' - cpu = 'i686' - - -class AndroidX8664(AndroidPlatformInfo): - name = 'android_x86_64' - arch = cpu = abi = 'x86_64' - arch_full = 'x86_64-linux-android' - - -class Android(MetaPlatformInfo): - name = "android" - compatible_hosts = ['fedora', 'debian'] - - @property - def subPlatformNames(self): - return ['android_{}'.format(arch) for arch in option('android_arch')] - - def add_targets(self, targetName, targets): - return super().add_targets(targetName, targets) - - def __str__(self): - return self.name - - def set_comp_flags(self, env): - pass diff --git a/kiwixbuild/platforms/armhf.py b/kiwixbuild/platforms/armhf.py deleted file mode 100644 index 834db05..0000000 --- a/kiwixbuild/platforms/armhf.py +++ /dev/null @@ -1,166 +0,0 @@ -from .base import PlatformInfo, MixedMixin - -from kiwixbuild.utils import pj -from kiwixbuild._global import get_target_step - - -# Base platform -class ArmPlatformInfo(PlatformInfo): - compatible_hosts = ['fedora', 'debian'] - - def get_cross_config(self): - return { - 'binaries': self.binaries, - 'exe_wrapper_def': '', - 'root_path': self.root_path, - 'extra_libs': [], - 'extra_cflags': ['-I{}'.format(include_dir) for include_dir in self.get_include_dirs()], - 'host_machine': { - 'system': 'linux', - 'lsystem': 'linux', - 'cpu_family': self.cpu_family, - 'cpu': self.cpu, - 'endian': 'little', - 'abi': '' - } - } - - @property - def libdir(self): - return "lib/{}".format(self.arch_full) - - @property - def toolchain(self): - return get_target_step(self.build, 'neutral') - - @property - def root_path(self): - return self.toolchain.build_path - - @property - def binaries(self): - binaries = ((k,'{}-{}'.format(self.arch_full, v)) - for k, v in (('CC', 'gcc'), - ('CXX', 'g++'), - ('AR', 'ar'), - ('STRIP', 'strip'), - ('WINDRES', 'windres'), - ('RANLIB', 'ranlib'), - ('LD', 'ld'), - ('LDSHARED', 'g++ -shared') - ) - ) - binaries = {k:pj(self.root_path, 'bin', v) - for k,v in binaries} - binaries['PKGCONFIG'] = 'pkg-config' - return binaries - - @property - def exe_wrapper_def(self): - try: - which('qemu-arm') - except subprocess.CalledProcessError: - return "" - else: - return "exe_wrapper = 'qemu-arm'" - - @property - def configure_options(self): - yield '--host={}'.format(self.arch_full) - - def get_bin_dir(self): - return [pj(self.root_path, 'bin')] - - def get_env(self): - env = super().get_env() - env['LD_LIBRARY_PATH'] = ':'.join([ - pj(self.root_path, self.arch_full, 'lib64'), - pj(self.root_path, 'lib'), - env['LD_LIBRARY_PATH'] - ]) - env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') - env['QEMU_LD_PREFIX'] = pj(self.root_path, self.arch_full, "libc") - env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format( - ':'.join([ - pj(self.root_path, self.arch_full, "lib"), - env['LD_LIBRARY_PATH'] - ])) - return env - - def set_comp_flags(self, env): - super().set_comp_flags(env) - env['CFLAGS'] = " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS'] - env['CXXFLAGS'] = " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS'] - - def set_compiler(self, env): - for k, v in self.binaries.items(): - env[k] = v - - def finalize_setup(self): - super().finalize_setup() - self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') - self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') - -class Armv6(ArmPlatformInfo): - build = "armv6" - arch_full = "armv6-rpi-linux-gnueabihf" - toolchain_names = ['armv6'] - cpu_family = 'arm' - cpu = 'armv6' - -class Armv6Dyn(Armv6): - name = 'armv6_dyn' - static = False - -class Armv6Static(Armv6): - name = 'armv6_static' - static = True - -class Armv6Mixed(MixedMixin('armv6_static'), Armv6): - name = 'armv6_mixed' - static = False - - -class Armv8(ArmPlatformInfo): - build = "armv8" - arch_full = "armv8-rpi3-linux-gnueabihf" - toolchain_names = ['armv8'] - cpu_family = 'arm' - cpu = 'armv8' - -class Armv8Dyn(Armv8): - name = 'armv8_dyn' - static = False - -class Armv8Static(Armv8): - name = 'armv8_static' - static = True - -class Armv8Mixed(MixedMixin('armv8_static'), Armv8): - name = 'armv8_mixed' - static = False - - -class Aarch64(ArmPlatformInfo): - build = 'aarch64' - arch_full = 'aarch64-linux-gnu' - toolchain_names = ['aarch64'] - cpu_family = 'aarch64' - cpu = 'aarch64' - - @property - def root_path(self): - return self.toolchain.build_path - -class Aarch64Dyn(Aarch64): - name = 'aarch64_dyn' - static = False - -class Aarch64Static(Aarch64): - name = 'aarch64_static' - static = True - - -class Aarch64Mixed(MixedMixin('aarch64_static'), Aarch64): - name = 'aarch64_mixed' - static = False diff --git a/kiwixbuild/platforms/flatpak.py b/kiwixbuild/platforms/flatpak.py deleted file mode 100644 index 4cd8342..0000000 --- a/kiwixbuild/platforms/flatpak.py +++ /dev/null @@ -1,18 +0,0 @@ -from .base import PlatformInfo -from kiwixbuild._global import option, neutralEnv - -class FlatpakPlatformInfo(PlatformInfo): - name = 'flatpak' - build = 'flatpak' - static = '' - toolchain_names = ['org.kde', 'io.qt.qtwebengine'] - compatible_hosts = ['debian', 'fedora'] - - def __str__(self): - return "flatpak" - - def get_env(self): - env = super().get_env() - env['FLATPAK_USER_DIR'] = self.buildEnv.build_dir - return env - diff --git a/kiwixbuild/platforms/i586.py b/kiwixbuild/platforms/i586.py deleted file mode 100644 index 71ae316..0000000 --- a/kiwixbuild/platforms/i586.py +++ /dev/null @@ -1,70 +0,0 @@ -import os - -from .base import PlatformInfo -from kiwixbuild.utils import which, pj - - -class I586PlatformInfo(PlatformInfo): - build = 'i586' - arch_full = 'i586-linux-gnu' - compatible_hosts = ['fedora', 'debian'] - - def get_cross_config(self): - return { - 'binaries': self.binaries, - 'exe_wrapper_def': '', - 'extra_libs': ['-m32', '-march=i586', '-mno-sse'], - 'extra_cflags': [ - '-m32', - '-march=i586', - '-mno-sse', - *('-I{}'.format(include_dir) for include_dir in self.get_include_dirs()) - ], - 'host_machine': { - 'system': 'linux', - 'lsystem': 'linux', - 'cpu_family': 'x86', - 'cpu': 'i586', - 'endian': 'little', - 'abi': '' - } - } - - @property - def configure_options(self): - yield f'--host={self.arch_full}' - - @property - def binaries(self): - return {k:which(v) - for k, v in (('CC', os.environ.get('CC', 'gcc')), - ('CXX', os.environ.get('CXX', 'g++')), - ('AR', 'ar'), - ('STRIP', 'strip'), - ('RANLIB', 'ranlib'), - ('LD', 'ld'), - ('PKGCONFIG', 'pkg-config')) - } - - def set_comp_flags(self, env): - super().set_comp_flags(env) - env['CFLAGS'] = "-m32 -march=i586 -mno-sse "+env['CFLAGS'] - env['CXXFLAGS'] = "-m32 -march=i586 -mno-sse "+env['CXXFLAGS'] - env['LDFLAGS'] = "-m32 -march=i586 -mno-sse "+env['LDFLAGS'] - - def get_bin_dir(self): - return [] - - - def finalize_setup(self): - super().finalize_setup() - self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_i586_cross_file.txt', 'cmake_cross_file.txt') - self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') - -class I586Dyn(I586PlatformInfo): - name = 'i586_dyn' - static = False - -class I586Static(I586PlatformInfo): - name = 'i586_static' - static = True diff --git a/kiwixbuild/platforms/ios.py b/kiwixbuild/platforms/ios.py deleted file mode 100644 index 89964b9..0000000 --- a/kiwixbuild/platforms/ios.py +++ /dev/null @@ -1,210 +0,0 @@ -import subprocess - -from kiwixbuild._global import option -from kiwixbuild.utils import pj, xrun_find -from .base import PlatformInfo, MetaPlatformInfo, MixedMixin -from kiwixbuild.dependencies.apple_xcframework import AppleXCFramework - - -MIN_MACOS_VERSION = '12.0' - -class ApplePlatformInfo(PlatformInfo): - build = 'iOS' - static = True - compatible_hosts = ['Darwin'] - arch = None - host = None - target = None - sdk_name = None - min_iphoneos_version = None - min_macos_version = None - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._root_path = None - - @property - def root_path(self): - if self._root_path is None: - command = "xcrun --sdk {} --show-sdk-path".format(self.sdk_name) - self._root_path = subprocess.check_output(command, shell=True)[:-1].decode() - return self._root_path - - def __str__(self): - return "iOS" - - def finalize_setup(self): - super().finalize_setup() - self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_ios_cross_file.txt', 'cmake_cross_file.txt') - self.buildEnv.meson_crossfile = self._gen_crossfile('meson_ios_cross_file.txt', 'meson_cross_file.txt') - - def get_cross_config(self): - config = { - 'root_path': self.root_path, - 'binaries': self.binaries, - 'exe_wrapper_def': '', - 'extra_libs': [ - '-isysroot', self.root_path, - '-arch', self.arch, - '-target', self.target, - ], - 'extra_cflags': [ - '-isysroot', self.root_path, - '-arch', self.arch, - '-target', self.target, - *('-I{}'.format(include_dir) for include_dir in self.get_include_dirs()) - ], - 'host_machine': { - 'system': 'Darwin', - 'lsystem': 'darwin', - 'cpu_family': self.arch, - 'cpu': self.cpu, - 'endian': '', - 'abi': '' - } - } - if self.min_iphoneos_version: - config['extra_libs'].append('-miphoneos-version-min={}'.format(self.min_iphoneos_version)) - config['extra_cflags'].append('-miphoneos-version-min={}'.format(self.min_iphoneos_version)) - if self.min_macos_version: - config['extra_libs'].append('-mmacosx-version-min={}'.format(self.min_macos_version)) - config['extra_cflags'].append('-mmacosx-version-min={}'.format(self.min_macos_version)) - return config - - def get_env(self): - env = super().get_env() - cflags = [env['CFLAGS']] - if self.min_iphoneos_version: - cflags.append('-miphoneos-version-min={}'.format(self.min_iphoneos_version)) - if self.min_macos_version: - cflags.append('-mmacosx-version-min={}'.format(self.min_macos_version)) - env['CFLAGS'] = ' '.join(cflags) - return env - - def set_comp_flags(self, env): - super().set_comp_flags(env) - cflags = [ - '-isysroot {}'.format(self.root_path), - '-arch {}'.format(self.arch), - '-target {}'.format(self.target), - env['CFLAGS'], - ] - if self.min_iphoneos_version: - cflags.append('-miphoneos-version-min={}'.format(self.min_iphoneos_version)) - env['CFLAGS'] = ' '.join(cflags) - env['CXXFLAGS'] = ' '.join([ - env['CFLAGS'], - '-std=c++11', - env['CXXFLAGS'], - ]) - env['LDFLAGS'] = ' '.join([ - ' -arch {}'.format(self.arch), - '-isysroot {}'.format(self.root_path), - ]) - - def get_bin_dir(self): - return [pj(self.root_path, 'bin')] - - @property - def binaries(self): - return { - 'CC': xrun_find('clang'), - 'CXX': xrun_find('clang++'), - 'AR': xrun_find('ar'), - 'STRIP': xrun_find('strip'), - 'RANLIB': xrun_find('ranlib'), - 'LD': xrun_find('ld'), - 'PKGCONFIG': 'pkg-config', - } - - @property - def configure_options(self): - yield f'--host={self.host}' - - -class iOSArm64(ApplePlatformInfo): - name = 'iOS_arm64' - arch = cpu = 'arm64' - host = 'arm-apple-darwin' - target = 'aarch64-apple-ios' - sdk_name = 'iphoneos' - min_iphoneos_version = '15.0' - - -class iOSx64Simulator(ApplePlatformInfo): - name = 'iOSSimulator_x86_64' - arch = cpu = 'x86_64' - host = 'x86_64-apple-darwin' - target = 'x86-apple-ios-simulator' - sdk_name = 'iphonesimulator' - min_iphoneos_version = '15.0' - - -class iOSArm64Simulator(ApplePlatformInfo): - name = 'iOSSimulator_arm64' - arch = cpu = 'arm64' - host = 'arm-apple-darwin' - target = 'aarch64-apple-ios-simulator' - sdk_name = 'iphonesimulator' - min_iphoneos_version = '15.0' - - -class macOSArm64(ApplePlatformInfo): - name = 'macOS_arm64_static' - arch = cpu = 'arm64' - host = 'aarch64-apple-darwin' - target = 'arm64-apple-macos' - sdk_name = 'macosx' - min_iphoneos_version = None - min_macos_version = MIN_MACOS_VERSION - - -class macOSArm64Mixed(MixedMixin('macOS_arm64_static'), ApplePlatformInfo): - name = 'macOS_arm64_mixed' - arch = cpu = 'arm64' - host = 'aarch64-apple-darwin' - target = 'arm64-apple-macos' - sdk_name = 'macosx' - min_iphoneos_version = None - min_macos_version = MIN_MACOS_VERSION - - -class macOSx64(ApplePlatformInfo): - name = 'macOS_x86_64' - arch = cpu = 'x86_64' - host = 'x86_64-apple-darwin' - target = 'x86_64-apple-macos' - sdk_name = 'macosx' - min_iphoneos_version = None - min_macos_version = MIN_MACOS_VERSION - - -class IOS(MetaPlatformInfo): - name = "iOS_multi" - compatible_hosts = ['Darwin'] - - @property - def subPlatformNames(self): - return ['iOS_{}'.format(arch) for arch in option('ios_arch')] - - def add_targets(self, targetName, targets): - super().add_targets(targetName, targets) - return PlatformInfo.add_targets(self, '_ios_fat_lib', targets) - - def __str__(self): - return self.name - -class AppleStaticAll(MetaPlatformInfo): - name = "apple_all_static" - compatible_hosts = ['Darwin'] - - @property - def subPlatformNames(self): - return AppleXCFramework.subPlatformNames - - def add_targets(self, targetName, targets): - super().add_targets(targetName, targets) - return PlatformInfo.add_targets(self, 'apple_xcframework', targets) - - def __str__(self): - return self.name diff --git a/kiwixbuild/platforms/musl.py b/kiwixbuild/platforms/musl.py deleted file mode 100644 index cd5fc46..0000000 --- a/kiwixbuild/platforms/musl.py +++ /dev/null @@ -1,148 +0,0 @@ -from .base import PlatformInfo, MixedMixin - -from kiwixbuild.utils import pj -from kiwixbuild._global import get_target_step - - -class MuslPlatformInfo(PlatformInfo): - compatible_hosts = ['fedora', 'debian'] - - def get_cross_config(self): - return { - 'binaries': self.binaries, - 'exe_wrapper_def': '', - 'root_path': self.root_path, - 'extra_libs': [], - 'extra_cflags': ['-I{}'.format(include_dir) for include_dir in self.get_include_dirs()], - 'host_machine': { - 'system': 'linux', - 'lsystem': 'linux', - 'cpu_family': self.cpu_family, - 'cpu': self.cpu, - 'endian': 'little', - 'abi': '' - } - } - - @property - def toolchain(self): - return get_target_step(self.build, 'neutral') - - @property - def root_path(self): - return self.toolchain.build_path - - @property - def binaries(self): - binaries = ((k,'{}-{}'.format(self.arch_full, v)) - for k, v in (('CC', 'gcc'), - ('CXX', 'g++'), - ('AR', 'ar'), - ('STRIP', 'strip'), - ('WINDRES', 'windres'), - ('RANLIB', 'ranlib'), - ('LD', 'ld'), - ('LDSHARED', 'g++ -shared') - ) - ) - binaries = {k:pj(self.root_path, 'bin', v) - for k,v in binaries} - binaries['PKGCONFIG'] = 'pkg-config' - return binaries - - @property - def exe_wrapper_def(self): - try: - which(self.qemu) - except subprocess.CalledProcessError: - return "" - except AttributeError: - return "" - else: - return f"exe_wrapper = '{self.qemu}'" - - @property - def configure_options(self): - return [f'--host={self.arch_full}'] - - def get_bin_dir(self): - return [pj(self.root_path, 'bin')] - - def get_env(self): - env = super().get_env() - env['LD_LIBRARY_PATH'] = ':'.join([ - pj(self.root_path, self.arch_full, 'lib64'), - pj(self.root_path, 'lib'), - env['LD_LIBRARY_PATH'] - ]) - env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') - env['QEMU_LD_PREFIX'] = pj(self.root_path, self.arch_full, "libc") - env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format( - ':'.join([ - pj(self.root_path, self.arch_full, "lib"), - env['LD_LIBRARY_PATH'] - ])) - return env - - def set_comp_flags(self, env): - super().set_comp_flags(env) - env['LD_LIBRARY_PATH'] = ':'.join([ - pj(self.root_path, self.arch_full, 'lib'), - env['LD_LIBRARY_PATH'] - ]) - env['CFLAGS'] = " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS'] - env['CXXFLAGS'] = " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS'] - - def set_compiler(self, env): - for k, v in self.binaries.items(): - env[k] = v - - def finalize_setup(self): - super().finalize_setup() - self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') - self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') - - -class Aarch64MuslPlatformInfo(MuslPlatformInfo): - build = 'aarch64_musl' - arch_full = 'aarch64-linux-musl' - toolchain_names = ['aarch64_musl'] - libdir = "lib/aarch64-linux-musl" - cpu_family = 'arm' - cpu = 'armhf' - qemu = 'qemu-arm' - - -class Aarch64MuslDyn(Aarch64MuslPlatformInfo): - name = 'aarch64_musl_dyn' - static = False - -class Aarch64MuslStatic(Aarch64MuslPlatformInfo): - name = 'aarch64_musl_static' - static = True - -class Aarch64MuslMixed(MixedMixin('aarch64_musl_static'), Aarch64MuslPlatformInfo): - name = 'aarch64_musl_mixed' - static = False - - -class X86_64MuslPlatformInfo(MuslPlatformInfo): - build = 'x86-64_musl' - arch_full = 'x86_64-linux-musl' - toolchain_names = ['x86-64_musl'] - libdir = "lib/x86_64-linux-musl" - cpu_family = 'x86_64' - cpu = 'x86_64' - - -class X86_64MuslDyn(X86_64MuslPlatformInfo): - name = 'x86-64_musl_dyn' - static = False - -class X86_64MuslStatic(X86_64MuslPlatformInfo): - name = 'x86-64_musl_static' - static = True - -class x86_64MuslMixed(MixedMixin('x86-64_musl_static'), X86_64MuslPlatformInfo): - name = 'x86-64_musl_mixed' - static = False diff --git a/kiwixbuild/platforms/native.py b/kiwixbuild/platforms/native.py deleted file mode 100644 index deac809..0000000 --- a/kiwixbuild/platforms/native.py +++ /dev/null @@ -1,32 +0,0 @@ -from .base import PlatformInfo, MixedMixin - -from kiwixbuild.utils import pj -from kiwixbuild._global import option, neutralEnv -from kiwixbuild.platforms.ios import MIN_MACOS_VERSION - -class NativePlatformInfo(PlatformInfo): - build = 'native' - - def get_env(self): - env = super().get_env() - if neutralEnv('distname') == 'fedora': - env['QT_SELECT'] = "5-64" - if neutralEnv('distname') == 'Darwin': - env['CFLAGS'] += f'-mmacosx-version-min={MIN_MACOS_VERSION}' - return env - - -class NativeDyn(NativePlatformInfo): - name = 'native_dyn' - static = False - compatible_hosts = ['fedora', 'debian', 'Darwin'] - -class NativeStatic(NativePlatformInfo): - name = 'native_static' - static = True - compatible_hosts = ['fedora', 'debian'] - -class NativeMixed(MixedMixin('native_static'), NativePlatformInfo): - name = 'native_mixed' - static = False - compatible_hosts = ['fedora', 'debian', 'Darwin'] diff --git a/kiwixbuild/platforms/neutral.py b/kiwixbuild/platforms/neutral.py deleted file mode 100644 index bbe339c..0000000 --- a/kiwixbuild/platforms/neutral.py +++ /dev/null @@ -1,9 +0,0 @@ -from .base import PlatformInfo - -class NeutralPlatformInfo(PlatformInfo): - name = 'neutral' - static = '' - compatible_hosts = ['fedora', 'debian', 'Darwin'] - - def __str__(self): - return "neutral" diff --git a/kiwixbuild/platforms/wasm.py b/kiwixbuild/platforms/wasm.py deleted file mode 100644 index 7b5cda0..0000000 --- a/kiwixbuild/platforms/wasm.py +++ /dev/null @@ -1,101 +0,0 @@ -from .base import PlatformInfo - -from kiwixbuild.utils import pj -from kiwixbuild._global import get_target_step - - -class WasmPlatformInfo(PlatformInfo): - name = 'wasm' - static = True - build = 'wasm' - arch_full = 'wasm64-emscripten' - libdir = "lib" - #arch_full = 'wasm64-linux' - toolchain_names = ['emsdk'] - compatible_hosts = ['fedora', 'debian'] - exe_wrapper_def = "" - - def get_cross_config(self): - return { - 'binaries': self.binaries, - 'exe_wrapper_def': '', - 'root_path': self.root_path, - 'extra_libs': [], - 'extra_cflags': [], - 'host_machine': { - 'system': 'emscripten', - 'lsystem': 'emscripten', - 'cpu_family': 'wasm64', - 'cpu': 'wasm64', - 'endian': 'little', - 'abi': '' - } - } - - @property - def wasm_sdk(self): - return get_target_step('emsdk', self.name) - - @property - def install_path(self): - return self.wasm_sdk.install_path - - @property - def root_path(self): - return self.install_path - - @property - def binaries(self): - binaries = (('CC', 'emcc'), - ('CXX', 'em++'), - ('AR', 'emar'), - ('STRIP', 'emstrip'), - ('WINDRES', 'windres'), - ('RANLIB', 'emranlib'), - ('LD', 'wasm-ld')) - binaries = {k:pj(self.install_path, 'upstream', 'emscripten', v) - for k,v in binaries} - binaries['PKGCONFIG'] = 'pkg-config' - return binaries - - @property - def configure_options(self): - #return "" - return [f'--host={self.arch_full}'] - - @property - def configure_wrapper(self): - return "emconfigure" - - @property - def make_wrapper(self): - return "emmake" - - def get_bin_dir(self): - return [pj(self.install_path, 'bin')] - - def get_env(self): - env = super().get_env() - env['PATH'] = ':'.join([ - env['PATH'], - self.install_path, - pj(self.install_path, 'upstream', 'emscripten'), - pj(self.install_path, 'node', '14.18.2_64bit', 'bin') - ]) - env['EMSDK'] = self.install_path - env['EMSDK_NODE'] = pj(self.install_path, 'node', '14.18.2_64bit', 'bin', 'node') - return env - - def set_comp_flags(self, env): - super().set_comp_flags(env) - env['CFLAGS'] = " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS'] - env['CXXFLAGS'] = " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS'] - - def set_compiler(self, env): - for k, v in self.binaries.items(): - env[k] = v - - def finalize_setup(self): - super().finalize_setup() - self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') - self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') diff --git a/kiwixbuild/platforms/win32.py b/kiwixbuild/platforms/win32.py deleted file mode 100644 index c5612d9..0000000 --- a/kiwixbuild/platforms/win32.py +++ /dev/null @@ -1,88 +0,0 @@ -import subprocess - -from .base import PlatformInfo -from kiwixbuild.utils import which, pj -from kiwixbuild._global import neutralEnv - - -class Win32PlatformInfo(PlatformInfo): - build = 'win32' - compatible_hosts = ['fedora', 'debian'] - arch_full = 'i686-w64-mingw32' - extra_libs = ['-lwinmm', '-lshlwapi', '-lws2_32', '-lssp'] - - def get_cross_config(self): - return { - 'exe_wrapper_def': self.exe_wrapper_def, - 'binaries': self.binaries, - 'root_path': self.root_path, - 'extra_libs': self.extra_libs, - 'extra_cflags': ['-DWIN32', *('-I{}'.format(include_dir) for include_dir in self.get_include_dirs())], - 'host_machine': { - 'system': 'Windows', - 'lsystem': 'windows', - 'cpu_family': 'x86', - 'cpu': 'i686', - 'endian': 'little', - 'abi': '' - } - } - - def finalize_setup(self): - super().finalize_setup() - self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') - self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') - - @property - def root_path(self): - root_paths = { - 'fedora': '/usr/i686-w64-mingw32/sys-root/mingw', - 'debian': '/usr/i686-w64-mingw32' - } - return root_paths[neutralEnv('distname')] - - @property - def binaries(self): - return {k:which('{}-{}'.format(self.arch_full, v)) - for k, v in (('CC', 'gcc'), - ('CXX', 'g++'), - ('AR', 'ar'), - ('STRIP', 'strip'), - ('WINDRES', 'windres'), - ('RANLIB', 'ranlib'), - ('PKGCONFIG', 'pkg-config')) - } - - @property - def exe_wrapper_def(self): - try: - which('wine') - except subprocess.CalledProcessError: - return "" - else: - return "exe_wrapper = 'wine'" - - @property - def configure_options(self): - yield f'--host={self.arch_full}' - - def set_compiler(self, env): - for k, v in self.binaries.items(): - env[k] = v - - def get_bin_dir(self): - return [pj(self.root_path, 'bin')] - - def get_env(self): - env = super().get_env() - env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') - env['LIBS'] = " ".join(self.extra_libs) + " " +env['LIBS'] - return env - -class Win32Dyn(Win32PlatformInfo): - name = 'win32_dyn' - static = False - -class Win32Static(Win32PlatformInfo): - name = 'win32_static' - static = True diff --git a/kiwixbuild/platforms/win64.py b/kiwixbuild/platforms/win64.py deleted file mode 100644 index f7c17c0..0000000 --- a/kiwixbuild/platforms/win64.py +++ /dev/null @@ -1,88 +0,0 @@ -import subprocess - -from .base import PlatformInfo -from kiwixbuild.utils import which, pj -from kiwixbuild._global import neutralEnv - - -class Win64PlatformInfo(PlatformInfo): - extra_libs = ['-lmingw32', '-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4', '-lmsvcr100', '-liphlpapi', '-lshell32', '-lkernel32'] - build = 'win64' - compatible_hosts = ['fedora', 'debian'] - arch_full = 'x86_64-w64-mingw32' - - def get_cross_config(self): - return { - 'exe_wrapper_def': self.exe_wrapper_def, - 'binaries': self.binaries, - 'root_path': self.root_path, - 'extra_libs': self.extra_libs, - 'extra_cflags': ['-DWIN32'], - 'host_machine': { - 'system': 'Windows', - 'lsystem': 'windows', - 'cpu_family': 'x86_64', - 'cpu': 'x86_64', - 'endian': 'little', - 'abi': '' - } - } - - def finalize_setup(self): - super().finalize_setup() - self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt') - self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') - - @property - def root_path(self): - root_paths = { - 'fedora': '/usr/x86_64-w64-mingw32/sys-root/mingw', - 'debian': '/usr/x86_64-w64-mingw32' - } - return root_paths[neutralEnv('distname')] - - @property - def binaries(self): - return {k:which('{}-{}'.format(self.arch_full, v)) - for k, v in (('CC', 'gcc'), - ('CXX', 'g++'), - ('AR', 'ar'), - ('STRIP', 'strip'), - ('WINDRES', 'windres'), - ('RANLIB', 'ranlib'), - ('PKGCONFIG', 'pkg-config')) - } - - @property - def exe_wrapper_def(self): - try: - which('wine') - except subprocess.CalledProcessError: - return "" - else: - return "exe_wrapper = 'wine'" - - @property - def configure_options(self): - return [f'--host={self.arch_full}'] - - def set_compiler(self, env): - for k, v in self.binaries.items(): - env[k] = v - - def get_bin_dir(self): - return [pj(self.root_path, 'bin')] - - def get_env(self): - env = super().get_env() - env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig') - env['LIBS'] = " ".join(self.extra_libs) + " " +env['LIBS'] - return env - -class Win64Dyn(Win64PlatformInfo): - name = 'win64_dyn' - static = False - -class Win64Static(Win64PlatformInfo): - name = 'win64_static' - static = True diff --git a/kiwixbuild/utils.py b/kiwixbuild/utils.py index bdf8550..1cdc990 100644 --- a/kiwixbuild/utils.py +++ b/kiwixbuild/utils.py @@ -17,16 +17,16 @@ pj = os.path.join COLORS = { - 'OK': '\033[92m', - 'WARNING': '\033[93m', - 'NEEDED': '\033[93m', - 'SKIP': '\033[34m', - 'ERROR': '\033[91m', - '': '\033[0m', + "OK": "\033[92m", + "WARNING": "\033[93m", + "NEEDED": "\033[93m", + "SKIP": "\033[34m", + "ERROR": "\033[91m", + "": "\033[0m", } -REMOTE_PREFIX = 'http://mirror.download.kiwix.org/dev/kiwix-build/' +REMOTE_PREFIX = "http://mirror.download.kiwix.org/dev/kiwix-build/" def which(name): @@ -34,16 +34,19 @@ def which(name): output = subprocess.check_output(command, shell=True) return output[:-1].decode() + def xrun_find(name): command = "xcrun -find {}".format(name) output = subprocess.check_output(command, shell=True) return output[:-1].decode() -regex_space = re.compile(r'((?= (3, 4, 3) else {} + extra_args = {"context": context} if sys.version_info >= (3, 4, 3) else {} progress_chars = "/-\|" try: - with urllib.request.urlopen(what.url, **extra_args) as resource, open(file_path, 'wb') as file: - tsize = resource.info().get('Content-Length', None) + with urllib.request.urlopen(what.url, **extra_args) as resource, open( + file_path, "wb" + ) as file: + tsize = resource.info().get("Content-Length", None) if tsize is not None: tsize = int(tsize) current = 0 @@ -147,17 +152,17 @@ def download_remote(what, where): break if tsize: current += batch_size - print_progress("{:.2%}".format(current/tsize)) + print_progress("{:.2%}".format(current / tsize)) else: - print_progress(progress_chars[current]) - current = (current+1)%4 + print_progress(progress_chars[current]) + current = (current + 1) % 4 file.write(batch) except urllib.error.URLError as e: print("Cannot download url {}:\n{}".format(what.url, e.reason)) raise StopBuild() if not what.sha256: - print('Sha256 for {} not set, do no verify download'.format(what.name)) + print("Sha256 for {} not set, do no verify download".format(what.name)) elif what.sha256 != get_sha256(file_path): os.remove(file_path) raise StopBuild("Sha 256 doesn't correspond") @@ -187,7 +192,7 @@ class StopBuild(BaseCommandResult): pass -class Remotefile(namedtuple('Remotefile', ('name', 'sha256', 'url'))): +class Remotefile(namedtuple("Remotefile", ("name", "sha256", "url"))): def __new__(cls, name, sha256, url=None): if url is None: url = REMOTE_PREFIX + name @@ -217,28 +222,28 @@ class Context: def _finalise(self): if self.autoskip_file is not None: os.makedirs(os.path.dirname(self.autoskip_file), exist_ok=True) - with open(self.autoskip_file, 'w'): + with open(self.autoskip_file, "w"): pass def extract_archive(archive_path, dest_dir, topdir=None, name=None): - is_zip_archive = archive_path.endswith('.zip') + is_zip_archive = archive_path.endswith(".zip") archive = None try: if is_zip_archive: archive = zipfile.ZipFile(archive_path) members = archive.infolist() - getname = lambda info : info.filename - isdir = lambda info: info.filename.endswith('/') + getname = lambda info: info.filename + isdir = lambda info: info.filename.endswith("/") else: archive = tarfile.open(archive_path) members = archive.getmembers() - getname = lambda info : getattr(info, 'name') + getname = lambda info: getattr(info, "name") isdir = lambda info: info.isdir() if not topdir: for d in (m for m in members if isdir(m)): _name = getname(d) - if _name.endswith('/'): + if _name.endswith("/"): _name = _name[:-1] if not os.path.dirname(_name): if topdir: @@ -249,9 +254,13 @@ def extract_archive(archive_path, dest_dir, topdir=None, name=None): break topdir = _name if topdir: - members_to_extract = [m for m in members if getname(m).startswith(topdir+'/')] + members_to_extract = [ + m for m in members if getname(m).startswith(topdir + "/") + ] os.makedirs(dest_dir, exist_ok=True) - with tempfile.TemporaryDirectory(prefix=os.path.basename(archive_path), dir=dest_dir) as tmpdir: + with tempfile.TemporaryDirectory( + prefix=os.path.basename(archive_path), dir=dest_dir + ) as tmpdir: if is_zip_archive: _members_to_extract = [getname(m) for m in members_to_extract] else: @@ -281,8 +290,8 @@ def run_command(command, cwd, context, *, env=None, input=None): env = DefaultEnv() log = None try: - if not option('verbose'): - log = open(context.log_file, 'w') + if not option("verbose"): + log = open(context.log_file, "w") print("run command '{}'".format(command), file=log) print("current directory is '{}'".format(cwd), file=log) print("env is :", file=log) @@ -293,8 +302,15 @@ def run_command(command, cwd, context, *, env=None, input=None): log.flush() kwargs = dict() if input: - kwargs['stdin'] = subprocess.PIPE - process = subprocess.Popen(command, cwd=cwd, env=env, stdout=log or sys.stdout, stderr=subprocess.STDOUT, **kwargs) + kwargs["stdin"] = subprocess.PIPE + process = subprocess.Popen( + command, + cwd=cwd, + env=env, + stdout=log or sys.stdout, + stderr=subprocess.STDOUT, + **kwargs + ) if input: input = input.encode() while True: @@ -308,7 +324,7 @@ def run_command(command, cwd, context, *, env=None, input=None): # `communicate` timeout (and we must set `input` to None # to not communicate again). input = None - print('.', end='', flush=True) + print(".", end="", flush=True) else: break if process.returncode: diff --git a/kiwixbuild/versions.py b/kiwixbuild/versions.py index c6c6aaa..5da3c41 100644 --- a/kiwixbuild/versions.py +++ b/kiwixbuild/versions.py @@ -1,12 +1,11 @@ - # This file reference all the versions of the depedencies we use in kiwix-build. main_project_versions = { - 'libzim': '9.1.0', - 'libkiwix': '13.0.0', - 'kiwix-tools': '3.6.0', - 'zim-tools': '3.4.0', - 'kiwix-desktop': '2.3.1' # Also change KIWIX_DESKTOP_VERSION and KIWIX_DESKTOP_RELEASE in appveyor.yml + "libzim": "9.1.0", + "libkiwix": "13.0.0", + "kiwix-tools": "3.6.0", + "zim-tools": "3.4.0", + "kiwix-desktop": "2.3.1", # Also change KIWIX_DESKTOP_VERSION and KIWIX_DESKTOP_RELEASE in appveyor.yml } # This dictionnary specify what we need to build at each release process. @@ -30,37 +29,37 @@ main_project_versions = { # - set KIWIX_DESKTOP_RELEASE to 0 release_versions = { - 'libzim': None, # Depends of base deps (was 0) - 'libkiwix': None, # Depends of libzim (was 1) - 'kiwix-tools': None, # Depends of libkiwix and libzim (was 0) - 'zim-tools': 0, # Depends of libzim (was None) - 'kiwix-desktop': None # Depends of libkiwix and libzim (was 4) + "libzim": None, # Depends of base deps (was 0) + "libkiwix": None, # Depends of libzim (was 1) + "kiwix-tools": None, # Depends of libkiwix and libzim (was 0) + "zim-tools": 0, # Depends of libzim (was None) + "kiwix-desktop": None, # Depends of libkiwix and libzim (was 4) } # This is the "version" of the whole base_deps_versions dict. # Change this when you change base_deps_versions. -base_deps_meta_version = '96' +base_deps_meta_version = "97" base_deps_versions = { - 'zlib' : '1.2.12', - 'lzma' : '5.2.6', - 'zstd' : '1.5.2', - 'docoptcpp' : '0.6.2', - 'uuid' : '1.43.4', - 'xapian-core' : '1.4.23', - 'mustache' : '4.1', - 'pugixml' : '1.2', - 'libmicrohttpd' : '0.9.76', - 'gumbo' : '0.10.1', - 'icu4c' : '73.2', - 'libaria2' : '1.37.0', - 'libmagic' : '5.44', - 'android-ndk' : 'r21e', - 'qt' : '5.10.1', - 'qtwebengine' : '5.10.1', - 'org.kde' : '5.15-21.08', - 'io.qt.qtwebengine' : '5.15-21.08', - 'zim-testing-suite': '0.3', - 'emsdk': '3.1.41', + "zlib": "1.2.12", + "lzma": "5.2.6", + "zstd": "1.5.2", + "docoptcpp": "0.6.2", + "uuid": "1.43.4", + "xapian-core": "1.4.23", + "mustache": "4.1", + "pugixml": "1.2", + "libmicrohttpd": "0.9.76", + "gumbo": "0.10.1", + "icu4c": "73.2", + "libaria2": "1.37.0", + "libmagic": "5.44", + "android-ndk": "r21e", + "qt": "5.10.1", + "qtwebengine": "5.10.1", + "org.kde": "5.15-21.08", + "io.qt.qtwebengine": "5.15-21.08", + "zim-testing-suite": "0.3", + "emsdk": "3.1.41", }