From 908b90190c57c0d767ce99a69154db13052669ac Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 30 Apr 2024 12:52:29 +0200 Subject: [PATCH] Use pathlib.Path instead of os.path --- kiwixbuild/__init__.py | 5 +- kiwixbuild/buildenv.py | 54 +++++----- kiwixbuild/configs/android.py | 18 ++-- kiwixbuild/configs/armhf.py | 20 ++-- kiwixbuild/configs/base.py | 35 +++--- kiwixbuild/configs/i586.py | 2 +- kiwixbuild/configs/ios.py | 11 +- kiwixbuild/configs/musl.py | 20 ++-- kiwixbuild/configs/native.py | 1 - kiwixbuild/configs/wasm.py | 18 ++-- kiwixbuild/configs/win32.py | 12 ++- kiwixbuild/configs/win64.py | 11 +- kiwixbuild/configs/winbash.py | 1 - kiwixbuild/dependencies/apple_xcframework.py | 25 +++-- kiwixbuild/dependencies/base.py | 64 +++++------ kiwixbuild/dependencies/icu4c.py | 29 +++-- kiwixbuild/dependencies/ios_fat_lib.py | 20 ++-- kiwixbuild/dependencies/libmagic.py | 4 +- kiwixbuild/dependencies/mustache.py | 6 +- kiwixbuild/dependencies/qt.py | 4 +- kiwixbuild/dependencies/tc_android_ndk.py | 33 +++--- kiwixbuild/dependencies/tc_emsdk.py | 7 +- kiwixbuild/dependencies/xapian.py | 4 +- kiwixbuild/flatpak_builder.py | 23 ++-- kiwixbuild/utils.py | 106 +++++++++---------- 25 files changed, 264 insertions(+), 269 deletions(-) diff --git a/kiwixbuild/__init__.py b/kiwixbuild/__init__.py index 03b2298..7990700 100644 --- a/kiwixbuild/__init__.py +++ b/kiwixbuild/__init__.py @@ -2,6 +2,7 @@ import os, sys import argparse +from pathlib import Path from .dependencies import Dependency from .configs import ConfigInfo @@ -22,6 +23,7 @@ def parse_args(): parser.add_argument( "--working-dir", default=".", + type=Path, help=( "Directory where kiwix-build puts all its files " "(source, archive and build)\n" @@ -31,6 +33,7 @@ def parse_args(): parser.add_argument( "--build-dir", default=".", + type=Path, help=( "Directory where kiwix-build puts all build files.\n" "build-dir can be absolute path or a relative (to working-dir) one." @@ -152,7 +155,7 @@ def parse_args(): def main(): options = parse_args() - options.working_dir = os.path.abspath(options.working_dir) + options.working_dir = options.working_dir.absolute() _global.set_options(options) neutralEnv = buildenv.NeutralEnv(options.get_build_dir) _global.set_neutralEnv(neutralEnv) diff --git a/kiwixbuild/buildenv.py b/kiwixbuild/buildenv.py index d75d610..f4aefd0 100644 --- a/kiwixbuild/buildenv.py +++ b/kiwixbuild/buildenv.py @@ -2,20 +2,21 @@ import os, sys, shutil import subprocess import platform import distro +from pathlib import Path -from .utils import pj, download_remote, escape_path +from .utils import download_remote, escape_path from ._global import neutralEnv, option class NeutralEnv: def __init__(self, dummy_run): - self.working_dir = option("working_dir") - self.source_dir = pj(self.working_dir, "SOURCE") - self.archive_dir = pj(self.working_dir, "ARCHIVE") - self.toolchain_dir = pj(self.working_dir, "TOOLCHAINS") - self.log_dir = pj(self.working_dir, "LOGS") + self.working_dir: Path = option("working_dir") + self.source_dir: Path = self.working_dir / "SOURCE" + self.archive_dir: Path = self.working_dir / "ARCHIVE" + self.toolchain_dir: Path = self.working_dir / "TOOLCHAINS" + self.log_dir: Path = self.working_dir / "LOGS" for d in (self.source_dir, self.archive_dir, self.toolchain_dir, self.log_dir): - os.makedirs(d, exist_ok=True) + d.mkdir(parents=True, exist_ok=True) self.detect_platform() if dummy_run: # If this is for a dummy run, we will not run anything. @@ -73,29 +74,28 @@ class NeutralEnv: class BuildEnv: def __init__(self, configInfo): self.configInfo = configInfo - self.base_build_dir = pj(option("working_dir"), option("build_dir")) + self.base_build_dir: Path = option("working_dir") / option("build_dir") build_dir = ( configInfo.arch_name if option("use_target_arch_name") else configInfo.name ) build_dir = f"BUILD_{build_dir}" - self.build_dir = pj(self.base_build_dir, build_dir) - self.install_dir = pj(self.build_dir, "INSTALL") - self.toolchain_dir = pj(self.build_dir, "TOOLCHAINS") - self.log_dir = pj(self.build_dir, "LOGS") + self.build_dir: Path = self.base_build_dir / build_dir + self.install_dir: Path = self.build_dir / "INSTALL" + self.toolchain_dir: Path = self.build_dir / "TOOLCHAINS" + self.log_dir: Path = self.build_dir / "LOGS" for d in (self.build_dir, self.install_dir, self.toolchain_dir, self.log_dir): - os.makedirs(d, exist_ok=True) + d.mkdir(parents=True, exist_ok=True) self.libprefix = option("libprefix") or self._detect_libdir() def clean_intermediate_directories(self): - for subdir in os.listdir(self.build_dir): - subpath = pj(self.build_dir, subdir) + for subpath in self.build_dir.iterdir(): if subpath == self.install_dir: continue - if os.path.isdir(subpath): + if subpath.isdir(): shutil.rmtree(subpath) else: - os.remove(subpath) + subpath.unlink() def _is_debianlike(self): return os.path.isfile("/etc/debian_version") @@ -122,35 +122,35 @@ class BuildEnv: def get_env(self, *, cross_comp_flags, cross_compilers, cross_path): env = self.configInfo.get_env() - pkgconfig_path = pj(self.install_dir, self.libprefix, "pkgconfig") + pkgconfig_path = self.install_dir / self.libprefix / "pkgconfig" env["PKG_CONFIG_PATH"].append(pkgconfig_path) - env["PATH"].insert(0, pj(self.install_dir, "bin")) + env["PATH"].insert(0, self.install_dir / "bin") env["LD_LIBRARY_PATH"].extend( [ - pj(self.install_dir, "lib"), - pj(self.install_dir, self.libprefix), + self.install_dir / "lib", + self.install_dir / self.libprefix, ] ) env["QMAKE_CXXFLAGS"] = " ".join( - [escape_path("-I" + pj(self.install_dir, "include")), env["QMAKE_CXXFLAGS"]] + [escape_path(f"-I{self.install_dir / 'include'}"), env["QMAKE_CXXFLAGS"]] ) env["CPPFLAGS"] = " ".join( - [escape_path("-I" + pj(self.install_dir, "include")), env["CPPFLAGS"]] + [escape_path(f"-I{self.install_dir / 'include'}"), env["CPPFLAGS"]] ) env["QMAKE_LFLAGS"] = " ".join( [ - escape_path("-L" + pj(self.install_dir, "lib")), - escape_path("-L" + pj(self.install_dir, self.libprefix)), + escape_path(f"-L{self.install_dir / 'lib'}"), + escape_path(f"-L{self.install_dir / self.libprefix}"), env["QMAKE_LFLAGS"], ] ) env["LDFLAGS"] = " ".join( [ - escape_path("-L" + pj(self.install_dir, "lib")), - escape_path("-L" + pj(self.install_dir, self.libprefix)), + escape_path(f"-L{self.install_dir / 'lib'}"), + escape_path(f"-L{self.install_dir / self.libprefix}"), env["LDFLAGS"], ] ) diff --git a/kiwixbuild/configs/android.py b/kiwixbuild/configs/android.py index 9374944..3f7d03e 100644 --- a/kiwixbuild/configs/android.py +++ b/kiwixbuild/configs/android.py @@ -1,5 +1,5 @@ +from pathlib import Path from .base import ConfigInfo, MetaConfigInfo -from kiwixbuild.utils import pj from kiwixbuild._global import get_target_step, option @@ -30,9 +30,7 @@ class AndroidConfigInfo(ConfigInfo): def binaries(self): install_path = self.install_path - binaries = { - k: pj(install_path, "bin", v) for k, v in self.binaries_name.items() - } + binaries = {k: install_path / "bin" / v for k, v in self.binaries_name.items()} binaries["PKGCONFIG"] = "pkg-config" return binaries @@ -41,7 +39,7 @@ class AndroidConfigInfo(ConfigInfo): return get_target_step("android-ndk", self.name) @property - def install_path(self): + def install_path(self) -> Path: return self.ndk_builder.install_path def get_cross_config(self): @@ -56,7 +54,7 @@ class AndroidConfigInfo(ConfigInfo): "exe_wrapper_def": "", "install_path": self.install_path, "binaries": self.binaries(), - "root_path": pj(self.install_path, "sysroot"), + "root_path": self.install_path / "sysroot", "extra_libs": extra_libs, "extra_cflags": extra_cflags, "host_machine": { @@ -71,17 +69,17 @@ class AndroidConfigInfo(ConfigInfo): def get_env(self): env = super().get_env() - root_path = pj(self.install_path, "sysroot") - env["PKG_CONFIG_LIBDIR"] = pj(root_path, "lib", "pkgconfig") + root_path = self.install_path / "sysroot" + env["PKG_CONFIG_LIBDIR"] = root_path / "lib" / "pkgconfig" env["NDK_DEBUG"] = "0" return env def get_bin_dir(self): - return [pj(self.install_path, "bin")] + return [self.install_path / "bin"] def set_comp_flags(self, env): super().set_comp_flags(env) - root_path = pj(self.install_path, "sysroot") + root_path = self.install_path / "sysroot" march = "-march={}".format(self.march) if hasattr(self, "march") else "" env["CFLAGS"] = ( "-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} {} ".format( diff --git a/kiwixbuild/configs/armhf.py b/kiwixbuild/configs/armhf.py index b46b2c4..f518c80 100644 --- a/kiwixbuild/configs/armhf.py +++ b/kiwixbuild/configs/armhf.py @@ -1,6 +1,6 @@ from .base import ConfigInfo, MixedMixin -from kiwixbuild.utils import pj +from pathlib import Path from kiwixbuild._global import get_target_step @@ -36,7 +36,7 @@ class ArmConfigInfo(ConfigInfo): return get_target_step(self.build, "neutral") @property - def root_path(self): + def root_path(self) -> Path: return self.toolchain.build_path @property @@ -54,7 +54,7 @@ class ArmConfigInfo(ConfigInfo): ("LDSHARED", "g++ -shared"), ) ) - binaries = {k: pj(self.root_path, "bin", v) for k, v in binaries} + binaries = {k: self.root_path / "bin" / v for k, v in binaries} binaries["PKGCONFIG"] = "pkg-config" return binaries @@ -72,19 +72,19 @@ class ArmConfigInfo(ConfigInfo): yield "--host={}".format(self.arch_full) def get_bin_dir(self): - return [pj(self.root_path, "bin")] + return [self.root_path / "bin"] def get_env(self): env = super().get_env() env["LD_LIBRARY_PATH"][0:0] = [ - pj(self.root_path, self.arch_full, "lib64"), - pj(self.root_path, "lib"), + self.root_path / self.arch_full / "lib64", + self.root_path / "lib", ] - env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig") - env["QEMU_LD_PREFIX"] = pj(self.root_path, self.arch_full, "libc") + env["PKG_CONFIG_LIBDIR"] = self.root_path / "lib" / "pkgconfig" + env["QEMU_LD_PREFIX"] = self.root_path / self.arch_full / "libc" env["QEMU_SET_ENV"] = "LD_LIBRARY_PATH={}".format( ":".join( - [pj(self.root_path, self.arch_full, "lib"), str(env["LD_LIBRARY_PATH"])] + [self.root_path / self.arch_full / "lib", str(env["LD_LIBRARY_PATH"])] ) ) return env @@ -164,7 +164,7 @@ class Aarch64(ArmConfigInfo): cpu = "aarch64" @property - def root_path(self): + def root_path(self) -> Path: return self.toolchain.build_path diff --git a/kiwixbuild/configs/base.py b/kiwixbuild/configs/base.py index a0a9c8a..65337fc 100644 --- a/kiwixbuild/configs/base.py +++ b/kiwixbuild/configs/base.py @@ -1,13 +1,14 @@ import os, sys import subprocess +from pathlib import Path from kiwixbuild.dependencies import Dependency -from kiwixbuild.utils import pj, remove_duplicates, DefaultEnv +from kiwixbuild.utils import remove_duplicates, DefaultEnv from kiwixbuild.buildenv import BuildEnv from kiwixbuild._global import neutralEnv, option, target_steps -_SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -TEMPLATES_DIR = pj(os.path.dirname(_SCRIPT_DIR), "templates") +_SCRIPT_DIR = Path(__file__).resolve().parent +TEMPLATES_DIR = _SCRIPT_DIR.parent / "templates" class _MetaConfig(type): @@ -81,7 +82,7 @@ class ConfigInfo(metaclass=_MetaConfig): return {} def get_include_dirs(self): - return [pj(self.buildEnv.install_dir, "include")] + return [self.buildEnv.install_dir / "include"] def get_env(self): return DefaultEnv() @@ -100,13 +101,11 @@ class ConfigInfo(metaclass=_MetaConfig): def _gen_crossfile(self, name, outname=None): if outname is None: outname = name - crossfile = pj(self.buildEnv.build_dir, outname) - template_file = pj(TEMPLATES_DIR, name) - with open(template_file, "r") as f: - template = f.read() + crossfile = self.buildEnv.build_dir / outname + template_file = TEMPLATES_DIR / name + template = template_file.read_text() content = template.format(**self.get_cross_config()) - with open(crossfile, "w") as outfile: - outfile.write(content) + crossfile.write_text(content) return crossfile def finalize_setup(self): @@ -155,22 +154,22 @@ def MixedMixin(static_name): def get_include_dirs(self): return [ - pj(self.buildEnv.install_dir, "include"), - pj(self.static_buildEnv.install_dir, "include"), + self.buildEnv.install_dir / "include", + self.static_buildEnv.install_dir / "include", ] def get_env(self): env = super().get_env() - env["PATH"].insert(0, pj(self.static_buildEnv.install_dir, "bin")) - pkgconfig_path = pj( - self.static_buildEnv.install_dir, - self.static_buildEnv.libprefix, - "pkgconfig", + env["PATH"].insert(0, self.static_buildEnv.install_dir / "bin") + pkgconfig_path = ( + self.static_buildEnv.install_dir + / self.static_buildEnv.libprefix + / "pkgconfig" ) env["PKG_CONFIG_PATH"].append(pkgconfig_path) env["CPPFLAGS"] = " ".join( [ - "-I" + pj(self.static_buildEnv.install_dir, "include"), + f"-I{self.static_buildEnv.install_dir / 'include'}", env["CPPFLAGS"], ] ) diff --git a/kiwixbuild/configs/i586.py b/kiwixbuild/configs/i586.py index 8c402a3..ea686ea 100644 --- a/kiwixbuild/configs/i586.py +++ b/kiwixbuild/configs/i586.py @@ -1,7 +1,7 @@ import os from .base import ConfigInfo -from kiwixbuild.utils import which, pj +from kiwixbuild.utils import which class I586ConfigInfo(ConfigInfo): diff --git a/kiwixbuild/configs/ios.py b/kiwixbuild/configs/ios.py index 62a9c1f..cffcdb7 100644 --- a/kiwixbuild/configs/ios.py +++ b/kiwixbuild/configs/ios.py @@ -1,7 +1,8 @@ +from pathlib import Path import subprocess from kiwixbuild._global import option -from kiwixbuild.utils import pj, xrun_find +from kiwixbuild.utils import xrun_find from .base import ConfigInfo, MetaConfigInfo, MixedMixin from kiwixbuild.dependencies.apple_xcframework import AppleXCFramework @@ -29,10 +30,12 @@ class AppleConfigInfo(ConfigInfo): return self.target @property - def root_path(self): + def root_path(self) -> Path: if self._root_path is None: command = "xcrun --sdk {} --show-sdk-path".format(self.sdk_name) - self._root_path = subprocess.check_output(command, shell=True)[:-1].decode() + self._root_path = Path( + subprocess.check_output(command, shell=True)[:-1].decode() + ) return self._root_path def __str__(self): @@ -133,7 +136,7 @@ class AppleConfigInfo(ConfigInfo): ) def get_bin_dir(self): - return [pj(self.root_path, "bin")] + return [self.root_path / "bin"] @property def binaries(self): diff --git a/kiwixbuild/configs/musl.py b/kiwixbuild/configs/musl.py index d6c3939..b2d7140 100644 --- a/kiwixbuild/configs/musl.py +++ b/kiwixbuild/configs/musl.py @@ -1,6 +1,6 @@ +from pathlib import Path from .base import ConfigInfo, MixedMixin -from kiwixbuild.utils import pj from kiwixbuild._global import get_target_step @@ -31,7 +31,7 @@ class MuslConfigInfo(ConfigInfo): return get_target_step(self.build, "neutral") @property - def root_path(self): + def root_path(self) -> Path: return self.toolchain.build_path @property @@ -49,7 +49,7 @@ class MuslConfigInfo(ConfigInfo): ("LDSHARED", "g++ -shared"), ) ) - binaries = {k: pj(self.root_path, "bin", v) for k, v in binaries} + binaries = {k: self.root_path / "bin" / v for k, v in binaries} binaries["PKGCONFIG"] = "pkg-config" return binaries @@ -69,26 +69,26 @@ class MuslConfigInfo(ConfigInfo): return [f"--host={self.arch_full}"] def get_bin_dir(self): - return [pj(self.root_path, "bin")] + return [self.root_path / "bin"] def get_env(self): env = super().get_env() env["LD_LIBRARY_PATH"][0:0] = [ - pj(self.root_path, self.arch_full, "lib64"), - pj(self.root_path, "lib"), + self.root_path / self.arch_full / "lib64", + self.root_path / "lib", ] - env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig") - env["QEMU_LD_PREFIX"] = pj(self.root_path, self.arch_full, "libc") + env["PKG_CONFIG_LIBDIR"] = self.root_path / "lib" / "pkgconfig" + env["QEMU_LD_PREFIX"] = self.root_path / self.arch_full / "libc" env["QEMU_SET_ENV"] = "LD_LIBRARY_PATH={}".format( ":".join( - [pj(self.root_path, self.arch_full, "lib"), str(env["LD_LIBRARY_PATH"])] + [self.root_path / self.arch_full / "lib", str(env["LD_LIBRARY_PATH"])] ) ) return env def set_comp_flags(self, env): super().set_comp_flags(env) - env["LD_LIBRARY_PATH"].insert(0, pj(self.root_path, self.arch_full, "lib")) + env["LD_LIBRARY_PATH"].insert(0, self.root_path / self.arch_full / "lib") env["CFLAGS"] = ( " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 " + env["CFLAGS"] diff --git a/kiwixbuild/configs/native.py b/kiwixbuild/configs/native.py index bbd4103..52f2728 100644 --- a/kiwixbuild/configs/native.py +++ b/kiwixbuild/configs/native.py @@ -1,6 +1,5 @@ from .base import ConfigInfo, MixedMixin -from kiwixbuild.utils import pj from kiwixbuild._global import option, neutralEnv from kiwixbuild.configs.ios import MIN_MACOS_VERSION import sysconfig diff --git a/kiwixbuild/configs/wasm.py b/kiwixbuild/configs/wasm.py index e986952..7963bc9 100644 --- a/kiwixbuild/configs/wasm.py +++ b/kiwixbuild/configs/wasm.py @@ -1,6 +1,6 @@ +from pathlib import Path from .base import ConfigInfo -from kiwixbuild.utils import pj from kiwixbuild._global import get_target_step @@ -37,11 +37,11 @@ class WasmConfigInfo(ConfigInfo): return get_target_step("emsdk", self.name) @property - def install_path(self): + def install_path(self) -> Path: return self.wasm_sdk.install_path @property - def root_path(self): + def root_path(self) -> Path: return self.install_path @property @@ -56,7 +56,7 @@ class WasmConfigInfo(ConfigInfo): ("LD", "wasm-ld"), ) binaries = { - k: pj(self.install_path, "upstream", "emscripten", v) for k, v in binaries + k: self.install_path / "upstream" / "emscripten" / v for k, v in binaries } binaries["PKGCONFIG"] = "pkg-config" return binaries @@ -75,20 +75,20 @@ class WasmConfigInfo(ConfigInfo): return "emmake" def get_bin_dir(self): - return [pj(self.install_path, "bin")] + return [self.install_path / "bin"] def get_env(self): env = super().get_env() env["PATH"].extend( [ self.install_path, - pj(self.install_path, "upstream", "emscripten"), - pj(self.install_path, "node", "14.18.2_64bit", "bin"), + self.install_path / "upstream" / "emscripten", + self.install_path / "node" / "14.18.2_64bit" / "bin", ] ) env["EMSDK"] = self.install_path - env["EMSDK_NODE"] = pj( - self.install_path, "node", "14.18.2_64bit", "bin", "node" + env["EMSDK_NODE"] = ( + self.install_path / "node" / "14.18.2_64bit" / "bin" / "node" ) return env diff --git a/kiwixbuild/configs/win32.py b/kiwixbuild/configs/win32.py index 6b665e7..0ed4939 100644 --- a/kiwixbuild/configs/win32.py +++ b/kiwixbuild/configs/win32.py @@ -1,7 +1,9 @@ import subprocess + +from pathlib import Path from .base import ConfigInfo -from kiwixbuild.utils import which, pj +from kiwixbuild.utils import which from kiwixbuild._global import neutralEnv @@ -40,12 +42,12 @@ class Win32ConfigInfo(ConfigInfo): self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt") @property - def root_path(self): + def root_path(self) -> Path: root_paths = { "fedora": "/usr/i686-w64-mingw32/sys-root/mingw", "debian": "/usr/i686-w64-mingw32", } - return root_paths[neutralEnv("distname")] + return Path(root_paths[neutralEnv("distname")]) @property def binaries(self): @@ -80,11 +82,11 @@ class Win32ConfigInfo(ConfigInfo): env[k] = v def get_bin_dir(self): - return [pj(self.root_path, "bin")] + return [self.root_path / "bin"] def get_env(self): env = super().get_env() - env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig") + env["PKG_CONFIG_LIBDIR"] = self.root_path / "lib" / "pkgconfig" env["LIBS"] = " ".join(self.extra_libs) + " " + env["LIBS"] return env diff --git a/kiwixbuild/configs/win64.py b/kiwixbuild/configs/win64.py index 10f215a..1193699 100644 --- a/kiwixbuild/configs/win64.py +++ b/kiwixbuild/configs/win64.py @@ -1,7 +1,8 @@ import subprocess +from pathlib import Path from .base import ConfigInfo -from kiwixbuild.utils import which, pj +from kiwixbuild.utils import which from kiwixbuild._global import neutralEnv @@ -44,12 +45,12 @@ class Win64ConfigInfo(ConfigInfo): self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt") @property - def root_path(self): + def root_path(self) -> Path: root_paths = { "fedora": "/usr/x86_64-w64-mingw32/sys-root/mingw", "debian": "/usr/x86_64-w64-mingw32", } - return root_paths[neutralEnv("distname")] + return Path(root_paths[neutralEnv("distname")]) @property def binaries(self): @@ -84,11 +85,11 @@ class Win64ConfigInfo(ConfigInfo): env[k] = v def get_bin_dir(self): - return [pj(self.root_path, "bin")] + return [self.root_path / "bin"] def get_env(self): env = super().get_env() - env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig") + env["PKG_CONFIG_LIBDIR"] = self.root_path / "lib" / "pkgconfig" env["LIBS"] = " ".join(self.extra_libs) + " " + env["LIBS"] return env diff --git a/kiwixbuild/configs/winbash.py b/kiwixbuild/configs/winbash.py index 20435b5..431511e 100644 --- a/kiwixbuild/configs/winbash.py +++ b/kiwixbuild/configs/winbash.py @@ -1,7 +1,6 @@ from .base import ConfigInfo import sysconfig -from kiwixbuild.utils import pj from kiwixbuild._global import get_target_step diff --git a/kiwixbuild/dependencies/apple_xcframework.py b/kiwixbuild/dependencies/apple_xcframework.py index 7f8b7f7..e16b8bf 100644 --- a/kiwixbuild/dependencies/apple_xcframework.py +++ b/kiwixbuild/dependencies/apple_xcframework.py @@ -1,9 +1,8 @@ -import os import shutil from pathlib import Path from kiwixbuild.configs import ConfigInfo -from kiwixbuild.utils import pj, run_command +from kiwixbuild.utils import run_command from .base import Dependency, NoopSource, Builder as BaseBuilder @@ -48,11 +47,11 @@ class AppleXCFramework(Dependency): return [(target, "libkiwix") for target in AppleXCFramework.subConfigNames] @property - def final_path(self): - return pj(self.buildEnv.install_dir, "lib", "CoreKiwix.xcframework") + def final_path(self) -> Path: + return self.buildEnv.install_dir / "lib" / "CoreKiwix.xcframework" def _remove_if_exists(self, context): - if not os.path.exists(self.final_path): + if not self.final_path.exists(): return shutil.rmtree(self.final_path) @@ -64,8 +63,8 @@ class AppleXCFramework(Dependency): static_ars = [] cfg = ConfigInfo.get_config(target) - lib_dir = pj(cfg.buildEnv.install_dir, "lib") - static_ars = [str(f) for f in Path(lib_dir).glob("*.a")] + lib_dir = cfg.buildEnv.install_dir / "lib" + static_ars = [str(f) for f in lib_dir.glob("*.a")] # create merged.a from all *.a in install_dir/lib command = ["libtool", "-static", "-o", "merged.a", *static_ars] @@ -73,7 +72,7 @@ class AppleXCFramework(Dependency): # will be included in xcframework if target in self.ios_subconfigs: - xcf_libs.append(pj(lib_dir, "merged.a")) + xcf_libs.append(lib_dir / "merged.a") return xcf_libs @@ -82,12 +81,12 @@ class AppleXCFramework(Dependency): libs = [] for target in configs: cfg = ConfigInfo.get_config(target) - libs.append(pj(cfg.buildEnv.install_dir, "lib", "merged.a")) + libs.append(cfg.buildEnv.install_dir / "lib" / "merged.a") - fat_dir = pj(self.buildEnv.build_dir, folder_name) - os.makedirs(fat_dir, exist_ok=True) + fat_dir = self.buildEnv.build_dir / folder_name + fad_dir.mkdir(parents=True, exist_ok=True) - output_merged = pj(fat_dir, "merged.a") + output_merged = fat_dir / "merged.a" command = ["lipo", "-create", "-output", output_merged, *libs] run_command(command, self.buildEnv.build_dir, context) @@ -102,7 +101,7 @@ class AppleXCFramework(Dependency): "-library", lib, "-headers", - pj(ref_conf.buildEnv.install_dir, "include"), + ref_conf.buildEnv.install_dir / "include", ] command += ["-output", self.final_path] run_command(command, self.buildEnv.build_dir, context) diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index ac3b8e7..c2f670c 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -1,10 +1,10 @@ import subprocess -import os import shutil import time +from pathlib import Path +from os import environ from kiwixbuild.utils import ( - pj, Context, SkipCommand, WarningMessage, @@ -17,7 +17,7 @@ from kiwixbuild.utils import ( from kiwixbuild.versions import main_project_versions, base_deps_versions from kiwixbuild._global import neutralEnv, option, get_target_step -SCRIPT_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) +SCRIPT_DIR = Path(__file__).resolve().parent.parent class _MetaDependency(type): @@ -71,23 +71,23 @@ class Source: return self.target.full_name() @property - def source_path(self): - return pj(neutralEnv("source_dir"), self.source_dir) + def source_path(self) -> Path: + return neutralEnv("source_dir") / self.source_dir @property - def _log_dir(self): + def _log_dir(self) -> Path: return neutralEnv("log_dir") def _patch(self, context): context.try_skip(self.source_path) for p in self.patches: - patch_file_path = pj(SCRIPT_DIR, "patches", p) + patch_file_path = SCRIPT_DIR / "patches" / p patch_command = [*neutralEnv("patch_command"), "-p1", "-i", patch_file_path] run_command(patch_command, self.source_path, context) def command(self, name, function, *args): print(" {} {} : ".format(name, self.name), end="", flush=True) - log = pj(self._log_dir, "cmd_{}_{}.log".format(name, self.name)) + log = self._log_dir / "cmd_{}_{}.log".format(name, self.name) context = Context(name, log, True) try: start_time = time.time() @@ -126,8 +126,8 @@ class ReleaseDownload(Source): return (self.archive,) @property - def extract_path(self): - return pj(neutralEnv("source_dir"), self.source_dir) + def extract_path(self) -> Path: + return neutralEnv("source_dir") / self.source_dir def _download(self, context): context.try_skip(neutralEnv("archive_dir"), self.full_name) @@ -145,11 +145,11 @@ class ReleaseDownload(Source): def _extract(self, context): context.try_skip(self.extract_path) - if os.path.exists(self.extract_path): + if self.extract_path.exists(): shutil.rmtree(self.extract_path) for archive in self.archives: extract_archive( - pj(neutralEnv("archive_dir"), archive.name), + neutralEnv("archive_dir") / archive.name, neutralEnv("source_dir"), topdir=self.archive_top_dir, name=self.source_dir, @@ -180,8 +180,8 @@ class GitClone(Source): return self.git_dir @property - def git_path(self): - return pj(neutralEnv("source_dir"), self.source_dir) + def git_path(self) -> Path: + return neutralEnv("source_dir") / self.source_dir @property def git_ref(self): @@ -228,7 +228,7 @@ class GitClone(Source): raise WarningMessage("Cannot update, please check log for information") def prepare(self): - if not os.path.exists(self.git_path): + if not self.git_path.exists(): self.command("gitinit", self._git_init) else: self.command("gitupdate", self._git_update) @@ -254,23 +254,23 @@ class Builder: return self.target.name @property - def source_path(self): + def source_path(self) -> Path: base_source_path = self.source.source_path if self.subsource_dir: - return pj(base_source_path, self.subsource_dir) + return base_source_path / self.subsource_dir return base_source_path @property - def build_path(self): - return pj(self.buildEnv.build_dir, self.target.full_name()) + def build_path(self) -> Path: + return self.buildEnv.build_dir / self.target.full_name() @property - def _log_dir(self): + def _log_dir(self) -> Path: return self.buildEnv.log_dir def command(self, name, function, *args): print(" {} {} : ".format(name, self.name), end="", flush=True) - log = pj(self._log_dir, "cmd_{}_{}.log".format(name, self.name)) + log = self._log_dir / "cmd_{}_{}.log".format(name, self.name) context = Context(name, log, self.target.force_native_build) if self.target.force_build: context.no_skip = True @@ -357,8 +357,8 @@ class TcCopyBuilder(Builder): src_subdir = None @property - def build_path(self): - return pj(self.buildEnv.toolchain_dir, self.target.full_name()) + def build_path(self) -> Path: + return self.buildEnv.toolchain_dir / self.target.full_name() def build(self): self.command("copy", self._copy) @@ -366,7 +366,7 @@ class TcCopyBuilder(Builder): def _copy(self, context): context.try_skip(self.build_path) if self.src_subdir: - source_path = pj(self.source_path, self.src_subdir) + source_path = self.source_path / self.src_subdir else: source_path = self.source_path copy_tree(source_path, self.build_path) @@ -406,7 +406,7 @@ class MakeBuilder(Builder): if not self.target.force_native_build: yield from self.buildEnv.configInfo.configure_options yield from ("--prefix", self.buildEnv.install_dir) - yield from ("--libdir", pj(self.buildEnv.install_dir, self.buildEnv.libprefix)) + yield from ("--libdir", self.buildEnv.install_dir / self.buildEnv.libprefix) def set_configure_env(self, env): dep_conf_env = self.configure_env @@ -423,7 +423,7 @@ class MakeBuilder(Builder): context.try_skip(self.build_path) command = [ *self.buildEnv.configure_wrapper, - pj(self.source_path, self.configure_script), + self.source_path / self.configure_script, *self.all_configure_options, ] env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True) @@ -494,10 +494,10 @@ class QMakeBuilder(MakeBuilder): @property def env_options(self): - if "QMAKE_CC" in os.environ: - yield f"QMAKE_CC={os.environ['QMAKE_CC']}" - if "QMAKE_CXX" in os.environ: - yield f"QMAKE_CXX={os.environ['QMAKE_CXX']}" + if "QMAKE_CC" in environ: + yield f"QMAKE_CC={environ['QMAKE_CC']}" + if "QMAKE_CXX" in environ: + yield f"QMAKE_CXX={environ['QMAKE_CXX']}" def _configure(self, context): context.try_skip(self.build_path) @@ -545,9 +545,9 @@ class MesonBuilder(Builder): def _configure(self, context): context.try_skip(self.build_path) - if os.path.exists(self.build_path): + if self.build_path.exists(): shutil.rmtree(self.build_path) - os.makedirs(self.build_path) + self.build_path.mkdir(parents=True) cross_options = [] if not self.target.force_native_build and self.buildEnv.meson_crossfile: cross_options += ["--cross-file", self.buildEnv.meson_crossfile] diff --git a/kiwixbuild/dependencies/icu4c.py b/kiwixbuild/dependencies/icu4c.py index 3c7125c..0588c0f 100644 --- a/kiwixbuild/dependencies/icu4c.py +++ b/kiwixbuild/dependencies/icu4c.py @@ -1,6 +1,7 @@ from .base import Dependency, ReleaseDownload, MakeBuilder, MesonBuilder -from kiwixbuild.utils import pj, Remotefile, extract_archive +from pathlib import Path +from kiwixbuild.utils import Remotefile, extract_archive from kiwixbuild._global import get_target_step, neutralEnv import os, shutil import fileinput @@ -31,25 +32,25 @@ class Icu(Dependency): def _extract(self, context): context.try_skip(self.extract_path) - if os.path.exists(self.extract_path): + if self.extract_path.exists(): shutil.rmtree(self.extract_path) extract_archive( - pj(neutralEnv("archive_dir"), self.archive_src.name), + neutralEnv("archive_dir") / self.archive_src.name, neutralEnv("source_dir"), topdir=None, name=self.source_dir, ) shutil.rmtree( - pj(neutralEnv("source_dir"), self.source_dir, "source", "data") + neutralEnv("source_dir") / self.source_dir / "source" / "data" ) extract_archive( - pj(neutralEnv("archive_dir"), self.archive_data.name), - pj(neutralEnv("source_dir"), self.source_dir, "source"), + neutralEnv("archive_dir") / self.archive_data.name, + neutralEnv("source_dir") / self.source_dir / "source", topdir="data", name="data", ) extract_archive( - pj(neutralEnv("archive_dir"), self.meson_patch.name), + neutralEnv("archive_dir") / self.meson_patch.name, neutralEnv("source_dir"), topdir="icu", name=self.source_dir, @@ -69,9 +70,8 @@ class Icu(Dependency): class Builder(MesonBuilder): def set_env(self, env): - env["ICU_DATA_FILTER_FILE"] = pj( - os.path.dirname(os.path.realpath(__file__)), - "icu4c_data_filter.json", + env["ICU_DATA_FILTER_FILE"] = ( + Path(__file__).resolve().parent / "icu4c_data_filter.json" ) else: @@ -105,18 +105,15 @@ class Icu(Dependency): yield "--with-data-packaging=archive" def set_env(self, env): - env["ICU_DATA_FILTER_FILE"] = pj( - os.path.dirname(os.path.realpath(__file__)), - "icu4c_data_filter.json", + env["ICU_DATA_FILTER_FILE"] = ( + Path(__file__).resolve().parent / "icu4c_data_filter.json" ) def _post_configure_script(self, context): if self.buildEnv.configInfo.build != "wasm": context.skip() context.try_skip(self.build_path) - for line in fileinput.input( - pj(self.build_path, "Makefile"), inplace=True - ): + for line in fileinput.input(self.build_path / "Makefile", inplace=True): if line == "#DATASUBDIR = data\n": print("DATASUBDIR = data") else: diff --git a/kiwixbuild/dependencies/ios_fat_lib.py b/kiwixbuild/dependencies/ios_fat_lib.py index a0b74f2..192aaa1 100644 --- a/kiwixbuild/dependencies/ios_fat_lib.py +++ b/kiwixbuild/dependencies/ios_fat_lib.py @@ -1,7 +1,7 @@ import os from kiwixbuild.configs import ConfigInfo -from kiwixbuild.utils import pj, copy_tree, run_command +from kiwixbuild.utils import copy_tree, run_command from kiwixbuild._global import option from .base import Dependency, NoopSource, Builder as BaseBuilder @@ -19,29 +19,29 @@ class IOSFatLib(Dependency): def _copy_headers(self, context): plt = ConfigInfo.get_config("iOS_{}".format(option("ios_arch")[0])) - include_src = pj(plt.buildEnv.install_dir, "include") - include_dst = pj(self.buildEnv.install_dir, "include") + include_src = plt.buildEnv.install_dir / "include" + include_dst = self.buildEnv.install_dir / "include" copy_tree(include_src, include_dst) def _merge_libs(self, context): lib_dirs = [] for arch in option("ios_arch"): plt = ConfigInfo.get_config("iOS_{}".format(arch)) - lib_dirs.append(pj(plt.buildEnv.install_dir, "lib")) + lib_dirs.append(plt.buildEnv.install_dir / "lib") libs = [] - for f in os.listdir(lib_dirs[0]): - if os.path.islink(pj(lib_dirs[0], f)): + for f in lib_dirs[0].iterdir(): + if f.is_symlink(): continue - if f.endswith(".a") or f.endswith(".dylib"): + if f.suffix in (".a", ".dylib"): libs.append(f) - os.makedirs(pj(self.buildEnv.install_dir, "lib"), exist_ok=True) + (self.buildEnv.install_dir / "lib").mkdir(parents=True, exist_ok=True) for l in libs: command = [ "lipo", "-create", - *[pj(d, l) for d in lib_dirs], + *[d / l for d in lib_dirs], "-output", - pj(self.buildEnv.install_dir, "lib", l), + self.buildEnv.install_dir / "lib" / l, ] run_command(command, self.buildEnv.install_dir, context) diff --git a/kiwixbuild/dependencies/libmagic.py b/kiwixbuild/dependencies/libmagic.py index 9f860b8..d1ad419 100644 --- a/kiwixbuild/dependencies/libmagic.py +++ b/kiwixbuild/dependencies/libmagic.py @@ -4,7 +4,7 @@ from .base import ( MakeBuilder, ) -from kiwixbuild.utils import Remotefile, pj, run_command +from kiwixbuild.utils import Remotefile, run_command from kiwixbuild._global import get_target_step @@ -44,5 +44,5 @@ class LibMagic(Dependency): cross_comp_flags=True, cross_compilers=True, cross_path=True ) libmagic_native_builder = get_target_step("libmagic", "native_static") - env["PATH"].insert(0, pj(libmagic_native_builder.build_path, "src")) + env["PATH"].insert(0, libmagic_native_builder.build_path / "src") run_command(command, self.build_path, context, env=env) diff --git a/kiwixbuild/dependencies/mustache.py b/kiwixbuild/dependencies/mustache.py index 08eb947..4c8edd1 100644 --- a/kiwixbuild/dependencies/mustache.py +++ b/kiwixbuild/dependencies/mustache.py @@ -1,6 +1,6 @@ from .base import Dependency, ReleaseDownload, Builder as BaseBuilder -from kiwixbuild.utils import Remotefile, pj +from kiwixbuild.utils import Remotefile from shutil import copy2 @@ -21,8 +21,8 @@ class Mustache(Dependency): def _copy_header(self, context): context.try_skip(self.build_path) copy2( - pj(self.source_path, "mustache.hpp"), - pj(self.buildEnv.install_dir, "include"), + self.source_path / "mustache.hpp", + self.buildEnv.install_dir / "include", ) def set_flatpak_buildsystem(self, module): diff --git a/kiwixbuild/dependencies/qt.py b/kiwixbuild/dependencies/qt.py index 2f31a92..d699bf1 100644 --- a/kiwixbuild/dependencies/qt.py +++ b/kiwixbuild/dependencies/qt.py @@ -1,6 +1,6 @@ from .base import Dependency, ReleaseDownload, MakeBuilder, QMakeBuilder -from kiwixbuild.utils import Remotefile, pj +from kiwixbuild.utils import Remotefile class Qt(Dependency): @@ -32,7 +32,7 @@ class Qt(Dependency): yield from ("-prefix", self.buildEnv.install_dir) yield from ( "-libdir", - pj(self.buildEnv.install_dir, self.buildEnv.libprefix), + self.buildEnv.install_dir / self.buildEnv.libprefix, ) @property diff --git a/kiwixbuild/dependencies/tc_android_ndk.py b/kiwixbuild/dependencies/tc_android_ndk.py index c65768f..8770bc7 100644 --- a/kiwixbuild/dependencies/tc_android_ndk.py +++ b/kiwixbuild/dependencies/tc_android_ndk.py @@ -1,7 +1,6 @@ -import os - +from pathlib import Path from .base import Dependency, ReleaseDownload, Builder -from kiwixbuild.utils import Remotefile, add_execution_right, run_command, pj +from kiwixbuild.utils import Remotefile, add_execution_right, run_command class android_ndk(Dependency): @@ -24,8 +23,8 @@ class android_ndk(Dependency): class Builder(Builder): @property - def install_path(self): - return pj(self.buildEnv.toolchain_dir, self.target.full_name()) + def install_path(self) -> Path: + return self.buildEnv.toolchain_dir / self.target.full_name() @property def api(self): @@ -45,7 +44,7 @@ class android_ndk(Dependency): def _build_toolchain(self, context): context.try_skip(self.build_path) - script = pj(self.source_path, "build/tools/make_standalone_toolchain.py") + script = self.source_path / "build/tools/make_standalone_toolchain.py" add_execution_right(script) command = [ script, @@ -62,23 +61,21 @@ class android_ndk(Dependency): def _fix_permission_right(self, context): context.try_skip(self.build_path) bin_dirs = [ - pj(self.install_path, "bin"), - pj(self.install_path, self.arch_full, "bin"), - pj( - self.install_path, - "libexec", - "gcc", - self.arch_full, - self.target.gccver, - ), + self.install_path / "bin", + self.install_pat / self.arch_full / "bin", + self.install_path + / "libexec" + / "gcc" + / self.arch_full + / self.target.gccver, ] - for root, dirs, files in os.walk(self.install_path): + for root, dirs, files in self.install_path.walk(): if not root in bin_dirs: continue for file_ in files: - file_path = pj(root, file_) - if os.path.islink(file_path): + file_path = root / file_ + if file_path.is_symlink(): continue add_execution_right(file_path) diff --git a/kiwixbuild/dependencies/tc_emsdk.py b/kiwixbuild/dependencies/tc_emsdk.py index f47f9f7..6eec7a5 100644 --- a/kiwixbuild/dependencies/tc_emsdk.py +++ b/kiwixbuild/dependencies/tc_emsdk.py @@ -1,5 +1,6 @@ +from pathlib import Path from .base import Dependency, ReleaseDownload, Builder -from kiwixbuild.utils import pj, Remotefile, run_command, copy_tree +from kiwixbuild.utils import Remotefile, run_command, copy_tree class emsdk(Dependency): @@ -20,8 +21,8 @@ class emsdk(Dependency): class Builder(Builder): @property - def install_path(self): - return pj(self.buildEnv.toolchain_dir, self.target.full_name()) + def install_path(self) -> Path: + return self.buildEnv.toolchain_dir / self.target.full_name() def _copy_source(self, context): context.try_skip(self.build_path) diff --git a/kiwixbuild/dependencies/xapian.py b/kiwixbuild/dependencies/xapian.py index eba5271..d376152 100644 --- a/kiwixbuild/dependencies/xapian.py +++ b/kiwixbuild/dependencies/xapian.py @@ -1,6 +1,6 @@ from .base import Dependency, ReleaseDownload, MakeBuilder -from kiwixbuild.utils import Remotefile, pj +from kiwixbuild.utils import Remotefile from kiwixbuild._global import neutralEnv @@ -17,7 +17,7 @@ class Xapian(Dependency): @property def configure_options(self): if neutralEnv("distname") == "Windows": - compile_script = pj(self.source_path, "compile") + compile_script = self.source_path / "compile" return [ 'CC="cl -nologo"', 'CXX=f"{compile_script} cl -nologo"', diff --git a/kiwixbuild/flatpak_builder.py b/kiwixbuild/flatpak_builder.py index ea0d934..c2d32fd 100644 --- a/kiwixbuild/flatpak_builder.py +++ b/kiwixbuild/flatpak_builder.py @@ -206,9 +206,8 @@ class FlatpakBuilder: m["sources"] = temp manifest["modules"] = modules manifest_name = "{}.json".format(MANIFEST["app-id"]) - manifest_path = pj(self.config.buildEnv.build_dir, manifest_name) - with open(manifest_path, "w") as f: - f.write(json.dumps(manifest, indent=4)) + manifest_path = self.config.buildEnv.build_dir / manifest_name + manifest_path.write_text(json.dumps(manifest, indent=4)) def copy_patches(self): sourceDefs = (tDef for tDef in target_steps() if tDef[0] == "source") @@ -217,15 +216,15 @@ class FlatpakBuilder: if not hasattr(source, "patches"): continue for p in source.patches: - path = pj(SCRIPT_DIR, "patches", p) - os.makedirs( - pj(self.config.buildEnv.build_dir, "patches"), exist_ok=True + path = SCRIPT_DIR / "patches" / p + (self.config.buildEnv.build_dir / "patches").mkdir( + parents=True, exist_ok=True ) - dest = pj(self.config.buildEnv.build_dir, "patches", p) + dest = self.config.buildEnv.build_dir / "patches" / p copyfile(path, dest) def build(self): - log = pj(self.config.buildEnv.log_dir, "cmd_build_flatpak.log") + log = self.config.buildEnv.log_dir / "cmd_build_flatpak.log" context = Context("build", log, False) command = [ "flatpak-builder", @@ -247,12 +246,11 @@ class FlatpakBuilder: ) context._finalise() except subprocess.CalledProcessError: - with open(log, "r") as f: - print(f.read()) + print(log.read_text()) raise StopBuild() def bundle(self): - log = pj(self.config.buildEnv.log_dir, "cmd_bundle_flatpak.log") + log = self.config.buildEnv.log_dir / "cmd_bundle_flatpak.log" context = Context("bundle", log, False) app_id = MANIFEST["app-id"] command = ["flatpak", "build-bundle", "repo", f"{app_id}.flatpak", app_id] @@ -265,8 +263,7 @@ class FlatpakBuilder: ) context._finalise() except subprocess.CalledProcessError: - with open(log, "r") as f: - print(f.read()) + print(log.read_text()) raise StopBuild() def _get_packages(self): diff --git a/kiwixbuild/utils.py b/kiwixbuild/utils.py index cfb1219..5328607 100644 --- a/kiwixbuild/utils.py +++ b/kiwixbuild/utils.py @@ -1,4 +1,3 @@ -import os.path import hashlib import tarfile, zipfile import tempfile @@ -9,15 +8,12 @@ import urllib.error import ssl import subprocess import re +from pathlib import Path from collections import namedtuple, defaultdict from kiwixbuild._global import neutralEnv, option -def pj(*args): - return os.path.normpath(os.path.join(*args)) - - COLORS = { "OK": "\033[92m", "WARNING": "\033[93m", @@ -46,7 +42,7 @@ def xrun_find(name): regex_space = re.compile(r"((?= (3, 4, 3) else {} progress_chars = "/-\\|" try: - with urllib.request.urlopen(what.url, **extra_args) as resource, open( - file_path, "wb" + with urllib.request.urlopen(what.url, **extra_args) as resource, file_path.open( + "wb" ) as file: tsize = resource.info().get("Content-Length", None) if tsize is not None: @@ -190,7 +193,7 @@ def download_remote(what, where): if not what.sha256: print("Sha256 for {} not set, do no verify download".format(what.name)) elif what.sha256 != get_sha256(file_path): - os.remove(file_path) + file_path.unlink() raise StopBuild("Sha 256 doesn't correspond") @@ -218,13 +221,6 @@ class StopBuild(BaseCommandResult): pass -class Remotefile(namedtuple("Remotefile", ("name", "sha256", "url"))): - def __new__(cls, name, sha256, url=None): - if url is None: - url = REMOTE_PREFIX + name - return super().__new__(cls, name, sha256, url) - - class Context: def __init__(self, command_name, log_file, force_native_build): self.command_name = command_name @@ -236,24 +232,28 @@ class Context: def skip(self, msg=""): raise SkipCommand(msg) - def try_skip(self, path, extra_name=""): + def try_skip(self, path: Path, extra_name=""): if self.no_skip: return if extra_name: extra_name = "_{}".format(extra_name) - self.autoskip_file = pj(path, ".{}{}_ok".format(self.command_name, extra_name)) - if os.path.exists(self.autoskip_file): + self.autoskip_file = path / ".{}{}_ok".format(self.command_name, extra_name) + if self.autoskip_file.exists(): raise SkipCommand() def _finalise(self): if self.autoskip_file is not None: - os.makedirs(os.path.dirname(self.autoskip_file), exist_ok=True) - with open(self.autoskip_file, "w"): - pass + self.autoskip_file.parent.mkdir(parents=True, exist_ok=True) + self.autoskip_file.touch() -def extract_archive(archive_path, dest_dir, topdir=None, name=None): - is_zip_archive = archive_path.endswith(".zip") +def extract_archive(archive_path: Path, dest_dir: Path, topdir=None, name=None): + """Extract an archive to dest_dir. + This archive can be a zip or a tar archive. + If topdir is given, only this directory (and sub files/dirs) will be extracted. + If name is given, the topdir will be extracted under this name. + """ + is_zip_archive = archive_path.suffix == ".zip" archive = None try: if is_zip_archive: @@ -283,40 +283,39 @@ def extract_archive(archive_path, dest_dir, topdir=None, name=None): members_to_extract = [ m for m in members if getname(m).startswith(topdir + "/") ] - os.makedirs(dest_dir, exist_ok=True) + dest_dir.mkdir(parents=True, exist_ok=True) with tempfile.TemporaryDirectory( - prefix=os.path.basename(archive_path), dir=dest_dir + prefix=archive_path.name, dir=dest_dir ) as tmpdir: + tmpdir_path = Path(tmpdir) if is_zip_archive: _members_to_extract = [getname(m) for m in members_to_extract] else: _members_to_extract = members_to_extract - archive.extractall(path=tmpdir, members=_members_to_extract) + archive.extractall(path=tmpdir_path, members=_members_to_extract) if is_zip_archive: for member in members_to_extract: if isdir(member): continue perm = (member.external_attr >> 16) & 0x1FF if perm: - os.chmod(pj(tmpdir, getname(member)), perm) + (tmpdir_path / getname(member)).chmod(perm) name = name or topdir shutil.copytree( - pj(tmpdir, topdir), - pj(dest_dir, name), + tmpdir_path / topdir, + dest_dir / name, symlinks=True, dirs_exist_ok=True, ) # Be sure that all directory in tmpdir are writable to allow correct suppersion of it - for root, dirs, _files in os.walk(tmpdir): + for root, dirs, _files in tmpdir_path.walk(): for d in dirs: - os.chmod( - pj(root, d), stat.S_IWRITE | stat.S_IREAD | stat.S_IEXEC - ) + (root / d).chmod(stat.S_IWRITE | stat.S_IREAD | stat.S_IEXEC) else: if name: - dest_dir = pj(dest_dir, name) - os.makedirs(dest_dir) + dest_dir = dest_dir / name + dest_dir.mkdir(parents=True) archive.extractall(path=dest_dir) finally: if archive is not None: @@ -328,6 +327,7 @@ def run_command(command, cwd, context, *, env=None, input=None): if env is None: env = DefaultEnv() log = None + command = [str(v) for v in command] try: if not option("verbose"): log = open(context.log_file, "w")