smallsolar
/
kiwix-build
mirror of https://github.com/kiwix/kiwix-build.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use pathlib.Path instead of os.path

This commit is contained in:
Matthieu Gautier 2024-04-30 12:52:29 +02:00
parent 06cad66925
commit 908b90190c
25 changed files with 264 additions and 269 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
kiwixbuild/__init__.py
View File

@ -2,6 +2,7 @@
import os, sys
import argparse
from pathlib import Path
from .dependencies import Dependency
from .configs import ConfigInfo
@ -22,6 +23,7 @@ def parse_args():
parser.add_argument(
"--working-dir",
default=".",
type=Path,
help=(
"Directory where kiwix-build puts all its files "
"(source, archive and build)\n"
@ -31,6 +33,7 @@ def parse_args():
parser.add_argument(
"--build-dir",
default=".",
type=Path,
help=(
"Directory where kiwix-build puts all build files.\n"
"build-dir can be absolute path or a relative (to working-dir) one."
@ -152,7 +155,7 @@ def parse_args():
def main():
options = parse_args()
options.working_dir = os.path.abspath(options.working_dir)
options.working_dir = options.working_dir.absolute()
_global.set_options(options)
neutralEnv = buildenv.NeutralEnv(options.get_build_dir)
_global.set_neutralEnv(neutralEnv)

54
kiwixbuild/buildenv.py
View File

@ -2,20 +2,21 @@ import os, sys, shutil
import subprocess
import platform
import distro
from pathlib import Path
from .utils import pj, download_remote, escape_path
from .utils import download_remote, escape_path
from ._global import neutralEnv, option
class NeutralEnv:
def __init__(self, dummy_run):
self.working_dir = option("working_dir")
self.source_dir = pj(self.working_dir, "SOURCE")
self.archive_dir = pj(self.working_dir, "ARCHIVE")
self.toolchain_dir = pj(self.working_dir, "TOOLCHAINS")
self.log_dir = pj(self.working_dir, "LOGS")
self.working_dir: Path = option("working_dir")
self.source_dir: Path = self.working_dir / "SOURCE"
self.archive_dir: Path = self.working_dir / "ARCHIVE"
self.toolchain_dir: Path = self.working_dir / "TOOLCHAINS"
self.log_dir: Path = self.working_dir / "LOGS"
for d in (self.source_dir, self.archive_dir, self.toolchain_dir, self.log_dir):
os.makedirs(d, exist_ok=True)
d.mkdir(parents=True, exist_ok=True)
self.detect_platform()
if dummy_run:
# If this is for a dummy run, we will not run anything.
@ -73,29 +74,28 @@ class NeutralEnv:
class BuildEnv:
def __init__(self, configInfo):
self.configInfo = configInfo
self.base_build_dir = pj(option("working_dir"), option("build_dir"))
self.base_build_dir: Path = option("working_dir") / option("build_dir")
build_dir = (
configInfo.arch_name if option("use_target_arch_name") else configInfo.name
)
build_dir = f"BUILD_{build_dir}"
self.build_dir = pj(self.base_build_dir, build_dir)
self.install_dir = pj(self.build_dir, "INSTALL")
self.toolchain_dir = pj(self.build_dir, "TOOLCHAINS")
self.log_dir = pj(self.build_dir, "LOGS")
self.build_dir: Path = self.base_build_dir / build_dir
self.install_dir: Path = self.build_dir / "INSTALL"
self.toolchain_dir: Path = self.build_dir / "TOOLCHAINS"
self.log_dir: Path = self.build_dir / "LOGS"
for d in (self.build_dir, self.install_dir, self.toolchain_dir, self.log_dir):
os.makedirs(d, exist_ok=True)
d.mkdir(parents=True, exist_ok=True)
self.libprefix = option("libprefix") or self._detect_libdir()
def clean_intermediate_directories(self):
for subdir in os.listdir(self.build_dir):
subpath = pj(self.build_dir, subdir)
for subpath in self.build_dir.iterdir():
if subpath == self.install_dir:
continue
if os.path.isdir(subpath):
if subpath.isdir():
shutil.rmtree(subpath)
else:
os.remove(subpath)
subpath.unlink()
def _is_debianlike(self):
return os.path.isfile("/etc/debian_version")
@ -122,35 +122,35 @@ class BuildEnv:
def get_env(self, *, cross_comp_flags, cross_compilers, cross_path):
env = self.configInfo.get_env()
pkgconfig_path = pj(self.install_dir, self.libprefix, "pkgconfig")
pkgconfig_path = self.install_dir / self.libprefix / "pkgconfig"
env["PKG_CONFIG_PATH"].append(pkgconfig_path)
env["PATH"].insert(0, pj(self.install_dir, "bin"))
env["PATH"].insert(0, self.install_dir / "bin")
env["LD_LIBRARY_PATH"].extend(
[
pj(self.install_dir, "lib"),
pj(self.install_dir, self.libprefix),
self.install_dir / "lib",
self.install_dir / self.libprefix,
]
)
env["QMAKE_CXXFLAGS"] = " ".join(
[escape_path("-I" + pj(self.install_dir, "include")), env["QMAKE_CXXFLAGS"]]
[escape_path(f"-I{self.install_dir / 'include'}"), env["QMAKE_CXXFLAGS"]]
)
env["CPPFLAGS"] = " ".join(
[escape_path("-I" + pj(self.install_dir, "include")), env["CPPFLAGS"]]
[escape_path(f"-I{self.install_dir / 'include'}"), env["CPPFLAGS"]]
)
env["QMAKE_LFLAGS"] = " ".join(
[
escape_path("-L" + pj(self.install_dir, "lib")),
escape_path("-L" + pj(self.install_dir, self.libprefix)),
escape_path(f"-L{self.install_dir / 'lib'}"),
escape_path(f"-L{self.install_dir / self.libprefix}"),
env["QMAKE_LFLAGS"],
]
)
env["LDFLAGS"] = " ".join(
[
escape_path("-L" + pj(self.install_dir, "lib")),
escape_path("-L" + pj(self.install_dir, self.libprefix)),
escape_path(f"-L{self.install_dir / 'lib'}"),
escape_path(f"-L{self.install_dir / self.libprefix}"),
env["LDFLAGS"],
]
)

18
kiwixbuild/configs/android.py
View File

@ -1,5 +1,5 @@
from pathlib import Path
from .base import ConfigInfo, MetaConfigInfo
from kiwixbuild.utils import pj
from kiwixbuild._global import get_target_step, option
@ -30,9 +30,7 @@ class AndroidConfigInfo(ConfigInfo):
def binaries(self):
install_path = self.install_path
binaries = {
k: pj(install_path, "bin", v) for k, v in self.binaries_name.items()
}
binaries = {k: install_path / "bin" / v for k, v in self.binaries_name.items()}
binaries["PKGCONFIG"] = "pkg-config"
return binaries
@ -41,7 +39,7 @@ class AndroidConfigInfo(ConfigInfo):
return get_target_step("android-ndk", self.name)
@property
def install_path(self):
def install_path(self) -> Path:
return self.ndk_builder.install_path
def get_cross_config(self):
@ -56,7 +54,7 @@ class AndroidConfigInfo(ConfigInfo):
"exe_wrapper_def": "",
"install_path": self.install_path,
"binaries": self.binaries(),
"root_path": pj(self.install_path, "sysroot"),
"root_path": self.install_path / "sysroot",
"extra_libs": extra_libs,
"extra_cflags": extra_cflags,
"host_machine": {
@ -71,17 +69,17 @@ class AndroidConfigInfo(ConfigInfo):
def get_env(self):
env = super().get_env()
root_path = pj(self.install_path, "sysroot")
env["PKG_CONFIG_LIBDIR"] = pj(root_path, "lib", "pkgconfig")
root_path = self.install_path / "sysroot"
env["PKG_CONFIG_LIBDIR"] = root_path / "lib" / "pkgconfig"
env["NDK_DEBUG"] = "0"
return env
def get_bin_dir(self):
return [pj(self.install_path, "bin")]
return [self.install_path / "bin"]
def set_comp_flags(self, env):
super().set_comp_flags(env)
root_path = pj(self.install_path, "sysroot")
root_path = self.install_path / "sysroot"
march = "-march={}".format(self.march) if hasattr(self, "march") else ""
env["CFLAGS"] = (
"-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} {} ".format(

20
kiwixbuild/configs/armhf.py
View File

@ -1,6 +1,6 @@
from .base import ConfigInfo, MixedMixin
from kiwixbuild.utils import pj
from pathlib import Path
from kiwixbuild._global import get_target_step
@ -36,7 +36,7 @@ class ArmConfigInfo(ConfigInfo):
return get_target_step(self.build, "neutral")
@property
def root_path(self):
def root_path(self) -> Path:
return self.toolchain.build_path
@property
@ -54,7 +54,7 @@ class ArmConfigInfo(ConfigInfo):
("LDSHARED", "g++ -shared"),
)
)
binaries = {k: pj(self.root_path, "bin", v) for k, v in binaries}
binaries = {k: self.root_path / "bin" / v for k, v in binaries}
binaries["PKGCONFIG"] = "pkg-config"
return binaries
@ -72,19 +72,19 @@ class ArmConfigInfo(ConfigInfo):
yield "--host={}".format(self.arch_full)
def get_bin_dir(self):
return [pj(self.root_path, "bin")]
return [self.root_path / "bin"]
def get_env(self):
env = super().get_env()
env["LD_LIBRARY_PATH"][0:0] = [
pj(self.root_path, self.arch_full, "lib64"),
pj(self.root_path, "lib"),
self.root_path / self.arch_full / "lib64",
self.root_path / "lib",
]
env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig")
env["QEMU_LD_PREFIX"] = pj(self.root_path, self.arch_full, "libc")
env["PKG_CONFIG_LIBDIR"] = self.root_path / "lib" / "pkgconfig"
env["QEMU_LD_PREFIX"] = self.root_path / self.arch_full / "libc"
env["QEMU_SET_ENV"] = "LD_LIBRARY_PATH={}".format(
":".join(
[pj(self.root_path, self.arch_full, "lib"), str(env["LD_LIBRARY_PATH"])]
[self.root_path / self.arch_full / "lib", str(env["LD_LIBRARY_PATH"])]
)
)
return env
@ -164,7 +164,7 @@ class Aarch64(ArmConfigInfo):
cpu = "aarch64"
@property
def root_path(self):
def root_path(self) -> Path:
return self.toolchain.build_path

35
kiwixbuild/configs/base.py
View File

@ -1,13 +1,14 @@
import os, sys
import subprocess
from pathlib import Path
from kiwixbuild.dependencies import Dependency
from kiwixbuild.utils import pj, remove_duplicates, DefaultEnv
from kiwixbuild.utils import remove_duplicates, DefaultEnv
from kiwixbuild.buildenv import BuildEnv
from kiwixbuild._global import neutralEnv, option, target_steps
_SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
TEMPLATES_DIR = pj(os.path.dirname(_SCRIPT_DIR), "templates")
_SCRIPT_DIR = Path(__file__).resolve().parent
TEMPLATES_DIR = _SCRIPT_DIR.parent / "templates"
class _MetaConfig(type):
@ -81,7 +82,7 @@ class ConfigInfo(metaclass=_MetaConfig):
return {}
def get_include_dirs(self):
return [pj(self.buildEnv.install_dir, "include")]
return [self.buildEnv.install_dir / "include"]
def get_env(self):
return DefaultEnv()
@ -100,13 +101,11 @@ class ConfigInfo(metaclass=_MetaConfig):
def _gen_crossfile(self, name, outname=None):
if outname is None:
outname = name
crossfile = pj(self.buildEnv.build_dir, outname)
template_file = pj(TEMPLATES_DIR, name)
with open(template_file, "r") as f:
template = f.read()
crossfile = self.buildEnv.build_dir / outname
template_file = TEMPLATES_DIR / name
template = template_file.read_text()
content = template.format(**self.get_cross_config())
with open(crossfile, "w") as outfile:
outfile.write(content)
crossfile.write_text(content)
return crossfile
def finalize_setup(self):
@ -155,22 +154,22 @@ def MixedMixin(static_name):
def get_include_dirs(self):
return [
pj(self.buildEnv.install_dir, "include"),
pj(self.static_buildEnv.install_dir, "include"),
self.buildEnv.install_dir / "include",
self.static_buildEnv.install_dir / "include",
]
def get_env(self):
env = super().get_env()
env["PATH"].insert(0, pj(self.static_buildEnv.install_dir, "bin"))
pkgconfig_path = pj(
self.static_buildEnv.install_dir,
self.static_buildEnv.libprefix,
"pkgconfig",
env["PATH"].insert(0, self.static_buildEnv.install_dir / "bin")
pkgconfig_path = (
self.static_buildEnv.install_dir
/ self.static_buildEnv.libprefix
/ "pkgconfig"
)
env["PKG_CONFIG_PATH"].append(pkgconfig_path)
env["CPPFLAGS"] = " ".join(
[
"-I" + pj(self.static_buildEnv.install_dir, "include"),
f"-I{self.static_buildEnv.install_dir / 'include'}",
env["CPPFLAGS"],
]
)

2
kiwixbuild/configs/i586.py
View File

@ -1,7 +1,7 @@
import os
from .base import ConfigInfo
from kiwixbuild.utils import which, pj
from kiwixbuild.utils import which
class I586ConfigInfo(ConfigInfo):

11
kiwixbuild/configs/ios.py
View File

@ -1,7 +1,8 @@
from pathlib import Path
import subprocess
from kiwixbuild._global import option
from kiwixbuild.utils import pj, xrun_find
from kiwixbuild.utils import xrun_find
from .base import ConfigInfo, MetaConfigInfo, MixedMixin
from kiwixbuild.dependencies.apple_xcframework import AppleXCFramework
@ -29,10 +30,12 @@ class AppleConfigInfo(ConfigInfo):
return self.target
@property
def root_path(self):
def root_path(self) -> Path:
if self._root_path is None:
command = "xcrun --sdk {} --show-sdk-path".format(self.sdk_name)
self._root_path = subprocess.check_output(command, shell=True)[:-1].decode()
self._root_path = Path(
subprocess.check_output(command, shell=True)[:-1].decode()
)
return self._root_path
def __str__(self):
@ -133,7 +136,7 @@ class AppleConfigInfo(ConfigInfo):
)
def get_bin_dir(self):
return [pj(self.root_path, "bin")]
return [self.root_path / "bin"]
@property
def binaries(self):

20
kiwixbuild/configs/musl.py
View File

@ -1,6 +1,6 @@
from pathlib import Path
from .base import ConfigInfo, MixedMixin
from kiwixbuild.utils import pj
from kiwixbuild._global import get_target_step
@ -31,7 +31,7 @@ class MuslConfigInfo(ConfigInfo):
return get_target_step(self.build, "neutral")
@property
def root_path(self):
def root_path(self) -> Path:
return self.toolchain.build_path
@property
@ -49,7 +49,7 @@ class MuslConfigInfo(ConfigInfo):
("LDSHARED", "g++ -shared"),
)
)
binaries = {k: pj(self.root_path, "bin", v) for k, v in binaries}
binaries = {k: self.root_path / "bin" / v for k, v in binaries}
binaries["PKGCONFIG"] = "pkg-config"
return binaries
@ -69,26 +69,26 @@ class MuslConfigInfo(ConfigInfo):
return [f"--host={self.arch_full}"]
def get_bin_dir(self):
return [pj(self.root_path, "bin")]
return [self.root_path / "bin"]
def get_env(self):
env = super().get_env()
env["LD_LIBRARY_PATH"][0:0] = [
pj(self.root_path, self.arch_full, "lib64"),
pj(self.root_path, "lib"),
self.root_path / self.arch_full / "lib64",
self.root_path / "lib",
]
env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig")
env["QEMU_LD_PREFIX"] = pj(self.root_path, self.arch_full, "libc")
env["PKG_CONFIG_LIBDIR"] = self.root_path / "lib" / "pkgconfig"
env["QEMU_LD_PREFIX"] = self.root_path / self.arch_full / "libc"
env["QEMU_SET_ENV"] = "LD_LIBRARY_PATH={}".format(
":".join(
[pj(self.root_path, self.arch_full, "lib"), str(env["LD_LIBRARY_PATH"])]
[self.root_path / self.arch_full / "lib", str(env["LD_LIBRARY_PATH"])]
)
)
return env
def set_comp_flags(self, env):
super().set_comp_flags(env)
env["LD_LIBRARY_PATH"].insert(0, pj(self.root_path, self.arch_full, "lib"))
env["LD_LIBRARY_PATH"].insert(0, self.root_path / self.arch_full / "lib")
env["CFLAGS"] = (
" -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
+ env["CFLAGS"]

1
kiwixbuild/configs/native.py
View File

@ -1,6 +1,5 @@
from .base import ConfigInfo, MixedMixin
from kiwixbuild.utils import pj
from kiwixbuild._global import option, neutralEnv
from kiwixbuild.configs.ios import MIN_MACOS_VERSION
import sysconfig

18
kiwixbuild/configs/wasm.py
View File

@ -1,6 +1,6 @@
from pathlib import Path
from .base import ConfigInfo
from kiwixbuild.utils import pj
from kiwixbuild._global import get_target_step
@ -37,11 +37,11 @@ class WasmConfigInfo(ConfigInfo):
return get_target_step("emsdk", self.name)
@property
def install_path(self):
def install_path(self) -> Path:
return self.wasm_sdk.install_path
@property
def root_path(self):
def root_path(self) -> Path:
return self.install_path
@property
@ -56,7 +56,7 @@ class WasmConfigInfo(ConfigInfo):
("LD", "wasm-ld"),
)
binaries = {
k: pj(self.install_path, "upstream", "emscripten", v) for k, v in binaries
k: self.install_path / "upstream" / "emscripten" / v for k, v in binaries
}
binaries["PKGCONFIG"] = "pkg-config"
return binaries
@ -75,20 +75,20 @@ class WasmConfigInfo(ConfigInfo):
return "emmake"
def get_bin_dir(self):
return [pj(self.install_path, "bin")]
return [self.install_path / "bin"]
def get_env(self):
env = super().get_env()
env["PATH"].extend(
[
self.install_path,
pj(self.install_path, "upstream", "emscripten"),
pj(self.install_path, "node", "14.18.2_64bit", "bin"),
self.install_path / "upstream" / "emscripten",
self.install_path / "node" / "14.18.2_64bit" / "bin",
]
)
env["EMSDK"] = self.install_path
env["EMSDK_NODE"] = pj(
self.install_path, "node", "14.18.2_64bit", "bin", "node"
env["EMSDK_NODE"] = (
self.install_path / "node" / "14.18.2_64bit" / "bin" / "node"
)
return env

12
kiwixbuild/configs/win32.py
View File

@ -1,7 +1,9 @@
import subprocess
from pathlib import Path
from .base import ConfigInfo
from kiwixbuild.utils import which, pj
from kiwixbuild.utils import which
from kiwixbuild._global import neutralEnv
@ -40,12 +42,12 @@ class Win32ConfigInfo(ConfigInfo):
self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
@property
def root_path(self):
def root_path(self) -> Path:
root_paths = {
"fedora": "/usr/i686-w64-mingw32/sys-root/mingw",
"debian": "/usr/i686-w64-mingw32",
}
return root_paths[neutralEnv("distname")]
return Path(root_paths[neutralEnv("distname")])
@property
def binaries(self):
@ -80,11 +82,11 @@ class Win32ConfigInfo(ConfigInfo):
env[k] = v
def get_bin_dir(self):
return [pj(self.root_path, "bin")]
return [self.root_path / "bin"]
def get_env(self):
env = super().get_env()
env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig")
env["PKG_CONFIG_LIBDIR"] = self.root_path / "lib" / "pkgconfig"
env["LIBS"] = " ".join(self.extra_libs) + " " + env["LIBS"]
return env

11
kiwixbuild/configs/win64.py
View File

@ -1,7 +1,8 @@
import subprocess
from pathlib import Path
from .base import ConfigInfo
from kiwixbuild.utils import which, pj
from kiwixbuild.utils import which
from kiwixbuild._global import neutralEnv
@ -44,12 +45,12 @@ class Win64ConfigInfo(ConfigInfo):
self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
@property
def root_path(self):
def root_path(self) -> Path:
root_paths = {
"fedora": "/usr/x86_64-w64-mingw32/sys-root/mingw",
"debian": "/usr/x86_64-w64-mingw32",
}
return root_paths[neutralEnv("distname")]
return Path(root_paths[neutralEnv("distname")])
@property
def binaries(self):
@ -84,11 +85,11 @@ class Win64ConfigInfo(ConfigInfo):
env[k] = v
def get_bin_dir(self):
return [pj(self.root_path, "bin")]
return [self.root_path / "bin"]
def get_env(self):
env = super().get_env()
env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig")
env["PKG_CONFIG_LIBDIR"] = self.root_path / "lib" / "pkgconfig"
env["LIBS"] = " ".join(self.extra_libs) + " " + env["LIBS"]
return env

1
kiwixbuild/configs/winbash.py
View File

@ -1,7 +1,6 @@
from .base import ConfigInfo
import sysconfig
from kiwixbuild.utils import pj
from kiwixbuild._global import get_target_step

25
kiwixbuild/dependencies/apple_xcframework.py
View File

@ -1,9 +1,8 @@
import os
import shutil
from pathlib import Path
from kiwixbuild.configs import ConfigInfo
from kiwixbuild.utils import pj, run_command
from kiwixbuild.utils import run_command
from .base import Dependency, NoopSource, Builder as BaseBuilder
@ -48,11 +47,11 @@ class AppleXCFramework(Dependency):
return [(target, "libkiwix") for target in AppleXCFramework.subConfigNames]
@property
def final_path(self):
return pj(self.buildEnv.install_dir, "lib", "CoreKiwix.xcframework")
def final_path(self) -> Path:
return self.buildEnv.install_dir / "lib" / "CoreKiwix.xcframework"
def _remove_if_exists(self, context):
if not os.path.exists(self.final_path):
if not self.final_path.exists():
return
shutil.rmtree(self.final_path)
@ -64,8 +63,8 @@ class AppleXCFramework(Dependency):
static_ars = []
cfg = ConfigInfo.get_config(target)
lib_dir = pj(cfg.buildEnv.install_dir, "lib")
static_ars = [str(f) for f in Path(lib_dir).glob("*.a")]
lib_dir = cfg.buildEnv.install_dir / "lib"
static_ars = [str(f) for f in lib_dir.glob("*.a")]
# create merged.a from all *.a in install_dir/lib
command = ["libtool", "-static", "-o", "merged.a", *static_ars]
@ -73,7 +72,7 @@ class AppleXCFramework(Dependency):
# will be included in xcframework
if target in self.ios_subconfigs:
xcf_libs.append(pj(lib_dir, "merged.a"))
xcf_libs.append(lib_dir / "merged.a")
return xcf_libs
@ -82,12 +81,12 @@ class AppleXCFramework(Dependency):
libs = []
for target in configs:
cfg = ConfigInfo.get_config(target)
libs.append(pj(cfg.buildEnv.install_dir, "lib", "merged.a"))
libs.append(cfg.buildEnv.install_dir / "lib" / "merged.a")
fat_dir = pj(self.buildEnv.build_dir, folder_name)
os.makedirs(fat_dir, exist_ok=True)
fat_dir = self.buildEnv.build_dir / folder_name
fad_dir.mkdir(parents=True, exist_ok=True)
output_merged = pj(fat_dir, "merged.a")
output_merged = fat_dir / "merged.a"
command = ["lipo", "-create", "-output", output_merged, *libs]
run_command(command, self.buildEnv.build_dir, context)
@ -102,7 +101,7 @@ class AppleXCFramework(Dependency):
"-library",
lib,
"-headers",
pj(ref_conf.buildEnv.install_dir, "include"),
ref_conf.buildEnv.install_dir / "include",
]
command += ["-output", self.final_path]
run_command(command, self.buildEnv.build_dir, context)

64
kiwixbuild/dependencies/base.py
View File

@ -1,10 +1,10 @@
import subprocess
import os
import shutil
import time
from pathlib import Path
from os import environ
from kiwixbuild.utils import (
pj,
Context,
SkipCommand,
WarningMessage,
@ -17,7 +17,7 @@ from kiwixbuild.utils import (
from kiwixbuild.versions import main_project_versions, base_deps_versions
from kiwixbuild._global import neutralEnv, option, get_target_step
SCRIPT_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
SCRIPT_DIR = Path(__file__).resolve().parent.parent
class _MetaDependency(type):
@ -71,23 +71,23 @@ class Source:
return self.target.full_name()
@property
def source_path(self):
return pj(neutralEnv("source_dir"), self.source_dir)
def source_path(self) -> Path:
return neutralEnv("source_dir") / self.source_dir
@property
def _log_dir(self):
def _log_dir(self) -> Path:
return neutralEnv("log_dir")
def _patch(self, context):
context.try_skip(self.source_path)
for p in self.patches:
patch_file_path = pj(SCRIPT_DIR, "patches", p)
patch_file_path = SCRIPT_DIR / "patches" / p
patch_command = [*neutralEnv("patch_command"), "-p1", "-i", patch_file_path]
run_command(patch_command, self.source_path, context)
def command(self, name, function, *args):
print(" {} {} : ".format(name, self.name), end="", flush=True)
log = pj(self._log_dir, "cmd_{}_{}.log".format(name, self.name))
log = self._log_dir / "cmd_{}_{}.log".format(name, self.name)
context = Context(name, log, True)
try:
start_time = time.time()
@ -126,8 +126,8 @@ class ReleaseDownload(Source):
return (self.archive,)
@property
def extract_path(self):
return pj(neutralEnv("source_dir"), self.source_dir)
def extract_path(self) -> Path:
return neutralEnv("source_dir") / self.source_dir
def _download(self, context):
context.try_skip(neutralEnv("archive_dir"), self.full_name)
@ -145,11 +145,11 @@ class ReleaseDownload(Source):
def _extract(self, context):
context.try_skip(self.extract_path)
if os.path.exists(self.extract_path):
if self.extract_path.exists():
shutil.rmtree(self.extract_path)
for archive in self.archives:
extract_archive(
pj(neutralEnv("archive_dir"), archive.name),
neutralEnv("archive_dir") / archive.name,
neutralEnv("source_dir"),
topdir=self.archive_top_dir,
name=self.source_dir,
@ -180,8 +180,8 @@ class GitClone(Source):
return self.git_dir
@property
def git_path(self):
return pj(neutralEnv("source_dir"), self.source_dir)
def git_path(self) -> Path:
return neutralEnv("source_dir") / self.source_dir
@property
def git_ref(self):
@ -228,7 +228,7 @@ class GitClone(Source):
raise WarningMessage("Cannot update, please check log for information")
def prepare(self):
if not os.path.exists(self.git_path):
if not self.git_path.exists():
self.command("gitinit", self._git_init)
else:
self.command("gitupdate", self._git_update)
@ -254,23 +254,23 @@ class Builder:
return self.target.name
@property
def source_path(self):
def source_path(self) -> Path:
base_source_path = self.source.source_path
if self.subsource_dir:
return pj(base_source_path, self.subsource_dir)
return base_source_path / self.subsource_dir
return base_source_path
@property
def build_path(self):
return pj(self.buildEnv.build_dir, self.target.full_name())
def build_path(self) -> Path:
return self.buildEnv.build_dir / self.target.full_name()
@property
def _log_dir(self):
def _log_dir(self) -> Path:
return self.buildEnv.log_dir
def command(self, name, function, *args):
print(" {} {} : ".format(name, self.name), end="", flush=True)
log = pj(self._log_dir, "cmd_{}_{}.log".format(name, self.name))
log = self._log_dir / "cmd_{}_{}.log".format(name, self.name)
context = Context(name, log, self.target.force_native_build)
if self.target.force_build:
context.no_skip = True
@ -357,8 +357,8 @@ class TcCopyBuilder(Builder):
src_subdir = None
@property
def build_path(self):
return pj(self.buildEnv.toolchain_dir, self.target.full_name())
def build_path(self) -> Path:
return self.buildEnv.toolchain_dir / self.target.full_name()
def build(self):
self.command("copy", self._copy)
@ -366,7 +366,7 @@ class TcCopyBuilder(Builder):
def _copy(self, context):
context.try_skip(self.build_path)
if self.src_subdir:
source_path = pj(self.source_path, self.src_subdir)
source_path = self.source_path / self.src_subdir
else:
source_path = self.source_path
copy_tree(source_path, self.build_path)
@ -406,7 +406,7 @@ class MakeBuilder(Builder):
if not self.target.force_native_build:
yield from self.buildEnv.configInfo.configure_options
yield from ("--prefix", self.buildEnv.install_dir)
yield from ("--libdir", pj(self.buildEnv.install_dir, self.buildEnv.libprefix))
yield from ("--libdir", self.buildEnv.install_dir / self.buildEnv.libprefix)
def set_configure_env(self, env):
dep_conf_env = self.configure_env
@ -423,7 +423,7 @@ class MakeBuilder(Builder):
context.try_skip(self.build_path)
command = [
*self.buildEnv.configure_wrapper,
pj(self.source_path, self.configure_script),
self.source_path / self.configure_script,
*self.all_configure_options,
]
env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
@ -494,10 +494,10 @@ class QMakeBuilder(MakeBuilder):
@property
def env_options(self):
if "QMAKE_CC" in os.environ:
yield f"QMAKE_CC={os.environ['QMAKE_CC']}"
if "QMAKE_CXX" in os.environ:
yield f"QMAKE_CXX={os.environ['QMAKE_CXX']}"
if "QMAKE_CC" in environ:
yield f"QMAKE_CC={environ['QMAKE_CC']}"
if "QMAKE_CXX" in environ:
yield f"QMAKE_CXX={environ['QMAKE_CXX']}"
def _configure(self, context):
context.try_skip(self.build_path)
@ -545,9 +545,9 @@ class MesonBuilder(Builder):
def _configure(self, context):
context.try_skip(self.build_path)
if os.path.exists(self.build_path):
if self.build_path.exists():
shutil.rmtree(self.build_path)
os.makedirs(self.build_path)
self.build_path.mkdir(parents=True)
cross_options = []
if not self.target.force_native_build and self.buildEnv.meson_crossfile:
cross_options += ["--cross-file", self.buildEnv.meson_crossfile]

29
kiwixbuild/dependencies/icu4c.py
View File

@ -1,6 +1,7 @@
from .base import Dependency, ReleaseDownload, MakeBuilder, MesonBuilder
from kiwixbuild.utils import pj, Remotefile, extract_archive
from pathlib import Path
from kiwixbuild.utils import Remotefile, extract_archive
from kiwixbuild._global import get_target_step, neutralEnv
import os, shutil
import fileinput
@ -31,25 +32,25 @@ class Icu(Dependency):
def _extract(self, context):
context.try_skip(self.extract_path)
if os.path.exists(self.extract_path):
if self.extract_path.exists():
shutil.rmtree(self.extract_path)
extract_archive(
pj(neutralEnv("archive_dir"), self.archive_src.name),
neutralEnv("archive_dir") / self.archive_src.name,
neutralEnv("source_dir"),
topdir=None,
name=self.source_dir,
)
shutil.rmtree(
pj(neutralEnv("source_dir"), self.source_dir, "source", "data")
neutralEnv("source_dir") / self.source_dir / "source" / "data"
)
extract_archive(
pj(neutralEnv("archive_dir"), self.archive_data.name),
pj(neutralEnv("source_dir"), self.source_dir, "source"),
neutralEnv("archive_dir") / self.archive_data.name,
neutralEnv("source_dir") / self.source_dir / "source",
topdir="data",
name="data",
)
extract_archive(
pj(neutralEnv("archive_dir"), self.meson_patch.name),
neutralEnv("archive_dir") / self.meson_patch.name,
neutralEnv("source_dir"),
topdir="icu",
name=self.source_dir,
@ -69,9 +70,8 @@ class Icu(Dependency):
class Builder(MesonBuilder):
def set_env(self, env):
env["ICU_DATA_FILTER_FILE"] = pj(
os.path.dirname(os.path.realpath(__file__)),
"icu4c_data_filter.json",
env["ICU_DATA_FILTER_FILE"] = (
Path(__file__).resolve().parent / "icu4c_data_filter.json"
)
else:
@ -105,18 +105,15 @@ class Icu(Dependency):
yield "--with-data-packaging=archive"
def set_env(self, env):
env["ICU_DATA_FILTER_FILE"] = pj(
os.path.dirname(os.path.realpath(__file__)),
"icu4c_data_filter.json",
env["ICU_DATA_FILTER_FILE"] = (
Path(__file__).resolve().parent / "icu4c_data_filter.json"
)
def _post_configure_script(self, context):
if self.buildEnv.configInfo.build != "wasm":
context.skip()
context.try_skip(self.build_path)
for line in fileinput.input(
pj(self.build_path, "Makefile"), inplace=True
):
for line in fileinput.input(self.build_path / "Makefile", inplace=True):
if line == "#DATASUBDIR = data\n":
print("DATASUBDIR = data")
else:

20
kiwixbuild/dependencies/ios_fat_lib.py
View File

@ -1,7 +1,7 @@
import os
from kiwixbuild.configs import ConfigInfo
from kiwixbuild.utils import pj, copy_tree, run_command
from kiwixbuild.utils import copy_tree, run_command
from kiwixbuild._global import option
from .base import Dependency, NoopSource, Builder as BaseBuilder
@ -19,29 +19,29 @@ class IOSFatLib(Dependency):
def _copy_headers(self, context):
plt = ConfigInfo.get_config("iOS_{}".format(option("ios_arch")[0]))
include_src = pj(plt.buildEnv.install_dir, "include")
include_dst = pj(self.buildEnv.install_dir, "include")
include_src = plt.buildEnv.install_dir / "include"
include_dst = self.buildEnv.install_dir / "include"
copy_tree(include_src, include_dst)
def _merge_libs(self, context):
lib_dirs = []
for arch in option("ios_arch"):
plt = ConfigInfo.get_config("iOS_{}".format(arch))
lib_dirs.append(pj(plt.buildEnv.install_dir, "lib"))
lib_dirs.append(plt.buildEnv.install_dir / "lib")
libs = []
for f in os.listdir(lib_dirs[0]):
if os.path.islink(pj(lib_dirs[0], f)):
for f in lib_dirs[0].iterdir():
if f.is_symlink():
continue
if f.endswith(".a") or f.endswith(".dylib"):
if f.suffix in (".a", ".dylib"):
libs.append(f)
os.makedirs(pj(self.buildEnv.install_dir, "lib"), exist_ok=True)
(self.buildEnv.install_dir / "lib").mkdir(parents=True, exist_ok=True)
for l in libs:
command = [
"lipo",
"-create",
*[pj(d, l) for d in lib_dirs],
*[d / l for d in lib_dirs],
"-output",
pj(self.buildEnv.install_dir, "lib", l),
self.buildEnv.install_dir / "lib" / l,
]
run_command(command, self.buildEnv.install_dir, context)

4
kiwixbuild/dependencies/libmagic.py
View File

@ -4,7 +4,7 @@ from .base import (
MakeBuilder,
)
from kiwixbuild.utils import Remotefile, pj, run_command
from kiwixbuild.utils import Remotefile, run_command
from kiwixbuild._global import get_target_step
@ -44,5 +44,5 @@ class LibMagic(Dependency):
cross_comp_flags=True, cross_compilers=True, cross_path=True
)
libmagic_native_builder = get_target_step("libmagic", "native_static")
env["PATH"].insert(0, pj(libmagic_native_builder.build_path, "src"))
env["PATH"].insert(0, libmagic_native_builder.build_path / "src")
run_command(command, self.build_path, context, env=env)

6
kiwixbuild/dependencies/mustache.py
View File

@ -1,6 +1,6 @@
from .base import Dependency, ReleaseDownload, Builder as BaseBuilder
from kiwixbuild.utils import Remotefile, pj
from kiwixbuild.utils import Remotefile
from shutil import copy2
@ -21,8 +21,8 @@ class Mustache(Dependency):
def _copy_header(self, context):
context.try_skip(self.build_path)
copy2(
pj(self.source_path, "mustache.hpp"),
pj(self.buildEnv.install_dir, "include"),
self.source_path / "mustache.hpp",
self.buildEnv.install_dir / "include",
)
def set_flatpak_buildsystem(self, module):

4
kiwixbuild/dependencies/qt.py
View File

@ -1,6 +1,6 @@
from .base import Dependency, ReleaseDownload, MakeBuilder, QMakeBuilder
from kiwixbuild.utils import Remotefile, pj
from kiwixbuild.utils import Remotefile
class Qt(Dependency):
@ -32,7 +32,7 @@ class Qt(Dependency):
yield from ("-prefix", self.buildEnv.install_dir)
yield from (
"-libdir",
pj(self.buildEnv.install_dir, self.buildEnv.libprefix),
self.buildEnv.install_dir / self.buildEnv.libprefix,
)
@property

33
kiwixbuild/dependencies/tc_android_ndk.py
View File

@ -1,7 +1,6 @@
import os
from pathlib import Path
from .base import Dependency, ReleaseDownload, Builder
from kiwixbuild.utils import Remotefile, add_execution_right, run_command, pj
from kiwixbuild.utils import Remotefile, add_execution_right, run_command
class android_ndk(Dependency):
@ -24,8 +23,8 @@ class android_ndk(Dependency):
class Builder(Builder):
@property
def install_path(self):
return pj(self.buildEnv.toolchain_dir, self.target.full_name())
def install_path(self) -> Path:
return self.buildEnv.toolchain_dir / self.target.full_name()
@property
def api(self):
@ -45,7 +44,7 @@ class android_ndk(Dependency):
def _build_toolchain(self, context):
context.try_skip(self.build_path)
script = pj(self.source_path, "build/tools/make_standalone_toolchain.py")
script = self.source_path / "build/tools/make_standalone_toolchain.py"
add_execution_right(script)
command = [
script,
@ -62,23 +61,21 @@ class android_ndk(Dependency):
def _fix_permission_right(self, context):
context.try_skip(self.build_path)
bin_dirs = [
pj(self.install_path, "bin"),
pj(self.install_path, self.arch_full, "bin"),
pj(
self.install_path,
"libexec",
"gcc",
self.arch_full,
self.target.gccver,
),
self.install_path / "bin",
self.install_pat / self.arch_full / "bin",
self.install_path
/ "libexec"
/ "gcc"
/ self.arch_full
/ self.target.gccver,
]
for root, dirs, files in os.walk(self.install_path):
for root, dirs, files in self.install_path.walk():
if not root in bin_dirs:
continue
for file_ in files:
file_path = pj(root, file_)
if os.path.islink(file_path):
file_path = root / file_
if file_path.is_symlink():
continue
add_execution_right(file_path)

7
kiwixbuild/dependencies/tc_emsdk.py
View File

@ -1,5 +1,6 @@
from pathlib import Path
from .base import Dependency, ReleaseDownload, Builder
from kiwixbuild.utils import pj, Remotefile, run_command, copy_tree
from kiwixbuild.utils import Remotefile, run_command, copy_tree
class emsdk(Dependency):
@ -20,8 +21,8 @@ class emsdk(Dependency):
class Builder(Builder):
@property
def install_path(self):
return pj(self.buildEnv.toolchain_dir, self.target.full_name())
def install_path(self) -> Path:
return self.buildEnv.toolchain_dir / self.target.full_name()
def _copy_source(self, context):
context.try_skip(self.build_path)

4
kiwixbuild/dependencies/xapian.py
View File

@ -1,6 +1,6 @@
from .base import Dependency, ReleaseDownload, MakeBuilder
from kiwixbuild.utils import Remotefile, pj
from kiwixbuild.utils import Remotefile
from kiwixbuild._global import neutralEnv
@ -17,7 +17,7 @@ class Xapian(Dependency):
@property
def configure_options(self):
if neutralEnv("distname") == "Windows":
compile_script = pj(self.source_path, "compile")
compile_script = self.source_path / "compile"
return [
'CC="cl -nologo"',
'CXX=f"{compile_script} cl -nologo"',

23
kiwixbuild/flatpak_builder.py
View File

@ -206,9 +206,8 @@ class FlatpakBuilder:
m["sources"] = temp
manifest["modules"] = modules
manifest_name = "{}.json".format(MANIFEST["app-id"])
manifest_path = pj(self.config.buildEnv.build_dir, manifest_name)
with open(manifest_path, "w") as f:
f.write(json.dumps(manifest, indent=4))
manifest_path = self.config.buildEnv.build_dir / manifest_name
manifest_path.write_text(json.dumps(manifest, indent=4))
def copy_patches(self):
sourceDefs = (tDef for tDef in target_steps() if tDef[0] == "source")
@ -217,15 +216,15 @@ class FlatpakBuilder:
if not hasattr(source, "patches"):
continue
for p in source.patches:
path = pj(SCRIPT_DIR, "patches", p)
os.makedirs(
pj(self.config.buildEnv.build_dir, "patches"), exist_ok=True
path = SCRIPT_DIR / "patches" / p
(self.config.buildEnv.build_dir / "patches").mkdir(
parents=True, exist_ok=True
)
dest = pj(self.config.buildEnv.build_dir, "patches", p)
dest = self.config.buildEnv.build_dir / "patches" / p
copyfile(path, dest)
def build(self):
log = pj(self.config.buildEnv.log_dir, "cmd_build_flatpak.log")
log = self.config.buildEnv.log_dir / "cmd_build_flatpak.log"
context = Context("build", log, False)
command = [
"flatpak-builder",
@ -247,12 +246,11 @@ class FlatpakBuilder:
)
context._finalise()
except subprocess.CalledProcessError:
with open(log, "r") as f:
print(f.read())
print(log.read_text())
raise StopBuild()
def bundle(self):
log = pj(self.config.buildEnv.log_dir, "cmd_bundle_flatpak.log")
log = self.config.buildEnv.log_dir / "cmd_bundle_flatpak.log"
context = Context("bundle", log, False)
app_id = MANIFEST["app-id"]
command = ["flatpak", "build-bundle", "repo", f"{app_id}.flatpak", app_id]
@ -265,8 +263,7 @@ class FlatpakBuilder:
)
context._finalise()
except subprocess.CalledProcessError:
with open(log, "r") as f:
print(f.read())
print(log.read_text())
raise StopBuild()
def _get_packages(self):

106
kiwixbuild/utils.py
View File

@ -1,4 +1,3 @@
import os.path
import hashlib
import tarfile, zipfile
import tempfile
@ -9,15 +8,12 @@ import urllib.error
import ssl
import subprocess
import re
from pathlib import Path
from collections import namedtuple, defaultdict
from kiwixbuild._global import neutralEnv, option
def pj(*args):
return os.path.normpath(os.path.join(*args))
COLORS = {
"OK": "\033[92m",
"WARNING": "\033[93m",
@ -46,7 +42,7 @@ def xrun_find(name):
regex_space = re.compile(r"((?<!\\) )")
def escape_path(path):
def escape_path(path: Path):
path = str(path)
return regex_space.sub(r"\ ", path)
@ -87,7 +83,7 @@ class PathArray(list):
super().__init__(value.split(self.separator))
def __str__(self):
return self.separator.join(self)
return self.separator.join((str(v) for v in self))
def remove_duplicates(iterable, key_function=None):
@ -102,12 +98,12 @@ def remove_duplicates(iterable, key_function=None):
yield elem
def get_sha256(path):
def get_sha256(path: Path):
progress_chars = "/-\\|"
current = 0
batch_size = 1024 * 8
sha256 = hashlib.sha256()
with open(path, "br") as f:
with path.open("br") as f:
while True:
batch = f.read(batch_size)
if not batch:
@ -130,30 +126,37 @@ def print_progress(progress):
print(text, end="")
def add_execution_right(file_path):
def add_execution_right(file_path: Path):
current_permissions = stat.S_IMODE(os.lstat(file_path).st_mode)
os.chmod(file_path, current_permissions | stat.S_IXUSR)
file_path.chmod(current_permissions | stat.S_IXUSR)
def copy_tree(src, dst, post_copy_function=None):
os.makedirs(dst, exist_ok=True)
for root, dirs, files in os.walk(src):
r = os.path.relpath(root, src)
dstdir = pj(dst, r)
os.makedirs(dstdir, exist_ok=True)
def copy_tree(src: Path, dst: Path, post_copy_function=None):
dst.mkdir(parents=True, exist_ok=True)
for root, dirs, files in src.walk():
r = root.relative_to(src)
dstdir = dst / r
dstdir.mkdir(exist_ok=True)
for f in files:
dstfile = pj(dstdir, f)
shutil.copy2(pj(root, f), dstfile, follow_symlinks=False)
dstfile = dstdir / f
shutil.copy2(root / f, dstfile, follow_symlinks=False)
if post_copy_function is not None:
post_copy_function(dstfile)
def download_remote(what, where):
file_path = pj(where, what.name)
if os.path.exists(file_path):
class Remotefile(namedtuple("Remotefile", ("name", "sha256", "url"))):
def __new__(cls, name, sha256, url=None):
if url is None:
url = REMOTE_PREFIX + name
return super().__new__(cls, name, sha256, url)
def download_remote(what: Remotefile, where: Path):
file_path = where / what.name
if file_path.exists():
if what.sha256 == get_sha256(file_path):
raise SkipCommand()
os.remove(file_path)
file_path.unlink()
if option("no_cert_check"):
context = ssl.create_default_context()
@ -165,8 +168,8 @@ def download_remote(what, where):
extra_args = {"context": context} if sys.version_info >= (3, 4, 3) else {}
progress_chars = "/-\\|"
try:
with urllib.request.urlopen(what.url, **extra_args) as resource, open(
file_path, "wb"
with urllib.request.urlopen(what.url, **extra_args) as resource, file_path.open(
"wb"
) as file:
tsize = resource.info().get("Content-Length", None)
if tsize is not None:
@ -190,7 +193,7 @@ def download_remote(what, where):
if not what.sha256:
print("Sha256 for {} not set, do no verify download".format(what.name))
elif what.sha256 != get_sha256(file_path):
os.remove(file_path)
file_path.unlink()
raise StopBuild("Sha 256 doesn't correspond")
@ -218,13 +221,6 @@ class StopBuild(BaseCommandResult):
pass
class Remotefile(namedtuple("Remotefile", ("name", "sha256", "url"))):
def __new__(cls, name, sha256, url=None):
if url is None:
url = REMOTE_PREFIX + name
return super().__new__(cls, name, sha256, url)
class Context:
def __init__(self, command_name, log_file, force_native_build):
self.command_name = command_name
@ -236,24 +232,28 @@ class Context:
def skip(self, msg=""):
raise SkipCommand(msg)
def try_skip(self, path, extra_name=""):
def try_skip(self, path: Path, extra_name=""):
if self.no_skip:
return
if extra_name:
extra_name = "_{}".format(extra_name)
self.autoskip_file = pj(path, ".{}{}_ok".format(self.command_name, extra_name))
if os.path.exists(self.autoskip_file):
self.autoskip_file = path / ".{}{}_ok".format(self.command_name, extra_name)
if self.autoskip_file.exists():
raise SkipCommand()
def _finalise(self):
if self.autoskip_file is not None:
os.makedirs(os.path.dirname(self.autoskip_file), exist_ok=True)
with open(self.autoskip_file, "w"):
pass
self.autoskip_file.parent.mkdir(parents=True, exist_ok=True)
self.autoskip_file.touch()
def extract_archive(archive_path, dest_dir, topdir=None, name=None):
is_zip_archive = archive_path.endswith(".zip")
def extract_archive(archive_path: Path, dest_dir: Path, topdir=None, name=None):
"""Extract an archive to dest_dir.
This archive can be a zip or a tar archive.
If topdir is given, only this directory (and sub files/dirs) will be extracted.
If name is given, the topdir will be extracted under this name.
"""
is_zip_archive = archive_path.suffix == ".zip"
archive = None
try:
if is_zip_archive:
@ -283,40 +283,39 @@ def extract_archive(archive_path, dest_dir, topdir=None, name=None):
members_to_extract = [
m for m in members if getname(m).startswith(topdir + "/")
]
os.makedirs(dest_dir, exist_ok=True)
dest_dir.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(
prefix=os.path.basename(archive_path), dir=dest_dir
prefix=archive_path.name, dir=dest_dir
) as tmpdir:
tmpdir_path = Path(tmpdir)
if is_zip_archive:
_members_to_extract = [getname(m) for m in members_to_extract]
else:
_members_to_extract = members_to_extract
archive.extractall(path=tmpdir, members=_members_to_extract)
archive.extractall(path=tmpdir_path, members=_members_to_extract)
if is_zip_archive:
for member in members_to_extract:
if isdir(member):
continue
perm = (member.external_attr >> 16) & 0x1FF
if perm:
os.chmod(pj(tmpdir, getname(member)), perm)
(tmpdir_path / getname(member)).chmod(perm)
name = name or topdir
shutil.copytree(
pj(tmpdir, topdir),
pj(dest_dir, name),
tmpdir_path / topdir,
dest_dir / name,
symlinks=True,
dirs_exist_ok=True,
)
# Be sure that all directory in tmpdir are writable to allow correct suppersion of it
for root, dirs, _files in os.walk(tmpdir):
for root, dirs, _files in tmpdir_path.walk():
for d in dirs:
os.chmod(
pj(root, d), stat.S_IWRITE | stat.S_IREAD | stat.S_IEXEC
)
(root / d).chmod(stat.S_IWRITE | stat.S_IREAD | stat.S_IEXEC)
else:
if name:
dest_dir = pj(dest_dir, name)
os.makedirs(dest_dir)
dest_dir = dest_dir / name
dest_dir.mkdir(parents=True)
archive.extractall(path=dest_dir)
finally:
if archive is not None:
@ -328,6 +327,7 @@ def run_command(command, cwd, context, *, env=None, input=None):
if env is None:
env = DefaultEnv()
log = None
command = [str(v) for v in command]
try:
if not option("verbose"):
log = open(context.log_file, "w")