Make the dependency responsible to set the compilation env.

Instead of having the run_command function setting the
env from the buildEnv, this is the dependency that create the
env and then pass it to the run_command function.

This way, each dependency will be able to set a specific env.
This commit is contained in:
Matthieu Gautier 2020-02-20 16:37:42 +01:00
parent e27ede80cc
commit 7b6c79482a
17 changed files with 213 additions and 118 deletions

View File

@ -115,23 +115,12 @@ class BuildEnv:
return 'lib64'
return 'lib'
def _set_env(self, env, cross_compile_env, cross_compile_compiler, cross_compile_path):
if env is None:
env = Defaultdict(str, os.environ)
def get_env(self, *, cross_comp_flags, cross_compilers, cross_path):
env = self.platformInfo.get_env()
pkgconfig_path = pj(self.install_dir, self.libprefix, 'pkgconfig')
env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path])
# Add ccache path
for p in ('/usr/lib/ccache', '/usr/lib64/ccache'):
if os.path.isdir(p):
ccache_path = [p]
break
else:
ccache_path = []
env['PATH'] = ':'.join([pj(self.install_dir, 'bin')] +
ccache_path +
[env['PATH']])
env['PATH'] = ':'.join([pj(self.install_dir, 'bin'), env['PATH']])
env['LD_LIBRARY_PATH'] = ':'.join([env['LD_LIBRARY_PATH'],
pj(self.install_dir, 'lib'),
@ -147,16 +136,10 @@ class BuildEnv:
'-L'+pj(self.install_dir, self.libprefix),
env['LDFLAGS']])
if cross_compile_env:
for k, v in self.cross_config.get('env', {}).items():
if k.startswith('_format_'):
v = v.format(**self.cross_config)
k = k[8:]
env[k] = v
self.platformInfo.set_env(env)
if cross_compile_compiler:
if cross_comp_flags:
self.platformInfo.set_comp_flags(env)
if cross_compilers:
self.platformInfo.set_compiler(env)
if cross_compile_path:
bin_dirs = self.platformInfo.get_bind_dir()
env['PATH'] = ':'.join(bin_dirs + [env['PATH']])
if cross_path:
env['PATH'] = ':'.join(self.platformInfo.get_bin_dir() + [env['PATH']])
return env

View File

@ -52,8 +52,8 @@ class android_ndk(Dependency):
api=self.api,
install_dir=self.install_path
)
context.force_native_build = True
run_command(command, self.build_path, context, buildEnv=self.buildEnv)
env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False)
run_command(command, self.build_path, context, env=env)
def _fix_permission_right(self, context):
context.try_skip(self.build_path)

View File

@ -63,7 +63,6 @@ class Source:
def _patch(self, context):
context.try_skip(self.source_path)
context.force_native_build = True
for p in self.patches:
with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input:
run_command("patch -p1", self.source_path, context, input=patch_input.read())
@ -312,6 +311,18 @@ class MakeBuilder(Builder):
)
return option
def set_configure_env(self, env):
dep_conf_env = self.configure_env
if not dep_conf_env:
return
for k in list(dep_conf_env):
if k.startswith('_format_'):
v = dep_conf_env.pop(k)
v = v.format(buildEnv=self.buildEnv, env=env)
dep_conf_env[k[8:]] = v
env.update(dep_conf_env)
def _configure(self, context):
context.try_skip(self.build_path)
command = "{configure_script} {configure_option}"
@ -319,19 +330,9 @@ class MakeBuilder(Builder):
configure_script=pj(self.source_path, self.configure_script),
configure_option=self.all_configure_option
)
env = Defaultdict(str, os.environ)
if self.buildEnv.platformInfo.static:
env['CFLAGS'] = env['CFLAGS'] + ' -fPIC'
env['CXXFLAGS'] = env['CXXFLAGS'] + ' -fPIC'
dep_conf_env = self.configure_env
if dep_conf_env:
for k in list(dep_conf_env):
if k.startswith('_format_'):
v = dep_conf_env.pop(k)
v = v.format(buildEnv=self.buildEnv, env=env)
dep_conf_env[k[8:]] = v
env.update(dep_conf_env)
run_command(command, self.build_path, context, buildEnv=self.buildEnv, env=env)
env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
self.set_configure_env(env)
run_command(command, self.build_path, context, env=env)
def _compile(self, context):
context.try_skip(self.build_path)
@ -339,7 +340,8 @@ class MakeBuilder(Builder):
make_target=self.make_target,
make_option=self.make_option
)
run_command(command, self.build_path, context, buildEnv=self.buildEnv)
env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
run_command(command, self.build_path, context, env=env)
def _install(self, context):
context.try_skip(self.build_path)
@ -347,12 +349,14 @@ class MakeBuilder(Builder):
make_install_target=self.make_install_target,
make_option=self.make_option
)
run_command(command, self.build_path, context, buildEnv=self.buildEnv)
env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
run_command(command, self.build_path, context, env=env)
def _make_dist(self, context):
context.try_skip(self.build_path)
command = "make dist"
run_command(command, self.build_path, context, buildEnv=self.buildEnv)
env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
run_command(command, self.build_path, context, env=env)
class CMakeBuilder(MakeBuilder):
@ -376,18 +380,9 @@ class CMakeBuilder(MakeBuilder):
source_path=self.source_path,
cross_option=cross_option
)
env = Defaultdict(str, os.environ)
if self.buildEnv.platformInfo.static:
env['CFLAGS'] = env['CFLAGS'] + ' -fPIC'
env['CXXFLAGS'] = env['CXXFLAGS'] + ' -fPIC'
if self.configure_env:
for k in self.configure_env:
if k.startswith('_format_'):
v = self.configure_env.pop(k)
v = v.format(buildEnv=self.buildEnv, env=env)
self.configure_env[k[8:]] = v
env.update(self.configure_env)
run_command(command, self.build_path, context, env=env, buildEnv=self.buildEnv, cross_env_only=True)
env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=False, cross_path=True)
self.set_configure_env(env)
run_command(command, self.build_path, context, env=env)
def set_flatpak_buildsystem(self, module):
super().set_flatpak_buildsystem( module)
@ -420,15 +415,9 @@ class QMakeBuilder(MakeBuilder):
source_path=self.source_path,
cross_option=cross_option
)
run_command(command, self.build_path, context, buildEnv=self.buildEnv, cross_env_only=True)
def _install(self, context):
context.try_skip(self.build_path)
command = "make {make_install_target} {make_option}".format(
make_install_target=self.make_install_target,
make_option=self.make_option
)
run_command(command, self.build_path, context, buildEnv=self.buildEnv)
env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=False, cross_path=True)
self.set_configure_env(env)
run_command(command, self.build_path, context, env=env)
def _make_dist(self, context):
command = "git archive -o {build_dir}/{name}.tar.gz --prefix={name}/ HEAD"
@ -436,7 +425,7 @@ class QMakeBuilder(MakeBuilder):
build_dir = self.build_path,
name = self.target.full_name()
)
run_command(command, self.source_path, context, buildEnv=self.buildEnv)
run_command(command, self.source_path, context)
@ -484,11 +473,13 @@ class MesonBuilder(Builder):
buildEnv=self.buildEnv,
cross_option=cross_option
)
run_command(command, self.source_path, context, buildEnv=self.buildEnv, cross_env_only=True)
env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True)
run_command(command, self.source_path, context, env=env)
def _compile(self, context):
command = "{} -v".format(neutralEnv('ninja_command'))
run_command(command, self.build_path, context, buildEnv=self.buildEnv)
env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True)
run_command(command, self.build_path, context, env=env)
def _test(self, context):
if ( self.buildEnv.platformInfo.build == 'android'
@ -497,15 +488,18 @@ class MesonBuilder(Builder):
):
raise SkipCommand()
command = "{} --verbose {}".format(neutralEnv('mesontest_command'), self.test_option)
run_command(command, self.build_path, context, buildEnv=self.buildEnv)
env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True)
run_command(command, self.build_path, context, env=env)
def _install(self, context):
command = "{} -v install".format(neutralEnv('ninja_command'))
run_command(command, self.build_path, context, buildEnv=self.buildEnv)
env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True)
run_command(command, self.build_path, context, env=env)
def _make_dist(self, context):
command = "{} -v dist".format(neutralEnv('ninja_command'))
run_command(command, self.build_path, context, buildEnv=self.buildEnv)
env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True)
run_command(command, self.build_path, context, env=env)
class GradleBuilder(Builder):
@ -530,4 +524,5 @@ class GradleBuilder(Builder):
command = command.format(
gradle_target=self.gradle_target,
gradle_option=self.gradle_option)
run_command(command, self.build_path, context, buildEnv=self.buildEnv)
env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
run_command(command, self.build_path, context, env=env)

View File

@ -18,7 +18,8 @@ class org_kde(Dependency):
remote_name = 'flathub',
remote_url = 'https://flathub.org/repo/flathub.flatpakrepo'
)
run_command(command, self.buildEnv.build_dir, context, buildEnv=self.buildEnv)
env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False)
run_command(command, self.buildEnv.build_dir, context, env=env)
def _install_sdk(self, context):
command = "flatpak --user install -y {remote_name} {name}.Sdk//{version} {name}.Platform//{version}"
@ -27,7 +28,8 @@ class org_kde(Dependency):
name = self.target.name,
version = self.target.version()
)
run_command(command, self.buildEnv.build_dir, context, buildEnv=self.buildEnv)
env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False)
run_command(command, self.buildEnv.build_dir, context, env=env)
def build(self):
self.command('setup_remote', self._setup_remote)

View File

@ -36,7 +36,7 @@ class LibMagic(Dependency):
make_target=self.make_target,
make_option=self.make_option
)
env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
libmagic_native_builder = get_target_step('libmagic', 'native_static')
env = Defaultdict(str, os.environ)
env['PATH'] = ':'.join([pj(libmagic_native_builder.build_path, 'src'), env['PATH']])
run_command(command, self.build_path, context, buildEnv=self.buildEnv, env=env)
run_command(command, self.build_path, context, env=env)

View File

@ -224,7 +224,7 @@ class FlatpakBuilder:
command = "flatpak-builder --user --ccache --force-clean --repo=repo builddir {id}.json"
command = command.format(id = MANIFEST['app-id'])
try:
run_command(command, self.platform.buildEnv.build_dir, context, self.platform.buildEnv)
run_command(command, self.platform.buildEnv.build_dir, context, env=self.platform.get_env())
context._finalise()
except subprocess.CalledProcessError:
with open(log, 'r') as f:
@ -237,7 +237,7 @@ class FlatpakBuilder:
command = "flatpak build-bundle repo {id}.flatpak {id}"
command = command.format(id = MANIFEST['app-id'])
try:
run_command(command, self.platform.buildEnv.build_dir, context, self.platform.buildEnv)
run_command(command, self.platform.buildEnv.build_dir, context, env=self.platform.get_env())
context._finalise()
except subprocess.CalledProcessError:
with open(log, 'r') as f:

View File

@ -56,24 +56,25 @@ class AndroidPlatformInfo(PlatformInfo):
'cpu': self.cpu,
'endian': 'little',
'abi': self.abi
},
}
}
def get_env(self):
env = super().get_env()
root_path = pj(self.install_path, 'sysroot')
env['PKG_CONFIG_LIBDIR'] = pj(root_path, 'lib', 'pkgconfig')
env['NDK_DEBUG'] = '0'
return env
def get_bin_dir(self):
return [pj(self.install_path, 'bin')]
def set_env(self, env):
def set_comp_flags(self, env):
super().set_comp_flags(env)
root_path = pj(self.install_path, 'sysroot')
env['PKG_CONFIG_LIBDIR'] = pj(root_path, 'lib', 'pkgconfig')
env['CFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(root_path) + env['CFLAGS']
env['CXXFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(root_path) + env['CXXFLAGS']
env['LDFLAGS'] = '--sysroot={} '.format(root_path) + env['LDFLAGS']
#env['CFLAGS'] = ' -fPIC -D_FILE_OFFSET_BITS=64 -O3 '+env['CFLAGS']
#env['CXXFLAGS'] = (' -D__OPTIMIZE__ -fno-strict-aliasing '
# ' -DU_HAVE_NL_LANGINFO_CODESET=0 '
# '-DU_STATIC_IMPLEMENTATION -O3 '
# '-DU_HAVE_STD_STRING -DU_TIMEZONE=0 ')+env['CXXFLAGS']
env['NDK_DEBUG'] = '0'
def set_compiler(self, env):
binaries = self.binaries()
@ -140,5 +141,7 @@ class Android(MetaPlatformInfo):
def sdk_builder(self):
return get_target_step('android-sdk', 'neutral')
def set_env(self, env):
def get_env(self):
env = super().get_env()
env['ANDROID_HOME'] = self.sdk_builder.install_path
return env

View File

@ -70,16 +70,21 @@ class ArmhfPlatformInfo(PlatformInfo):
def get_bin_dir(self):
return [pj(self.root_path, 'bin')]
def set_env(self, env):
def get_env(self):
env = super().get_env()
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
env['CFLAGS'] = " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS']
env['CXXFLAGS'] = " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
env['QEMU_LD_PREFIX'] = pj(self.root_path, "arm-linux-gnueabihf", "libc")
env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format(
':'.join([
pj(self.root_path, self.arch_full, "lib"),
env['LD_LIBRARY_PATH']
]))
return env
def set_comp_flags(self, env):
super().set_comp_flags(env)
env['CFLAGS'] = " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS']
env['CXXFLAGS'] = " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
def set_compiler(self, env):
for k, v in self.binaries.items():

View File

@ -3,7 +3,7 @@ import os, sys
import subprocess
from kiwixbuild.dependencies import Dependency
from kiwixbuild.utils import pj, remove_duplicates
from kiwixbuild.utils import pj, remove_duplicates, DefaultEnv
from kiwixbuild.buildenv import BuildEnv
from kiwixbuild._global import neutralEnv, option, target_steps
@ -75,15 +75,25 @@ class PlatformInfo(metaclass=_MetaPlatform):
def get_cross_config(self):
return {}
def set_env(self, env):
pass
def get_bind_dir(self):
def get_env(self):
return DefaultEnv()
def get_bin_dir(self):
return []
def set_compiler(self, env):
pass
def set_comp_flags(self, env):
if self.static:
env['CFLAGS'] = env['CFLAGS'] + ' -fPIC'
env['CXXFLAGS'] = env['CXXFLAGS'] + ' -fPIC'
def _gen_crossfile(self, name):
crossfile = pj(self.buildEnv.build_dir, name)
template_file = pj(TEMPLATES_DIR, name)

View File

@ -1,6 +1,5 @@
from .base import PlatformInfo
from kiwixbuild._global import option, neutralEnv
from kiwixbuild.utils import run_command
class FlatpakPlatformInfo(PlatformInfo):
name = 'flatpak'
@ -12,6 +11,8 @@ class FlatpakPlatformInfo(PlatformInfo):
def __str__(self):
return "flatpak"
def set_env(self, env):
def get_env(self):
env = super().get_env()
env['FLATPAK_USER_DIR'] = self.buildEnv.build_dir
return env

View File

@ -41,7 +41,8 @@ class I586PlatformInfo(PlatformInfo):
('PKGCONFIG', 'pkg-config'))
}
def set_env(self, env):
def set_comp_flags(self, env):
super().set_comp_flags(env)
env['CFLAGS'] = "-m32 -march=i586 -mno-sse "+env['CFLAGS']
env['CXXFLAGS'] = "-m32 -march=i586 -mno-sse "+env['CXXFLAGS']
env['LDFLAGS'] = "-m32 -march=i586 -mno-sse "+env['LDFLAGS']

View File

@ -44,14 +44,20 @@ class iOSPlatformInfo(PlatformInfo):
'cpu': self.cpu,
'endian': '',
'abi': ''
},
}
}
def set_env(self, env):
def get_env(self):
env = super().get_env()
env['MACOSX_DEPLOYMENT_TARGET'] = '10.10'
return env
def set_comp_flags(self, env):
super().set_comp_flags(env)
env['CFLAGS'] = " -fembed-bitcode -isysroot {SDKROOT} -arch {arch} -miphoneos-version-min={min_iphoneos_version} ".format(SDKROOT=self.root_path, min_iphoneos_version=self.min_iphoneos_version, arch=self.arch) + env['CFLAGS']
env['CXXFLAGS'] = env['CFLAGS'] + " -stdlib=libc++ -std=c++11 "+env['CXXFLAGS']
env['LDFLAGS'] = " -arch {arch} -isysroot {SDKROOT} ".format(SDKROOT=self.root_path, arch=self.arch)
env['MACOSX_DEPLOYMENT_TARGET'] = "10.10"
def get_bin_dir(self):
return [pj(self.root_path, 'bin')]

View File

@ -38,12 +38,18 @@ class NativeMixed(NativePlatformInfo):
return 'native_mixed', dep
return 'native_static', dep
def set_env(self, env):
super().set_env(env)
def get_env(self):
env = super().get_env()
static_platform = self.get_platform('native_static')
static_buildEnv = static_platform.buildEnv
static_install_dir = static_buildEnv.install_dir
env['CPPFLAGS'] = " ".join(['-I'+pj(static_install_dir, 'include'), env['CPPFLAGS']])
env['PATH'] = ':'.join([pj(static_install_dir, 'bin')] + [env['PATH']])
pkgconfig_path = pj(static_install_dir, static_buildEnv.libprefix, 'pkgconfig')
env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path])
return env
def set_comp_flags(self, env):
super().set_comp_flags(env)
static_platform = self.get_platform('native_static')
static_install_dir = static_platform.buildEnv.install_dir
env['CPPFLAGS'] = " ".join(['-I'+pj(static_install_dir, 'include'), env['CPPFLAGS']])

View File

@ -73,9 +73,11 @@ class Win32PlatformInfo(PlatformInfo):
def get_bin_dir(self):
return [pj(self.root_path, 'bin')]
def set_env(self, env):
def get_env(self):
env = super().get_env()
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
env['LIBS'] = " ".join(self.extra_libs) + " " +env['LIBS']
return env
class Win32Dyn(Win32PlatformInfo):
name = 'win32_dyn'

View File

@ -0,0 +1,88 @@
import subprocess
from .base import PlatformInfo
from kiwixbuild.utils import which, pj
from kiwixbuild._global import neutralEnv
class Win64PlatformInfo(PlatformInfo):
extra_libs = ['-lmingw32', '-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4', '-lmsvcr100', '-liphlpapi', '-lshell32', '-lkernel32']
build = 'win64'
compatible_hosts = ['fedora', 'debian']
arch_full = 'x86_64-w64-mingw32'
def get_cross_config(self):
return {
'exe_wrapper_def': self.exe_wrapper_def,
'binaries': self.binaries,
'root_path': self.root_path,
'extra_libs': self.extra_libs,
'extra_cflags': ['-DWIN32'],
'host_machine': {
'system': 'Windows',
'lsystem': 'windows',
'cpu_family': 'x86_64',
'cpu': 'x86_64',
'endian': 'little',
'abi': ''
}
}
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
@property
def root_path(self):
root_paths = {
'fedora': '/usr/x86_64-w64-mingw32/sys-root/mingw',
'debian': '/usr/x86_64-w64-mingw32'
}
return root_paths[neutralEnv('distname')]
@property
def binaries(self):
return {k:which('{}-{}'.format(self.arch_full, v))
for k, v in (('CC', 'gcc'),
('CXX', 'g++'),
('AR', 'ar'),
('STRIP', 'strip'),
('WINDRES', 'windres'),
('RANLIB', 'ranlib'),
('PKGCONFIG', 'pkg-config'))
}
@property
def exe_wrapper_def(self):
try:
which('wine')
except subprocess.CalledProcessError:
return ""
else:
return "exe_wrapper = 'wine'"
@property
def configure_option(self):
return '--host={}'.format(self.arch_full)
def set_compiler(self, env):
for k, v in self.binaries.items():
env[k] = v
def get_bin_dir(self):
return [pj(self.root_path, 'bin')]
def get_env(self):
env = super().get_env()
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
env['LIBS'] = " ".join(self.extra_libs) + " " +env['LIBS']
return env
class Win64Dyn(Win64PlatformInfo):
name = 'win64_dyn'
static = False
class Win64Static(Win64PlatformInfo):
name = 'win64_static'
static = True

View File

@ -34,6 +34,10 @@ class Defaultdict(defaultdict):
return self[name]
def DefaultEnv():
return Defaultdict(str, os.environ)
def remove_duplicates(iterable, key_function=None):
seen = set()
if key_function is None:
@ -224,21 +228,10 @@ def extract_archive(archive_path, dest_dir, topdir=None, name=None):
archive.close()
def run_command(command, cwd, context, buildEnv=None, env=None, input=None, cross_env_only=False):
def run_command(command, cwd, context, *, env=None, input=None):
os.makedirs(cwd, exist_ok=True)
if env is None:
env = Defaultdict(str, os.environ)
if buildEnv is not None:
cross_compile_env = True
cross_compile_compiler = True
cross_compile_path = True
if context.force_native_build:
cross_compile_env = False
cross_compile_compiler = False
cross_compile_path = False
if cross_env_only:
cross_compile_compiler = False
env = buildEnv._set_env(env, cross_compile_env, cross_compile_compiler, cross_compile_path)
log = None
try:
if not option('verbose'):

View File

@ -42,7 +42,7 @@ release_versions = {
# This is the "version" of the whole base_deps_versions dict.
# Change this when you change base_deps_versions.
base_deps_meta_version = '60'
base_deps_meta_version = '61'
base_deps_versions = {
'zlib' : '1.2.8',