Move Builder and BuildEnv in specific module.

Introduce also a "NeutralEnv", a build environment independent of the
targeted platform. All `Source` now build using the neutralEnv.

Most of toolchains are also using neutralEnv except android_ndk who is
specific to a platform.

As toolchain are neutral, platform specific environment variables are now
set by the platformInfo directly instead of the toolchain.
This commit is contained in:
Matthieu Gautier
2018-05-22 16:36:54 +02:00
parent ac83dec674
commit b950feb893
15 changed files with 649 additions and 589 deletions

View File

@ -1,533 +1,12 @@
#!/usr/bin/env python3
import os, sys, shutil
import os, sys
import argparse
import ssl
import urllib.request
import subprocess
import platform
from collections import OrderedDict
from .toolchains import Toolchain
from .dependencies import Dependency
from .platforms import PlatformInfo
from .utils import (
pj,
remove_duplicates,
add_execution_right,
get_sha256,
print_progress,
setup_print_progress,
download_remote,
StopBuild,
SkipCommand,
Defaultdict,
Remotefile,
Context)
REMOTE_PREFIX = 'http://download.kiwix.org/dev/'
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
_fedora_common = ['automake', 'libtool', 'cmake', 'git', 'subversion', 'ccache', 'pkgconfig', 'gcc-c++', 'gettext-devel']
_debian_common = ['automake', 'libtool', 'cmake', 'git', 'subversion', 'ccache', 'pkg-config', 'gcc', 'autopoint']
PACKAGE_NAME_MAPPERS = {
'fedora_native_dyn': {
'COMMON': _fedora_common,
'uuid': ['libuuid-devel'],
'xapian-core': None, # Not the right version on fedora 25
'ctpp2': None,
'pugixml': None, # ['pugixml-devel'] but package doesn't provide pkg-config file
'libmicrohttpd': ['libmicrohttpd-devel'],
'zlib': ['zlib-devel'],
'lzma': ['xz-devel'],
'icu4c': None,
'zimlib': None,
'file' : ['file-devel'],
'gumbo' : ['gumbo-parser-devel'],
},
'fedora_native_static': {
'COMMON': _fedora_common + ['glibc-static', 'libstdc++-static'],
'zlib': ['zlib-devel', 'zlib-static'],
'lzma': ['xz-devel', 'xz-static']
# Either there is no packages, or no static or too old
},
'fedora_i586_dyn': {
'COMMON': _fedora_common + ['glibc-devel.i686', 'libstdc++-devel.i686'],
},
'fedora_i586_static': {
'COMMON': _fedora_common + ['glibc-devel.i686'],
},
'fedora_win32_dyn': {
'COMMON': _fedora_common + ['mingw32-gcc-c++', 'mingw32-bzip2', 'mingw32-win-iconv', 'mingw32-winpthreads', 'wine'],
'zlib': ['mingw32-zlib'],
'lzma': ['mingw32-xz-libs'],
'libmicrohttpd': ['mingw32-libmicrohttpd'],
},
'fedora_win32_static': {
'COMMON': _fedora_common + ['mingw32-gcc-c++', 'mingw32-bzip2-static', 'mingw32-win-iconv-static', 'mingw32-winpthreads-static', 'wine'],
'zlib': ['mingw32-zlib-static'],
'lzma': ['mingw32-xz-libs-static'],
'libmicrohttpd': None, # ['mingw32-libmicrohttpd-static'] packaging dependecy seems buggy, and some static lib are name libfoo.dll.a and
# gcc cannot found them.
},
'fedora_armhf_static': {
'COMMON': _fedora_common
},
'fedora_armhf_dyn': {
'COMMON': _fedora_common
},
'fedora_android': {
'COMMON': _fedora_common + ['java-1.8.0-openjdk-devel']
},
'debian_native_dyn': {
'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'],
'zlib': ['zlib1g-dev'],
'uuid': ['uuid-dev'],
'ctpp2': ['libctpp2-dev'],
'ctpp2c': ['ctpp2-utils'],
'libmicrohttpd': ['libmicrohttpd-dev', 'ccache']
},
'debian_native_static': {
'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'],
'zlib': ['zlib1g-dev'],
'uuid': ['uuid-dev'],
'ctpp2': ['libctpp2-dev'],
'ctpp2c': ['ctpp2-utils'],
},
'debian_i586_dyn': {
'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib'],
},
'debian_i586_static': {
'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib'],
},
'debian_win32_dyn': {
'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'],
'ctpp2c': ['ctpp2-utils'],
},
'debian_win32_static': {
'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'],
'ctpp2c': ['ctpp2-utils'],
},
'debian_armhf_static': {
'COMMON': _debian_common,
'ctpp2c': ['ctpp2-utils'],
},
'debian_armhf_dyn': {
'COMMON': _debian_common,
'ctpp2c': ['ctpp2-utils'],
},
'debian_android': {
'COMMON': _debian_common + ['default-jdk'],
'ctpp2c': ['ctpp2-utils'],
},
'Darwin_native_dyn': {
'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'],
'file': ['libmagic']
},
'Darwin_iOS': {
'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'],
'file': ['libmagic']
},
}
class BuildEnv:
def __init__(self, options, targetsDict):
self.source_dir = pj(options.working_dir, "SOURCE")
build_dir = "BUILD_{}".format(options.target_platform)
self.build_dir = pj(options.working_dir, build_dir)
self.archive_dir = pj(options.working_dir, "ARCHIVE")
self.toolchain_dir = pj(options.working_dir, "TOOLCHAINS")
self.log_dir = pj(self.build_dir, 'LOGS')
self.install_dir = pj(self.build_dir, "INSTALL")
for d in (self.source_dir,
self.build_dir,
self.archive_dir,
self.toolchain_dir,
self.log_dir,
self.install_dir):
os.makedirs(d, exist_ok=True)
self.detect_platform()
self.ninja_command = self._detect_ninja()
if not self.ninja_command:
sys.exit("ERROR: ninja command not found")
self.meson_command = self._detect_meson()
if not self.meson_command:
sys.exit("ERROR: meson command not fount")
self.mesontest_command = "meson test"
self.setup_build(options.target_platform)
self.setup_toolchains()
self.options = options
self.libprefix = options.libprefix or self._detect_libdir()
self.targetsDict = targetsDict
def clean_intermediate_directories(self):
for subdir in os.listdir(self.build_dir):
subpath = pj(self.build_dir, subdir)
if subpath == self.install_dir:
continue
if os.path.isdir(subpath):
shutil.rmtree(subpath)
else:
os.remove(subpath)
def detect_platform(self):
_platform = platform.system()
self.distname = _platform
if _platform == 'Windows':
print('ERROR: kiwix-build is not intented to run on Windows platform.\n'
'It should probably not work, but well, you still can have a try.')
cont = input('Do you want to continue ? [y/N]')
if cont.lower() != 'y':
sys.exit(0)
if _platform == 'Linux':
self.distname, _, _ = platform.linux_distribution()
self.distname = self.distname.lower()
if self.distname == 'ubuntu':
self.distname = 'debian'
def setup_build(self, target_platform):
self.platform_info = PlatformInfo.all_platforms[target_platform]
if self.distname not in self.platform_info.compatible_hosts:
print(('ERROR: The target {} cannot be build on host {}.\n'
'Select another target platform, or change your host system.'
).format(target_platform, self.distname))
sys.exit(-1)
self.cross_config = self.platform_info.get_cross_config(self.distname)
def setup_toolchains(self):
toolchain_names = self.platform_info.toolchains
self.toolchains =[Toolchain.all_toolchains[toolchain_name](self)
for toolchain_name in toolchain_names]
def finalize_setup(self):
getattr(self, 'setup_{}'.format(self.platform_info.build))()
def setup_native(self):
self.cmake_crossfile = None
self.meson_crossfile = None
def _gen_crossfile(self, name):
crossfile = pj(self.build_dir, name)
template_file = pj(SCRIPT_DIR, 'templates', name)
with open(template_file, 'r') as f:
template = f.read()
content = template.format(
toolchain=self.toolchains[0],
**self.cross_config
)
with open(crossfile, 'w') as outfile:
outfile.write(content)
return crossfile
def setup_win32(self):
self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def setup_android(self):
self.cmake_crossfile = self._gen_crossfile('cmake_android_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def setup_armhf(self):
self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def setup_iOS(self):
self.cmake_crossfile = self._gen_crossfile('cmake_ios_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def setup_i586(self):
self.cmake_crossfile = self._gen_crossfile('cmake_i586_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def __getattr__(self, name):
return getattr(self.options, name)
def _is_debianlike(self):
return os.path.isfile('/etc/debian_version')
def _detect_libdir(self):
if self._is_debianlike():
try:
pc = subprocess.Popen(['dpkg-architecture', '-qDEB_HOST_MULTIARCH'],
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL)
(stdo, _) = pc.communicate()
if pc.returncode == 0:
archpath = stdo.decode().strip()
return 'lib/' + archpath
except Exception:
pass
if os.path.isdir('/usr/lib64') and not os.path.islink('/usr/lib64'):
return 'lib64'
return 'lib'
def _detect_ninja(self):
for n in ['ninja', 'ninja-build']:
try:
retcode = subprocess.check_call([n, '--version'],
stdout=subprocess.DEVNULL)
except (FileNotFoundError, PermissionError):
# Doesn't exist in PATH or isn't executable
continue
if retcode == 0:
return n
def _detect_meson(self):
for n in ['meson.py', 'meson']:
try:
retcode = subprocess.check_call([n, '--version'],
stdout=subprocess.DEVNULL)
except (FileNotFoundError, PermissionError):
# Doesn't exist in PATH or isn't executable
continue
if retcode == 0:
return n
@property
def configure_option(self):
configure_options = [tlc.configure_option for tlc in self.toolchains]
return " ".join(configure_options)
@property
def cmake_option(self):
cmake_options = [tlc.cmake_option for tlc in self.toolchains]
return " ".join(cmake_options)
def _set_env(self, env, cross_compile_env, cross_compile_compiler, cross_compile_path):
if env is None:
env = Defaultdict(str, os.environ)
bin_dirs = []
if cross_compile_env:
for k, v in self.cross_config.get('env', {}).items():
if k.startswith('_format_'):
v = v.format(**self.cross_config)
k = k[8:]
env[k] = v
for toolchain in self.toolchains:
toolchain.set_env(env)
if cross_compile_compiler:
for toolchain in self.toolchains:
toolchain.set_compiler(env)
if cross_compile_path:
for tlc in self.toolchains:
bin_dirs += tlc.get_bin_dir()
pkgconfig_path = pj(self.install_dir, self.libprefix, 'pkgconfig')
env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path])
# Add ccache path
for p in ('/usr/lib/ccache', '/usr/lib64/ccache'):
if os.path.isdir(p):
ccache_path = [p]
break
else:
ccache_path = []
env['PATH'] = ':'.join(bin_dirs +
[pj(self.install_dir, 'bin')] +
ccache_path +
[env['PATH']])
env['LD_LIBRARY_PATH'] = ':'.join([env['LD_LIBRARY_PATH'],
pj(self.install_dir, 'lib'),
pj(self.install_dir, self.libprefix)
])
env['CPPFLAGS'] = " ".join(['-I'+pj(self.install_dir, 'include'), env['CPPFLAGS']])
env['LDFLAGS'] = " ".join(['-L'+pj(self.install_dir, 'lib'),
'-L'+pj(self.install_dir, self.libprefix),
env['LDFLAGS']])
return env
def run_command(self, command, cwd, context, env=None, input=None, cross_env_only=False):
os.makedirs(cwd, exist_ok=True)
cross_compile_env = True
cross_compile_compiler = True
cross_compile_path = True
if context.force_native_build:
cross_compile_env = False
cross_compile_compiler = False
cross_compile_path = False
if cross_env_only:
cross_compile_compiler = False
env = self._set_env(env, cross_compile_env, cross_compile_compiler, cross_compile_path)
log = None
try:
if not self.options.verbose:
log = open(context.log_file, 'w')
print("run command '{}'".format(command), file=log)
print("current directory is '{}'".format(cwd), file=log)
print("env is :", file=log)
for k, v in env.items():
print(" {} : {!r}".format(k, v), file=log)
kwargs = dict()
if input:
kwargs['stdin'] = subprocess.PIPE
process = subprocess.Popen(command, shell=True, cwd=cwd, env=env, stdout=log or sys.stdout, stderr=subprocess.STDOUT, **kwargs)
if input:
process.communicate(input.encode())
retcode = process.wait()
if retcode:
raise subprocess.CalledProcessError(retcode, command)
finally:
if log:
log.close()
def download(self, what, where=None):
where = where or self.archive_dir
download_remote(what, where, not self.options.no_cert_check)
def install_packages(self):
autoskip_file = pj(self.build_dir, ".install_packages_ok")
if self.distname in ('fedora', 'redhat', 'centos'):
package_installer = 'sudo dnf install {}'
package_checker = 'rpm -q --quiet {}'
elif self.distname in ('debian', 'Ubuntu'):
package_installer = 'sudo apt-get install {}'
package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1'
elif self.distname == 'Darwin':
package_installer = 'brew install {}'
package_checker = 'brew list -1 | grep -q {}'
mapper_name = "{host}_{target}".format(
host=self.distname,
target=self.platform_info)
try:
package_name_mapper = PACKAGE_NAME_MAPPERS[mapper_name]
except KeyError:
print("SKIP : We don't know which packages we must install to compile"
" a {target} {build_type} version on a {host} host.".format(
target=self.platform_info,
host=self.distname))
return
packages_list = package_name_mapper.get('COMMON', [])
for dep in self.targetsDict.values():
packages = package_name_mapper.get(dep.name)
if packages:
packages_list += packages
dep.skip = True
for dep in self.targetsDict.values():
packages = getattr(dep, 'extra_packages', [])
for package in packages:
packages_list += package_name_mapper.get(package, [])
if not self.options.force_install_packages and os.path.exists(autoskip_file):
print("SKIP")
return
packages_to_install = []
for package in packages_list:
print(" - {} : ".format(package), end="")
command = package_checker.format(package)
try:
subprocess.check_call(command, shell=True)
except subprocess.CalledProcessError:
print("NEEDED")
packages_to_install.append(package)
else:
print("SKIP")
if packages_to_install:
command = package_installer.format(" ".join(packages_to_install))
print(command)
subprocess.check_call(command, shell=True)
else:
print("SKIP, No package to install.")
with open(autoskip_file, 'w'):
pass
class Builder:
def __init__(self, options):
self.options = options
self.targets = OrderedDict()
self.buildEnv = buildEnv = BuildEnv(options, self.targets)
_targets = {}
targetDef = options.targets
self.add_targets(targetDef, _targets)
dependencies = self.order_dependencies(_targets, targetDef)
dependencies = list(remove_duplicates(dependencies))
if options.build_nodeps:
self.targets[targetDef] = _targets[targetDef]
else:
for dep in dependencies:
if self.options.build_deps_only and dep == targetDef:
continue
self.targets[dep] = _targets[dep]
def add_targets(self, targetName, targets):
if targetName in targets:
return
targetClass = Dependency.all_deps[targetName]
target = targetClass(self.buildEnv)
targets[targetName] = target
for dep in target.dependencies:
self.add_targets(dep, targets)
def order_dependencies(self, _targets, targetName):
target = _targets[targetName]
for depName in target.dependencies:
yield from self.order_dependencies(_targets, depName)
yield targetName
def prepare_sources(self):
if self.options.skip_source_prepare:
print("SKIP")
return
toolchain_sources = (tlc.source for tlc in self.buildEnv.toolchains if tlc.source)
for toolchain_source in toolchain_sources:
print("prepare sources for toolchain {} :".format(toolchain_source.name))
toolchain_source.prepare()
sources = (dep.source for dep in self.targets.values() if not dep.skip)
sources = remove_duplicates(sources, lambda s: s.__class__)
for source in sources:
print("prepare sources {} :".format(source.name))
source.prepare()
def build(self):
toolchain_builders = (tlc.builder for tlc in self.buildEnv.toolchains if tlc.builder)
for toolchain_builder in toolchain_builders:
print("build toolchain {} :".format(toolchain_builder.name))
toolchain_builder.build()
builders = (dep.builder for dep in self.targets.values() if (dep.builder and not dep.skip))
for builder in builders:
if self.options.make_dist and builder.name == self.options.targets:
continue
print("build {} :".format(builder.name))
builder.build()
if self.options.make_dist:
dep = self.targets[self.options.targets]
builder = dep.builder
print("make dist {}:".format(builder.name))
builder.make_dist()
def run(self):
try:
print("[INSTALL PACKAGES]")
self.buildEnv.install_packages()
self.buildEnv.finalize_setup()
print("[PREPARE]")
self.prepare_sources()
print("[BUILD]")
self.build()
# No error, clean intermediate file at end of build if needed.
print("[CLEAN]")
if self.buildEnv.options.clean_at_end:
self.buildEnv.clean_intermediate_directories()
else:
print("SKIP")
except StopBuild:
sys.exit("Stopping build due to errors")
from .builder import Builder
from .utils import setup_print_progress
def parse_args():
parser = argparse.ArgumentParser()