Move Builder and BuildEnv in specific module.

Introduce also a "NeutralEnv", a build environment independent of the
targeted platform. All `Source` now build using the neutralEnv.

Most of toolchains are also using neutralEnv except android_ndk who is
specific to a platform.

As toolchain are neutral, platform specific environment variables are now
set by the platformInfo directly instead of the toolchain.
This commit is contained in:
Matthieu Gautier 2018-05-22 16:36:54 +02:00
parent ac83dec674
commit b950feb893
15 changed files with 649 additions and 589 deletions

View File

@ -1,533 +1,12 @@
#!/usr/bin/env python3
import os, sys, shutil
import os, sys
import argparse
import ssl
import urllib.request
import subprocess
import platform
from collections import OrderedDict
from .toolchains import Toolchain
from .dependencies import Dependency
from .platforms import PlatformInfo
from .utils import (
pj,
remove_duplicates,
add_execution_right,
get_sha256,
print_progress,
setup_print_progress,
download_remote,
StopBuild,
SkipCommand,
Defaultdict,
Remotefile,
Context)
REMOTE_PREFIX = 'http://download.kiwix.org/dev/'
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
_fedora_common = ['automake', 'libtool', 'cmake', 'git', 'subversion', 'ccache', 'pkgconfig', 'gcc-c++', 'gettext-devel']
_debian_common = ['automake', 'libtool', 'cmake', 'git', 'subversion', 'ccache', 'pkg-config', 'gcc', 'autopoint']
PACKAGE_NAME_MAPPERS = {
'fedora_native_dyn': {
'COMMON': _fedora_common,
'uuid': ['libuuid-devel'],
'xapian-core': None, # Not the right version on fedora 25
'ctpp2': None,
'pugixml': None, # ['pugixml-devel'] but package doesn't provide pkg-config file
'libmicrohttpd': ['libmicrohttpd-devel'],
'zlib': ['zlib-devel'],
'lzma': ['xz-devel'],
'icu4c': None,
'zimlib': None,
'file' : ['file-devel'],
'gumbo' : ['gumbo-parser-devel'],
},
'fedora_native_static': {
'COMMON': _fedora_common + ['glibc-static', 'libstdc++-static'],
'zlib': ['zlib-devel', 'zlib-static'],
'lzma': ['xz-devel', 'xz-static']
# Either there is no packages, or no static or too old
},
'fedora_i586_dyn': {
'COMMON': _fedora_common + ['glibc-devel.i686', 'libstdc++-devel.i686'],
},
'fedora_i586_static': {
'COMMON': _fedora_common + ['glibc-devel.i686'],
},
'fedora_win32_dyn': {
'COMMON': _fedora_common + ['mingw32-gcc-c++', 'mingw32-bzip2', 'mingw32-win-iconv', 'mingw32-winpthreads', 'wine'],
'zlib': ['mingw32-zlib'],
'lzma': ['mingw32-xz-libs'],
'libmicrohttpd': ['mingw32-libmicrohttpd'],
},
'fedora_win32_static': {
'COMMON': _fedora_common + ['mingw32-gcc-c++', 'mingw32-bzip2-static', 'mingw32-win-iconv-static', 'mingw32-winpthreads-static', 'wine'],
'zlib': ['mingw32-zlib-static'],
'lzma': ['mingw32-xz-libs-static'],
'libmicrohttpd': None, # ['mingw32-libmicrohttpd-static'] packaging dependecy seems buggy, and some static lib are name libfoo.dll.a and
# gcc cannot found them.
},
'fedora_armhf_static': {
'COMMON': _fedora_common
},
'fedora_armhf_dyn': {
'COMMON': _fedora_common
},
'fedora_android': {
'COMMON': _fedora_common + ['java-1.8.0-openjdk-devel']
},
'debian_native_dyn': {
'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'],
'zlib': ['zlib1g-dev'],
'uuid': ['uuid-dev'],
'ctpp2': ['libctpp2-dev'],
'ctpp2c': ['ctpp2-utils'],
'libmicrohttpd': ['libmicrohttpd-dev', 'ccache']
},
'debian_native_static': {
'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'],
'zlib': ['zlib1g-dev'],
'uuid': ['uuid-dev'],
'ctpp2': ['libctpp2-dev'],
'ctpp2c': ['ctpp2-utils'],
},
'debian_i586_dyn': {
'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib'],
},
'debian_i586_static': {
'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib'],
},
'debian_win32_dyn': {
'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'],
'ctpp2c': ['ctpp2-utils'],
},
'debian_win32_static': {
'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'],
'ctpp2c': ['ctpp2-utils'],
},
'debian_armhf_static': {
'COMMON': _debian_common,
'ctpp2c': ['ctpp2-utils'],
},
'debian_armhf_dyn': {
'COMMON': _debian_common,
'ctpp2c': ['ctpp2-utils'],
},
'debian_android': {
'COMMON': _debian_common + ['default-jdk'],
'ctpp2c': ['ctpp2-utils'],
},
'Darwin_native_dyn': {
'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'],
'file': ['libmagic']
},
'Darwin_iOS': {
'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'],
'file': ['libmagic']
},
}
class BuildEnv:
def __init__(self, options, targetsDict):
self.source_dir = pj(options.working_dir, "SOURCE")
build_dir = "BUILD_{}".format(options.target_platform)
self.build_dir = pj(options.working_dir, build_dir)
self.archive_dir = pj(options.working_dir, "ARCHIVE")
self.toolchain_dir = pj(options.working_dir, "TOOLCHAINS")
self.log_dir = pj(self.build_dir, 'LOGS')
self.install_dir = pj(self.build_dir, "INSTALL")
for d in (self.source_dir,
self.build_dir,
self.archive_dir,
self.toolchain_dir,
self.log_dir,
self.install_dir):
os.makedirs(d, exist_ok=True)
self.detect_platform()
self.ninja_command = self._detect_ninja()
if not self.ninja_command:
sys.exit("ERROR: ninja command not found")
self.meson_command = self._detect_meson()
if not self.meson_command:
sys.exit("ERROR: meson command not fount")
self.mesontest_command = "meson test"
self.setup_build(options.target_platform)
self.setup_toolchains()
self.options = options
self.libprefix = options.libprefix or self._detect_libdir()
self.targetsDict = targetsDict
def clean_intermediate_directories(self):
for subdir in os.listdir(self.build_dir):
subpath = pj(self.build_dir, subdir)
if subpath == self.install_dir:
continue
if os.path.isdir(subpath):
shutil.rmtree(subpath)
else:
os.remove(subpath)
def detect_platform(self):
_platform = platform.system()
self.distname = _platform
if _platform == 'Windows':
print('ERROR: kiwix-build is not intented to run on Windows platform.\n'
'It should probably not work, but well, you still can have a try.')
cont = input('Do you want to continue ? [y/N]')
if cont.lower() != 'y':
sys.exit(0)
if _platform == 'Linux':
self.distname, _, _ = platform.linux_distribution()
self.distname = self.distname.lower()
if self.distname == 'ubuntu':
self.distname = 'debian'
def setup_build(self, target_platform):
self.platform_info = PlatformInfo.all_platforms[target_platform]
if self.distname not in self.platform_info.compatible_hosts:
print(('ERROR: The target {} cannot be build on host {}.\n'
'Select another target platform, or change your host system.'
).format(target_platform, self.distname))
sys.exit(-1)
self.cross_config = self.platform_info.get_cross_config(self.distname)
def setup_toolchains(self):
toolchain_names = self.platform_info.toolchains
self.toolchains =[Toolchain.all_toolchains[toolchain_name](self)
for toolchain_name in toolchain_names]
def finalize_setup(self):
getattr(self, 'setup_{}'.format(self.platform_info.build))()
def setup_native(self):
self.cmake_crossfile = None
self.meson_crossfile = None
def _gen_crossfile(self, name):
crossfile = pj(self.build_dir, name)
template_file = pj(SCRIPT_DIR, 'templates', name)
with open(template_file, 'r') as f:
template = f.read()
content = template.format(
toolchain=self.toolchains[0],
**self.cross_config
)
with open(crossfile, 'w') as outfile:
outfile.write(content)
return crossfile
def setup_win32(self):
self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def setup_android(self):
self.cmake_crossfile = self._gen_crossfile('cmake_android_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def setup_armhf(self):
self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def setup_iOS(self):
self.cmake_crossfile = self._gen_crossfile('cmake_ios_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def setup_i586(self):
self.cmake_crossfile = self._gen_crossfile('cmake_i586_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def __getattr__(self, name):
return getattr(self.options, name)
def _is_debianlike(self):
return os.path.isfile('/etc/debian_version')
def _detect_libdir(self):
if self._is_debianlike():
try:
pc = subprocess.Popen(['dpkg-architecture', '-qDEB_HOST_MULTIARCH'],
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL)
(stdo, _) = pc.communicate()
if pc.returncode == 0:
archpath = stdo.decode().strip()
return 'lib/' + archpath
except Exception:
pass
if os.path.isdir('/usr/lib64') and not os.path.islink('/usr/lib64'):
return 'lib64'
return 'lib'
def _detect_ninja(self):
for n in ['ninja', 'ninja-build']:
try:
retcode = subprocess.check_call([n, '--version'],
stdout=subprocess.DEVNULL)
except (FileNotFoundError, PermissionError):
# Doesn't exist in PATH or isn't executable
continue
if retcode == 0:
return n
def _detect_meson(self):
for n in ['meson.py', 'meson']:
try:
retcode = subprocess.check_call([n, '--version'],
stdout=subprocess.DEVNULL)
except (FileNotFoundError, PermissionError):
# Doesn't exist in PATH or isn't executable
continue
if retcode == 0:
return n
@property
def configure_option(self):
configure_options = [tlc.configure_option for tlc in self.toolchains]
return " ".join(configure_options)
@property
def cmake_option(self):
cmake_options = [tlc.cmake_option for tlc in self.toolchains]
return " ".join(cmake_options)
def _set_env(self, env, cross_compile_env, cross_compile_compiler, cross_compile_path):
if env is None:
env = Defaultdict(str, os.environ)
bin_dirs = []
if cross_compile_env:
for k, v in self.cross_config.get('env', {}).items():
if k.startswith('_format_'):
v = v.format(**self.cross_config)
k = k[8:]
env[k] = v
for toolchain in self.toolchains:
toolchain.set_env(env)
if cross_compile_compiler:
for toolchain in self.toolchains:
toolchain.set_compiler(env)
if cross_compile_path:
for tlc in self.toolchains:
bin_dirs += tlc.get_bin_dir()
pkgconfig_path = pj(self.install_dir, self.libprefix, 'pkgconfig')
env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path])
# Add ccache path
for p in ('/usr/lib/ccache', '/usr/lib64/ccache'):
if os.path.isdir(p):
ccache_path = [p]
break
else:
ccache_path = []
env['PATH'] = ':'.join(bin_dirs +
[pj(self.install_dir, 'bin')] +
ccache_path +
[env['PATH']])
env['LD_LIBRARY_PATH'] = ':'.join([env['LD_LIBRARY_PATH'],
pj(self.install_dir, 'lib'),
pj(self.install_dir, self.libprefix)
])
env['CPPFLAGS'] = " ".join(['-I'+pj(self.install_dir, 'include'), env['CPPFLAGS']])
env['LDFLAGS'] = " ".join(['-L'+pj(self.install_dir, 'lib'),
'-L'+pj(self.install_dir, self.libprefix),
env['LDFLAGS']])
return env
def run_command(self, command, cwd, context, env=None, input=None, cross_env_only=False):
os.makedirs(cwd, exist_ok=True)
cross_compile_env = True
cross_compile_compiler = True
cross_compile_path = True
if context.force_native_build:
cross_compile_env = False
cross_compile_compiler = False
cross_compile_path = False
if cross_env_only:
cross_compile_compiler = False
env = self._set_env(env, cross_compile_env, cross_compile_compiler, cross_compile_path)
log = None
try:
if not self.options.verbose:
log = open(context.log_file, 'w')
print("run command '{}'".format(command), file=log)
print("current directory is '{}'".format(cwd), file=log)
print("env is :", file=log)
for k, v in env.items():
print(" {} : {!r}".format(k, v), file=log)
kwargs = dict()
if input:
kwargs['stdin'] = subprocess.PIPE
process = subprocess.Popen(command, shell=True, cwd=cwd, env=env, stdout=log or sys.stdout, stderr=subprocess.STDOUT, **kwargs)
if input:
process.communicate(input.encode())
retcode = process.wait()
if retcode:
raise subprocess.CalledProcessError(retcode, command)
finally:
if log:
log.close()
def download(self, what, where=None):
where = where or self.archive_dir
download_remote(what, where, not self.options.no_cert_check)
def install_packages(self):
autoskip_file = pj(self.build_dir, ".install_packages_ok")
if self.distname in ('fedora', 'redhat', 'centos'):
package_installer = 'sudo dnf install {}'
package_checker = 'rpm -q --quiet {}'
elif self.distname in ('debian', 'Ubuntu'):
package_installer = 'sudo apt-get install {}'
package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1'
elif self.distname == 'Darwin':
package_installer = 'brew install {}'
package_checker = 'brew list -1 | grep -q {}'
mapper_name = "{host}_{target}".format(
host=self.distname,
target=self.platform_info)
try:
package_name_mapper = PACKAGE_NAME_MAPPERS[mapper_name]
except KeyError:
print("SKIP : We don't know which packages we must install to compile"
" a {target} {build_type} version on a {host} host.".format(
target=self.platform_info,
host=self.distname))
return
packages_list = package_name_mapper.get('COMMON', [])
for dep in self.targetsDict.values():
packages = package_name_mapper.get(dep.name)
if packages:
packages_list += packages
dep.skip = True
for dep in self.targetsDict.values():
packages = getattr(dep, 'extra_packages', [])
for package in packages:
packages_list += package_name_mapper.get(package, [])
if not self.options.force_install_packages and os.path.exists(autoskip_file):
print("SKIP")
return
packages_to_install = []
for package in packages_list:
print(" - {} : ".format(package), end="")
command = package_checker.format(package)
try:
subprocess.check_call(command, shell=True)
except subprocess.CalledProcessError:
print("NEEDED")
packages_to_install.append(package)
else:
print("SKIP")
if packages_to_install:
command = package_installer.format(" ".join(packages_to_install))
print(command)
subprocess.check_call(command, shell=True)
else:
print("SKIP, No package to install.")
with open(autoskip_file, 'w'):
pass
class Builder:
def __init__(self, options):
self.options = options
self.targets = OrderedDict()
self.buildEnv = buildEnv = BuildEnv(options, self.targets)
_targets = {}
targetDef = options.targets
self.add_targets(targetDef, _targets)
dependencies = self.order_dependencies(_targets, targetDef)
dependencies = list(remove_duplicates(dependencies))
if options.build_nodeps:
self.targets[targetDef] = _targets[targetDef]
else:
for dep in dependencies:
if self.options.build_deps_only and dep == targetDef:
continue
self.targets[dep] = _targets[dep]
def add_targets(self, targetName, targets):
if targetName in targets:
return
targetClass = Dependency.all_deps[targetName]
target = targetClass(self.buildEnv)
targets[targetName] = target
for dep in target.dependencies:
self.add_targets(dep, targets)
def order_dependencies(self, _targets, targetName):
target = _targets[targetName]
for depName in target.dependencies:
yield from self.order_dependencies(_targets, depName)
yield targetName
def prepare_sources(self):
if self.options.skip_source_prepare:
print("SKIP")
return
toolchain_sources = (tlc.source for tlc in self.buildEnv.toolchains if tlc.source)
for toolchain_source in toolchain_sources:
print("prepare sources for toolchain {} :".format(toolchain_source.name))
toolchain_source.prepare()
sources = (dep.source for dep in self.targets.values() if not dep.skip)
sources = remove_duplicates(sources, lambda s: s.__class__)
for source in sources:
print("prepare sources {} :".format(source.name))
source.prepare()
def build(self):
toolchain_builders = (tlc.builder for tlc in self.buildEnv.toolchains if tlc.builder)
for toolchain_builder in toolchain_builders:
print("build toolchain {} :".format(toolchain_builder.name))
toolchain_builder.build()
builders = (dep.builder for dep in self.targets.values() if (dep.builder and not dep.skip))
for builder in builders:
if self.options.make_dist and builder.name == self.options.targets:
continue
print("build {} :".format(builder.name))
builder.build()
if self.options.make_dist:
dep = self.targets[self.options.targets]
builder = dep.builder
print("make dist {}:".format(builder.name))
builder.make_dist()
def run(self):
try:
print("[INSTALL PACKAGES]")
self.buildEnv.install_packages()
self.buildEnv.finalize_setup()
print("[PREPARE]")
self.prepare_sources()
print("[BUILD]")
self.build()
# No error, clean intermediate file at end of build if needed.
print("[CLEAN]")
if self.buildEnv.options.clean_at_end:
self.buildEnv.clean_intermediate_directories()
else:
print("SKIP")
except StopBuild:
sys.exit("Stopping build due to errors")
from .builder import Builder
from .utils import setup_print_progress
def parse_args():
parser = argparse.ArgumentParser()

376
kiwixbuild/buildenv.py Normal file
View File

@ -0,0 +1,376 @@
import os, sys, shutil
import subprocess
import platform
from .platforms import PlatformInfo
from .toolchains import Toolchain
from .packages import PACKAGE_NAME_MAPPERS
from .utils import (
pj,
download_remote,
Defaultdict)
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
class PlatformNeutralEnv:
def __init__(self, options):
self.options = options
self.source_dir = pj(options.working_dir, "SOURCE")
self.archive_dir = pj(options.working_dir, "ARCHIVE")
self.toolchain_dir = pj(options.working_dir, "TOOLCHAINS")
self.log_dir = pj(self.working_dir, 'LOGS')
for d in (self.source_dir,
self.archive_dir,
self.toolchain_dir,
self.log_dir):
os.makedirs(d, exist_ok=True)
self.toolchains = {}
self.detect_platform()
self.ninja_command = self._detect_ninja()
if not self.ninja_command:
sys.exit("ERROR: ninja command not found")
self.meson_command = self._detect_meson()
if not self.meson_command:
sys.exit("ERROR: meson command not fount")
self.mesontest_command = "{} test".format(self.meson_command)
def run_command(self, command, cwd, context, env=None, input=None):
os.makedirs(cwd, exist_ok=True)
if env is None:
env = Defaultdict(str, os.environ)
log = None
try:
if not self.options.verbose:
log = open(context.log_file, 'w')
print("run command '{}'".format(command), file=log)
print("current directory is '{}'".format(cwd), file=log)
print("env is :", file=log)
for k, v in env.items():
print(" {} : {!r}".format(k, v), file=log)
kwargs = dict()
if input:
kwargs['stdin'] = subprocess.PIPE
process = subprocess.Popen(command, shell=True, cwd=cwd, env=env, stdout=log or sys.stdout, stderr=subprocess.STDOUT, **kwargs)
if input:
process.communicate(input.encode())
retcode = process.wait()
if retcode:
raise subprocess.CalledProcessError(retcode, command)
finally:
if log:
log.close()
def detect_platform(self):
_platform = platform.system()
self.distname = _platform
if _platform == 'Windows':
print('ERROR: kiwix-build is not intented to run on Windows platform.\n'
'It should probably not work, but well, you still can have a try.')
cont = input('Do you want to continue ? [y/N]')
if cont.lower() != 'y':
sys.exit(0)
if _platform == 'Linux':
self.distname, _, _ = platform.linux_distribution()
self.distname = self.distname.lower()
if self.distname == 'ubuntu':
self.distname = 'debian'
def download(self, what, where=None):
where = where or self.archive_dir
download_remote(what, where, not self.options.no_cert_check)
def _detect_ninja(self):
for n in ['ninja', 'ninja-build']:
try:
retcode = subprocess.check_call([n, '--version'],
stdout=subprocess.DEVNULL)
except (FileNotFoundError, PermissionError):
# Doesn't exist in PATH or isn't executable
continue
if retcode == 0:
return n
def add_toolchain(self, toolchain_name):
if toolchain_name not in self.toolchains:
ToolchainClass = Toolchain.all_toolchains[toolchain_name]
self.toolchains[toolchain_name] = ToolchainClass(self)
return self.toolchains[toolchain_name]
def _detect_meson(self):
for n in ['meson.py', 'meson']:
try:
retcode = subprocess.check_call([n, '--version'],
stdout=subprocess.DEVNULL)
except (FileNotFoundError, PermissionError):
# Doesn't exist in PATH or isn't executable
continue
if retcode == 0:
return n
def __getattr__(self, name):
return getattr(self.options, name)
class BuildEnv:
def __init__(self, options, neutralEnv, targetsDict):
build_dir = "BUILD_{}".format(options.target_platform)
self.neutralEnv = neutralEnv
self.build_dir = pj(options.working_dir, build_dir)
self.install_dir = pj(self.build_dir, "INSTALL")
for d in (self.build_dir,
self.install_dir):
os.makedirs(d, exist_ok=True)
self.setup_build(options.target_platform)
self.setup_toolchains()
self.options = options
self.libprefix = options.libprefix or self._detect_libdir()
self.targetsDict = targetsDict
def clean_intermediate_directories(self):
for subdir in os.listdir(self.build_dir):
subpath = pj(self.build_dir, subdir)
if subpath == self.install_dir:
continue
if os.path.isdir(subpath):
shutil.rmtree(subpath)
else:
os.remove(subpath)
def setup_build(self, target_platform):
self.platform_info = PlatformInfo.all_platforms[target_platform]
if self.distname not in self.platform_info.compatible_hosts:
print(('ERROR: The target {} cannot be build on host {}.\n'
'Select another target platform, or change your host system.'
).format(target_platform, self.distname))
sys.exit(-1)
self.cross_config = self.platform_info.get_cross_config(self.distname)
def setup_toolchains(self):
toolchain_names = self.platform_info.toolchains
self.toolchains = []
for toolchain_name in toolchain_names:
ToolchainClass = Toolchain.all_toolchains[toolchain_name]
if ToolchainClass.neutral:
self.toolchains.append(
self.neutralEnv.add_toolchain(toolchain_name)
)
else:
self.toolchains.append(ToolchainClass(self))
def finalize_setup(self):
getattr(self, 'setup_{}'.format(self.platform_info.build))()
def setup_native(self):
self.cmake_crossfile = None
self.meson_crossfile = None
def _gen_crossfile(self, name):
crossfile = pj(self.build_dir, name)
template_file = pj(SCRIPT_DIR, 'templates', name)
with open(template_file, 'r') as f:
template = f.read()
content = template.format(
toolchain=self.toolchains[0],
**self.cross_config
)
with open(crossfile, 'w') as outfile:
outfile.write(content)
return crossfile
def setup_win32(self):
self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def setup_android(self):
self.cmake_crossfile = self._gen_crossfile('cmake_android_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def setup_armhf(self):
self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def setup_iOS(self):
self.cmake_crossfile = self._gen_crossfile('cmake_ios_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def setup_i586(self):
self.cmake_crossfile = self._gen_crossfile('cmake_i586_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def __getattr__(self, name):
return getattr(self.neutralEnv, name)
def _is_debianlike(self):
return os.path.isfile('/etc/debian_version')
def _detect_libdir(self):
if self._is_debianlike():
try:
pc = subprocess.Popen(['dpkg-architecture', '-qDEB_HOST_MULTIARCH'],
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL)
(stdo, _) = pc.communicate()
if pc.returncode == 0:
archpath = stdo.decode().strip()
return 'lib/' + archpath
except Exception:
pass
if os.path.isdir('/usr/lib64') and not os.path.islink('/usr/lib64'):
return 'lib64'
return 'lib'
@property
def configure_option(self):
configure_options = [tlc.configure_option for tlc in self.toolchains]
return " ".join(configure_options)
@property
def cmake_option(self):
cmake_options = [tlc.cmake_option for tlc in self.toolchains]
return " ".join(cmake_options)
def _set_env(self, env, cross_compile_env, cross_compile_compiler, cross_compile_path):
if env is None:
env = Defaultdict(str, os.environ)
pkgconfig_path = pj(self.install_dir, self.libprefix, 'pkgconfig')
env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path])
# Add ccache path
for p in ('/usr/lib/ccache', '/usr/lib64/ccache'):
if os.path.isdir(p):
ccache_path = [p]
break
else:
ccache_path = []
env['PATH'] = ':'.join([pj(self.install_dir, 'bin')] +
ccache_path +
[env['PATH']])
env['LD_LIBRARY_PATH'] = ':'.join([env['LD_LIBRARY_PATH'],
pj(self.install_dir, 'lib'),
pj(self.install_dir, self.libprefix)
])
env['CPPFLAGS'] = " ".join(['-I'+pj(self.install_dir, 'include'), env['CPPFLAGS']])
env['LDFLAGS'] = " ".join(['-L'+pj(self.install_dir, 'lib'),
'-L'+pj(self.install_dir, self.libprefix),
env['LDFLAGS']])
if cross_compile_env:
for k, v in self.cross_config.get('env', {}).items():
if k.startswith('_format_'):
v = v.format(**self.cross_config)
k = k[8:]
env[k] = v
for toolchain in self.toolchains:
toolchain.set_env(env)
self.platform_info.set_env(env)
if cross_compile_compiler:
for toolchain in self.toolchains:
toolchain.set_compiler(env)
if cross_compile_path:
bin_dirs = []
for tlc in self.toolchains:
bin_dirs += tlc.get_bin_dir()
bin_dirs += self.platform_info.get_bind_dir()
env['PATH'] = ':'.join(bin_dirs + [env['PATH']])
return env
def run_command(self, command, cwd, context, env=None, input=None, cross_env_only=False):
os.makedirs(cwd, exist_ok=True)
cross_compile_env = True
cross_compile_compiler = True
cross_compile_path = True
if context.force_native_build:
cross_compile_env = False
cross_compile_compiler = False
cross_compile_path = False
if cross_env_only:
cross_compile_compiler = False
env = self._set_env(env, cross_compile_env, cross_compile_compiler, cross_compile_path)
log = None
try:
if not self.options.verbose:
log = open(context.log_file, 'w')
print("run command '{}'".format(command), file=log)
print("current directory is '{}'".format(cwd), file=log)
print("env is :", file=log)
for k, v in env.items():
print(" {} : {!r}".format(k, v), file=log)
kwargs = dict()
if input:
kwargs['stdin'] = subprocess.PIPE
process = subprocess.Popen(command, shell=True, cwd=cwd, env=env, stdout=log or sys.stdout, stderr=subprocess.STDOUT, **kwargs)
if input:
process.communicate(input.encode())
retcode = process.wait()
if retcode:
raise subprocess.CalledProcessError(retcode, command)
finally:
if log:
log.close()
def install_packages(self):
autoskip_file = pj(self.build_dir, ".install_packages_ok")
if self.distname in ('fedora', 'redhat', 'centos'):
package_installer = 'sudo dnf install {}'
package_checker = 'rpm -q --quiet {}'
elif self.distname in ('debian', 'Ubuntu'):
package_installer = 'sudo apt-get install {}'
package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1'
elif self.distname == 'Darwin':
package_installer = 'brew install {}'
package_checker = 'brew list -1 | grep -q {}'
mapper_name = "{host}_{target}".format(
host=self.distname,
target=self.platform_info)
try:
package_name_mapper = PACKAGE_NAME_MAPPERS[mapper_name]
except KeyError:
print("SKIP : We don't know which packages we must install to compile"
" a {target} {build_type} version on a {host} host.".format(
target=self.platform_info,
host=self.distname))
return
packages_list = package_name_mapper.get('COMMON', [])
for dep in self.targetsDict.values():
packages = package_name_mapper.get(dep.name)
if packages:
packages_list += packages
dep.skip = True
for dep in self.targetsDict.values():
packages = getattr(dep, 'extra_packages', [])
for package in packages:
packages_list += package_name_mapper.get(package, [])
if not self.options.force_install_packages and os.path.exists(autoskip_file):
print("SKIP")
return
packages_to_install = []
for package in packages_list:
print(" - {} : ".format(package), end="")
command = package_checker.format(package)
try:
subprocess.check_call(command, shell=True)
except subprocess.CalledProcessError:
print("NEEDED")
packages_to_install.append(package)
else:
print("SKIP")
if packages_to_install:
command = package_installer.format(" ".join(packages_to_install))
print(command)
subprocess.check_call(command, shell=True)
else:
print("SKIP, No package to install.")
with open(autoskip_file, 'w'):
pass

97
kiwixbuild/builder.py Normal file
View File

@ -0,0 +1,97 @@
import sys
from collections import OrderedDict
from .buildenv import *
from .utils import remove_duplicates, StopBuild
from .dependencies import Dependency
class Builder:
def __init__(self, options):
self.options = options
self.targets = OrderedDict()
self.neutralEnv = PlatformNeutralEnv(options)
self.buildEnv = BuildEnv(options, self.neutralEnv, self.targets)
_targets = {}
targetDef = options.targets
self.add_targets(targetDef, _targets)
dependencies = self.order_dependencies(_targets, targetDef)
dependencies = list(remove_duplicates(dependencies))
if options.build_nodeps:
self.targets[targetDef] = _targets[targetDef]
else:
for dep in dependencies:
if self.options.build_deps_only and dep == targetDef:
continue
self.targets[dep] = _targets[dep]
def add_targets(self, targetName, targets):
if targetName in targets:
return
targetClass = Dependency.all_deps[targetName]
target = targetClass(self.neutralEnv, self.buildEnv)
targets[targetName] = target
for dep in target.dependencies:
self.add_targets(dep, targets)
def order_dependencies(self, _targets, targetName):
target = _targets[targetName]
for depName in target.dependencies:
yield from self.order_dependencies(_targets, depName)
yield targetName
def prepare_sources(self):
if self.options.skip_source_prepare:
print("SKIP")
return
toolchain_sources = (tlc.source for tlc in self.buildEnv.toolchains if tlc.source)
for toolchain_source in toolchain_sources:
print("prepare sources for toolchain {} :".format(toolchain_source.name))
toolchain_source.prepare()
sources = (dep.source for dep in self.targets.values() if not dep.skip)
sources = remove_duplicates(sources, lambda s: s.__class__)
for source in sources:
print("prepare sources {} :".format(source.name))
source.prepare()
def build(self):
toolchain_builders = (tlc.builder for tlc in self.buildEnv.toolchains if tlc.builder)
for toolchain_builder in toolchain_builders:
print("build toolchain {} :".format(toolchain_builder.name))
toolchain_builder.build()
builders = (dep.builder for dep in self.targets.values() if (dep.builder and not dep.skip))
for builder in builders:
if self.options.make_dist and builder.name == self.options.targets:
continue
print("build {} :".format(builder.name))
builder.build()
if self.options.make_dist:
dep = self.targets[self.options.targets]
builder = dep.builder
print("make dist {}:".format(builder.name))
builder.make_dist()
def run(self):
try:
print("[INSTALL PACKAGES]")
self.buildEnv.install_packages()
self.buildEnv.finalize_setup()
print("[PREPARE]")
self.prepare_sources()
print("[BUILD]")
self.build()
# No error, clean intermediate file at end of build if needed.
print("[CLEAN]")
if self.buildEnv.options.clean_at_end:
self.buildEnv.clean_intermediate_directories()
else:
print("SKIP")
except StopBuild:
sys.exit("Stopping build due to errors")

View File

@ -22,7 +22,8 @@ class Dependency(metaclass=_MetaDependency):
dependencies = []
force_native_build = False
def __init__(self, buildEnv):
def __init__(self, neutralEnv, buildEnv):
self.neutralEnv = neutralEnv
self.buildEnv = buildEnv
self.source = self.Source(self)
self.builder = self.Builder(self)
@ -40,7 +41,7 @@ class Dependency(metaclass=_MetaDependency):
@property
def source_path(self):
return pj(self.buildEnv.source_dir, self.source.source_dir)
return pj(self.neutralEnv.source_dir, self.source.source_dir)
@property
def _log_dir(self):
@ -73,10 +74,10 @@ class Dependency(metaclass=_MetaDependency):
class Source:
"""Base Class to the real preparator
A source preparator must install source in the self.source_dir attribute
inside the buildEnv.source_dir."""
inside the neutralEnv.source_dir."""
def __init__(self, target):
self.target = target
self.buildEnv = target.buildEnv
self.neutralEnv = target.neutralEnv
@property
def name(self):
@ -87,12 +88,12 @@ class Source:
return self.target.full_name
def _patch(self, context):
source_path = pj(self.buildEnv.source_dir, self.source_dir)
source_path = pj(self.neutralEnv.source_dir, self.source_dir)
context.try_skip(source_path)
context.force_native_build = True
for p in self.patches:
with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input:
self.buildEnv.run_command("patch -p1", source_path, context, input=patch_input.read())
self.neutralEnv.run_command("patch -p1", source_path, context, input=patch_input.read())
def command(self, *args, **kwargs):
return self.target.command(*args, **kwargs)
@ -108,18 +109,18 @@ class ReleaseDownload(Source):
@property
def extract_path(self):
return pj(self.buildEnv.source_dir, self.source_dir)
return pj(self.neutralEnv.source_dir, self.source_dir)
def _download(self, context):
context.try_skip(self.buildEnv.archive_dir, self.name)
self.buildEnv.download(self.archive)
context.try_skip(self.neutralEnv.archive_dir, self.name)
self.neutralEnv.download(self.archive)
def _extract(self, context):
context.try_skip(self.extract_path)
if os.path.exists(self.extract_path):
shutil.rmtree(self.extract_path)
extract_archive(pj(self.buildEnv.archive_dir, self.archive.name),
self.buildEnv.source_dir,
extract_archive(pj(self.neutralEnv.archive_dir, self.archive.name),
self.neutralEnv.source_dir,
topdir=self.archive_top_dir,
name=self.source_dir)
@ -141,36 +142,34 @@ class GitClone(Source):
@property
def source_dir(self):
if self.buildEnv.make_release:
if self.neutralEnv.make_release:
return "{}_release".format(self.git_dir)
else:
return self.git_dir
@property
def git_path(self):
return pj(self.buildEnv.source_dir, self.source_dir)
return pj(self.neutralEnv.source_dir, self.source_dir)
@property
def git_ref(self):
if self.buildEnv.make_release:
if self.neutralEnv.make_release:
return self.release_git_ref
else:
return self.base_git_ref
def _git_clone(self, context):
context.force_native_build = True
if os.path.exists(self.git_path):
raise SkipCommand()
command = "git clone --depth=1 --branch {} {} {}".format(
self.git_ref, self.git_remote, self.source_dir)
self.buildEnv.run_command(command, self.buildEnv.source_dir, context)
self.neutralEnv.run_command(command, self.neutralEnv.source_dir, context)
def _git_update(self, context):
context.force_native_build = True
command = "git fetch origin {}".format(
self.git_ref)
self.buildEnv.run_command(command, self.git_path, context)
self.buildEnv.run_command("git checkout "+self.git_ref, self.git_path, context)
self.neutralEnv.run_command(command, self.git_path, context)
self.neutralEnv.run_command("git checkout "+self.git_ref, self.git_path, context)
def prepare(self):
self.command('gitclone', self._git_clone)
@ -186,19 +185,17 @@ class SvnClone(Source):
@property
def svn_path(self):
return pj(self.buildEnv.source_dir, self.svn_dir)
return pj(self.neutralEnv.source_dir, self.svn_dir)
def _svn_checkout(self, context):
context.force_native_build = True
if os.path.exists(self.svn_path):
raise SkipCommand()
command = "svn checkout {} {}".format(self.svn_remote, self.svn_dir)
self.buildEnv.run_command(command, self.buildEnv.source_dir, context)
self.neutralEnv.run_command(command, self.neutralEnv.source_dir, context)
def _svn_update(self, context):
context.try_skip(self.svn_path)
context.force_native_build = True
self.buildEnv.run_command("svn update", self.svn_path, context)
self.neutralEnv.run_command("svn update", self.svn_path, context)
def prepare(self):
self.command('svncheckout', self._svn_checkout)
@ -358,6 +355,12 @@ class MesonBuilder(Builder):
configure_option = ""
test_option = ""
def __init__(self, target):
super().__init__(target)
self.meson_command = self.buildEnv.neutralEnv.meson_command
self.mesontest_command = self.buildEnv.neutralEnv.mesontest_command
self.ninja_command = self.buildEnv.neutralEnv.ninja_command
@property
def library_type(self):
return 'static' if self.buildEnv.platform_info.static else 'shared'
@ -379,7 +382,7 @@ class MesonBuilder(Builder):
" --libdir={buildEnv.libprefix}"
" {cross_option}")
command = command.format(
command=self.buildEnv.meson_command,
command=self.meson_command,
library_type=self.library_type,
configure_option=configure_option,
build_path=self.build_path,
@ -389,7 +392,7 @@ class MesonBuilder(Builder):
self.buildEnv.run_command(command, self.source_path, context, cross_env_only=True)
def _compile(self, context):
command = "{} -v".format(self.buildEnv.ninja_command)
command = "{} -v".format(self.ninja_command)
self.buildEnv.run_command(command, self.build_path, context)
def _test(self, context):
@ -398,15 +401,15 @@ class MesonBuilder(Builder):
and not self.buildEnv.platform_info.static)
):
raise SkipCommand()
command = "{} --verbose {}".format(self.buildEnv.mesontest_command, self.test_option)
command = "{} --verbose {}".format(self.mesontest_command, self.test_option)
self.buildEnv.run_command(command, self.build_path, context)
def _install(self, context):
command = "{} -v install".format(self.buildEnv.ninja_command)
command = "{} -v install".format(self.ninja_command)
self.buildEnv.run_command(command, self.build_path, context)
def _make_dist(self, context):
command = "{} -v dist".format(self.buildEnv.ninja_command)
command = "{} -v dist".format(self.ninja_command)
self.buildEnv.run_command(command, self.build_path, context)

View File

@ -20,7 +20,7 @@ class Aria2(Dependency):
def _post_prepare_script(self, context):
context.try_skip(self.extract_path)
command = "autoreconf -i"
self.buildEnv.run_command(command, self.extract_path, context)
self.neutralEnv.run_command(command, self.extract_path, context)
class Builder(MakeBuilder):
configure_option = "--enable-libaria2 --disable-ssl --disable-bittorent --disable-metalink --without-sqlite3 --without-libxml2 --without-libexpat"

103
kiwixbuild/packages.py Normal file
View File

@ -0,0 +1,103 @@
_fedora_common = ['automake', 'libtool', 'cmake', 'git', 'subversion', 'ccache', 'pkgconfig', 'gcc-c++', 'gettext-devel']
_debian_common = ['automake', 'libtool', 'cmake', 'git', 'subversion', 'ccache', 'pkg-config', 'gcc', 'autopoint']
PACKAGE_NAME_MAPPERS = {
'fedora_native_dyn': {
'COMMON': _fedora_common,
'uuid': ['libuuid-devel'],
'xapian-core': None, # Not the right version on fedora 25
'ctpp2': None,
'pugixml': None, # ['pugixml-devel'] but package doesn't provide pkg-config file
'libmicrohttpd': ['libmicrohttpd-devel'],
'zlib': ['zlib-devel'],
'lzma': ['xz-devel'],
'icu4c': None,
'zimlib': None,
'file' : ['file-devel'],
'gumbo' : ['gumbo-parser-devel'],
},
'fedora_native_static': {
'COMMON': _fedora_common + ['glibc-static', 'libstdc++-static'],
'zlib': ['zlib-devel', 'zlib-static'],
'lzma': ['xz-devel', 'xz-static']
# Either there is no packages, or no static or too old
},
'fedora_i586_dyn': {
'COMMON': _fedora_common + ['glibc-devel.i686', 'libstdc++-devel.i686'],
},
'fedora_i586_static': {
'COMMON': _fedora_common + ['glibc-devel.i686'],
},
'fedora_win32_dyn': {
'COMMON': _fedora_common + ['mingw32-gcc-c++', 'mingw32-bzip2', 'mingw32-win-iconv', 'mingw32-winpthreads', 'wine'],
'zlib': ['mingw32-zlib'],
'lzma': ['mingw32-xz-libs'],
'libmicrohttpd': ['mingw32-libmicrohttpd'],
},
'fedora_win32_static': {
'COMMON': _fedora_common + ['mingw32-gcc-c++', 'mingw32-bzip2-static', 'mingw32-win-iconv-static', 'mingw32-winpthreads-static', 'wine'],
'zlib': ['mingw32-zlib-static'],
'lzma': ['mingw32-xz-libs-static'],
'libmicrohttpd': None, # ['mingw32-libmicrohttpd-static'] packaging dependecy seems buggy, and some static lib are name libfoo.dll.a and
# gcc cannot found them.
},
'fedora_armhf_static': {
'COMMON': _fedora_common
},
'fedora_armhf_dyn': {
'COMMON': _fedora_common
},
'fedora_android': {
'COMMON': _fedora_common + ['java-1.8.0-openjdk-devel']
},
'debian_native_dyn': {
'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'],
'zlib': ['zlib1g-dev'],
'uuid': ['uuid-dev'],
'ctpp2': ['libctpp2-dev'],
'ctpp2c': ['ctpp2-utils'],
'libmicrohttpd': ['libmicrohttpd-dev', 'ccache']
},
'debian_native_static': {
'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'],
'zlib': ['zlib1g-dev'],
'uuid': ['uuid-dev'],
'ctpp2': ['libctpp2-dev'],
'ctpp2c': ['ctpp2-utils'],
},
'debian_i586_dyn': {
'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib'],
},
'debian_i586_static': {
'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib'],
},
'debian_win32_dyn': {
'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'],
'ctpp2c': ['ctpp2-utils'],
},
'debian_win32_static': {
'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'],
'ctpp2c': ['ctpp2-utils'],
},
'debian_armhf_static': {
'COMMON': _debian_common,
'ctpp2c': ['ctpp2-utils'],
},
'debian_armhf_dyn': {
'COMMON': _debian_common,
'ctpp2c': ['ctpp2-utils'],
},
'debian_android': {
'COMMON': _debian_common + ['default-jdk'],
'ctpp2c': ['ctpp2-utils'],
},
'Darwin_native_dyn': {
'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'],
'file': ['libmagic']
},
'Darwin_iOS': {
'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'],
'file': ['libmagic']
},
}

View File

@ -13,3 +13,9 @@ class PlatformInfo:
def __str__(self):
return "{}_{}".format(self.build, 'static' if self.static else 'dyn')
def set_env(self, env):
pass
def get_bind_dir(self):
return []

View File

@ -43,6 +43,15 @@ class iOSPlatformInfo(PlatformInfo):
},
}
def set_env(self, env):
env['CFLAGS'] = " -fembed-bitcode -isysroot {SDKROOT} -arch {arch} -miphoneos-version-min=9.0 ".format(SDKROOT=self.root_path, arch=self.arch) + env['CFLAGS']
env['CXXFLAGS'] = env['CFLAGS'] + " -stdlib=libc++ -std=c++11 "+env['CXXFLAGS']
env['LDFLAGS'] = " -arch {arch} -isysroot {SDKROOT} ".format(SDKROOT=self.root_path, arch=self.arch)
env['MACOSX_DEPLOYMENT_TARGET'] = "10.7"
def get_bin_dir(self):
return [pj(self.root_path, 'bin')]
iOSPlatformInfo('iOS_armv7', 'armv7')
iOSPlatformInfo('iOS_arm64', 'arm64')
iOSPlatformInfo('iOS_i386', 'i386')

View File

@ -4,15 +4,17 @@ from .base import PlatformInfo
class Win32PlatformInfo(PlatformInfo):
def __init__(self, name, static):
super().__init__(name, 'win32', static, ['mingw32_toolchain'], ['fedora', 'debian'])
self.extra_libs = ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4', '-lmsvcr90', '-liphlpapi']
def get_cross_config(self, host):
root_paths = {
'fedora': '/usr/i686-w64-mingw32/sys-root/mingw',
'debian': '/usr/i686-w64-mingw32'
}
self.root_path = root_paths[host]
return {
'root_path': root_paths[host],
'extra_libs': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4', '-lmsvcr90', '-liphlpapi'],
'root_path': self.root_path,
'extra_libs': self.extra_libs,
'extra_cflags': ['-DWIN32'],
'host_machine': {
'system': 'Windows',
@ -24,6 +26,12 @@ class Win32PlatformInfo(PlatformInfo):
}
}
def get_bin_dir(self):
return [pj(self.root_path, 'bin')]
def set_env(self, env):
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
env['LIBS'] = " ".join(self.extra_libs) + " " +env['LIBS']
Win32PlatformInfo('win32_dyn', False)
Win32PlatformInfo('win32_static', True)

View File

@ -7,6 +7,7 @@ from kiwixbuild.utils import Remotefile, add_execution_right
pj = os.path.join
class android_ndk(Toolchain):
neutral = False
name = 'android-ndk'
version = 'r13b'
gccver = '4.9.x'

View File

@ -50,8 +50,5 @@ class android_sdk(Toolchain):
self.command('build_platform', self._build_platform)
self.command('fix_licenses', self._fix_licenses)
def get_bin_dir(self):
return []
def set_env(self, env):
env['ANDROID_HOME'] = self.builder.install_path

View File

@ -3,7 +3,7 @@ import subprocess
from .base_toolchain import Toolchain
from kiwixbuild.dependencies import GitClone
from kiwixbuild.utils import Remotefile, which
from kiwixbuild.utils import which
pj = os.path.join
class armhf_toolchain(Toolchain):
@ -52,13 +52,11 @@ class armhf_toolchain(Toolchain):
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
env['CFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS']
env['CXXFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
env['LIBS'] = " ".join(self.buildEnv.cross_config['extra_libs']) + " " +env['LIBS']
env['QEMU_LD_PREFIX'] = pj(self.root_path, "arm-linux-gnueabihf", "libc")
env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format(
':'.join([
pj(self.root_path, "arm-linux-gnueabihf", "lib"),
pj(self.buildEnv.install_dir, 'lib'),
pj(self.buildEnv.install_dir, self.buildEnv.libprefix)
env['LD_LIBRARY_PATH']
]))
def set_compiler(self, env):

View File

@ -14,6 +14,7 @@ class _MetaToolchain(type):
class Toolchain(metaclass=_MetaToolchain):
neutral = True
all_toolchains = {}
configure_option = ""
cmake_option = ""
@ -21,8 +22,9 @@ class Toolchain(metaclass=_MetaToolchain):
Builder = None
Source = None
def __init__(self, buildEnv):
self.buildEnv = buildEnv
def __init__(self, neutralEnv):
self.neutralEnv = neutralEnv
self.buildEnv = neutralEnv
self.source = self.Source(self) if self.Source else None
self.builder = self.Builder(self) if self.Builder else None
@ -34,11 +36,11 @@ class Toolchain(metaclass=_MetaToolchain):
@property
def source_path(self):
return pj(self.buildEnv.source_dir, self.source.source_dir)
return pj(self.neutralEnv.source_dir, self.source.source_dir)
@property
def _log_dir(self):
return self.buildEnv.log_dir
return self.neutralEnv.log_dir
def set_env(self, env):
pass
@ -46,6 +48,9 @@ class Toolchain(metaclass=_MetaToolchain):
def set_compiler(self, env):
pass
def get_bin_dir(self):
return []
def command(self, name, function, *args):
print(" {} {} : ".format(name, self.name), end="", flush=True)
log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))

View File

@ -3,10 +3,6 @@ from .base_toolchain import Toolchain
from kiwixbuild.utils import pj, xrun_find
class iOS_sdk(Toolchain):
@property
def root_path(self):
return self.buildEnv.platform_info.root_path
@property
def binaries(self):
return {
@ -22,16 +18,6 @@ class iOS_sdk(Toolchain):
def configure_option(self):
return '--host=arm-apple-darwin'
def get_bin_dir(self):
return [pj(self.root_path, 'bin')]
def set_env(self, env):
arch = self.buildEnv.platform_info.arch
env['CFLAGS'] = " -fembed-bitcode -isysroot {SDKROOT} -arch {arch} -miphoneos-version-min=9.0 ".format(SDKROOT=self.root_path, arch=arch) + env['CFLAGS']
env['CXXFLAGS'] = env['CFLAGS'] + " -stdlib=libc++ -std=c++11 "+env['CXXFLAGS']
env['LDFLAGS'] = " -arch {arch} -isysroot {SDKROOT} ".format(SDKROOT=self.root_path, arch=arch)
env['MACOSX_DEPLOYMENT_TARGET'] = "10.7"
def set_compiler(self, env):
env['CC'] = self.binaries['CC']
env['CXX'] = self.binaries['CXX']

View File

@ -34,18 +34,10 @@ class mingw32_toolchain(Toolchain):
else:
return "exec_wrapper = 'wine'"
@property
def configure_option(self):
return '--host={}'.format(self.arch_full)
def get_bin_dir(self):
return [pj(self.root_path, 'bin')]
def set_env(self, env):
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
env['LIBS'] = " ".join(self.buildEnv.cross_config['extra_libs']) + " " +env['LIBS']
def set_compiler(self, env):
for k, v in self.binaries.items():
env[k] = v