Move Builder and BuildEnv in specific module.
Introduce also a "NeutralEnv", a build environment independent of the targeted platform. All `Source` now build using the neutralEnv. Most of toolchains are also using neutralEnv except android_ndk who is specific to a platform. As toolchain are neutral, platform specific environment variables are now set by the platformInfo directly instead of the toolchain.
This commit is contained in:
Matthieu Gautier
parent
ac83dec674
commit
b950feb893
|
|
@ -1,533 +1,12 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import os, sys, shutil
|
||||
import os, sys
|
||||
import argparse
|
||||
import ssl
|
||||
import urllib.request
|
||||
import subprocess
|
||||
import platform
|
||||
from collections import OrderedDict
|
||||
|
||||
from .toolchains import Toolchain
|
||||
from .dependencies import Dependency
|
||||
from .platforms import PlatformInfo
|
||||
from .utils import (
|
||||
pj,
|
||||
remove_duplicates,
|
||||
add_execution_right,
|
||||
get_sha256,
|
||||
print_progress,
|
||||
setup_print_progress,
|
||||
download_remote,
|
||||
StopBuild,
|
||||
SkipCommand,
|
||||
Defaultdict,
|
||||
Remotefile,
|
||||
Context)
|
||||
|
||||
REMOTE_PREFIX = 'http://download.kiwix.org/dev/'
|
||||
|
||||
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
|
||||
|
||||
|
||||
_fedora_common = ['automake', 'libtool', 'cmake', 'git', 'subversion', 'ccache', 'pkgconfig', 'gcc-c++', 'gettext-devel']
|
||||
_debian_common = ['automake', 'libtool', 'cmake', 'git', 'subversion', 'ccache', 'pkg-config', 'gcc', 'autopoint']
|
||||
PACKAGE_NAME_MAPPERS = {
|
||||
'fedora_native_dyn': {
|
||||
'COMMON': _fedora_common,
|
||||
'uuid': ['libuuid-devel'],
|
||||
'xapian-core': None, # Not the right version on fedora 25
|
||||
'ctpp2': None,
|
||||
'pugixml': None, # ['pugixml-devel'] but package doesn't provide pkg-config file
|
||||
'libmicrohttpd': ['libmicrohttpd-devel'],
|
||||
'zlib': ['zlib-devel'],
|
||||
'lzma': ['xz-devel'],
|
||||
'icu4c': None,
|
||||
'zimlib': None,
|
||||
'file' : ['file-devel'],
|
||||
'gumbo' : ['gumbo-parser-devel'],
|
||||
},
|
||||
'fedora_native_static': {
|
||||
'COMMON': _fedora_common + ['glibc-static', 'libstdc++-static'],
|
||||
'zlib': ['zlib-devel', 'zlib-static'],
|
||||
'lzma': ['xz-devel', 'xz-static']
|
||||
# Either there is no packages, or no static or too old
|
||||
},
|
||||
'fedora_i586_dyn': {
|
||||
'COMMON': _fedora_common + ['glibc-devel.i686', 'libstdc++-devel.i686'],
|
||||
},
|
||||
'fedora_i586_static': {
|
||||
'COMMON': _fedora_common + ['glibc-devel.i686'],
|
||||
},
|
||||
'fedora_win32_dyn': {
|
||||
'COMMON': _fedora_common + ['mingw32-gcc-c++', 'mingw32-bzip2', 'mingw32-win-iconv', 'mingw32-winpthreads', 'wine'],
|
||||
'zlib': ['mingw32-zlib'],
|
||||
'lzma': ['mingw32-xz-libs'],
|
||||
'libmicrohttpd': ['mingw32-libmicrohttpd'],
|
||||
},
|
||||
'fedora_win32_static': {
|
||||
'COMMON': _fedora_common + ['mingw32-gcc-c++', 'mingw32-bzip2-static', 'mingw32-win-iconv-static', 'mingw32-winpthreads-static', 'wine'],
|
||||
'zlib': ['mingw32-zlib-static'],
|
||||
'lzma': ['mingw32-xz-libs-static'],
|
||||
'libmicrohttpd': None, # ['mingw32-libmicrohttpd-static'] packaging dependecy seems buggy, and some static lib are name libfoo.dll.a and
|
||||
# gcc cannot found them.
|
||||
},
|
||||
'fedora_armhf_static': {
|
||||
'COMMON': _fedora_common
|
||||
},
|
||||
'fedora_armhf_dyn': {
|
||||
'COMMON': _fedora_common
|
||||
},
|
||||
'fedora_android': {
|
||||
'COMMON': _fedora_common + ['java-1.8.0-openjdk-devel']
|
||||
},
|
||||
'debian_native_dyn': {
|
||||
'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'],
|
||||
'zlib': ['zlib1g-dev'],
|
||||
'uuid': ['uuid-dev'],
|
||||
'ctpp2': ['libctpp2-dev'],
|
||||
'ctpp2c': ['ctpp2-utils'],
|
||||
'libmicrohttpd': ['libmicrohttpd-dev', 'ccache']
|
||||
},
|
||||
'debian_native_static': {
|
||||
'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'],
|
||||
'zlib': ['zlib1g-dev'],
|
||||
'uuid': ['uuid-dev'],
|
||||
'ctpp2': ['libctpp2-dev'],
|
||||
'ctpp2c': ['ctpp2-utils'],
|
||||
},
|
||||
'debian_i586_dyn': {
|
||||
'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib'],
|
||||
},
|
||||
'debian_i586_static': {
|
||||
'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib'],
|
||||
},
|
||||
'debian_win32_dyn': {
|
||||
'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'],
|
||||
'ctpp2c': ['ctpp2-utils'],
|
||||
},
|
||||
'debian_win32_static': {
|
||||
'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'],
|
||||
'ctpp2c': ['ctpp2-utils'],
|
||||
},
|
||||
'debian_armhf_static': {
|
||||
'COMMON': _debian_common,
|
||||
'ctpp2c': ['ctpp2-utils'],
|
||||
},
|
||||
'debian_armhf_dyn': {
|
||||
'COMMON': _debian_common,
|
||||
'ctpp2c': ['ctpp2-utils'],
|
||||
},
|
||||
'debian_android': {
|
||||
'COMMON': _debian_common + ['default-jdk'],
|
||||
'ctpp2c': ['ctpp2-utils'],
|
||||
},
|
||||
'Darwin_native_dyn': {
|
||||
'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'],
|
||||
'file': ['libmagic']
|
||||
},
|
||||
'Darwin_iOS': {
|
||||
'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'],
|
||||
'file': ['libmagic']
|
||||
},
|
||||
}
|
||||
|
||||
class BuildEnv:
|
||||
def __init__(self, options, targetsDict):
|
||||
self.source_dir = pj(options.working_dir, "SOURCE")
|
||||
build_dir = "BUILD_{}".format(options.target_platform)
|
||||
self.build_dir = pj(options.working_dir, build_dir)
|
||||
self.archive_dir = pj(options.working_dir, "ARCHIVE")
|
||||
self.toolchain_dir = pj(options.working_dir, "TOOLCHAINS")
|
||||
self.log_dir = pj(self.build_dir, 'LOGS')
|
||||
self.install_dir = pj(self.build_dir, "INSTALL")
|
||||
for d in (self.source_dir,
|
||||
self.build_dir,
|
||||
self.archive_dir,
|
||||
self.toolchain_dir,
|
||||
self.log_dir,
|
||||
self.install_dir):
|
||||
os.makedirs(d, exist_ok=True)
|
||||
self.detect_platform()
|
||||
self.ninja_command = self._detect_ninja()
|
||||
if not self.ninja_command:
|
||||
sys.exit("ERROR: ninja command not found")
|
||||
self.meson_command = self._detect_meson()
|
||||
if not self.meson_command:
|
||||
sys.exit("ERROR: meson command not fount")
|
||||
self.mesontest_command = "meson test"
|
||||
self.setup_build(options.target_platform)
|
||||
self.setup_toolchains()
|
||||
self.options = options
|
||||
self.libprefix = options.libprefix or self._detect_libdir()
|
||||
self.targetsDict = targetsDict
|
||||
|
||||
def clean_intermediate_directories(self):
|
||||
for subdir in os.listdir(self.build_dir):
|
||||
subpath = pj(self.build_dir, subdir)
|
||||
if subpath == self.install_dir:
|
||||
continue
|
||||
if os.path.isdir(subpath):
|
||||
shutil.rmtree(subpath)
|
||||
else:
|
||||
os.remove(subpath)
|
||||
|
||||
def detect_platform(self):
|
||||
_platform = platform.system()
|
||||
self.distname = _platform
|
||||
if _platform == 'Windows':
|
||||
print('ERROR: kiwix-build is not intented to run on Windows platform.\n'
|
||||
'It should probably not work, but well, you still can have a try.')
|
||||
cont = input('Do you want to continue ? [y/N]')
|
||||
if cont.lower() != 'y':
|
||||
sys.exit(0)
|
||||
if _platform == 'Linux':
|
||||
self.distname, _, _ = platform.linux_distribution()
|
||||
self.distname = self.distname.lower()
|
||||
if self.distname == 'ubuntu':
|
||||
self.distname = 'debian'
|
||||
|
||||
def setup_build(self, target_platform):
|
||||
self.platform_info = PlatformInfo.all_platforms[target_platform]
|
||||
if self.distname not in self.platform_info.compatible_hosts:
|
||||
print(('ERROR: The target {} cannot be build on host {}.\n'
|
||||
'Select another target platform, or change your host system.'
|
||||
).format(target_platform, self.distname))
|
||||
sys.exit(-1)
|
||||
self.cross_config = self.platform_info.get_cross_config(self.distname)
|
||||
|
||||
def setup_toolchains(self):
|
||||
toolchain_names = self.platform_info.toolchains
|
||||
self.toolchains =[Toolchain.all_toolchains[toolchain_name](self)
|
||||
for toolchain_name in toolchain_names]
|
||||
|
||||
def finalize_setup(self):
|
||||
getattr(self, 'setup_{}'.format(self.platform_info.build))()
|
||||
|
||||
def setup_native(self):
|
||||
self.cmake_crossfile = None
|
||||
self.meson_crossfile = None
|
||||
|
||||
def _gen_crossfile(self, name):
|
||||
crossfile = pj(self.build_dir, name)
|
||||
template_file = pj(SCRIPT_DIR, 'templates', name)
|
||||
with open(template_file, 'r') as f:
|
||||
template = f.read()
|
||||
content = template.format(
|
||||
toolchain=self.toolchains[0],
|
||||
**self.cross_config
|
||||
)
|
||||
with open(crossfile, 'w') as outfile:
|
||||
outfile.write(content)
|
||||
return crossfile
|
||||
|
||||
def setup_win32(self):
|
||||
self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
|
||||
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
|
||||
|
||||
def setup_android(self):
|
||||
self.cmake_crossfile = self._gen_crossfile('cmake_android_cross_file.txt')
|
||||
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
|
||||
|
||||
def setup_armhf(self):
|
||||
self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
|
||||
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
|
||||
|
||||
def setup_iOS(self):
|
||||
self.cmake_crossfile = self._gen_crossfile('cmake_ios_cross_file.txt')
|
||||
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
|
||||
|
||||
def setup_i586(self):
|
||||
self.cmake_crossfile = self._gen_crossfile('cmake_i586_cross_file.txt')
|
||||
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(self.options, name)
|
||||
|
||||
def _is_debianlike(self):
|
||||
return os.path.isfile('/etc/debian_version')
|
||||
|
||||
def _detect_libdir(self):
|
||||
if self._is_debianlike():
|
||||
try:
|
||||
pc = subprocess.Popen(['dpkg-architecture', '-qDEB_HOST_MULTIARCH'],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL)
|
||||
(stdo, _) = pc.communicate()
|
||||
if pc.returncode == 0:
|
||||
archpath = stdo.decode().strip()
|
||||
return 'lib/' + archpath
|
||||
except Exception:
|
||||
pass
|
||||
if os.path.isdir('/usr/lib64') and not os.path.islink('/usr/lib64'):
|
||||
return 'lib64'
|
||||
return 'lib'
|
||||
|
||||
def _detect_ninja(self):
|
||||
for n in ['ninja', 'ninja-build']:
|
||||
try:
|
||||
retcode = subprocess.check_call([n, '--version'],
|
||||
stdout=subprocess.DEVNULL)
|
||||
except (FileNotFoundError, PermissionError):
|
||||
# Doesn't exist in PATH or isn't executable
|
||||
continue
|
||||
if retcode == 0:
|
||||
return n
|
||||
|
||||
def _detect_meson(self):
|
||||
for n in ['meson.py', 'meson']:
|
||||
try:
|
||||
retcode = subprocess.check_call([n, '--version'],
|
||||
stdout=subprocess.DEVNULL)
|
||||
except (FileNotFoundError, PermissionError):
|
||||
# Doesn't exist in PATH or isn't executable
|
||||
continue
|
||||
if retcode == 0:
|
||||
return n
|
||||
|
||||
@property
|
||||
def configure_option(self):
|
||||
configure_options = [tlc.configure_option for tlc in self.toolchains]
|
||||
return " ".join(configure_options)
|
||||
|
||||
@property
|
||||
def cmake_option(self):
|
||||
cmake_options = [tlc.cmake_option for tlc in self.toolchains]
|
||||
return " ".join(cmake_options)
|
||||
|
||||
def _set_env(self, env, cross_compile_env, cross_compile_compiler, cross_compile_path):
|
||||
if env is None:
|
||||
env = Defaultdict(str, os.environ)
|
||||
|
||||
bin_dirs = []
|
||||
if cross_compile_env:
|
||||
for k, v in self.cross_config.get('env', {}).items():
|
||||
if k.startswith('_format_'):
|
||||
v = v.format(**self.cross_config)
|
||||
k = k[8:]
|
||||
env[k] = v
|
||||
for toolchain in self.toolchains:
|
||||
toolchain.set_env(env)
|
||||
if cross_compile_compiler:
|
||||
for toolchain in self.toolchains:
|
||||
toolchain.set_compiler(env)
|
||||
if cross_compile_path:
|
||||
for tlc in self.toolchains:
|
||||
bin_dirs += tlc.get_bin_dir()
|
||||
|
||||
pkgconfig_path = pj(self.install_dir, self.libprefix, 'pkgconfig')
|
||||
env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path])
|
||||
|
||||
# Add ccache path
|
||||
for p in ('/usr/lib/ccache', '/usr/lib64/ccache'):
|
||||
if os.path.isdir(p):
|
||||
ccache_path = [p]
|
||||
break
|
||||
else:
|
||||
ccache_path = []
|
||||
env['PATH'] = ':'.join(bin_dirs +
|
||||
[pj(self.install_dir, 'bin')] +
|
||||
ccache_path +
|
||||
[env['PATH']])
|
||||
|
||||
env['LD_LIBRARY_PATH'] = ':'.join([env['LD_LIBRARY_PATH'],
|
||||
pj(self.install_dir, 'lib'),
|
||||
pj(self.install_dir, self.libprefix)
|
||||
])
|
||||
|
||||
env['CPPFLAGS'] = " ".join(['-I'+pj(self.install_dir, 'include'), env['CPPFLAGS']])
|
||||
env['LDFLAGS'] = " ".join(['-L'+pj(self.install_dir, 'lib'),
|
||||
'-L'+pj(self.install_dir, self.libprefix),
|
||||
env['LDFLAGS']])
|
||||
return env
|
||||
|
||||
def run_command(self, command, cwd, context, env=None, input=None, cross_env_only=False):
|
||||
os.makedirs(cwd, exist_ok=True)
|
||||
cross_compile_env = True
|
||||
cross_compile_compiler = True
|
||||
cross_compile_path = True
|
||||
if context.force_native_build:
|
||||
cross_compile_env = False
|
||||
cross_compile_compiler = False
|
||||
cross_compile_path = False
|
||||
if cross_env_only:
|
||||
cross_compile_compiler = False
|
||||
env = self._set_env(env, cross_compile_env, cross_compile_compiler, cross_compile_path)
|
||||
log = None
|
||||
try:
|
||||
if not self.options.verbose:
|
||||
log = open(context.log_file, 'w')
|
||||
print("run command '{}'".format(command), file=log)
|
||||
print("current directory is '{}'".format(cwd), file=log)
|
||||
print("env is :", file=log)
|
||||
for k, v in env.items():
|
||||
print(" {} : {!r}".format(k, v), file=log)
|
||||
|
||||
kwargs = dict()
|
||||
if input:
|
||||
kwargs['stdin'] = subprocess.PIPE
|
||||
process = subprocess.Popen(command, shell=True, cwd=cwd, env=env, stdout=log or sys.stdout, stderr=subprocess.STDOUT, **kwargs)
|
||||
if input:
|
||||
process.communicate(input.encode())
|
||||
retcode = process.wait()
|
||||
if retcode:
|
||||
raise subprocess.CalledProcessError(retcode, command)
|
||||
finally:
|
||||
if log:
|
||||
log.close()
|
||||
|
||||
def download(self, what, where=None):
|
||||
where = where or self.archive_dir
|
||||
download_remote(what, where, not self.options.no_cert_check)
|
||||
|
||||
def install_packages(self):
|
||||
autoskip_file = pj(self.build_dir, ".install_packages_ok")
|
||||
if self.distname in ('fedora', 'redhat', 'centos'):
|
||||
package_installer = 'sudo dnf install {}'
|
||||
package_checker = 'rpm -q --quiet {}'
|
||||
elif self.distname in ('debian', 'Ubuntu'):
|
||||
package_installer = 'sudo apt-get install {}'
|
||||
package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1'
|
||||
elif self.distname == 'Darwin':
|
||||
package_installer = 'brew install {}'
|
||||
package_checker = 'brew list -1 | grep -q {}'
|
||||
mapper_name = "{host}_{target}".format(
|
||||
host=self.distname,
|
||||
target=self.platform_info)
|
||||
try:
|
||||
package_name_mapper = PACKAGE_NAME_MAPPERS[mapper_name]
|
||||
except KeyError:
|
||||
print("SKIP : We don't know which packages we must install to compile"
|
||||
" a {target} {build_type} version on a {host} host.".format(
|
||||
target=self.platform_info,
|
||||
host=self.distname))
|
||||
return
|
||||
|
||||
packages_list = package_name_mapper.get('COMMON', [])
|
||||
for dep in self.targetsDict.values():
|
||||
packages = package_name_mapper.get(dep.name)
|
||||
if packages:
|
||||
packages_list += packages
|
||||
dep.skip = True
|
||||
for dep in self.targetsDict.values():
|
||||
packages = getattr(dep, 'extra_packages', [])
|
||||
for package in packages:
|
||||
packages_list += package_name_mapper.get(package, [])
|
||||
if not self.options.force_install_packages and os.path.exists(autoskip_file):
|
||||
print("SKIP")
|
||||
return
|
||||
|
||||
packages_to_install = []
|
||||
for package in packages_list:
|
||||
print(" - {} : ".format(package), end="")
|
||||
command = package_checker.format(package)
|
||||
try:
|
||||
subprocess.check_call(command, shell=True)
|
||||
except subprocess.CalledProcessError:
|
||||
print("NEEDED")
|
||||
packages_to_install.append(package)
|
||||
else:
|
||||
print("SKIP")
|
||||
|
||||
if packages_to_install:
|
||||
command = package_installer.format(" ".join(packages_to_install))
|
||||
print(command)
|
||||
subprocess.check_call(command, shell=True)
|
||||
else:
|
||||
print("SKIP, No package to install.")
|
||||
|
||||
with open(autoskip_file, 'w'):
|
||||
pass
|
||||
|
||||
|
||||
class Builder:
|
||||
def __init__(self, options):
|
||||
self.options = options
|
||||
self.targets = OrderedDict()
|
||||
self.buildEnv = buildEnv = BuildEnv(options, self.targets)
|
||||
|
||||
_targets = {}
|
||||
targetDef = options.targets
|
||||
self.add_targets(targetDef, _targets)
|
||||
dependencies = self.order_dependencies(_targets, targetDef)
|
||||
dependencies = list(remove_duplicates(dependencies))
|
||||
|
||||
if options.build_nodeps:
|
||||
self.targets[targetDef] = _targets[targetDef]
|
||||
else:
|
||||
for dep in dependencies:
|
||||
if self.options.build_deps_only and dep == targetDef:
|
||||
continue
|
||||
self.targets[dep] = _targets[dep]
|
||||
|
||||
def add_targets(self, targetName, targets):
|
||||
if targetName in targets:
|
||||
return
|
||||
targetClass = Dependency.all_deps[targetName]
|
||||
target = targetClass(self.buildEnv)
|
||||
targets[targetName] = target
|
||||
for dep in target.dependencies:
|
||||
self.add_targets(dep, targets)
|
||||
|
||||
def order_dependencies(self, _targets, targetName):
|
||||
target = _targets[targetName]
|
||||
for depName in target.dependencies:
|
||||
yield from self.order_dependencies(_targets, depName)
|
||||
yield targetName
|
||||
|
||||
def prepare_sources(self):
|
||||
if self.options.skip_source_prepare:
|
||||
print("SKIP")
|
||||
return
|
||||
|
||||
toolchain_sources = (tlc.source for tlc in self.buildEnv.toolchains if tlc.source)
|
||||
for toolchain_source in toolchain_sources:
|
||||
print("prepare sources for toolchain {} :".format(toolchain_source.name))
|
||||
toolchain_source.prepare()
|
||||
|
||||
sources = (dep.source for dep in self.targets.values() if not dep.skip)
|
||||
sources = remove_duplicates(sources, lambda s: s.__class__)
|
||||
for source in sources:
|
||||
print("prepare sources {} :".format(source.name))
|
||||
source.prepare()
|
||||
|
||||
def build(self):
|
||||
toolchain_builders = (tlc.builder for tlc in self.buildEnv.toolchains if tlc.builder)
|
||||
for toolchain_builder in toolchain_builders:
|
||||
print("build toolchain {} :".format(toolchain_builder.name))
|
||||
toolchain_builder.build()
|
||||
|
||||
builders = (dep.builder for dep in self.targets.values() if (dep.builder and not dep.skip))
|
||||
for builder in builders:
|
||||
if self.options.make_dist and builder.name == self.options.targets:
|
||||
continue
|
||||
print("build {} :".format(builder.name))
|
||||
builder.build()
|
||||
|
||||
if self.options.make_dist:
|
||||
dep = self.targets[self.options.targets]
|
||||
builder = dep.builder
|
||||
print("make dist {}:".format(builder.name))
|
||||
builder.make_dist()
|
||||
|
||||
def run(self):
|
||||
try:
|
||||
print("[INSTALL PACKAGES]")
|
||||
self.buildEnv.install_packages()
|
||||
self.buildEnv.finalize_setup()
|
||||
print("[PREPARE]")
|
||||
self.prepare_sources()
|
||||
print("[BUILD]")
|
||||
self.build()
|
||||
# No error, clean intermediate file at end of build if needed.
|
||||
print("[CLEAN]")
|
||||
if self.buildEnv.options.clean_at_end:
|
||||
self.buildEnv.clean_intermediate_directories()
|
||||
else:
|
||||
print("SKIP")
|
||||
except StopBuild:
|
||||
sys.exit("Stopping build due to errors")
|
||||
|
||||
from .builder import Builder
|
||||
from .utils import setup_print_progress
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,376 @@
|
|||
|
||||
import os, sys, shutil
|
||||
import subprocess
|
||||
import platform
|
||||
|
||||
from .platforms import PlatformInfo
|
||||
from .toolchains import Toolchain
|
||||
from .packages import PACKAGE_NAME_MAPPERS
|
||||
from .utils import (
|
||||
pj,
|
||||
download_remote,
|
||||
Defaultdict)
|
||||
|
||||
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
|
||||
|
||||
|
||||
class PlatformNeutralEnv:
|
||||
def __init__(self, options):
|
||||
self.options = options
|
||||
self.source_dir = pj(options.working_dir, "SOURCE")
|
||||
self.archive_dir = pj(options.working_dir, "ARCHIVE")
|
||||
self.toolchain_dir = pj(options.working_dir, "TOOLCHAINS")
|
||||
self.log_dir = pj(self.working_dir, 'LOGS')
|
||||
for d in (self.source_dir,
|
||||
self.archive_dir,
|
||||
self.toolchain_dir,
|
||||
self.log_dir):
|
||||
os.makedirs(d, exist_ok=True)
|
||||
self.toolchains = {}
|
||||
self.detect_platform()
|
||||
self.ninja_command = self._detect_ninja()
|
||||
if not self.ninja_command:
|
||||
sys.exit("ERROR: ninja command not found")
|
||||
self.meson_command = self._detect_meson()
|
||||
if not self.meson_command:
|
||||
sys.exit("ERROR: meson command not fount")
|
||||
self.mesontest_command = "{} test".format(self.meson_command)
|
||||
|
||||
def run_command(self, command, cwd, context, env=None, input=None):
|
||||
os.makedirs(cwd, exist_ok=True)
|
||||
if env is None:
|
||||
env = Defaultdict(str, os.environ)
|
||||
log = None
|
||||
try:
|
||||
if not self.options.verbose:
|
||||
log = open(context.log_file, 'w')
|
||||
print("run command '{}'".format(command), file=log)
|
||||
print("current directory is '{}'".format(cwd), file=log)
|
||||
print("env is :", file=log)
|
||||
for k, v in env.items():
|
||||
print(" {} : {!r}".format(k, v), file=log)
|
||||
|
||||
kwargs = dict()
|
||||
if input:
|
||||
kwargs['stdin'] = subprocess.PIPE
|
||||
process = subprocess.Popen(command, shell=True, cwd=cwd, env=env, stdout=log or sys.stdout, stderr=subprocess.STDOUT, **kwargs)
|
||||
if input:
|
||||
process.communicate(input.encode())
|
||||
retcode = process.wait()
|
||||
if retcode:
|
||||
raise subprocess.CalledProcessError(retcode, command)
|
||||
finally:
|
||||
if log:
|
||||
log.close()
|
||||
|
||||
def detect_platform(self):
|
||||
_platform = platform.system()
|
||||
self.distname = _platform
|
||||
if _platform == 'Windows':
|
||||
print('ERROR: kiwix-build is not intented to run on Windows platform.\n'
|
||||
'It should probably not work, but well, you still can have a try.')
|
||||
cont = input('Do you want to continue ? [y/N]')
|
||||
if cont.lower() != 'y':
|
||||
sys.exit(0)
|
||||
if _platform == 'Linux':
|
||||
self.distname, _, _ = platform.linux_distribution()
|
||||
self.distname = self.distname.lower()
|
||||
if self.distname == 'ubuntu':
|
||||
self.distname = 'debian'
|
||||
|
||||
def download(self, what, where=None):
|
||||
where = where or self.archive_dir
|
||||
download_remote(what, where, not self.options.no_cert_check)
|
||||
|
||||
def _detect_ninja(self):
|
||||
for n in ['ninja', 'ninja-build']:
|
||||
try:
|
||||
retcode = subprocess.check_call([n, '--version'],
|
||||
stdout=subprocess.DEVNULL)
|
||||
except (FileNotFoundError, PermissionError):
|
||||
# Doesn't exist in PATH or isn't executable
|
||||
continue
|
||||
if retcode == 0:
|
||||
return n
|
||||
|
||||
def add_toolchain(self, toolchain_name):
|
||||
if toolchain_name not in self.toolchains:
|
||||
ToolchainClass = Toolchain.all_toolchains[toolchain_name]
|
||||
self.toolchains[toolchain_name] = ToolchainClass(self)
|
||||
return self.toolchains[toolchain_name]
|
||||
|
||||
def _detect_meson(self):
|
||||
for n in ['meson.py', 'meson']:
|
||||
try:
|
||||
retcode = subprocess.check_call([n, '--version'],
|
||||
stdout=subprocess.DEVNULL)
|
||||
except (FileNotFoundError, PermissionError):
|
||||
# Doesn't exist in PATH or isn't executable
|
||||
continue
|
||||
if retcode == 0:
|
||||
return n
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(self.options, name)
|
||||
|
||||
|
||||
class BuildEnv:
|
||||
def __init__(self, options, neutralEnv, targetsDict):
|
||||
build_dir = "BUILD_{}".format(options.target_platform)
|
||||
self.neutralEnv = neutralEnv
|
||||
self.build_dir = pj(options.working_dir, build_dir)
|
||||
self.install_dir = pj(self.build_dir, "INSTALL")
|
||||
for d in (self.build_dir,
|
||||
self.install_dir):
|
||||
os.makedirs(d, exist_ok=True)
|
||||
|
||||
self.setup_build(options.target_platform)
|
||||
self.setup_toolchains()
|
||||
self.options = options
|
||||
self.libprefix = options.libprefix or self._detect_libdir()
|
||||
self.targetsDict = targetsDict
|
||||
|
||||
def clean_intermediate_directories(self):
|
||||
for subdir in os.listdir(self.build_dir):
|
||||
subpath = pj(self.build_dir, subdir)
|
||||
if subpath == self.install_dir:
|
||||
continue
|
||||
if os.path.isdir(subpath):
|
||||
shutil.rmtree(subpath)
|
||||
else:
|
||||
os.remove(subpath)
|
||||
|
||||
def setup_build(self, target_platform):
|
||||
self.platform_info = PlatformInfo.all_platforms[target_platform]
|
||||
if self.distname not in self.platform_info.compatible_hosts:
|
||||
print(('ERROR: The target {} cannot be build on host {}.\n'
|
||||
'Select another target platform, or change your host system.'
|
||||
).format(target_platform, self.distname))
|
||||
sys.exit(-1)
|
||||
self.cross_config = self.platform_info.get_cross_config(self.distname)
|
||||
|
||||
def setup_toolchains(self):
|
||||
toolchain_names = self.platform_info.toolchains
|
||||
self.toolchains = []
|
||||
for toolchain_name in toolchain_names:
|
||||
ToolchainClass = Toolchain.all_toolchains[toolchain_name]
|
||||
if ToolchainClass.neutral:
|
||||
self.toolchains.append(
|
||||
self.neutralEnv.add_toolchain(toolchain_name)
|
||||
)
|
||||
else:
|
||||
self.toolchains.append(ToolchainClass(self))
|
||||
|
||||
def finalize_setup(self):
|
||||
getattr(self, 'setup_{}'.format(self.platform_info.build))()
|
||||
|
||||
def setup_native(self):
|
||||
self.cmake_crossfile = None
|
||||
self.meson_crossfile = None
|
||||
|
||||
def _gen_crossfile(self, name):
|
||||
crossfile = pj(self.build_dir, name)
|
||||
template_file = pj(SCRIPT_DIR, 'templates', name)
|
||||
with open(template_file, 'r') as f:
|
||||
template = f.read()
|
||||
content = template.format(
|
||||
toolchain=self.toolchains[0],
|
||||
**self.cross_config
|
||||
)
|
||||
with open(crossfile, 'w') as outfile:
|
||||
outfile.write(content)
|
||||
return crossfile
|
||||
|
||||
def setup_win32(self):
|
||||
self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
|
||||
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
|
||||
|
||||
def setup_android(self):
|
||||
self.cmake_crossfile = self._gen_crossfile('cmake_android_cross_file.txt')
|
||||
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
|
||||
|
||||
def setup_armhf(self):
|
||||
self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
|
||||
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
|
||||
|
||||
def setup_iOS(self):
|
||||
self.cmake_crossfile = self._gen_crossfile('cmake_ios_cross_file.txt')
|
||||
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
|
||||
|
||||
def setup_i586(self):
|
||||
self.cmake_crossfile = self._gen_crossfile('cmake_i586_cross_file.txt')
|
||||
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(self.neutralEnv, name)
|
||||
|
||||
def _is_debianlike(self):
|
||||
return os.path.isfile('/etc/debian_version')
|
||||
|
||||
def _detect_libdir(self):
|
||||
if self._is_debianlike():
|
||||
try:
|
||||
pc = subprocess.Popen(['dpkg-architecture', '-qDEB_HOST_MULTIARCH'],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL)
|
||||
(stdo, _) = pc.communicate()
|
||||
if pc.returncode == 0:
|
||||
archpath = stdo.decode().strip()
|
||||
return 'lib/' + archpath
|
||||
except Exception:
|
||||
pass
|
||||
if os.path.isdir('/usr/lib64') and not os.path.islink('/usr/lib64'):
|
||||
return 'lib64'
|
||||
return 'lib'
|
||||
|
||||
@property
|
||||
def configure_option(self):
|
||||
configure_options = [tlc.configure_option for tlc in self.toolchains]
|
||||
return " ".join(configure_options)
|
||||
|
||||
@property
|
||||
def cmake_option(self):
|
||||
cmake_options = [tlc.cmake_option for tlc in self.toolchains]
|
||||
return " ".join(cmake_options)
|
||||
|
||||
def _set_env(self, env, cross_compile_env, cross_compile_compiler, cross_compile_path):
|
||||
if env is None:
|
||||
env = Defaultdict(str, os.environ)
|
||||
|
||||
pkgconfig_path = pj(self.install_dir, self.libprefix, 'pkgconfig')
|
||||
env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path])
|
||||
|
||||
# Add ccache path
|
||||
for p in ('/usr/lib/ccache', '/usr/lib64/ccache'):
|
||||
if os.path.isdir(p):
|
||||
ccache_path = [p]
|
||||
break
|
||||
else:
|
||||
ccache_path = []
|
||||
env['PATH'] = ':'.join([pj(self.install_dir, 'bin')] +
|
||||
ccache_path +
|
||||
[env['PATH']])
|
||||
|
||||
env['LD_LIBRARY_PATH'] = ':'.join([env['LD_LIBRARY_PATH'],
|
||||
pj(self.install_dir, 'lib'),
|
||||
pj(self.install_dir, self.libprefix)
|
||||
])
|
||||
|
||||
env['CPPFLAGS'] = " ".join(['-I'+pj(self.install_dir, 'include'), env['CPPFLAGS']])
|
||||
env['LDFLAGS'] = " ".join(['-L'+pj(self.install_dir, 'lib'),
|
||||
'-L'+pj(self.install_dir, self.libprefix),
|
||||
env['LDFLAGS']])
|
||||
|
||||
if cross_compile_env:
|
||||
for k, v in self.cross_config.get('env', {}).items():
|
||||
if k.startswith('_format_'):
|
||||
v = v.format(**self.cross_config)
|
||||
k = k[8:]
|
||||
env[k] = v
|
||||
for toolchain in self.toolchains:
|
||||
toolchain.set_env(env)
|
||||
self.platform_info.set_env(env)
|
||||
if cross_compile_compiler:
|
||||
for toolchain in self.toolchains:
|
||||
toolchain.set_compiler(env)
|
||||
if cross_compile_path:
|
||||
bin_dirs = []
|
||||
for tlc in self.toolchains:
|
||||
bin_dirs += tlc.get_bin_dir()
|
||||
bin_dirs += self.platform_info.get_bind_dir()
|
||||
env['PATH'] = ':'.join(bin_dirs + [env['PATH']])
|
||||
return env
|
||||
|
||||
def run_command(self, command, cwd, context, env=None, input=None, cross_env_only=False):
|
||||
os.makedirs(cwd, exist_ok=True)
|
||||
cross_compile_env = True
|
||||
cross_compile_compiler = True
|
||||
cross_compile_path = True
|
||||
if context.force_native_build:
|
||||
cross_compile_env = False
|
||||
cross_compile_compiler = False
|
||||
cross_compile_path = False
|
||||
if cross_env_only:
|
||||
cross_compile_compiler = False
|
||||
env = self._set_env(env, cross_compile_env, cross_compile_compiler, cross_compile_path)
|
||||
log = None
|
||||
try:
|
||||
if not self.options.verbose:
|
||||
log = open(context.log_file, 'w')
|
||||
print("run command '{}'".format(command), file=log)
|
||||
print("current directory is '{}'".format(cwd), file=log)
|
||||
print("env is :", file=log)
|
||||
for k, v in env.items():
|
||||
print(" {} : {!r}".format(k, v), file=log)
|
||||
|
||||
kwargs = dict()
|
||||
if input:
|
||||
kwargs['stdin'] = subprocess.PIPE
|
||||
process = subprocess.Popen(command, shell=True, cwd=cwd, env=env, stdout=log or sys.stdout, stderr=subprocess.STDOUT, **kwargs)
|
||||
if input:
|
||||
process.communicate(input.encode())
|
||||
retcode = process.wait()
|
||||
if retcode:
|
||||
raise subprocess.CalledProcessError(retcode, command)
|
||||
finally:
|
||||
if log:
|
||||
log.close()
|
||||
|
||||
def install_packages(self):
|
||||
autoskip_file = pj(self.build_dir, ".install_packages_ok")
|
||||
if self.distname in ('fedora', 'redhat', 'centos'):
|
||||
package_installer = 'sudo dnf install {}'
|
||||
package_checker = 'rpm -q --quiet {}'
|
||||
elif self.distname in ('debian', 'Ubuntu'):
|
||||
package_installer = 'sudo apt-get install {}'
|
||||
package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1'
|
||||
elif self.distname == 'Darwin':
|
||||
package_installer = 'brew install {}'
|
||||
package_checker = 'brew list -1 | grep -q {}'
|
||||
mapper_name = "{host}_{target}".format(
|
||||
host=self.distname,
|
||||
target=self.platform_info)
|
||||
try:
|
||||
package_name_mapper = PACKAGE_NAME_MAPPERS[mapper_name]
|
||||
except KeyError:
|
||||
print("SKIP : We don't know which packages we must install to compile"
|
||||
" a {target} {build_type} version on a {host} host.".format(
|
||||
target=self.platform_info,
|
||||
host=self.distname))
|
||||
return
|
||||
|
||||
packages_list = package_name_mapper.get('COMMON', [])
|
||||
for dep in self.targetsDict.values():
|
||||
packages = package_name_mapper.get(dep.name)
|
||||
if packages:
|
||||
packages_list += packages
|
||||
dep.skip = True
|
||||
for dep in self.targetsDict.values():
|
||||
packages = getattr(dep, 'extra_packages', [])
|
||||
for package in packages:
|
||||
packages_list += package_name_mapper.get(package, [])
|
||||
if not self.options.force_install_packages and os.path.exists(autoskip_file):
|
||||
print("SKIP")
|
||||
return
|
||||
|
||||
packages_to_install = []
|
||||
for package in packages_list:
|
||||
print(" - {} : ".format(package), end="")
|
||||
command = package_checker.format(package)
|
||||
try:
|
||||
subprocess.check_call(command, shell=True)
|
||||
except subprocess.CalledProcessError:
|
||||
print("NEEDED")
|
||||
packages_to_install.append(package)
|
||||
else:
|
||||
print("SKIP")
|
||||
|
||||
if packages_to_install:
|
||||
command = package_installer.format(" ".join(packages_to_install))
|
||||
print(command)
|
||||
subprocess.check_call(command, shell=True)
|
||||
else:
|
||||
print("SKIP, No package to install.")
|
||||
|
||||
with open(autoskip_file, 'w'):
|
||||
pass
|
||||
|
|
@ -0,0 +1,97 @@
|
|||
|
||||
import sys
|
||||
from collections import OrderedDict
|
||||
from .buildenv import *
|
||||
|
||||
from .utils import remove_duplicates, StopBuild
|
||||
from .dependencies import Dependency
|
||||
|
||||
class Builder:
|
||||
def __init__(self, options):
|
||||
self.options = options
|
||||
self.targets = OrderedDict()
|
||||
self.neutralEnv = PlatformNeutralEnv(options)
|
||||
self.buildEnv = BuildEnv(options, self.neutralEnv, self.targets)
|
||||
|
||||
_targets = {}
|
||||
targetDef = options.targets
|
||||
self.add_targets(targetDef, _targets)
|
||||
dependencies = self.order_dependencies(_targets, targetDef)
|
||||
dependencies = list(remove_duplicates(dependencies))
|
||||
|
||||
if options.build_nodeps:
|
||||
self.targets[targetDef] = _targets[targetDef]
|
||||
else:
|
||||
for dep in dependencies:
|
||||
if self.options.build_deps_only and dep == targetDef:
|
||||
continue
|
||||
self.targets[dep] = _targets[dep]
|
||||
|
||||
def add_targets(self, targetName, targets):
|
||||
if targetName in targets:
|
||||
return
|
||||
targetClass = Dependency.all_deps[targetName]
|
||||
target = targetClass(self.neutralEnv, self.buildEnv)
|
||||
targets[targetName] = target
|
||||
for dep in target.dependencies:
|
||||
self.add_targets(dep, targets)
|
||||
|
||||
def order_dependencies(self, _targets, targetName):
|
||||
target = _targets[targetName]
|
||||
for depName in target.dependencies:
|
||||
yield from self.order_dependencies(_targets, depName)
|
||||
yield targetName
|
||||
|
||||
def prepare_sources(self):
|
||||
if self.options.skip_source_prepare:
|
||||
print("SKIP")
|
||||
return
|
||||
|
||||
toolchain_sources = (tlc.source for tlc in self.buildEnv.toolchains if tlc.source)
|
||||
for toolchain_source in toolchain_sources:
|
||||
print("prepare sources for toolchain {} :".format(toolchain_source.name))
|
||||
toolchain_source.prepare()
|
||||
|
||||
sources = (dep.source for dep in self.targets.values() if not dep.skip)
|
||||
sources = remove_duplicates(sources, lambda s: s.__class__)
|
||||
for source in sources:
|
||||
print("prepare sources {} :".format(source.name))
|
||||
source.prepare()
|
||||
|
||||
def build(self):
|
||||
toolchain_builders = (tlc.builder for tlc in self.buildEnv.toolchains if tlc.builder)
|
||||
for toolchain_builder in toolchain_builders:
|
||||
print("build toolchain {} :".format(toolchain_builder.name))
|
||||
toolchain_builder.build()
|
||||
|
||||
builders = (dep.builder for dep in self.targets.values() if (dep.builder and not dep.skip))
|
||||
for builder in builders:
|
||||
if self.options.make_dist and builder.name == self.options.targets:
|
||||
continue
|
||||
print("build {} :".format(builder.name))
|
||||
builder.build()
|
||||
|
||||
if self.options.make_dist:
|
||||
dep = self.targets[self.options.targets]
|
||||
builder = dep.builder
|
||||
print("make dist {}:".format(builder.name))
|
||||
builder.make_dist()
|
||||
|
||||
def run(self):
|
||||
try:
|
||||
print("[INSTALL PACKAGES]")
|
||||
self.buildEnv.install_packages()
|
||||
self.buildEnv.finalize_setup()
|
||||
print("[PREPARE]")
|
||||
self.prepare_sources()
|
||||
print("[BUILD]")
|
||||
self.build()
|
||||
# No error, clean intermediate file at end of build if needed.
|
||||
print("[CLEAN]")
|
||||
if self.buildEnv.options.clean_at_end:
|
||||
self.buildEnv.clean_intermediate_directories()
|
||||
else:
|
||||
print("SKIP")
|
||||
except StopBuild:
|
||||
sys.exit("Stopping build due to errors")
|
||||
|
||||
|
|
@ -22,7 +22,8 @@ class Dependency(metaclass=_MetaDependency):
|
|||
dependencies = []
|
||||
force_native_build = False
|
||||
|
||||
def __init__(self, buildEnv):
|
||||
def __init__(self, neutralEnv, buildEnv):
|
||||
self.neutralEnv = neutralEnv
|
||||
self.buildEnv = buildEnv
|
||||
self.source = self.Source(self)
|
||||
self.builder = self.Builder(self)
|
||||
|
|
@ -40,7 +41,7 @@ class Dependency(metaclass=_MetaDependency):
|
|||
|
||||
@property
|
||||
def source_path(self):
|
||||
return pj(self.buildEnv.source_dir, self.source.source_dir)
|
||||
return pj(self.neutralEnv.source_dir, self.source.source_dir)
|
||||
|
||||
@property
|
||||
def _log_dir(self):
|
||||
|
|
@ -73,10 +74,10 @@ class Dependency(metaclass=_MetaDependency):
|
|||
class Source:
|
||||
"""Base Class to the real preparator
|
||||
A source preparator must install source in the self.source_dir attribute
|
||||
inside the buildEnv.source_dir."""
|
||||
inside the neutralEnv.source_dir."""
|
||||
def __init__(self, target):
|
||||
self.target = target
|
||||
self.buildEnv = target.buildEnv
|
||||
self.neutralEnv = target.neutralEnv
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
|
|
@ -87,12 +88,12 @@ class Source:
|
|||
return self.target.full_name
|
||||
|
||||
def _patch(self, context):
|
||||
source_path = pj(self.buildEnv.source_dir, self.source_dir)
|
||||
source_path = pj(self.neutralEnv.source_dir, self.source_dir)
|
||||
context.try_skip(source_path)
|
||||
context.force_native_build = True
|
||||
for p in self.patches:
|
||||
with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input:
|
||||
self.buildEnv.run_command("patch -p1", source_path, context, input=patch_input.read())
|
||||
self.neutralEnv.run_command("patch -p1", source_path, context, input=patch_input.read())
|
||||
|
||||
def command(self, *args, **kwargs):
|
||||
return self.target.command(*args, **kwargs)
|
||||
|
|
@ -108,18 +109,18 @@ class ReleaseDownload(Source):
|
|||
|
||||
@property
|
||||
def extract_path(self):
|
||||
return pj(self.buildEnv.source_dir, self.source_dir)
|
||||
return pj(self.neutralEnv.source_dir, self.source_dir)
|
||||
|
||||
def _download(self, context):
|
||||
context.try_skip(self.buildEnv.archive_dir, self.name)
|
||||
self.buildEnv.download(self.archive)
|
||||
context.try_skip(self.neutralEnv.archive_dir, self.name)
|
||||
self.neutralEnv.download(self.archive)
|
||||
|
||||
def _extract(self, context):
|
||||
context.try_skip(self.extract_path)
|
||||
if os.path.exists(self.extract_path):
|
||||
shutil.rmtree(self.extract_path)
|
||||
extract_archive(pj(self.buildEnv.archive_dir, self.archive.name),
|
||||
self.buildEnv.source_dir,
|
||||
extract_archive(pj(self.neutralEnv.archive_dir, self.archive.name),
|
||||
self.neutralEnv.source_dir,
|
||||
topdir=self.archive_top_dir,
|
||||
name=self.source_dir)
|
||||
|
||||
|
|
@ -141,36 +142,34 @@ class GitClone(Source):
|
|||
|
||||
@property
|
||||
def source_dir(self):
|
||||
if self.buildEnv.make_release:
|
||||
if self.neutralEnv.make_release:
|
||||
return "{}_release".format(self.git_dir)
|
||||
else:
|
||||
return self.git_dir
|
||||
|
||||
@property
|
||||
def git_path(self):
|
||||
return pj(self.buildEnv.source_dir, self.source_dir)
|
||||
return pj(self.neutralEnv.source_dir, self.source_dir)
|
||||
|
||||
@property
|
||||
def git_ref(self):
|
||||
if self.buildEnv.make_release:
|
||||
if self.neutralEnv.make_release:
|
||||
return self.release_git_ref
|
||||
else:
|
||||
return self.base_git_ref
|
||||
|
||||
def _git_clone(self, context):
|
||||
context.force_native_build = True
|
||||
if os.path.exists(self.git_path):
|
||||
raise SkipCommand()
|
||||
command = "git clone --depth=1 --branch {} {} {}".format(
|
||||
self.git_ref, self.git_remote, self.source_dir)
|
||||
self.buildEnv.run_command(command, self.buildEnv.source_dir, context)
|
||||
self.neutralEnv.run_command(command, self.neutralEnv.source_dir, context)
|
||||
|
||||
def _git_update(self, context):
|
||||
context.force_native_build = True
|
||||
command = "git fetch origin {}".format(
|
||||
self.git_ref)
|
||||
self.buildEnv.run_command(command, self.git_path, context)
|
||||
self.buildEnv.run_command("git checkout "+self.git_ref, self.git_path, context)
|
||||
self.neutralEnv.run_command(command, self.git_path, context)
|
||||
self.neutralEnv.run_command("git checkout "+self.git_ref, self.git_path, context)
|
||||
|
||||
def prepare(self):
|
||||
self.command('gitclone', self._git_clone)
|
||||
|
|
@ -186,19 +185,17 @@ class SvnClone(Source):
|
|||
|
||||
@property
|
||||
def svn_path(self):
|
||||
return pj(self.buildEnv.source_dir, self.svn_dir)
|
||||
return pj(self.neutralEnv.source_dir, self.svn_dir)
|
||||
|
||||
def _svn_checkout(self, context):
|
||||
context.force_native_build = True
|
||||
if os.path.exists(self.svn_path):
|
||||
raise SkipCommand()
|
||||
command = "svn checkout {} {}".format(self.svn_remote, self.svn_dir)
|
||||
self.buildEnv.run_command(command, self.buildEnv.source_dir, context)
|
||||
self.neutralEnv.run_command(command, self.neutralEnv.source_dir, context)
|
||||
|
||||
def _svn_update(self, context):
|
||||
context.try_skip(self.svn_path)
|
||||
context.force_native_build = True
|
||||
self.buildEnv.run_command("svn update", self.svn_path, context)
|
||||
self.neutralEnv.run_command("svn update", self.svn_path, context)
|
||||
|
||||
def prepare(self):
|
||||
self.command('svncheckout', self._svn_checkout)
|
||||
|
|
@ -358,6 +355,12 @@ class MesonBuilder(Builder):
|
|||
configure_option = ""
|
||||
test_option = ""
|
||||
|
||||
def __init__(self, target):
|
||||
super().__init__(target)
|
||||
self.meson_command = self.buildEnv.neutralEnv.meson_command
|
||||
self.mesontest_command = self.buildEnv.neutralEnv.mesontest_command
|
||||
self.ninja_command = self.buildEnv.neutralEnv.ninja_command
|
||||
|
||||
@property
|
||||
def library_type(self):
|
||||
return 'static' if self.buildEnv.platform_info.static else 'shared'
|
||||
|
|
@ -379,7 +382,7 @@ class MesonBuilder(Builder):
|
|||
" --libdir={buildEnv.libprefix}"
|
||||
" {cross_option}")
|
||||
command = command.format(
|
||||
command=self.buildEnv.meson_command,
|
||||
command=self.meson_command,
|
||||
library_type=self.library_type,
|
||||
configure_option=configure_option,
|
||||
build_path=self.build_path,
|
||||
|
|
@ -389,7 +392,7 @@ class MesonBuilder(Builder):
|
|||
self.buildEnv.run_command(command, self.source_path, context, cross_env_only=True)
|
||||
|
||||
def _compile(self, context):
|
||||
command = "{} -v".format(self.buildEnv.ninja_command)
|
||||
command = "{} -v".format(self.ninja_command)
|
||||
self.buildEnv.run_command(command, self.build_path, context)
|
||||
|
||||
def _test(self, context):
|
||||
|
|
@ -398,15 +401,15 @@ class MesonBuilder(Builder):
|
|||
and not self.buildEnv.platform_info.static)
|
||||
):
|
||||
raise SkipCommand()
|
||||
command = "{} --verbose {}".format(self.buildEnv.mesontest_command, self.test_option)
|
||||
command = "{} --verbose {}".format(self.mesontest_command, self.test_option)
|
||||
self.buildEnv.run_command(command, self.build_path, context)
|
||||
|
||||
def _install(self, context):
|
||||
command = "{} -v install".format(self.buildEnv.ninja_command)
|
||||
command = "{} -v install".format(self.ninja_command)
|
||||
self.buildEnv.run_command(command, self.build_path, context)
|
||||
|
||||
def _make_dist(self, context):
|
||||
command = "{} -v dist".format(self.buildEnv.ninja_command)
|
||||
command = "{} -v dist".format(self.ninja_command)
|
||||
self.buildEnv.run_command(command, self.build_path, context)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ class Aria2(Dependency):
|
|||
def _post_prepare_script(self, context):
|
||||
context.try_skip(self.extract_path)
|
||||
command = "autoreconf -i"
|
||||
self.buildEnv.run_command(command, self.extract_path, context)
|
||||
self.neutralEnv.run_command(command, self.extract_path, context)
|
||||
|
||||
class Builder(MakeBuilder):
|
||||
configure_option = "--enable-libaria2 --disable-ssl --disable-bittorent --disable-metalink --without-sqlite3 --without-libxml2 --without-libexpat"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,103 @@
|
|||
|
||||
|
||||
_fedora_common = ['automake', 'libtool', 'cmake', 'git', 'subversion', 'ccache', 'pkgconfig', 'gcc-c++', 'gettext-devel']
|
||||
_debian_common = ['automake', 'libtool', 'cmake', 'git', 'subversion', 'ccache', 'pkg-config', 'gcc', 'autopoint']
|
||||
PACKAGE_NAME_MAPPERS = {
|
||||
'fedora_native_dyn': {
|
||||
'COMMON': _fedora_common,
|
||||
'uuid': ['libuuid-devel'],
|
||||
'xapian-core': None, # Not the right version on fedora 25
|
||||
'ctpp2': None,
|
||||
'pugixml': None, # ['pugixml-devel'] but package doesn't provide pkg-config file
|
||||
'libmicrohttpd': ['libmicrohttpd-devel'],
|
||||
'zlib': ['zlib-devel'],
|
||||
'lzma': ['xz-devel'],
|
||||
'icu4c': None,
|
||||
'zimlib': None,
|
||||
'file' : ['file-devel'],
|
||||
'gumbo' : ['gumbo-parser-devel'],
|
||||
},
|
||||
'fedora_native_static': {
|
||||
'COMMON': _fedora_common + ['glibc-static', 'libstdc++-static'],
|
||||
'zlib': ['zlib-devel', 'zlib-static'],
|
||||
'lzma': ['xz-devel', 'xz-static']
|
||||
# Either there is no packages, or no static or too old
|
||||
},
|
||||
'fedora_i586_dyn': {
|
||||
'COMMON': _fedora_common + ['glibc-devel.i686', 'libstdc++-devel.i686'],
|
||||
},
|
||||
'fedora_i586_static': {
|
||||
'COMMON': _fedora_common + ['glibc-devel.i686'],
|
||||
},
|
||||
'fedora_win32_dyn': {
|
||||
'COMMON': _fedora_common + ['mingw32-gcc-c++', 'mingw32-bzip2', 'mingw32-win-iconv', 'mingw32-winpthreads', 'wine'],
|
||||
'zlib': ['mingw32-zlib'],
|
||||
'lzma': ['mingw32-xz-libs'],
|
||||
'libmicrohttpd': ['mingw32-libmicrohttpd'],
|
||||
},
|
||||
'fedora_win32_static': {
|
||||
'COMMON': _fedora_common + ['mingw32-gcc-c++', 'mingw32-bzip2-static', 'mingw32-win-iconv-static', 'mingw32-winpthreads-static', 'wine'],
|
||||
'zlib': ['mingw32-zlib-static'],
|
||||
'lzma': ['mingw32-xz-libs-static'],
|
||||
'libmicrohttpd': None, # ['mingw32-libmicrohttpd-static'] packaging dependecy seems buggy, and some static lib are name libfoo.dll.a and
|
||||
# gcc cannot found them.
|
||||
},
|
||||
'fedora_armhf_static': {
|
||||
'COMMON': _fedora_common
|
||||
},
|
||||
'fedora_armhf_dyn': {
|
||||
'COMMON': _fedora_common
|
||||
},
|
||||
'fedora_android': {
|
||||
'COMMON': _fedora_common + ['java-1.8.0-openjdk-devel']
|
||||
},
|
||||
'debian_native_dyn': {
|
||||
'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'],
|
||||
'zlib': ['zlib1g-dev'],
|
||||
'uuid': ['uuid-dev'],
|
||||
'ctpp2': ['libctpp2-dev'],
|
||||
'ctpp2c': ['ctpp2-utils'],
|
||||
'libmicrohttpd': ['libmicrohttpd-dev', 'ccache']
|
||||
},
|
||||
'debian_native_static': {
|
||||
'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'],
|
||||
'zlib': ['zlib1g-dev'],
|
||||
'uuid': ['uuid-dev'],
|
||||
'ctpp2': ['libctpp2-dev'],
|
||||
'ctpp2c': ['ctpp2-utils'],
|
||||
},
|
||||
'debian_i586_dyn': {
|
||||
'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib'],
|
||||
},
|
||||
'debian_i586_static': {
|
||||
'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib'],
|
||||
},
|
||||
'debian_win32_dyn': {
|
||||
'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'],
|
||||
'ctpp2c': ['ctpp2-utils'],
|
||||
},
|
||||
'debian_win32_static': {
|
||||
'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'],
|
||||
'ctpp2c': ['ctpp2-utils'],
|
||||
},
|
||||
'debian_armhf_static': {
|
||||
'COMMON': _debian_common,
|
||||
'ctpp2c': ['ctpp2-utils'],
|
||||
},
|
||||
'debian_armhf_dyn': {
|
||||
'COMMON': _debian_common,
|
||||
'ctpp2c': ['ctpp2-utils'],
|
||||
},
|
||||
'debian_android': {
|
||||
'COMMON': _debian_common + ['default-jdk'],
|
||||
'ctpp2c': ['ctpp2-utils'],
|
||||
},
|
||||
'Darwin_native_dyn': {
|
||||
'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'],
|
||||
'file': ['libmagic']
|
||||
},
|
||||
'Darwin_iOS': {
|
||||
'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'],
|
||||
'file': ['libmagic']
|
||||
},
|
||||
}
|
||||
|
|
@ -13,3 +13,9 @@ class PlatformInfo:
|
|||
|
||||
def __str__(self):
|
||||
return "{}_{}".format(self.build, 'static' if self.static else 'dyn')
|
||||
|
||||
def set_env(self, env):
|
||||
pass
|
||||
|
||||
def get_bind_dir(self):
|
||||
return []
|
||||
|
|
|
|||
|
|
@ -43,6 +43,15 @@ class iOSPlatformInfo(PlatformInfo):
|
|||
},
|
||||
}
|
||||
|
||||
def set_env(self, env):
|
||||
env['CFLAGS'] = " -fembed-bitcode -isysroot {SDKROOT} -arch {arch} -miphoneos-version-min=9.0 ".format(SDKROOT=self.root_path, arch=self.arch) + env['CFLAGS']
|
||||
env['CXXFLAGS'] = env['CFLAGS'] + " -stdlib=libc++ -std=c++11 "+env['CXXFLAGS']
|
||||
env['LDFLAGS'] = " -arch {arch} -isysroot {SDKROOT} ".format(SDKROOT=self.root_path, arch=self.arch)
|
||||
env['MACOSX_DEPLOYMENT_TARGET'] = "10.7"
|
||||
|
||||
def get_bin_dir(self):
|
||||
return [pj(self.root_path, 'bin')]
|
||||
|
||||
iOSPlatformInfo('iOS_armv7', 'armv7')
|
||||
iOSPlatformInfo('iOS_arm64', 'arm64')
|
||||
iOSPlatformInfo('iOS_i386', 'i386')
|
||||
|
|
|
|||
|
|
@ -4,15 +4,17 @@ from .base import PlatformInfo
|
|||
class Win32PlatformInfo(PlatformInfo):
|
||||
def __init__(self, name, static):
|
||||
super().__init__(name, 'win32', static, ['mingw32_toolchain'], ['fedora', 'debian'])
|
||||
self.extra_libs = ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4', '-lmsvcr90', '-liphlpapi']
|
||||
|
||||
def get_cross_config(self, host):
|
||||
root_paths = {
|
||||
'fedora': '/usr/i686-w64-mingw32/sys-root/mingw',
|
||||
'debian': '/usr/i686-w64-mingw32'
|
||||
}
|
||||
self.root_path = root_paths[host]
|
||||
return {
|
||||
'root_path': root_paths[host],
|
||||
'extra_libs': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4', '-lmsvcr90', '-liphlpapi'],
|
||||
'root_path': self.root_path,
|
||||
'extra_libs': self.extra_libs,
|
||||
'extra_cflags': ['-DWIN32'],
|
||||
'host_machine': {
|
||||
'system': 'Windows',
|
||||
|
|
@ -24,6 +26,12 @@ class Win32PlatformInfo(PlatformInfo):
|
|||
}
|
||||
}
|
||||
|
||||
def get_bin_dir(self):
|
||||
return [pj(self.root_path, 'bin')]
|
||||
|
||||
def set_env(self, env):
|
||||
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
|
||||
env['LIBS'] = " ".join(self.extra_libs) + " " +env['LIBS']
|
||||
|
||||
Win32PlatformInfo('win32_dyn', False)
|
||||
Win32PlatformInfo('win32_static', True)
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from kiwixbuild.utils import Remotefile, add_execution_right
|
|||
pj = os.path.join
|
||||
|
||||
class android_ndk(Toolchain):
|
||||
neutral = False
|
||||
name = 'android-ndk'
|
||||
version = 'r13b'
|
||||
gccver = '4.9.x'
|
||||
|
|
|
|||
|
|
@ -50,8 +50,5 @@ class android_sdk(Toolchain):
|
|||
self.command('build_platform', self._build_platform)
|
||||
self.command('fix_licenses', self._fix_licenses)
|
||||
|
||||
def get_bin_dir(self):
|
||||
return []
|
||||
|
||||
def set_env(self, env):
|
||||
env['ANDROID_HOME'] = self.builder.install_path
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import subprocess
|
|||
|
||||
from .base_toolchain import Toolchain
|
||||
from kiwixbuild.dependencies import GitClone
|
||||
from kiwixbuild.utils import Remotefile, which
|
||||
from kiwixbuild.utils import which
|
||||
pj = os.path.join
|
||||
|
||||
class armhf_toolchain(Toolchain):
|
||||
|
|
@ -52,13 +52,11 @@ class armhf_toolchain(Toolchain):
|
|||
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
|
||||
env['CFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS']
|
||||
env['CXXFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
|
||||
env['LIBS'] = " ".join(self.buildEnv.cross_config['extra_libs']) + " " +env['LIBS']
|
||||
env['QEMU_LD_PREFIX'] = pj(self.root_path, "arm-linux-gnueabihf", "libc")
|
||||
env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format(
|
||||
':'.join([
|
||||
pj(self.root_path, "arm-linux-gnueabihf", "lib"),
|
||||
pj(self.buildEnv.install_dir, 'lib'),
|
||||
pj(self.buildEnv.install_dir, self.buildEnv.libprefix)
|
||||
env['LD_LIBRARY_PATH']
|
||||
]))
|
||||
|
||||
def set_compiler(self, env):
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ class _MetaToolchain(type):
|
|||
|
||||
|
||||
class Toolchain(metaclass=_MetaToolchain):
|
||||
neutral = True
|
||||
all_toolchains = {}
|
||||
configure_option = ""
|
||||
cmake_option = ""
|
||||
|
|
@ -21,8 +22,9 @@ class Toolchain(metaclass=_MetaToolchain):
|
|||
Builder = None
|
||||
Source = None
|
||||
|
||||
def __init__(self, buildEnv):
|
||||
self.buildEnv = buildEnv
|
||||
def __init__(self, neutralEnv):
|
||||
self.neutralEnv = neutralEnv
|
||||
self.buildEnv = neutralEnv
|
||||
self.source = self.Source(self) if self.Source else None
|
||||
self.builder = self.Builder(self) if self.Builder else None
|
||||
|
||||
|
|
@ -34,11 +36,11 @@ class Toolchain(metaclass=_MetaToolchain):
|
|||
|
||||
@property
|
||||
def source_path(self):
|
||||
return pj(self.buildEnv.source_dir, self.source.source_dir)
|
||||
return pj(self.neutralEnv.source_dir, self.source.source_dir)
|
||||
|
||||
@property
|
||||
def _log_dir(self):
|
||||
return self.buildEnv.log_dir
|
||||
return self.neutralEnv.log_dir
|
||||
|
||||
def set_env(self, env):
|
||||
pass
|
||||
|
|
@ -46,6 +48,9 @@ class Toolchain(metaclass=_MetaToolchain):
|
|||
def set_compiler(self, env):
|
||||
pass
|
||||
|
||||
def get_bin_dir(self):
|
||||
return []
|
||||
|
||||
def command(self, name, function, *args):
|
||||
print(" {} {} : ".format(name, self.name), end="", flush=True)
|
||||
log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
|
||||
|
|
|
|||
|
|
@ -3,10 +3,6 @@ from .base_toolchain import Toolchain
|
|||
from kiwixbuild.utils import pj, xrun_find
|
||||
|
||||
class iOS_sdk(Toolchain):
|
||||
@property
|
||||
def root_path(self):
|
||||
return self.buildEnv.platform_info.root_path
|
||||
|
||||
@property
|
||||
def binaries(self):
|
||||
return {
|
||||
|
|
@ -22,16 +18,6 @@ class iOS_sdk(Toolchain):
|
|||
def configure_option(self):
|
||||
return '--host=arm-apple-darwin'
|
||||
|
||||
def get_bin_dir(self):
|
||||
return [pj(self.root_path, 'bin')]
|
||||
|
||||
def set_env(self, env):
|
||||
arch = self.buildEnv.platform_info.arch
|
||||
env['CFLAGS'] = " -fembed-bitcode -isysroot {SDKROOT} -arch {arch} -miphoneos-version-min=9.0 ".format(SDKROOT=self.root_path, arch=arch) + env['CFLAGS']
|
||||
env['CXXFLAGS'] = env['CFLAGS'] + " -stdlib=libc++ -std=c++11 "+env['CXXFLAGS']
|
||||
env['LDFLAGS'] = " -arch {arch} -isysroot {SDKROOT} ".format(SDKROOT=self.root_path, arch=arch)
|
||||
env['MACOSX_DEPLOYMENT_TARGET'] = "10.7"
|
||||
|
||||
def set_compiler(self, env):
|
||||
env['CC'] = self.binaries['CC']
|
||||
env['CXX'] = self.binaries['CXX']
|
||||
|
|
|
|||
|
|
@ -34,18 +34,10 @@ class mingw32_toolchain(Toolchain):
|
|||
else:
|
||||
return "exec_wrapper = 'wine'"
|
||||
|
||||
|
||||
@property
|
||||
def configure_option(self):
|
||||
return '--host={}'.format(self.arch_full)
|
||||
|
||||
def get_bin_dir(self):
|
||||
return [pj(self.root_path, 'bin')]
|
||||
|
||||
def set_env(self, env):
|
||||
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
|
||||
env['LIBS'] = " ".join(self.buildEnv.cross_config['extra_libs']) + " " +env['LIBS']
|
||||
|
||||
def set_compiler(self, env):
|
||||
for k, v in self.binaries.items():
|
||||
env[k] = v
|
||||
|
|
|
|||
Loading…
Reference in New Issue