mirror of
https://github.com/kiwix/kiwix-build.git
synced 2025-06-26 10:11:27 +00:00
a9eaa71357
Not all target platform can be compiled on all platform. For instance, all `static` target or `win32`, `armhf` and `android` targets cannot be compiled on macOS. Simply check that the current host is supported by TargetInfo and exit nicely if needed.
991 lines
36 KiB
Python
Executable File
991 lines
36 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
|
||
import os, sys, shutil
|
||
import argparse
|
||
import ssl
|
||
import urllib.request
|
||
import subprocess
|
||
import platform
|
||
from collections import OrderedDict
|
||
|
||
from dependencies import Dependency
|
||
from dependency_utils import ReleaseDownload, Builder, GitClone
|
||
from utils import (
|
||
pj,
|
||
remove_duplicates,
|
||
add_execution_right,
|
||
get_sha256,
|
||
print_progress,
|
||
setup_print_progress,
|
||
download_remote,
|
||
StopBuild,
|
||
SkipCommand,
|
||
Defaultdict,
|
||
Remotefile,
|
||
Context)
|
||
|
||
REMOTE_PREFIX = 'http://download.kiwix.org/dev/'
|
||
|
||
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
|
||
|
||
|
||
_fedora_common = ['automake', 'cmake', 'git', 'subversion', 'ccache', 'pkgconfig', 'gcc-c++']
|
||
_debian_common = ['automake', 'cmake', 'git', 'subversion', 'ccache', 'pkg-config', 'gcc']
|
||
PACKAGE_NAME_MAPPERS = {
|
||
'fedora_native_dyn': {
|
||
'COMMON': _fedora_common,
|
||
'uuid': ['libuuid-devel'],
|
||
'xapian-core': None, # Not the right version on fedora 25
|
||
'ctpp2': None,
|
||
'pugixml': None, # ['pugixml-devel'] but package doesn't provide pkg-config file
|
||
'libmicrohttpd': ['libmicrohttpd-devel'],
|
||
'zlib': ['zlib-devel'],
|
||
'lzma': ['xz-devel'],
|
||
'icu4c': None,
|
||
'zimlib': None,
|
||
'file' : ['file-devel'],
|
||
'gumbo' : ['gumbo-parser-devel'],
|
||
},
|
||
'fedora_native_static': {
|
||
'COMMON': _fedora_common + ['glibc-static', 'libstdc++-static'],
|
||
'zlib': ['zlib-devel', 'zlib-static'],
|
||
'lzma': ['xz-devel', 'xz-static']
|
||
# Either there is no packages, or no static or too old
|
||
},
|
||
'fedora_win32_dyn': {
|
||
'COMMON': _fedora_common + ['mingw32-gcc-c++', 'mingw32-bzip2', 'mingw32-win-iconv', 'mingw32-winpthreads', 'wine'],
|
||
'zlib': ['mingw32-zlib'],
|
||
'lzma': ['mingw32-xz-libs'],
|
||
'libmicrohttpd': ['mingw32-libmicrohttpd'],
|
||
},
|
||
'fedora_win32_static': {
|
||
'COMMON': _fedora_common + ['mingw32-gcc-c++', 'mingw32-bzip2-static', 'mingw32-win-iconv-static', 'mingw32-winpthreads-static', 'wine'],
|
||
'zlib': ['mingw32-zlib-static'],
|
||
'lzma': ['mingw32-xz-libs-static'],
|
||
'libmicrohttpd': None, # ['mingw32-libmicrohttpd-static'] packaging dependecy seems buggy, and some static lib are name libfoo.dll.a and
|
||
# gcc cannot found them.
|
||
},
|
||
'fedora_armhf_static': {
|
||
'COMMON': _fedora_common
|
||
},
|
||
'fedora_armhf_dyn': {
|
||
'COMMON': _fedora_common
|
||
},
|
||
'fedora_android': {
|
||
'COMMON': _fedora_common + ['java-1.8.0-openjdk-devel']
|
||
},
|
||
'debian_native_dyn': {
|
||
'COMMON': _debian_common + ['libbz2-dev'],
|
||
'zlib': ['zlib1g-dev'],
|
||
'uuid': ['uuid-dev'],
|
||
'ctpp2': ['libctpp2-dev'],
|
||
'ctpp2c': ['ctpp2-utils'],
|
||
'libmicrohttpd': ['libmicrohttpd-dev', 'ccache']
|
||
},
|
||
'debian_native_static': {
|
||
'COMMON': _debian_common + ['libbz2-dev'],
|
||
'zlib': ['zlib1g-dev'],
|
||
'uuid': ['uuid-dev'],
|
||
'ctpp2': ['libctpp2-dev'],
|
||
'ctpp2c': ['ctpp2-utils'],
|
||
},
|
||
'debian_win32_dyn': {
|
||
'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'],
|
||
'ctpp2c': ['ctpp2-utils'],
|
||
},
|
||
'debian_win32_static': {
|
||
'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'],
|
||
'ctpp2c': ['ctpp2-utils'],
|
||
},
|
||
'debian_armhf_static': {
|
||
'COMMON': _debian_common,
|
||
'ctpp2c': ['ctpp2-utils'],
|
||
},
|
||
'debian_armhf_dyn': {
|
||
'COMMON': _debian_common,
|
||
'ctpp2c': ['ctpp2-utils'],
|
||
},
|
||
'debian_android': {
|
||
'COMMON': _debian_common + ['default-jdk'],
|
||
'ctpp2c': ['ctpp2-utils'],
|
||
},
|
||
'Darwin_native_dyn': {
|
||
'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'],
|
||
},
|
||
'Darwin_iOS_static': {
|
||
'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'],
|
||
},
|
||
}
|
||
|
||
|
||
def which(name):
|
||
command = "which {}".format(name)
|
||
output = subprocess.check_output(command, shell=True)
|
||
return output[:-1].decode()
|
||
|
||
|
||
class TargetInfo:
|
||
def __init__(self, build, static, toolchains, hosts=None):
|
||
self.build = build
|
||
self.static = static
|
||
self.toolchains = toolchains
|
||
self.compatible_hosts = hosts
|
||
|
||
def __str__(self):
|
||
return "{}_{}".format(self.build, 'static' if self.static else 'dyn')
|
||
|
||
def get_cross_config(self, host):
|
||
if self.build == 'native':
|
||
return {}
|
||
elif self.build == 'win32':
|
||
root_paths = {
|
||
'fedora': '/usr/i686-w64-mingw32/sys-root/mingw',
|
||
'debian': '/usr/i686-w64-mingw32'
|
||
}
|
||
return {
|
||
'root_path': root_paths[host],
|
||
'extra_libs': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4', '-lmsvcr90'],
|
||
'extra_cflags': ['-DWIN32'],
|
||
'host_machine': {
|
||
'system': 'Windows',
|
||
'lsystem': 'windows',
|
||
'cpu_family': 'x86',
|
||
'cpu': 'i686',
|
||
'endian': 'little',
|
||
'abi': ''
|
||
}
|
||
}
|
||
elif self.build == 'armhf':
|
||
return {
|
||
'extra_libs': [],
|
||
'extra_cflags': [],
|
||
'host_machine': {
|
||
'system': 'linux',
|
||
'lsystem': 'linux',
|
||
'cpu_family': 'arm',
|
||
'cpu': 'armhf',
|
||
'endian': 'little',
|
||
'abi': ''
|
||
}
|
||
}
|
||
|
||
class AndroidTargetInfo(TargetInfo):
|
||
__arch_infos = {
|
||
'arm' : ('arm-linux-androideabi', 'arm', 'armeabi'),
|
||
'arm64': ('aarch64-linux-android', 'aarch64', 'arm64-v8a'),
|
||
'mips': ('mipsel-linux-android', 'mipsel', 'mips'),
|
||
'mips64': ('mips64el-linux-android', 'mips64el', 'mips64'),
|
||
'x86': ('i686-linux-android', 'i686', 'x86'),
|
||
'x86_64': ('x86_64-linux-android', 'x86_64', 'x86_64'),
|
||
}
|
||
|
||
def __init__(self, arch):
|
||
super().__init__('android', True, ['android_ndk', 'android_sdk'],
|
||
hosts=['fedora', 'debian'])
|
||
self.arch = arch
|
||
self.arch_full, self.cpu, self.abi = self.__arch_infos[arch]
|
||
|
||
def __str__(self):
|
||
return "android"
|
||
|
||
def get_cross_config(self, host):
|
||
return {
|
||
'extra_libs': [],
|
||
'extra_cflags': [],
|
||
'host_machine': {
|
||
'system': 'Android',
|
||
'lsystem': 'android',
|
||
'cpu_family': self.arch,
|
||
'cpu': self.cpu,
|
||
'endian': 'little',
|
||
'abi': self.abi
|
||
},
|
||
}
|
||
|
||
class iOSTargetInfo(TargetInfo):
|
||
def __init__(self, arch):
|
||
super().__init__('iOS', True, ['iOS_sdk'],
|
||
hosts=['Darwin'])
|
||
self.arch = arch
|
||
|
||
|
||
class BuildEnv:
|
||
target_platforms = {
|
||
'native_dyn': TargetInfo('native', False, [],
|
||
hosts=['fedora', 'debian', 'Darwin']),
|
||
'native_static': TargetInfo('native', True, [],
|
||
hosts=['fedora', 'debian']),
|
||
'win32_dyn': TargetInfo('win32', False, ['mingw32_toolchain'],
|
||
hosts=['fedora', 'debian']),
|
||
'win32_static': TargetInfo('win32', True, ['mingw32_toolchain'],
|
||
hosts=['fedora', 'debian']),
|
||
'armhf_dyn': TargetInfo('armhf', False, ['armhf_toolchain'],
|
||
hosts=['fedora', 'debian']),
|
||
'armhf_static': TargetInfo('armhf', True, ['armhf_toolchain'],
|
||
hosts=['fedora', 'debian']),
|
||
'android_arm': AndroidTargetInfo('arm'),
|
||
'android_arm64': AndroidTargetInfo('arm64'),
|
||
'android_mips': AndroidTargetInfo('mips'),
|
||
'android_mips64': AndroidTargetInfo('mips64'),
|
||
'android_x86': AndroidTargetInfo('x86'),
|
||
'android_x86_64': AndroidTargetInfo('x86_64'),
|
||
'iOS_arm64': iOSTargetInfo('arm64'),
|
||
}
|
||
|
||
def __init__(self, options, targetsDict):
|
||
self.source_dir = pj(options.working_dir, "SOURCE")
|
||
build_dir = "BUILD_{}".format(options.target_platform)
|
||
self.build_dir = pj(options.working_dir, build_dir)
|
||
self.archive_dir = pj(options.working_dir, "ARCHIVE")
|
||
self.toolchain_dir = pj(options.working_dir, "TOOLCHAINS")
|
||
self.log_dir = pj(self.build_dir, 'LOGS')
|
||
self.install_dir = pj(self.build_dir, "INSTALL")
|
||
for d in (self.source_dir,
|
||
self.build_dir,
|
||
self.archive_dir,
|
||
self.toolchain_dir,
|
||
self.log_dir,
|
||
self.install_dir):
|
||
os.makedirs(d, exist_ok=True)
|
||
self.detect_platform()
|
||
self.ninja_command = self._detect_ninja()
|
||
if not self.ninja_command:
|
||
sys.exit("ERROR: ninja command not found")
|
||
self.meson_command = self._detect_meson()
|
||
if not self.meson_command:
|
||
sys.exit("ERROR: meson command not fount")
|
||
self.setup_build(options.target_platform)
|
||
self.setup_toolchains()
|
||
self.options = options
|
||
self.libprefix = options.libprefix or self._detect_libdir()
|
||
self.targetsDict = targetsDict
|
||
|
||
def clean_intermediate_directories(self):
|
||
for subdir in os.listdir(self.build_dir):
|
||
subpath = pj(self.build_dir, subdir)
|
||
if subpath == self.install_dir:
|
||
continue
|
||
if os.path.isdir(subpath):
|
||
shutil.rmtree(subpath)
|
||
else:
|
||
os.remove(subpath)
|
||
|
||
def detect_platform(self):
|
||
_platform = platform.system()
|
||
self.distname = _platform
|
||
if _platform == 'Windows':
|
||
print('ERROR: kiwix-build is not intented to run on Windows platform.\n'
|
||
'It should probably not work, but well, you still can have a try.')
|
||
cont = input('Do you want to continue ? [y/N]')
|
||
if cont.lower() != 'y':
|
||
sys.exit(0)
|
||
if _platform == 'Darwin':
|
||
print('WARNING: kiwix-build has not been tested on MacOS platfrom.\n'
|
||
'Tests, bug reports and patches are welcomed.')
|
||
if _platform == 'Linux':
|
||
self.distname, _, _ = platform.linux_distribution()
|
||
self.distname = self.distname.lower()
|
||
if self.distname == 'ubuntu':
|
||
self.distname = 'debian'
|
||
|
||
def setup_build(self, target_platform):
|
||
self.platform_info = self.target_platforms[target_platform]
|
||
if self.distname not in self.platform_info.compatible_hosts:
|
||
print(('ERROR: The target {} cannot be build on host {}.\n'
|
||
'Select another target platform, or change your host system.'
|
||
).format(target_platform, self.distname))
|
||
sys.exit(-1)
|
||
self.cross_config = self.platform_info.get_cross_config(self.distname)
|
||
|
||
def setup_toolchains(self):
|
||
toolchain_names = self.platform_info.toolchains
|
||
self.toolchains =[Toolchain.all_toolchains[toolchain_name](self)
|
||
for toolchain_name in toolchain_names]
|
||
|
||
def finalize_setup(self):
|
||
getattr(self, 'setup_{}'.format(self.platform_info.build))()
|
||
|
||
def setup_native(self):
|
||
self.cmake_crossfile = None
|
||
self.meson_crossfile = None
|
||
|
||
def _gen_crossfile(self, name):
|
||
crossfile = pj(self.build_dir, name)
|
||
template_file = pj(SCRIPT_DIR, 'templates', name)
|
||
with open(template_file, 'r') as f:
|
||
template = f.read()
|
||
content = template.format(
|
||
toolchain=self.toolchains[0],
|
||
**self.cross_config
|
||
)
|
||
with open(crossfile, 'w') as outfile:
|
||
outfile.write(content)
|
||
return crossfile
|
||
|
||
def setup_win32(self):
|
||
self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
|
||
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
|
||
|
||
def setup_android(self):
|
||
self.cmake_crossfile = self._gen_crossfile('cmake_android_cross_file.txt')
|
||
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
|
||
|
||
def setup_armhf(self):
|
||
self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
|
||
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
|
||
|
||
def setup_iOS(self):
|
||
pass
|
||
|
||
def __getattr__(self, name):
|
||
return getattr(self.options, name)
|
||
|
||
def _is_debianlike(self):
|
||
return os.path.isfile('/etc/debian_version')
|
||
|
||
def _detect_libdir(self):
|
||
if self._is_debianlike():
|
||
try:
|
||
pc = subprocess.Popen(['dpkg-architecture', '-qDEB_HOST_MULTIARCH'],
|
||
stdout=subprocess.PIPE,
|
||
stderr=subprocess.DEVNULL)
|
||
(stdo, _) = pc.communicate()
|
||
if pc.returncode == 0:
|
||
archpath = stdo.decode().strip()
|
||
return 'lib/' + archpath
|
||
except Exception:
|
||
pass
|
||
if os.path.isdir('/usr/lib64') and not os.path.islink('/usr/lib64'):
|
||
return 'lib64'
|
||
return 'lib'
|
||
|
||
def _detect_ninja(self):
|
||
for n in ['ninja', 'ninja-build']:
|
||
try:
|
||
retcode = subprocess.check_call([n, '--version'],
|
||
stdout=subprocess.DEVNULL)
|
||
except (FileNotFoundError, PermissionError):
|
||
# Doesn't exist in PATH or isn't executable
|
||
continue
|
||
if retcode == 0:
|
||
return n
|
||
|
||
def _detect_meson(self):
|
||
for n in ['meson.py', 'meson']:
|
||
try:
|
||
retcode = subprocess.check_call([n, '--version'],
|
||
stdout=subprocess.DEVNULL)
|
||
except (FileNotFoundError, PermissionError):
|
||
# Doesn't exist in PATH or isn't executable
|
||
continue
|
||
if retcode == 0:
|
||
return n
|
||
|
||
@property
|
||
def configure_option(self):
|
||
configure_options = [tlc.configure_option for tlc in self.toolchains]
|
||
return " ".join(configure_options)
|
||
|
||
@property
|
||
def cmake_option(self):
|
||
cmake_options = [tlc.cmake_option for tlc in self.toolchains]
|
||
return " ".join(cmake_options)
|
||
|
||
def _set_env(self, env, cross_compile_env, cross_compile_compiler, cross_compile_path):
|
||
if env is None:
|
||
env = Defaultdict(str, os.environ)
|
||
|
||
bin_dirs = []
|
||
if cross_compile_env:
|
||
for k, v in self.cross_config.get('env', {}).items():
|
||
if k.startswith('_format_'):
|
||
v = v.format(**self.cross_config)
|
||
k = k[8:]
|
||
env[k] = v
|
||
for toolchain in self.toolchains:
|
||
toolchain.set_env(env)
|
||
if cross_compile_compiler:
|
||
for toolchain in self.toolchains:
|
||
toolchain.set_compiler(env)
|
||
if cross_compile_path:
|
||
for tlc in self.toolchains:
|
||
bin_dirs += tlc.get_bin_dir()
|
||
|
||
pkgconfig_path = pj(self.install_dir, self.libprefix, 'pkgconfig')
|
||
env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path])
|
||
|
||
# Add ccache path
|
||
for p in ('/usr/lib/ccache', '/usr/lib64/ccache'):
|
||
if os.path.isdir(p):
|
||
ccache_path = [p]
|
||
break
|
||
else:
|
||
ccache_path = []
|
||
env['PATH'] = ':'.join(bin_dirs +
|
||
[pj(self.install_dir, 'bin')] +
|
||
ccache_path +
|
||
[env['PATH']])
|
||
|
||
env['LD_LIBRARY_PATH'] = ':'.join([env['LD_LIBRARY_PATH'],
|
||
pj(self.install_dir, 'lib'),
|
||
pj(self.install_dir, 'lib64')
|
||
])
|
||
|
||
env['CPPFLAGS'] = " ".join(['-I'+pj(self.install_dir, 'include'), env['CPPFLAGS']])
|
||
env['LDFLAGS'] = " ".join(['-L'+pj(self.install_dir, 'lib'),
|
||
'-L'+pj(self.install_dir, 'lib64'),
|
||
env['LDFLAGS']])
|
||
return env
|
||
|
||
def run_command(self, command, cwd, context, env=None, input=None, cross_env_only=False):
|
||
os.makedirs(cwd, exist_ok=True)
|
||
cross_compile_env = True
|
||
cross_compile_compiler = True
|
||
cross_compile_path = True
|
||
if context.force_native_build:
|
||
cross_compile_env = False
|
||
cross_compile_compiler = False
|
||
cross_compile_path = False
|
||
if cross_env_only:
|
||
cross_compile_compiler = False
|
||
env = self._set_env(env, cross_compile_env, cross_compile_compiler, cross_compile_path)
|
||
log = None
|
||
try:
|
||
if not self.options.verbose:
|
||
log = open(context.log_file, 'w')
|
||
print("run command '{}'".format(command), file=log)
|
||
print("current directory is '{}'".format(cwd), file=log)
|
||
print("env is :", file=log)
|
||
for k, v in env.items():
|
||
print(" {} : {!r}".format(k, v), file=log)
|
||
|
||
kwargs = dict()
|
||
if input:
|
||
kwargs['stdin'] = subprocess.PIPE
|
||
process = subprocess.Popen(command, shell=True, cwd=cwd, env=env, stdout=log or sys.stdout, stderr=subprocess.STDOUT, **kwargs)
|
||
if input:
|
||
process.communicate(input.encode())
|
||
retcode = process.wait()
|
||
if retcode:
|
||
raise subprocess.CalledProcessError(retcode, command)
|
||
finally:
|
||
if log:
|
||
log.close()
|
||
|
||
def download(self, what, where=None):
|
||
where = where or self.archive_dir
|
||
download_remote(what, where, not self.options.no_cert_check)
|
||
|
||
def install_packages(self):
|
||
autoskip_file = pj(self.build_dir, ".install_packages_ok")
|
||
if self.distname in ('fedora', 'redhat', 'centos'):
|
||
package_installer = 'sudo dnf install {}'
|
||
package_checker = 'rpm -q --quiet {}'
|
||
elif self.distname in ('debian', 'Ubuntu'):
|
||
package_installer = 'sudo apt-get install {}'
|
||
package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1'
|
||
elif self.distname == 'Darwin':
|
||
package_installer = 'brew install {}'
|
||
package_checker = 'brew list -1 | grep -q {}'
|
||
mapper_name = "{host}_{target}".format(
|
||
host=self.distname,
|
||
target=self.platform_info)
|
||
try:
|
||
package_name_mapper = PACKAGE_NAME_MAPPERS[mapper_name]
|
||
except KeyError:
|
||
print("SKIP : We don't know which packages we must install to compile"
|
||
" a {target} {build_type} version on a {host} host.".format(
|
||
target=self.platform_info,
|
||
host=self.distname))
|
||
return
|
||
|
||
packages_list = package_name_mapper.get('COMMON', [])
|
||
for dep in self.targetsDict.values():
|
||
packages = package_name_mapper.get(dep.name)
|
||
if packages:
|
||
packages_list += packages
|
||
dep.skip = True
|
||
for dep in self.targetsDict.values():
|
||
packages = getattr(dep, 'extra_packages', [])
|
||
for package in packages:
|
||
packages_list += package_name_mapper.get(package, [])
|
||
if os.path.exists(autoskip_file):
|
||
print("SKIP")
|
||
return
|
||
|
||
packages_to_install = []
|
||
for package in packages_list:
|
||
print(" - {} : ".format(package), end="")
|
||
command = package_checker.format(package)
|
||
try:
|
||
subprocess.check_call(command, shell=True)
|
||
except subprocess.CalledProcessError:
|
||
print("NEEDED")
|
||
packages_to_install.append(package)
|
||
else:
|
||
print("SKIP")
|
||
|
||
if packages_to_install:
|
||
command = package_installer.format(" ".join(packages_to_install))
|
||
print(command)
|
||
subprocess.check_call(command, shell=True)
|
||
else:
|
||
print("SKIP, No package to install.")
|
||
|
||
with open(autoskip_file, 'w'):
|
||
pass
|
||
|
||
class _MetaToolchain(type):
|
||
def __new__(cls, name, bases, dct):
|
||
_class = type.__new__(cls, name, bases, dct)
|
||
if name != 'Toolchain':
|
||
Toolchain.all_toolchains[name] = _class
|
||
return _class
|
||
|
||
|
||
class Toolchain(metaclass=_MetaToolchain):
|
||
all_toolchains = {}
|
||
configure_option = ""
|
||
cmake_option = ""
|
||
Builder = None
|
||
Source = None
|
||
|
||
def __init__(self, buildEnv):
|
||
self.buildEnv = buildEnv
|
||
self.source = self.Source(self) if self.Source else None
|
||
self.builder = self.Builder(self) if self.Builder else None
|
||
|
||
@property
|
||
def full_name(self):
|
||
return "{name}-{version}".format(
|
||
name = self.name,
|
||
version = self.version)
|
||
|
||
@property
|
||
def source_path(self):
|
||
return pj(self.buildEnv.source_dir, self.source.source_dir)
|
||
|
||
@property
|
||
def _log_dir(self):
|
||
return self.buildEnv.log_dir
|
||
|
||
def set_env(self, env):
|
||
pass
|
||
|
||
def set_compiler(self, env):
|
||
pass
|
||
|
||
def command(self, name, function, *args):
|
||
print(" {} {} : ".format(name, self.name), end="", flush=True)
|
||
log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
|
||
context = Context(name, log, True)
|
||
try:
|
||
ret = function(*args, context=context)
|
||
context._finalise()
|
||
print("OK")
|
||
return ret
|
||
except SkipCommand:
|
||
print("SKIP")
|
||
except subprocess.CalledProcessError:
|
||
print("ERROR")
|
||
try:
|
||
with open(log, 'r') as f:
|
||
print(f.read())
|
||
except:
|
||
pass
|
||
raise StopBuild()
|
||
except:
|
||
print("ERROR")
|
||
raise
|
||
|
||
|
||
class mingw32_toolchain(Toolchain):
|
||
name = 'mingw32'
|
||
arch_full = 'i686-w64-mingw32'
|
||
|
||
@property
|
||
def root_path(self):
|
||
return self.buildEnv.cross_config['root_path']
|
||
|
||
@property
|
||
def binaries(self):
|
||
return {k:which('{}-{}'.format(self.arch_full, v))
|
||
for k, v in (('CC', 'gcc'),
|
||
('CXX', 'g++'),
|
||
('AR', 'ar'),
|
||
('STRIP', 'strip'),
|
||
('WINDRES', 'windres'),
|
||
('RANLIB', 'ranlib'))
|
||
}
|
||
|
||
@property
|
||
def configure_option(self):
|
||
return '--host={}'.format(self.arch_full)
|
||
|
||
def get_bin_dir(self):
|
||
return [pj(self.root_path, 'bin')]
|
||
|
||
def set_env(self, env):
|
||
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
|
||
env['LIBS'] = " ".join(self.buildEnv.cross_config['extra_libs']) + " " +env['LIBS']
|
||
|
||
def set_compiler(self, env):
|
||
for k, v in self.binaries.items():
|
||
env[k] = v
|
||
|
||
|
||
class android_ndk(Toolchain):
|
||
name = 'android-ndk'
|
||
version = 'r13b'
|
||
gccver = '4.9.x'
|
||
|
||
@property
|
||
def api(self):
|
||
return '21' if self.arch in ('arm64', 'mips64', 'x86_64') else '14'
|
||
|
||
@property
|
||
def platform(self):
|
||
return 'android-'+self.api
|
||
|
||
@property
|
||
def arch(self):
|
||
return self.buildEnv.platform_info.arch
|
||
|
||
@property
|
||
def arch_full(self):
|
||
return self.buildEnv.platform_info.arch_full
|
||
|
||
@property
|
||
def toolchain(self):
|
||
return self.arch_full+"-4.9"
|
||
|
||
@property
|
||
def root_path(self):
|
||
return pj(self.builder.install_path, 'sysroot')
|
||
|
||
@property
|
||
def binaries(self):
|
||
binaries = ((k,'{}-{}'.format(self.arch_full, v))
|
||
for k, v in (('CC', 'gcc'),
|
||
('CXX', 'g++'),
|
||
('AR', 'ar'),
|
||
('STRIP', 'strip'),
|
||
('WINDRES', 'windres'),
|
||
('RANLIB', 'ranlib'),
|
||
('LD', 'ld'))
|
||
)
|
||
return {k:pj(self.builder.install_path, 'bin', v)
|
||
for k,v in binaries}
|
||
|
||
@property
|
||
def configure_option(self):
|
||
return '--host={}'.format(self.arch_full)
|
||
|
||
@property
|
||
def full_name(self):
|
||
return "{name}-{version}-{arch}-{api}".format(
|
||
name = self.name,
|
||
version = self.version,
|
||
arch = self.arch,
|
||
api = self.api)
|
||
|
||
class Source(ReleaseDownload):
|
||
archive = Remotefile('android-ndk-r13b-linux-x86_64.zip',
|
||
'3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c',
|
||
'https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip')
|
||
|
||
@property
|
||
def source_dir(self):
|
||
return "{}-{}".format(
|
||
self.target.name,
|
||
self.target.version)
|
||
|
||
|
||
class Builder(Builder):
|
||
|
||
@property
|
||
def install_path(self):
|
||
return self.build_path
|
||
|
||
def _build_platform(self, context):
|
||
context.try_skip(self.build_path)
|
||
script = pj(self.source_path, 'build/tools/make_standalone_toolchain.py')
|
||
add_execution_right(script)
|
||
command = '{script} --arch={arch} --api={api} --install-dir={install_dir} --force'
|
||
command = command.format(
|
||
script=script,
|
||
arch=self.target.arch,
|
||
api=self.target.api,
|
||
install_dir=self.install_path
|
||
)
|
||
self.buildEnv.run_command(command, self.build_path, context)
|
||
|
||
def _fix_permission_right(self, context):
|
||
context.try_skip(self.build_path)
|
||
bin_dirs = [pj(self.install_path, 'bin'),
|
||
pj(self.install_path, self.target.arch_full, 'bin'),
|
||
pj(self.install_path, 'libexec', 'gcc', self.target.arch_full, self.target.gccver)
|
||
]
|
||
for root, dirs, files in os.walk(self.install_path):
|
||
if not root in bin_dirs:
|
||
continue
|
||
|
||
for file_ in files:
|
||
file_path = pj(root, file_)
|
||
if os.path.islink(file_path):
|
||
continue
|
||
add_execution_right(file_path)
|
||
|
||
def build(self):
|
||
self.command('build_platform', self._build_platform)
|
||
self.command('fix_permission_right', self._fix_permission_right)
|
||
|
||
def get_bin_dir(self):
|
||
return [pj(self.builder.install_path, 'bin')]
|
||
|
||
def set_env(self, env):
|
||
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
|
||
env['CFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(self.root_path) + env['CFLAGS']
|
||
env['CXXFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(self.root_path) + env['CXXFLAGS']
|
||
env['LDFLAGS'] = '--sysroot={} '.format(self.root_path) + env['LDFLAGS']
|
||
#env['CFLAGS'] = ' -fPIC -D_FILE_OFFSET_BITS=64 -O3 '+env['CFLAGS']
|
||
#env['CXXFLAGS'] = (' -D__OPTIMIZE__ -fno-strict-aliasing '
|
||
# ' -DU_HAVE_NL_LANGINFO_CODESET=0 '
|
||
# '-DU_STATIC_IMPLEMENTATION -O3 '
|
||
# '-DU_HAVE_STD_STRING -DU_TIMEZONE=0 ')+env['CXXFLAGS']
|
||
env['NDK_DEBUG'] = '0'
|
||
|
||
def set_compiler(self, env):
|
||
env['CC'] = self.binaries['CC']
|
||
env['CXX'] = self.binaries['CXX']
|
||
|
||
|
||
class android_sdk(Toolchain):
|
||
name = 'android-sdk'
|
||
version = 'r25.2.3'
|
||
|
||
class Source(ReleaseDownload):
|
||
archive = Remotefile('tools_r25.2.3-linux.zip',
|
||
'1b35bcb94e9a686dff6460c8bca903aa0281c6696001067f34ec00093145b560',
|
||
'https://dl.google.com/android/repository/tools_r25.2.3-linux.zip')
|
||
|
||
class Builder(Builder):
|
||
|
||
@property
|
||
def install_path(self):
|
||
return pj(self.buildEnv.toolchain_dir, self.target.full_name)
|
||
|
||
def _build_platform(self, context):
|
||
context.try_skip(self.install_path)
|
||
tools_dir = pj(self.install_path, 'tools')
|
||
shutil.copytree(self.source_path, tools_dir)
|
||
script = pj(tools_dir, 'android')
|
||
command = '{script} --verbose update sdk -a --no-ui --filter {packages}'
|
||
command = command.format(
|
||
script=script,
|
||
packages = ','.join(str(i) for i in [1,2,8,34,162])
|
||
)
|
||
# packages correspond to :
|
||
# - 1 : Android SDK Tools, revision 25.2.5
|
||
# - 2 : Android SDK Platform-tools, revision 25.0.3
|
||
# - 8 : Android SDK Build-tools, revision 24.0.1
|
||
# - 34 : SDK Platform Android 7.0, API 24, revision 2
|
||
# - 162 : Android Support Repository, revision 44
|
||
self.buildEnv.run_command(command, self.install_path, context, input="y\n")
|
||
|
||
def _fix_licenses(self, context):
|
||
context.try_skip(self.install_path)
|
||
os.makedirs(pj(self.install_path, 'licenses'), exist_ok=True)
|
||
with open(pj(self.install_path, 'licenses', 'android-sdk-license'), 'w') as f:
|
||
f.write("\n8933bad161af4178b1185d1a37fbf41ea5269c55")
|
||
|
||
def build(self):
|
||
self.command('build_platform', self._build_platform)
|
||
self.command('fix_licenses', self._fix_licenses)
|
||
|
||
def get_bin_dir(self):
|
||
return []
|
||
|
||
def set_env(self, env):
|
||
env['ANDROID_HOME'] = self.builder.install_path
|
||
|
||
|
||
class armhf_toolchain(Toolchain):
|
||
name = 'armhf'
|
||
arch_full = 'arm-linux-gnueabihf'
|
||
|
||
class Source(GitClone):
|
||
git_remote = "https://github.com/raspberrypi/tools"
|
||
git_dir = "raspberrypi-tools"
|
||
|
||
@property
|
||
def root_path(self):
|
||
return pj(self.source_path, 'arm-bcm2708', 'gcc-linaro-arm-linux-gnueabihf-raspbian-x64')
|
||
|
||
@property
|
||
def binaries(self):
|
||
binaries = ((k,'{}-{}'.format(self.arch_full, v))
|
||
for k, v in (('CC', 'gcc'),
|
||
('CXX', 'g++'),
|
||
('AR', 'ar'),
|
||
('STRIP', 'strip'),
|
||
('WINDRES', 'windres'),
|
||
('RANLIB', 'ranlib'),
|
||
('LD', 'ld'))
|
||
)
|
||
return {k:pj(self.root_path, 'bin', v)
|
||
for k,v in binaries}
|
||
|
||
@property
|
||
def configure_option(self):
|
||
return '--host={}'.format(self.arch_full)
|
||
|
||
def get_bin_dir(self):
|
||
return [pj(self.root_path, 'bin')]
|
||
|
||
def set_env(self, env):
|
||
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
|
||
env['CFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS']
|
||
env['CXXFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
|
||
env['LIBS'] = " ".join(self.buildEnv.cross_config['extra_libs']) + " " +env['LIBS']
|
||
|
||
def set_compiler(self, env):
|
||
env['CC'] = self.binaries['CC']
|
||
env['CXX'] = self.binaries['CXX']
|
||
|
||
|
||
class iOS_sdk(Toolchain):
|
||
pass
|
||
|
||
|
||
class Builder:
|
||
def __init__(self, options):
|
||
self.options = options
|
||
self.targets = OrderedDict()
|
||
self.buildEnv = buildEnv = BuildEnv(options, self.targets)
|
||
|
||
_targets = {}
|
||
targetDef = options.targets
|
||
self.add_targets(targetDef, _targets)
|
||
dependencies = self.order_dependencies(_targets, targetDef)
|
||
dependencies = list(remove_duplicates(dependencies))
|
||
|
||
for dep in dependencies:
|
||
if self.options.build_deps_only and dep == targetDef:
|
||
continue
|
||
self.targets[dep] = _targets[dep]
|
||
|
||
def add_targets(self, targetName, targets):
|
||
if targetName in targets:
|
||
return
|
||
targetClass = Dependency.all_deps[targetName]
|
||
target = targetClass(self.buildEnv)
|
||
targets[targetName] = target
|
||
for dep in target.dependencies:
|
||
self.add_targets(dep, targets)
|
||
|
||
def order_dependencies(self, _targets, targetName):
|
||
target = _targets[targetName]
|
||
for depName in target.dependencies:
|
||
yield from self.order_dependencies(_targets, depName)
|
||
yield targetName
|
||
|
||
def prepare_sources(self):
|
||
if self.options.skip_source_prepare:
|
||
print("SKIP")
|
||
return
|
||
|
||
toolchain_sources = (tlc.source for tlc in self.buildEnv.toolchains if tlc.source)
|
||
for toolchain_source in toolchain_sources:
|
||
print("prepare sources for toolchain {} :".format(toolchain_source.name))
|
||
toolchain_source.prepare()
|
||
|
||
sources = (dep.source for dep in self.targets.values() if not dep.skip)
|
||
sources = remove_duplicates(sources, lambda s: s.__class__)
|
||
for source in sources:
|
||
print("prepare sources {} :".format(source.name))
|
||
source.prepare()
|
||
|
||
def build(self):
|
||
toolchain_builders = (tlc.builder for tlc in self.buildEnv.toolchains if tlc.builder)
|
||
for toolchain_builder in toolchain_builders:
|
||
print("build toolchain {} :".format(toolchain_builder.name))
|
||
toolchain_builder.build()
|
||
|
||
builders = (dep.builder for dep in self.targets.values() if (dep.builder and not dep.skip))
|
||
for builder in builders:
|
||
print("build {} :".format(builder.name))
|
||
builder.build()
|
||
|
||
def run(self):
|
||
try:
|
||
print("[INSTALL PACKAGES]")
|
||
self.buildEnv.install_packages()
|
||
self.buildEnv.finalize_setup()
|
||
print("[PREPARE]")
|
||
self.prepare_sources()
|
||
print("[BUILD]")
|
||
self.build()
|
||
# No error, clean intermediate file at end of build if needed.
|
||
print("[CLEAN]")
|
||
if self.buildEnv.options.clean_at_end:
|
||
self.buildEnv.clean_intermediate_directories()
|
||
else:
|
||
print("SKIP")
|
||
except StopBuild:
|
||
sys.exit("Stopping build due to errors")
|
||
|
||
|
||
def parse_args():
|
||
parser = argparse.ArgumentParser()
|
||
parser.add_argument('targets', default='kiwix-tools', nargs='?', metavar='TARGET',
|
||
choices=Dependency.all_deps.keys())
|
||
parser.add_argument('--working-dir', default=".")
|
||
parser.add_argument('--libprefix', default=None)
|
||
parser.add_argument('--target-platform', default="native_dyn", choices=BuildEnv.target_platforms)
|
||
parser.add_argument('--verbose', '-v', action="store_true",
|
||
help=("Print all logs on stdout instead of in specific"
|
||
" log files per commands"))
|
||
parser.add_argument('--hide-progress', action='store_false', dest='show_progress',
|
||
help="Hide intermediate progress information.")
|
||
parser.add_argument('--skip-source-prepare', action='store_true',
|
||
help="Skip the source download part")
|
||
parser.add_argument('--build-deps-only', action='store_true',
|
||
help=("Build only the dependencies of the specified targets."))
|
||
subgroup = parser.add_argument_group('advanced')
|
||
subgroup.add_argument('--no-cert-check', action='store_true',
|
||
help="Skip SSL certificate verification during download")
|
||
subgroup.add_argument('--clean-at-end', action='store_true',
|
||
help="Clean all intermediate files after the (successfull) build")
|
||
subgroup = parser.add_argument_group('custom app',
|
||
description="Android custom app specific options")
|
||
subgroup.add_argument('--android-custom-app',
|
||
help="The custom android app to build")
|
||
subgroup.add_argument('--zim-file-url',
|
||
help="The url of the zim file to download")
|
||
subgroup.add_argument('--zim-file-size',
|
||
help="The size of the zim file.")
|
||
options = parser.parse_args()
|
||
|
||
if options.targets == 'kiwix-android-custom':
|
||
err = False
|
||
if not options.android_custom_app:
|
||
print("You need to specify ANDROID_CUSTOM_APP if you "
|
||
"want to build a kiwix-android-custom target")
|
||
err = True
|
||
if not options.zim_file_url and not options.zim_file_size:
|
||
print("You need to specify ZIM_FILE_SIZE or ZIM_FILE_URL if you "
|
||
"want to build a kiwix-android-custom target")
|
||
err = True
|
||
if err:
|
||
sys.exit(1)
|
||
return options
|
||
|
||
|
||
if __name__ == "__main__":
|
||
options = parse_args()
|
||
options.working_dir = os.path.abspath(options.working_dir)
|
||
setup_print_progress(options.show_progress)
|
||
builder = Builder(options)
|
||
builder.run()
|