smallsolar
/
kiwix-build
mirror of https://github.com/kiwix/kiwix-build.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Make the platform responsible to create the buildenv. 

It is now the platform who is responsible to:
- correctly add the toolchains as needed
- setup the cross environment
- create the cross files.
This commit is contained in:
Matthieu Gautier 2018-05-28 14:18:51 +02:00
parent 115fbfa147
commit 754df8df1d
29 changed files with 407 additions and 394 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

156
kiwixbuild/buildenv.py
View File

@ -3,13 +3,9 @@ import os, sys, shutil
import subprocess
import platform
from .platforms import PlatformInfo
from .toolchains import Toolchain
from .packages import PACKAGE_NAME_MAPPERS
from .utils import pj, download_remote, Defaultdict
from . import _global
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
from ._global import neutralEnv
class PlatformNeutralEnv:
@ -64,12 +60,6 @@ class PlatformNeutralEnv:
if retcode == 0:
return n
def add_toolchain(self, toolchain_name):
if toolchain_name not in self.toolchains:
ToolchainClass = Toolchain.all_toolchains[toolchain_name]
self.toolchains[toolchain_name] = ToolchainClass(self)
return self.toolchains[toolchain_name]
def _detect_meson(self):
for n in ['meson.py', 'meson']:
try:
@ -86,18 +76,16 @@ class PlatformNeutralEnv:
class BuildEnv:
def __init__(self, options, targetsDict):
build_dir = "BUILD_{}".format(options.target_platform)
self.build_dir = pj(options.working_dir, build_dir)
def __init__(self, platformInfo, targetsDict):
build_dir = "BUILD_{}".format(platformInfo.name)
self.platformInfo = platformInfo
self.build_dir = pj(neutralEnv('working_dir'), build_dir)
self.install_dir = pj(self.build_dir, "INSTALL")
for d in (self.build_dir,
self.install_dir):
os.makedirs(d, exist_ok=True)
self.setup_build(options.target_platform)
self.setup_toolchains()
self.options = options
self.libprefix = options.libprefix or self._detect_libdir()
self.libprefix = neutralEnv('libprefix') or self._detect_libdir()
self.targetsDict = targetsDict
def clean_intermediate_directories(self):
@ -110,67 +98,6 @@ class BuildEnv:
else:
os.remove(subpath)
def setup_build(self, target_platform):
self.platform_info = PlatformInfo.all_platforms[target_platform]
if self.distname not in self.platform_info.compatible_hosts:
print(('ERROR: The target {} cannot be build on host {}.\n'
'Select another target platform, or change your host system.'
).format(target_platform, self.distname))
sys.exit(-1)
self.cross_config = self.platform_info.get_cross_config()
def setup_toolchains(self):
toolchain_names = self.platform_info.toolchains
self.toolchains = []
for toolchain_name in toolchain_names:
ToolchainClass = Toolchain.all_toolchains[toolchain_name]
if ToolchainClass.neutral:
self.toolchains.append(
_global._neutralEnv.add_toolchain(toolchain_name)
)
else:
self.toolchains.append(ToolchainClass(self))
def finalize_setup(self):
getattr(self, 'setup_{}'.format(self.platform_info.build))()
def setup_native(self):
self.cmake_crossfile = None
self.meson_crossfile = None
def _gen_crossfile(self, name):
crossfile = pj(self.build_dir, name)
template_file = pj(SCRIPT_DIR, 'templates', name)
with open(template_file, 'r') as f:
template = f.read()
content = template.format(
toolchain=self.toolchains[0],
**self.cross_config
)
with open(crossfile, 'w') as outfile:
outfile.write(content)
return crossfile
def setup_win32(self):
self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def setup_android(self):
self.cmake_crossfile = self._gen_crossfile('cmake_android_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def setup_armhf(self):
self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def setup_iOS(self):
self.cmake_crossfile = self._gen_crossfile('cmake_ios_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def setup_i586(self):
self.cmake_crossfile = self._gen_crossfile('cmake_i586_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def __getattr__(self, name):
return _global.neutralEnv(name)
@ -237,75 +164,10 @@ class BuildEnv:
v = v.format(**self.cross_config)
k = k[8:]
env[k] = v
for toolchain in self.toolchains:
toolchain.set_env(env)
self.platform_info.set_env(env)
self.platformInfo.set_env(env)
if cross_compile_compiler:
for toolchain in self.toolchains:
toolchain.set_compiler(env)
self.platformInfo.set_compiler(env)
if cross_compile_path:
bin_dirs = []
for tlc in self.toolchains:
bin_dirs += tlc.get_bin_dir()
bin_dirs += self.platform_info.get_bind_dir()
bin_dirs = self.platformInfo.get_bind_dir()
env['PATH'] = ':'.join(bin_dirs + [env['PATH']])
return env
def install_packages(self):
autoskip_file = pj(self.build_dir, ".install_packages_ok")
if self.distname in ('fedora', 'redhat', 'centos'):
package_installer = 'sudo dnf install {}'
package_checker = 'rpm -q --quiet {}'
elif self.distname in ('debian', 'Ubuntu'):
package_installer = 'sudo apt-get install {}'
package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1'
elif self.distname == 'Darwin':
package_installer = 'brew install {}'
package_checker = 'brew list -1 | grep -q {}'
mapper_name = "{host}_{target}".format(
host=self.distname,
target=self.platform_info)
try:
package_name_mapper = PACKAGE_NAME_MAPPERS[mapper_name]
except KeyError:
print("SKIP : We don't know which packages we must install to compile"
" a {target} {build_type} version on a {host} host.".format(
target=self.platform_info,
host=self.distname))
return
packages_list = package_name_mapper.get('COMMON', [])
for dep in self.targetsDict.values():
packages = package_name_mapper.get(dep.name)
if packages:
packages_list += packages
dep.skip = True
for dep in self.targetsDict.values():
packages = getattr(dep, 'extra_packages', [])
for package in packages:
packages_list += package_name_mapper.get(package, [])
if not self.options.force_install_packages and os.path.exists(autoskip_file):
print("SKIP")
return
packages_to_install = []
for package in packages_list:
print(" - {} : ".format(package), end="")
command = package_checker.format(package)
try:
subprocess.check_call(command, shell=True)
except subprocess.CalledProcessError:
print("NEEDED")
packages_to_install.append(package)
else:
print("SKIP")
if packages_to_install:
command = package_installer.format(" ".join(packages_to_install))
print(command)
subprocess.check_call(command, shell=True)
else:
print("SKIP, No package to install.")
with open(autoskip_file, 'w'):
pass

22
kiwixbuild/builder.py
View File

@ -9,8 +9,14 @@ from .dependencies import Dependency
class Builder:
def __init__(self, options):
self.options = options
platformClass = PlatformInfo.all_platforms[options.target_platform]
if neutralEnv('distname') not in platformClass.compatible_hosts:
print(('ERROR: The target platform {} cannot be build on host {}.\n'
'Select another target platform, or change your host system.'
).format(options.target_platform, self.distname))
sys.exit(-1)
self.targets = OrderedDict()
self.buildEnv = BuildEnv(options, self.targets)
self.platform = platform = platformClass(self.targets)
_targets = {}
targetDef = options.targets
@ -30,7 +36,7 @@ class Builder:
if targetName in targets:
return
targetClass = Dependency.all_deps[targetName]
target = targetClass(self.buildEnv)
target = targetClass(self.platform.buildEnv)
targets[targetName] = target
for dep in target.builder.get_dependencies(self.platform):
self.add_targets(dep, targets)
@ -46,7 +52,7 @@ class Builder:
print("SKIP")
return
toolchain_sources = (tlc.source for tlc in self.buildEnv.toolchains if tlc.source)
toolchain_sources = (tlc.source for tlc in self.platform.buildEnv.toolchains if tlc.source)
for toolchain_source in toolchain_sources:
print("prepare sources for toolchain {} :".format(toolchain_source.name))
toolchain_source.prepare()
@ -58,7 +64,7 @@ class Builder:
source.prepare()
def build(self):
toolchain_builders = (tlc.builder for tlc in self.buildEnv.toolchains if tlc.builder)
toolchain_builders = (tlc.builder for tlc in self.platform.buildEnv.toolchains if tlc.builder)
for toolchain_builder in toolchain_builders:
print("build toolchain {} :".format(toolchain_builder.name))
toolchain_builder.build()
@ -79,16 +85,16 @@ class Builder:
def run(self):
try:
print("[INSTALL PACKAGES]")
self.buildEnv.install_packages()
self.buildEnv.finalize_setup()
self.platform.install_packages()
self.platform.finalize_setup()
print("[PREPARE]")
self.prepare_sources()
print("[BUILD]")
self.build()
# No error, clean intermediate file at end of build if needed.
print("[CLEAN]")
if self.buildEnv.options.clean_at_end:
self.buildEnv.clean_intermediate_directories()
if self.options.clean_at_end:
self.platform.clean_intermediate_directories()
else:
print("SKIP")
except StopBuild:

2
kiwixbuild/dependencies/icu4c.py
View File

@ -28,7 +28,7 @@ class Icu(Dependency):
@property
def configure_option(self):
options = "--disable-samples --disable-tests --disable-extras --disable-dyload --enable-rpath"
if self.buildEnv.platform_info.build == 'android':
if self.buildEnv.platformInfo.build == 'android':
options += " --with-data-packaging=archive"
return options

4
kiwixbuild/dependencies/kiwix_lib.py
View File

@ -27,12 +27,12 @@ class Kiwixlib(Dependency):
@property
def configure_option(self):
base_option = "-Dctpp2-install-prefix={buildEnv.install_dir}"
if self.buildEnv.platform_info.build == 'android':
if self.buildEnv.platformInfo.build == 'android':
base_option += ' -Dandroid=true'
return base_option
@property
def library_type(self):
if self.buildEnv.platform_info.build == 'android':
if self.buildEnv.platformInfo.build == 'android':
return 'shared'
return super().library_type

2
kiwixbuild/dependencies/kiwix_tools.py
View File

@ -15,6 +15,6 @@ class KiwixTools(Dependency):
@property
def configure_option(self):
if self.buildEnv.platform_info.static:
if self.buildEnv.platformInfo.static:
return "-Dstatic-linkage=true"
return ""

2
kiwixbuild/dependencies/zim_tools.py
View File

@ -15,6 +15,6 @@ class ZimTools(Dependency):
@property
def configure_option(self):
if self.buildEnv.platform_info.static:
if self.buildEnv.platformInfo.static:
return "-Dstatic-linkage=true"
return ""

2
kiwixbuild/dependencies/zimwriterfs.py
View File

@ -23,6 +23,6 @@ class Zimwriterfs(Dependency):
@property
def configure_option(self):
base_option = "-Dmagic-install-prefix={buildEnv.install_dir}"
if self.buildEnv.platform_info.static:
if self.buildEnv.platformInfo.static:
base_option += " -Dstatic-linkage=true"
return base_option

6
kiwixbuild/dependencies/zlib.py
View File

@ -28,16 +28,16 @@ class zlib(Dependency):
def _configure(self, context):
if self.buildEnv.platform_info.build == 'win32':
if self.buildEnv.platformInfo.build == 'win32':
raise SkipCommand()
return super()._configure(context)
@property
def make_option(self):
if self.buildEnv.platform_info.build == 'win32':
if self.buildEnv.platformInfo.build == 'win32':
return "--makefile win32/Makefile.gcc PREFIX={host}- SHARED_MODE={static} INCLUDE_PATH={include_path} LIBRARY_PATH={library_path} BINARY_PATH={binary_path}".format(
host='i686-w64-mingw32',
static="0" if self.buildEnv.platform_info.static else "1",
static="0" if self.buildEnv.platformInfo.static else "1",
include_path=pj(self.buildEnv.install_dir, 'include'),
library_path=pj(self.buildEnv.install_dir, self.buildEnv.libprefix),
binary_path=pj(self.buildEnv.install_dir, 'bin'),

41
kiwixbuild/platforms/android.py
View File

@ -20,8 +20,34 @@ class AndroidPlatformInfo(PlatformInfo):
def __str__(self):
return "android"
def binaries(self, install_path):
binaries = ((k,'{}-{}'.format(self.arch_full, v))
for k, v in (('CC', 'gcc'),
('CXX', 'g++'),
('AR', 'ar'),
('STRIP', 'strip'),
('WINDRES', 'windres'),
('RANLIB', 'ranlib'),
('LD', 'ld'))
)
return {k:pj(install_path, 'bin', v)
for k,v in binaries}
@property
def ndk_builder(self):
return self.toolchains[0].builder
@property
def sdk_builder(self):
return self.toolchains[1].builder
def get_cross_config(self):
install_path = self.ndk_builder.install_path
return {
'exec_wrapper_def': '',
'install_path': install_path,
'binaries': self.binaries(install_path),
'root_path': pj(install_path, 'sysroot'),
'extra_libs': [],
'extra_cflags': [],
'host_machine': {
@ -34,7 +60,22 @@ class AndroidPlatformInfo(PlatformInfo):
},
}
def get_bin_dir(self):
return [pj(self.ndk_builder.install_path, 'bin')]
def set_env(self, env):
root_path = pj(self.ndk_builder.install_path, 'sysroot')
env['PKG_CONFIG_LIBDIR'] = pj(root_path, 'lib', 'pkgconfig')
env['CFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(root_path) + env['CFLAGS']
env['CXXFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(root_path) + env['CXXFLAGS']
env['LDFLAGS'] = '--sysroot={} '.format(root_path) + env['LDFLAGS']
#env['CFLAGS'] = ' -fPIC -D_FILE_OFFSET_BITS=64 -O3 '+env['CFLAGS']
#env['CXXFLAGS'] = (' -D__OPTIMIZE__ -fno-strict-aliasing '
# ' -DU_HAVE_NL_LANGINFO_CODESET=0 '
# '-DU_STATIC_IMPLEMENTATION -O3 '
# '-DU_HAVE_STD_STRING -DU_TIMEZONE=0 ')+env['CXXFLAGS']
env['NDK_DEBUG'] = '0'
env['ANDROID_HOME'] = self.sdk_builder.install_path
AndroidPlatformInfo('android_arm', 'arm')
AndroidPlatformInfo('android_arm64', 'arm64')

63
kiwixbuild/platforms/armhf.py
View File

@ -2,11 +2,15 @@ from .base import PlatformInfo
class ArmhfPlatformInfo(PlatformInfo):
arch_full = 'arm-linux-gnueabihf'
def __init__(self, name, static):
super().__init__(name, 'armhf', static, ['armhf_toolchain'], ['fedora', 'debian'])
def get_cross_config(self):
return {
'binaries': self.binaries,
'exec_wrapper_def': '',
'root_path': self.root_path,
'extra_libs': [],
'extra_cflags': [],
'host_machine': {
@ -19,6 +23,65 @@ class ArmhfPlatformInfo(PlatformInfo):
}
}
@property
def tlc_source(self):
return self.toolchains[0].source
@property
def root_path(self):
return pj(self.tlc_source.source_path,
'arm-bcm2708',
'gcc-linaro-{}-raspbian-x64'.format(self.arch_full))
@property
def binaries(self):
binaries = ((k,'{}-{}'.format(self.arch_full, v))
for k, v in (('CC', 'gcc'),
('CXX', 'g++'),
('AR', 'ar'),
('STRIP', 'strip'),
('WINDRES', 'windres'),
('RANLIB', 'ranlib'),
('LD', 'ld'))
)
return {k:pj(self.root_path, 'bin', v)
for k,v in binaries}
@property
def exec_wrapper_def(self):
try:
which('qemu-arm')
except subprocess.CalledProcessError:
return ""
else:
return "exec_wrapper = 'qemu-arm'"
@property
def configure_option(self):
return '--host={}'.format(self.arch_full)
def get_bin_dir(self):
return [pj(self.root_path, 'bin')]
def set_env(self, env):
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
env['CFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS']
env['CXXFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
env['QEMU_LD_PREFIX'] = pj(self.root_path, "arm-linux-gnueabihf", "libc")
env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format(
':'.join([
pj(self.root_path, self.arch_full, "lib"),
env['LD_LIBRARY_PATH']
]))
def set_compiler(self, env):
env['CC'] = self.binaries['CC']
env['CXX'] = self.binaries['CXX']
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
ArmhfPlatformInfo('armhf_dyn', False)
ArmhfPlatformInfo('armhf_static', True)

125
kiwixbuild/platforms/base.py
View File

@ -1,5 +1,15 @@
import os
import subprocess
from kiwixbuild.toolchains import Toolchain
from kiwixbuild.packages import PACKAGE_NAME_MAPPERS
from kiwixbuild.utils import pj, remove_duplicates
from kiwixbuild.buildenv import BuildEnv
from kiwixbuild._global import neutralEnv
_SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
TEMPLATES_DIR = pj(os.path.dirname(_SCRIPT_DIR), 'templates')
class PlatformInfo:
all_platforms = {}
@ -10,12 +20,127 @@ class PlatformInfo:
self.static = static
self.toolchains = toolchains
self.compatible_hosts = hosts
self.buildEnv = BuildEnv(self)
self.setup_toolchains()
def __str__(self):
return "{}_{}".format(self.build, 'static' if self.static else 'dyn')
def setup_toolchains(self):
self.toolchains = {}
for tlc_name in self.toolchains:
ToolchainClass = Toolchain.all_toolchains[tlc_name]
self.toolchains[tlc_name] = ToolchainClass()
def get_cross_config(self):
return {}
def set_env(self, env):
pass
def get_bind_dir(self):
return []
def set_compiler(self, env):
pass
def _gen_crossfile(self, name):
crossfile = pj(self.buildEnv.build_dir, name)
template_file = pj(TEMPLATES_DIR, name)
with open(template_file, 'r') as f:
template = f.read()
content = template.format(
**self.get_cross_config()
)
with open(crossfile, 'w') as outfile:
outfile.write(content)
return crossfile
def finalize_setup(self):
self.buildEnv.toolchains = [
self.toolchains[tlc_name] for tlc_name in self.toolchain_names]
self.buildEnv.cross_config = self.get_cross_config()
self.buildEnv.meson_crossfile = None
self.buildEnv.cmake_crossfile = None
def get_packages(self):
mapper_name = "{host}_{target}".format(
host=neutralEnv('distname'),
target=self)
try:
package_name_mapper = PACKAGE_NAME_MAPPERS[mapper_name]
except KeyError:
print("SKIP : We don't know which packages we must install to compile"
" a {target} {build_type} version on a {host} host.".format(
target=self.platform_info,
host=neutralEnv('distname')))
return
packages_list = package_name_mapper.get('COMMON', [])
to_drop = []
build_steps = (step for step in target_steps() if step[0] == self.name)
for dep in build_steps:
packages = package_name_mapper.get(dep[1])
if packages:
packages_list += packages
to_drop.append(dep)
for dep in to_drop:
del target_steps()[dep]
source_dep = [dep[1] for dep in target_steps() if dep[0] == 'source']
for dep in source_dep:
it_build = (True for dep in target_steps() if dep[0] != 'source' and dep[1] == dep)
if next(it_build, False):
print("removing source ", dep)
del target_steps()[('source', dep)]
build_steps = (step for step in target_steps() if step[0] == self.name)
for dep in build_steps:
packages = getattr(dep, 'extra_packages', [])
for package in packages:
packages_list += package_name_mapper.get(package, [])
return packages_list
def install_packages(self):
autoskip_file = pj(self.buildEnv.build_dir, ".install_packages_ok")
packages_to_have = self.get_packages()
packages_to_have = remove_duplicates(packages_to_have)
if not neutralEnv('force_install_packages') and os.path.exists(autoskip_file):
print("SKIP")
return
distname = neutralEnv('distname')
if distname in ('fedora', 'redhat', 'centos'):
package_installer = 'sudo dnf install {}'
package_checker = 'rpm -q --quiet {}'
elif distname in ('debian', 'Ubuntu'):
package_installer = 'sudo apt-get install {}'
package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1'
elif distname == 'Darwin':
package_installer = 'brew install {}'
package_checker = 'brew list -1 | grep -q {}'
packages_to_install = []
for package in packages_to_have:
print(" - {} : ".format(package), end="")
command = package_checker.format(package)
try:
subprocess.check_call(command, shell=True)
except subprocess.CalledProcessError:
print("NEEDED")
packages_to_install.append(package)
else:
print("SKIP")
if packages_to_install:
command = package_installer.format(" ".join(packages_to_install))
print(command)
subprocess.check_call(command, shell=True)
else:
print("SKIP, No package to install.")
with open(autoskip_file, 'w'):
pass
def clean_intermediate_directories(self):
self.buildEnv.clean_intermediate_directories()

33
kiwixbuild/platforms/i586.py
View File

@ -3,11 +3,14 @@ from .base import PlatformInfo
class I586PlatformInfo(PlatformInfo):
arch_full = 'i586-linux-gnu'
def __init__(self, name, static):
super().__init__(name, 'i586', static, ['linux_i586_toolchain'], ['fedora', 'debian'])
super().__init__(name, 'i586', static, [], ['fedora', 'debian'])
def get_cross_config(self):
return {
'binaries': self.binaries,
'exec_wrapper_def': '',
'extra_libs': ['-m32', '-march=i586', '-mno-sse'],
'extra_cflags': ['-m32', '-march=i586', '-mno-sse'],
'host_machine': {
@ -20,6 +23,34 @@ class I586PlatformInfo(PlatformInfo):
}
}
@property
def configure_option(self):
return '--host={}'.format(self.arch_full)
@property
def binaries(self):
return {k:which(v)
for k, v in (('CC', os.environ.get('CC', 'gcc')),
('CXX', os.environ.get('CXX', 'g++')),
('AR', 'ar'),
('STRIP', 'strip'),
('RANLIB', 'ranlib'),
('LD', 'ld'))
}
def set_env(self, env):
env['CFLAGS'] = "-m32 -march=i586 -mno-sse "+env['CFLAGS']
env['CXXFLAGS'] = "-m32 -march=i586 -mno-sse "+env['CXXFLAGS']
env['LDFLAGS'] = "-m32 -march=i586 -mno-sse "+env['LDFLAGS']
def get_bin_dir(self):
return []
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_i586_cross_file.txt')
self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
I586PlatformInfo('i586_dyn', False)
I586PlatformInfo('i586_static', True)

32
kiwixbuild/platforms/ios.py
View File

@ -1,7 +1,8 @@
from .base import PlatformInfo
import subprocess
from .base import PlatformInfo
from kiwixbuild.utils import pj, xrun_find
class iOSPlatformInfo(PlatformInfo):
@ -13,7 +14,7 @@ class iOSPlatformInfo(PlatformInfo):
}
def __init__(self, name, arch):
super().__init__(name, 'iOS', True, ['iOS_sdk'],
super().__init__(name, 'iOS', True, [],
hosts=['Darwin'])
self.arch = arch
self.arch_full, self.cpu, self.sdk_name = self.__arch_infos[arch]
@ -29,9 +30,16 @@ class iOSPlatformInfo(PlatformInfo):
def __str__(self):
return "iOS"
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_ios_cross_file.txt')
self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def get_cross_config(self):
return {
'root_path': self.root_path,
'binaries': self.binaries,
'exec_wrapper_def': '',
'extra_libs': ['-fembed-bitcode', '-isysroot', self.root_path, '-arch', self.arch, '-miphoneos-version-min=9.0', '-stdlib=libc++'],
'extra_cflags': ['-fembed-bitcode', '-isysroot', self.root_path, '-arch', self.arch, '-miphoneos-version-min=9.0', '-stdlib=libc++'],
'host_machine': {
@ -53,6 +61,26 @@ class iOSPlatformInfo(PlatformInfo):
def get_bin_dir(self):
return [pj(self.root_path, 'bin')]
@property
def binaries(self):
return {
'CC': xrun_find('clang'),
'CXX': xrun_find('clang++'),
'AR': '/usr/bin/ar',
'STRIP': '/usr/bin/strip',
'RANLIB': '/usr/bin/ranlib',
'LD': '/usr/bin/ld',
}
@property
def configure_option(self):
return '--host={}'.format(self.arch_full)
def set_compiler(self, env):
env['CC'] = self.binaries['CC']
env['CXX'] = self.binaries['CXX']
iOSPlatformInfo('iOS_armv7', 'armv7')
iOSPlatformInfo('iOS_arm64', 'arm64')
iOSPlatformInfo('iOS_i386', 'i386')

3
kiwixbuild/platforms/native.py
View File

@ -5,9 +5,6 @@ class NativePlatformInfo(PlatformInfo):
def __init__(self, name, static, hosts):
super().__init__(name, 'native', static, [], hosts)
def get_cross_config(self):
return {}
NativePlatformInfo('native_dyn', False, ['fedora', 'debian', 'Darwin'])
NativePlatformInfo('native_static', True, ['fedora', 'debian'])

58
kiwixbuild/platforms/win32.py
View File

@ -1,13 +1,21 @@
import subprocess
from .base import PlatformInfo
from kiwixbuild.utils import which, pj
from kiwixbuild._global import neutralEnv
class Win32PlatformInfo(PlatformInfo):
extra_libs = ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4', '-lmsvcr90', '-liphlpapi']
arch_full = 'i686-w64-mingw32'
def __init__(self, name, static):
super().__init__(name, 'win32', static, ['mingw32_toolchain'], ['fedora', 'debian'])
super().__init__(name, 'win32', static, [], ['fedora', 'debian'])
def get_cross_config(self):
return {
'exec_wrapper_def': self.exec_wrapper_def,
'binaries': self.binaries,
'root_path': self.root_path,
'extra_libs': self.extra_libs,
'extra_cflags': ['-DWIN32'],
'host_machine': {
@ -20,5 +28,53 @@ class Win32PlatformInfo(PlatformInfo):
}
}
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
@property
def root_path(self):
root_paths = {
'fedora': '/usr/i686-w64-mingw32/sys-root/mingw',
'debian': '/usr/i686-w64-mingw32'
}
return root_paths[neutralEnv('distname')]
@property
def binaries(self):
return {k:which('{}-{}'.format(self.arch_full, v))
for k, v in (('CC', 'gcc'),
('CXX', 'g++'),
('AR', 'ar'),
('STRIP', 'strip'),
('WINDRES', 'windres'),
('RANLIB', 'ranlib'))
}
@property
def exec_wrapper_def(self):
try:
which('wine')
except subprocess.CalledProcessError:
return ""
else:
return "exec_wrapper = 'wine'"
@property
def configure_option(self):
return '--host={}'.format(self.arch_full)
def set_compiler(self, env):
for k, v in self.binaries.items():
env[k] = v
def get_bin_dir(self):
return [pj(self.root_path, 'bin')]
def set_env(self, env):
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
env['LIBS'] = " ".join(self.extra_libs) + " " +env['LIBS']
Win32PlatformInfo('win32_dyn', False)
Win32PlatformInfo('win32_static', True)

9
kiwixbuild/templates/cmake_android_cross_file.txt
View File

@ -1,7 +1,8 @@
SET(CMAKE_SYSTEM_NAME {host_machine[system]})
SET(CMAKE_ANDROID_STANDALONE_TOOLCHAIN {toolchain.builder.install_path})
SET(CMAKE_C_COMPILER "{toolchain.binaries[CC]}")
SET(CMAKE_CXX_COMPILER "{toolchain.binaries[CXX]}")
SET(CMAKE_ANDROID_STANDALONE_TOOLCHAIN {install_path})
SET(CMAKE_FIND_ROOT_PATH {toolchain.root_path})
SET(CMAKE_C_COMPILER "{binaries[CC]}")
SET(CMAKE_CXX_COMPILER "{binaries[CXX]}")
SET(CMAKE_FIND_ROOT_PATH {root_path})

12
kiwixbuild/templates/cmake_cross_file.txt
View File

@ -2,11 +2,11 @@ SET(CMAKE_SYSTEM_NAME {host_machine[system]})
SET(CMAKE_SYSTEM_PROCESSOR {host_machine[cpu_family]})
# specify the cross compiler
SET(CMAKE_C_COMPILER "{toolchain.binaries[CC]}")
SET(CMAKE_CXX_COMPILER "{toolchain.binaries[CXX]}")
SET(CMAKE_RC_COMPILER {toolchain.binaries[WINDRES]})
SET(CMAKE_AR:FILEPATH {toolchain.binaries[AR]})
SET(CMAKE_RANLIB:FILEPATH {toolchain.binaries[RANLIB]})
SET(CMAKE_C_COMPILER "{binaries[CC]}")
SET(CMAKE_CXX_COMPILER "{binaries[CXX]}")
SET(CMAKE_RC_COMPILER {binaries[WINDRES]})
SET(CMAKE_AR:FILEPATH {binaries[AR]})
SET(CMAKE_RANLIB:FILEPATH {binaries[RANLIB]})
find_program(CCACHE_FOUND ccache)
if(CCACHE_FOUND)
@ -15,7 +15,7 @@ if(CCACHE_FOUND)
endif(CCACHE_FOUND)
# where is the target environment
SET(CMAKE_FIND_ROOT_PATH {toolchain.root_path})
SET(CMAKE_FIND_ROOT_PATH {root_path})
# search for programs in the build host directories
SET(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)

8
kiwixbuild/templates/cmake_i586_cross_file.txt
View File

@ -2,13 +2,13 @@ SET(CMAKE_SYSTEM_NAME {host_machine[system]})
SET(CMAKE_SYSTEM_PROCESSOR {host_machine[cpu_family]})
# specify the cross compiler
SET(CMAKE_C_COMPILER "{toolchain.binaries[CC]}")
SET(CMAKE_CXX_COMPILER "{toolchain.binaries[CXX]}")
SET(CMAKE_C_COMPILER "{binaries[CC]}")
SET(CMAKE_CXX_COMPILER "{binaries[CXX]}")
SET(C_FLAGS "-m32 -march=i586")
SET(CXX_FLAGS "-m32 -march=i586")
SET(CMAKE_LD_FLAGS "-m32 -march=i586")
SET(CMAKE_AR:FILEPATH {toolchain.binaries[AR]})
SET(CMAKE_RANLIB:FILEPATH {toolchain.binaries[RANLIB]})
SET(CMAKE_AR:FILEPATH {binaries[AR]})
SET(CMAKE_RANLIB:FILEPATH {binaries[RANLIB]})
find_program(CCACHE_FOUND ccache)
if(CCACHE_FOUND)

4
kiwixbuild/templates/cmake_ios_cross_file.txt
View File

@ -1,7 +1,7 @@
SET(CMAKE_SYSTEM_NAME {host_machine[system]})
SET(CMAKE_C_COMPILER "{toolchain.binaries[CC]}")
SET(CMAKE_CXX_COMPILER "{toolchain.binaries[CXX]}")
SET(CMAKE_C_COMPILER "{binaries[CC]}")
SET(CMAKE_CXX_COMPILER "{binaries[CXX]}")
SET(CMAKE_FIND_ROOT_PATH {root_path})

8
kiwixbuild/templates/meson_android_cross_file.txt
View File

@ -1,9 +1,9 @@
[binaries]
pkgconfig = 'pkg-config'
c = '{toolchain.binaries[CC]}'
ar = '{toolchain.binaries[AR]}'
cpp = '{toolchain.binaries[CXX]}'
strip = '{toolchain.binaries[STRIP]}'
c = '{binaries[CC]}'
ar = '{binaries[AR]}'
cpp = '{binaries[CXX]}'
strip = '{binaries[STRIP]}'
[properties]
c_link_args = {extra_libs!r}

10
kiwixbuild/templates/meson_cross_file.txt
View File

@ -1,10 +1,10 @@
[binaries]
pkgconfig = 'pkg-config'
c = '{toolchain.binaries[CC]}'
ar = '{toolchain.binaries[AR]}'
cpp = '{toolchain.binaries[CXX]}'
strip = '{toolchain.binaries[STRIP]}'
{toolchain.exec_wrapper_def}
c = '{binaries[CC]}'
ar = '{binaries[AR]}'
cpp = '{binaries[CXX]}'
strip = '{binaries[STRIP]}'
{exec_wrapper_def}
[properties]
c_link_args = {extra_libs!r}

2
kiwixbuild/toolchains/__init__.py
View File

@ -1,4 +1,4 @@
from . import android_ndk, android_sdk, armhf, ios, mingw32, linux_i586
from . import android_ndk, android_sdk, armhf
from .base_toolchain import Toolchain

19
kiwixbuild/toolchains/android_ndk.py
View File

@ -113,22 +113,3 @@ class android_ndk(Toolchain):
self.command('build_platform', self._build_platform)
self.command('fix_permission_right', self._fix_permission_right)
def get_bin_dir(self):
return [pj(self.builder.install_path, 'bin')]
def set_env(self, env):
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
env['CFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(self.root_path) + env['CFLAGS']
env['CXXFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(self.root_path) + env['CXXFLAGS']
env['LDFLAGS'] = '--sysroot={} '.format(self.root_path) + env['LDFLAGS']
#env['CFLAGS'] = ' -fPIC -D_FILE_OFFSET_BITS=64 -O3 '+env['CFLAGS']
#env['CXXFLAGS'] = (' -D__OPTIMIZE__ -fno-strict-aliasing '
# ' -DU_HAVE_NL_LANGINFO_CODESET=0 '
# '-DU_STATIC_IMPLEMENTATION -O3 '
# '-DU_HAVE_STD_STRING -DU_TIMEZONE=0 ')+env['CXXFLAGS']
env['NDK_DEBUG'] = '0'
def set_compiler(self, env):
env['CC'] = self.binaries['CC']
env['CXX'] = self.binaries['CXX']

3
kiwixbuild/toolchains/android_sdk.py
View File

@ -49,6 +49,3 @@ class android_sdk(Toolchain):
def build(self):
self.command('build_platform', self._build_platform)
self.command('fix_licenses', self._fix_licenses)
def set_env(self, env):
env['ANDROID_HOME'] = self.builder.install_path

50
kiwixbuild/toolchains/armhf.py
View File

@ -8,57 +8,7 @@ pj = os.path.join
class armhf_toolchain(Toolchain):
name = 'armhf'
arch_full = 'arm-linux-gnueabihf'
class Source(GitClone):
git_remote = "https://github.com/raspberrypi/tools"
git_dir = "raspberrypi-tools"
@property
def root_path(self):
return pj(self.source_path, 'arm-bcm2708', 'gcc-linaro-arm-linux-gnueabihf-raspbian-x64')
@property
def binaries(self):
binaries = ((k,'{}-{}'.format(self.arch_full, v))
for k, v in (('CC', 'gcc'),
('CXX', 'g++'),
('AR', 'ar'),
('STRIP', 'strip'),
('WINDRES', 'windres'),
('RANLIB', 'ranlib'),
('LD', 'ld'))
)
return {k:pj(self.root_path, 'bin', v)
for k,v in binaries}
@property
def exec_wrapper_def(self):
try:
which('qemu-arm')
except subprocess.CalledProcessError:
return ""
else:
return "exec_wrapper = 'qemu-arm'"
@property
def configure_option(self):
return '--host={}'.format(self.arch_full)
def get_bin_dir(self):
return [pj(self.root_path, 'bin')]
def set_env(self, env):
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
env['CFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS']
env['CXXFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
env['QEMU_LD_PREFIX'] = pj(self.root_path, "arm-linux-gnueabihf", "libc")
env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format(
':'.join([
pj(self.root_path, "arm-linux-gnueabihf", "lib"),
env['LD_LIBRARY_PATH']
]))
def set_compiler(self, env):
env['CC'] = self.binaries['CC']
env['CXX'] = self.binaries['CXX']

9
kiwixbuild/toolchains/base_toolchain.py
View File

@ -41,15 +41,6 @@ class Toolchain(metaclass=_MetaToolchain):
def _log_dir(self):
return neutralEnv('log_dir')
def set_env(self, env):
pass
def set_compiler(self, env):
pass
def get_bin_dir(self):
return []
def command(self, name, function, *args):
print(" {} {} : ".format(name, self.name), end="", flush=True)
log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))

23
kiwixbuild/toolchains/ios.py
View File

@ -1,23 +0,0 @@
from .base_toolchain import Toolchain
from kiwixbuild.utils import pj, xrun_find
class iOS_sdk(Toolchain):
@property
def binaries(self):
return {
'CC': xrun_find('clang'),
'CXX': xrun_find('clang++'),
'AR': '/usr/bin/ar',
'STRIP': '/usr/bin/strip',
'RANLIB': '/usr/bin/ranlib',
'LD': '/usr/bin/ld',
}
@property
def configure_option(self):
return '--host=arm-apple-darwin'
def set_compiler(self, env):
env['CC'] = self.binaries['CC']
env['CXX'] = self.binaries['CXX']

37
kiwixbuild/toolchains/linux_i586.py
View File

@ -1,37 +0,0 @@
import os
from .base_toolchain import Toolchain
from kiwixbuild.dependencies import GitClone
from kiwixbuild.utils import which
pj = os.path.join
class linux_i586_toolchain(Toolchain):
name = 'linux_i586'
arch_full = 'i586-linux-gnu'
@property
def configure_option(self):
return '--host={}'.format(self.arch_full)
@property
def binaries(self):
return {k:which(v)
for k, v in (('CC', os.environ.get('CC', 'gcc')),
('CXX', os.environ.get('CXX', 'g++')),
('AR', 'ar'),
('STRIP', 'strip'),
('RANLIB', 'ranlib'),
('LD', 'ld'))
}
@property
def configure_option(self):
return '--host={}'.format(self.arch_full)
def set_env(self, env):
env['CFLAGS'] = "-m32 -march=i586 -mno-sse "+env['CFLAGS']
env['CXXFLAGS'] = "-m32 -march=i586 -mno-sse "+env['CXXFLAGS']
env['LDFLAGS'] = "-m32 -march=i586 -mno-sse "+env['LDFLAGS']
def get_bin_dir(self):
return []

56
kiwixbuild/toolchains/mingw32.py
View File

@ -1,56 +0,0 @@
import os
import subprocess
from .base_toolchain import Toolchain
from kiwixbuild.utils import which
from kiwixbuild._global import neutralEnv
pj = os.path.join
class mingw32_toolchain(Toolchain):
name = 'mingw32'
arch_full = 'i686-w64-mingw32'
extra_libs = ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4', '-lmsvcr90', '-liphlpapi']
@property
def root_path(self):
root_paths = {
'fedora': '/usr/i686-w64-mingw32/sys-root/mingw',
'debian': '/usr/i686-w64-mingw32'
}
return root_paths[neutralEnv('distname')]
@property
def binaries(self):
return {k:which('{}-{}'.format(self.arch_full, v))
for k, v in (('CC', 'gcc'),
('CXX', 'g++'),
('AR', 'ar'),
('STRIP', 'strip'),
('WINDRES', 'windres'),
('RANLIB', 'ranlib'))
}
@property
def exec_wrapper_def(self):
try:
which('wine')
except subprocess.CalledProcessError:
return ""
else:
return "exec_wrapper = 'wine'"
@property
def configure_option(self):
return '--host={}'.format(self.arch_full)
def set_compiler(self, env):
for k, v in self.binaries.items():
env[k] = v
def get_bin_dir(self):
return [pj(self.root_path, 'bin')]
def set_env(self, env):
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
env['LIBS'] = " ".join(self.extra_libs) + " " +env['LIBS']