smallsolar
/
kiwix-build
mirror of https://github.com/kiwix/kiwix-build.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #175 from kiwix/multi-arch 

Multi arch
This commit is contained in:
Matthieu Gautier 2018-06-08 14:33:15 +02:00 committed by GitHub
parent f44042f910 4dea44d305
commit 312e734678
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
53 changed files with 1272 additions and 1061 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.travis.yml
View File

@ -54,6 +54,7 @@ env:
- PLATFORM="armhf_static"
- PLATFORM="i586_dyn"
- PLATFORM="i586_static"
- PLATFORM="android"
- PLATFORM="android_arm"
- PLATFORM="android_arm64"
- PLATFORM="android_x86"
@ -66,6 +67,7 @@ matrix:
- env: PLATFORM="android_mips64"
- env: PLATFORM="android_x86"
- env: PLATFORM="android_x86_64"
- env: PLATFORM="android"
include:
- env: PLATFORM="android_mips"
if: type!=cron AND type!=pull_request

58
README.md
View File

@ -71,8 +71,10 @@ By default, `kiwix-build` will build `kiwix-tools` .
## Target platform
By default, `kiwix-build` will build everything for the current (native)
platform using dynamic linkage (`native_dyn`).
If no target platform is specified, a default one will be infered from
the specified target :
- `kiwix-android` will be build using the platform `android`
- Other targets will be build using the platform `native_dyn`
But you can select another target platform using the option
`--target-platform`. For now, there is ten different supported
@ -82,6 +84,7 @@ platforms :
- native_static
- win32_dyn
- win32_static
- android
- android_arm
- android_arm64
- android_mips
@ -95,11 +98,58 @@ So, if you want to compile `kiwix-tools` for win32 using static linkage:
$ kiwix-build --target-platform win32_dyn
```
Or, android apk for android_arm :
## Android
Android apk (kiwix-android) is a bit a special case.
`kiwix-android` itself is architecture independent (it is written in
java) but it use `kiwix-lib` who is architecture dependent.
When building `kiwix-lib`, you should directly use the
target-platform `android_<arch>`:
```
$ kiwix-build --target-platform android_arm kiwix-android
$ kiwix-build kiwix-android --target-platform android_arm
```
But, `kiwix-android` apk can also be multi arch (ie, it includes
`kiwix-lib` for several architectures). To do so, you must ask to build
`kiwix-android` using the `android` platform:
```
$ kiwix-build --target-platform android kiwix-android
$ kiwix-build kiwix-android # because `android` platform is the default `kiwix-android`
```
By default, when using platform `android`, `kiwix-lib` will be build for
all architectures. This can be change by using the option `--android-arch` :
```
$ kiwix-build kiwix-android # apk for all architectures
$ kiwix-build kiwix-android --android-arch arm # apk for arm architectures (equivalent to `kiwix-android --target-platform android_arm`)
$ kiwix-build kiwix-anrdoid --android-arch arm --android-arch arm64 # apk for arm and arm64 architectures
```
## IOS
When building for ios, we may want to compile a "fat library", a library
for several architectures.
To do so, you should directly use the target-platfrom `ios_multi`.
As for `android`, `kiwix-build` will build the library several times
(once for each platform) and then create the fat library.
```
$ kiwix-build --target-platform iOS_multi kiwix-lib
```
You can specify the supported architectures with the option `--ios-arch`:
```
$ kiwix-build --target-platform iOS_multi kiwix-lib # all architetures
$ kiwix-build --target-platform iOS_multi --ios-arch arm --ios-arch arm64 # arm and arm64 arch only
```
# Outputs
Kiwix-build.py will create several directories:

33
kiwixbuild/__init__.py
View File

@ -6,15 +6,15 @@ import argparse
from .dependencies import Dependency
from .platforms import PlatformInfo
from .builder import Builder
from .utils import setup_print_progress
from . import _global
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument('targets', default='kiwix-tools', nargs='?', metavar='TARGET',
parser.add_argument('target', default='kiwix-tools', nargs='?', metavar='TARGET',
choices=Dependency.all_deps.keys())
parser.add_argument('--working-dir', default=".")
parser.add_argument('--libprefix', default=None)
parser.add_argument('--target-platform', default="native_dyn", choices=PlatformInfo.all_platforms)
parser.add_argument('--target-platform', choices=PlatformInfo.all_platforms)
parser.add_argument('--verbose', '-v', action="store_true",
help=("Print all logs on stdout instead of in specific"
" log files per commands"))
@ -23,7 +23,7 @@ def parse_args():
parser.add_argument('--skip-source-prepare', action='store_true',
help="Skip the source download part")
parser.add_argument('--build-deps-only', action='store_true',
help="Build only the dependencies of the specified targets.")
help="Build only the dependencies of the specified target.")
parser.add_argument('--build-nodeps', action='store_true',
help="Build only the target, not its dependencies.")
parser.add_argument('--make-dist', action='store_true',
@ -35,8 +35,16 @@ def parse_args():
help="Skip SSL certificate verification during download")
subgroup.add_argument('--clean-at-end', action='store_true',
help="Clean all intermediate files after the (successfull) build")
subgroup.add_argument('--force-install-packages', action='store_true',
help="Allways check for needed packages before compiling")
subgroup.add_argument('--dont-install-packages', action='store_true',
help="Do not try to install packages before compiling")
subgroup.add_argument('--android-arch', action='append',
help=("Specify the architecture to build for android application/libraries.\n"
"Can be specified several times to build for several architectures.\n"
"If not specified, all architectures will be build."))
subgroup.add_argument('--ios-arch', action='append',
help=("Specify the architecture to build for ios application/libraries.\n"
"Can be specified several times to build for several architectures.\n"
"If not specified, all architectures will be build."))
subgroup = parser.add_argument_group('custom app',
description="Android custom app specific options")
subgroup.add_argument('--android-custom-app',
@ -47,7 +55,7 @@ def parse_args():
help="The size of the zim file.")
options = parser.parse_args()
if options.targets == 'kiwix-android-custom':
if options.target == 'kiwix-android-custom':
err = False
if not options.android_custom_app:
print("You need to specify ANDROID_CUSTOM_APP if you "
@ -59,12 +67,19 @@ def parse_args():
err = True
if err:
sys.exit(1)
if not options.android_arch:
options.android_arch = ['arm', 'arm64', 'mips', 'mips64', 'x86', 'x86_64']
if not options.ios_arch:
options.ios_arch = ['armv7', 'arm64', 'i386', 'x86_64']
return options
def main():
options = parse_args()
options.working_dir = os.path.abspath(options.working_dir)
setup_print_progress(options.show_progress)
builder = Builder(options)
_global.set_options(options)
neutralEnv = buildenv.PlatformNeutralEnv()
_global.set_neutralEnv(neutralEnv)
builder = Builder()
builder.run()

32
kiwixbuild/_global.py Normal file
View File

@ -0,0 +1,32 @@
from collections import OrderedDict as _OrderedDict
_neutralEnv = None
_options = None
_target_steps = _OrderedDict()
def set_neutralEnv(env):
global _neutralEnv
_neutralEnv = env
def neutralEnv(what):
return getattr(_neutralEnv, what)
def set_options(options):
global _options
_options = options
def option(what):
return getattr(_options, what)
def add_target_step(key, what):
_target_steps[key] = what
def get_target_step(key, default_context=None):
try:
context, target = key
except ValueError:
context, target = default_context, key
return _target_steps[(context, target)]
def target_steps():
return _target_steps

260
kiwixbuild/buildenv.py
View File

@ -3,30 +3,22 @@ import os, sys, shutil
import subprocess
import platform
from .platforms import PlatformInfo
from .toolchains import Toolchain
from .packages import PACKAGE_NAME_MAPPERS
from .utils import (
pj,
download_remote,
Defaultdict)
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
from .utils import pj, download_remote, Defaultdict
from ._global import neutralEnv, option
class PlatformNeutralEnv:
def __init__(self, options):
self.options = options
self.source_dir = pj(options.working_dir, "SOURCE")
self.archive_dir = pj(options.working_dir, "ARCHIVE")
self.toolchain_dir = pj(options.working_dir, "TOOLCHAINS")
def __init__(self):
self.working_dir = option('working_dir')
self.source_dir = pj(self.working_dir, "SOURCE")
self.archive_dir = pj(self.working_dir, "ARCHIVE")
self.toolchain_dir = pj(self.working_dir, "TOOLCHAINS")
self.log_dir = pj(self.working_dir, 'LOGS')
for d in (self.source_dir,
self.archive_dir,
self.toolchain_dir,
self.log_dir):
os.makedirs(d, exist_ok=True)
self.toolchains = {}
self.detect_platform()
self.ninja_command = self._detect_ninja()
if not self.ninja_command:
@ -36,33 +28,6 @@ class PlatformNeutralEnv:
sys.exit("ERROR: meson command not fount")
self.mesontest_command = "{} test".format(self.meson_command)
def run_command(self, command, cwd, context, env=None, input=None):
os.makedirs(cwd, exist_ok=True)
if env is None:
env = Defaultdict(str, os.environ)
log = None
try:
if not self.options.verbose:
log = open(context.log_file, 'w')
print("run command '{}'".format(command), file=log)
print("current directory is '{}'".format(cwd), file=log)
print("env is :", file=log)
for k, v in env.items():
print(" {} : {!r}".format(k, v), file=log)
kwargs = dict()
if input:
kwargs['stdin'] = subprocess.PIPE
process = subprocess.Popen(command, shell=True, cwd=cwd, env=env, stdout=log or sys.stdout, stderr=subprocess.STDOUT, **kwargs)
if input:
process.communicate(input.encode())
retcode = process.wait()
if retcode:
raise subprocess.CalledProcessError(retcode, command)
finally:
if log:
log.close()
def detect_platform(self):
_platform = platform.system()
self.distname = _platform
@ -80,7 +45,7 @@ class PlatformNeutralEnv:
def download(self, what, where=None):
where = where or self.archive_dir
download_remote(what, where, not self.options.no_cert_check)
download_remote(what, where)
def _detect_ninja(self):
for n in ['ninja', 'ninja-build']:
@ -93,12 +58,6 @@ class PlatformNeutralEnv:
if retcode == 0:
return n
def add_toolchain(self, toolchain_name):
if toolchain_name not in self.toolchains:
ToolchainClass = Toolchain.all_toolchains[toolchain_name]
self.toolchains[toolchain_name] = ToolchainClass(self)
return self.toolchains[toolchain_name]
def _detect_meson(self):
for n in ['meson.py', 'meson']:
try:
@ -110,25 +69,22 @@ class PlatformNeutralEnv:
if retcode == 0:
return n
def __getattr__(self, name):
return getattr(self.options, name)
class BuildEnv:
def __init__(self, options, neutralEnv, targetsDict):
build_dir = "BUILD_{}".format(options.target_platform)
self.neutralEnv = neutralEnv
self.build_dir = pj(options.working_dir, build_dir)
def __init__(self, platformInfo):
build_dir = "BUILD_{}".format(platformInfo.name)
self.platformInfo = platformInfo
self.build_dir = pj(option('working_dir'), build_dir)
self.install_dir = pj(self.build_dir, "INSTALL")
self.toolchain_dir = pj(self.build_dir, "TOOLCHAINS")
self.log_dir = pj(self.build_dir, 'LOGS')
for d in (self.build_dir,
self.install_dir):
self.install_dir,
self.toolchain_dir,
self.log_dir):
os.makedirs(d, exist_ok=True)
self.setup_build(options.target_platform)
self.setup_toolchains()
self.options = options
self.libprefix = options.libprefix or self._detect_libdir()
self.targetsDict = targetsDict
self.libprefix = option('libprefix') or self._detect_libdir()
def clean_intermediate_directories(self):
for subdir in os.listdir(self.build_dir):
@ -140,70 +96,6 @@ class BuildEnv:
else:
os.remove(subpath)
def setup_build(self, target_platform):
self.platform_info = PlatformInfo.all_platforms[target_platform]
if self.distname not in self.platform_info.compatible_hosts:
print(('ERROR: The target {} cannot be build on host {}.\n'
'Select another target platform, or change your host system.'
).format(target_platform, self.distname))
sys.exit(-1)
self.cross_config = self.platform_info.get_cross_config()
def setup_toolchains(self):
toolchain_names = self.platform_info.toolchains
self.toolchains = []
for toolchain_name in toolchain_names:
ToolchainClass = Toolchain.all_toolchains[toolchain_name]
if ToolchainClass.neutral:
self.toolchains.append(
self.neutralEnv.add_toolchain(toolchain_name)
)
else:
self.toolchains.append(ToolchainClass(self))
def finalize_setup(self):
getattr(self, 'setup_{}'.format(self.platform_info.build))()
def setup_native(self):
self.cmake_crossfile = None
self.meson_crossfile = None
def _gen_crossfile(self, name):
crossfile = pj(self.build_dir, name)
template_file = pj(SCRIPT_DIR, 'templates', name)
with open(template_file, 'r') as f:
template = f.read()
content = template.format(
toolchain=self.toolchains[0],
**self.cross_config
)
with open(crossfile, 'w') as outfile:
outfile.write(content)
return crossfile
def setup_win32(self):
self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def setup_android(self):
self.cmake_crossfile = self._gen_crossfile('cmake_android_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def setup_armhf(self):
self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def setup_iOS(self):
self.cmake_crossfile = self._gen_crossfile('cmake_ios_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def setup_i586(self):
self.cmake_crossfile = self._gen_crossfile('cmake_i586_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def __getattr__(self, name):
return getattr(self.neutralEnv, name)
def _is_debianlike(self):
return os.path.isfile('/etc/debian_version')
@ -223,16 +115,6 @@ class BuildEnv:
return 'lib64'
return 'lib'
@property
def configure_option(self):
configure_options = [tlc.configure_option for tlc in self.toolchains]
return " ".join(configure_options)
@property
def cmake_option(self):
cmake_options = [tlc.cmake_option for tlc in self.toolchains]
return " ".join(cmake_options)
def _set_env(self, env, cross_compile_env, cross_compile_compiler, cross_compile_path):
if env is None:
env = Defaultdict(str, os.environ)
@ -267,110 +149,10 @@ class BuildEnv:
v = v.format(**self.cross_config)
k = k[8:]
env[k] = v
for toolchain in self.toolchains:
toolchain.set_env(env)
self.platform_info.set_env(env)
self.platformInfo.set_env(env)
if cross_compile_compiler:
for toolchain in self.toolchains:
toolchain.set_compiler(env)
self.platformInfo.set_compiler(env)
if cross_compile_path:
bin_dirs = []
for tlc in self.toolchains:
bin_dirs += tlc.get_bin_dir()
bin_dirs += self.platform_info.get_bind_dir()
bin_dirs = self.platformInfo.get_bind_dir()
env['PATH'] = ':'.join(bin_dirs + [env['PATH']])
return env
def run_command(self, command, cwd, context, env=None, input=None, cross_env_only=False):
os.makedirs(cwd, exist_ok=True)
cross_compile_env = True
cross_compile_compiler = True
cross_compile_path = True
if context.force_native_build:
cross_compile_env = False
cross_compile_compiler = False
cross_compile_path = False
if cross_env_only:
cross_compile_compiler = False
env = self._set_env(env, cross_compile_env, cross_compile_compiler, cross_compile_path)
log = None
try:
if not self.options.verbose:
log = open(context.log_file, 'w')
print("run command '{}'".format(command), file=log)
print("current directory is '{}'".format(cwd), file=log)
print("env is :", file=log)
for k, v in env.items():
print(" {} : {!r}".format(k, v), file=log)
kwargs = dict()
if input:
kwargs['stdin'] = subprocess.PIPE
process = subprocess.Popen(command, shell=True, cwd=cwd, env=env, stdout=log or sys.stdout, stderr=subprocess.STDOUT, **kwargs)
if input:
process.communicate(input.encode())
retcode = process.wait()
if retcode:
raise subprocess.CalledProcessError(retcode, command)
finally:
if log:
log.close()
def install_packages(self):
autoskip_file = pj(self.build_dir, ".install_packages_ok")
if self.distname in ('fedora', 'redhat', 'centos'):
package_installer = 'sudo dnf install {}'
package_checker = 'rpm -q --quiet {}'
elif self.distname in ('debian', 'Ubuntu'):
package_installer = 'sudo apt-get install {}'
package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1'
elif self.distname == 'Darwin':
package_installer = 'brew install {}'
package_checker = 'brew list -1 | grep -q {}'
mapper_name = "{host}_{target}".format(
host=self.distname,
target=self.platform_info)
try:
package_name_mapper = PACKAGE_NAME_MAPPERS[mapper_name]
except KeyError:
print("SKIP : We don't know which packages we must install to compile"
" a {target} {build_type} version on a {host} host.".format(
target=self.platform_info,
host=self.distname))
return
packages_list = package_name_mapper.get('COMMON', [])
for dep in self.targetsDict.values():
packages = package_name_mapper.get(dep.name)
if packages:
packages_list += packages
dep.skip = True
for dep in self.targetsDict.values():
packages = getattr(dep, 'extra_packages', [])
for package in packages:
packages_list += package_name_mapper.get(package, [])
if not self.options.force_install_packages and os.path.exists(autoskip_file):
print("SKIP")
return
packages_to_install = []
for package in packages_list:
print(" - {} : ".format(package), end="")
command = package_checker.format(package)
try:
subprocess.check_call(command, shell=True)
except subprocess.CalledProcessError:
print("NEEDED")
packages_to_install.append(package)
else:
print("SKIP")
if packages_to_install:
command = package_installer.format(" ".join(packages_to_install))
print(command)
subprocess.check_call(command, shell=True)
else:
print("SKIP, No package to install.")
with open(autoskip_file, 'w'):
pass

211
kiwixbuild/builder.py
View File

@ -3,93 +3,194 @@ import sys
from collections import OrderedDict
from .buildenv import *
from .platforms import PlatformInfo
from .utils import remove_duplicates, StopBuild
from .dependencies import Dependency
from .packages import PACKAGE_NAME_MAPPERS
from ._global import (
neutralEnv, option,
add_target_step, get_target_step, target_steps)
from . import _global
class Builder:
def __init__(self, options):
self.options = options
self.targets = OrderedDict()
self.neutralEnv = PlatformNeutralEnv(options)
self.buildEnv = BuildEnv(options, self.neutralEnv, self.targets)
def __init__(self):
self._targets = {}
PlatformInfo.get_platform('neutral', self._targets)
_targets = {}
targetDef = options.targets
self.add_targets(targetDef, _targets)
dependencies = self.order_dependencies(_targets, targetDef)
dependencies = list(remove_duplicates(dependencies))
target_platform = option('target_platform')
if not target_platform:
if option('target') == 'kiwix-android':
target_platform = 'android'
else:
target_platform = 'native_dyn'
platform = PlatformInfo.get_platform(target_platform, self._targets)
if neutralEnv('distname') not in platform.compatible_hosts:
print(('ERROR: The target platform {} cannot be build on host {}.\n'
'Select another target platform or change your host system.'
).format(platform.name, neutralEnv('distname')))
self.targetDefs = platform.add_targets(option('target'), self._targets)
if options.build_nodeps:
self.targets[targetDef] = _targets[targetDef]
def finalize_target_steps(self):
steps = []
for targetDef in self.targetDefs:
steps += self.order_steps(targetDef)
steps = list(remove_duplicates(steps))
if option('build_nodeps'):
add_target_step(targetDef, self._targets[targetDef])
else:
for dep in dependencies:
if self.options.build_deps_only and dep == targetDef:
for dep in steps:
if option('build_deps_only') and dep == targetDef:
continue
self.targets[dep] = _targets[dep]
add_target_step(dep, self._targets[dep])
self.instanciate_steps()
def add_targets(self, targetName, targets):
if targetName in targets:
def order_steps(self, targetDef):
for pltName in PlatformInfo.all_running_platforms:
plt = PlatformInfo.all_platforms[pltName]
for tlcName in plt.toolchain_names:
tlc = Dependency.all_deps[tlcName]
yield('source', tlcName)
yield('neutral' if tlc.neutral else pltName, tlcName)
_targets =dict(self._targets)
yield from self.order_dependencies(targetDef, _targets)
def order_dependencies(self, targetDef, targets):
targetPlatformName, targetName = targetDef
if targetPlatformName == 'source':
# Do not try to order sources, they will be added as dep by the
# build step two lines later.
return
try:
target = targets.pop(targetDef)
except KeyError:
return
targetClass = Dependency.all_deps[targetName]
target = targetClass(self.neutralEnv, self.buildEnv)
targets[targetName] = target
for dep in target.dependencies:
self.add_targets(dep, targets)
def order_dependencies(self, _targets, targetName):
target = _targets[targetName]
for depName in target.dependencies:
yield from self.order_dependencies(_targets, depName)
yield targetName
targetPlatform = PlatformInfo.get_platform(targetPlatformName)
for dep in target.get_dependencies(targetPlatform, True):
try:
depPlatform, depName = dep
except ValueError:
depPlatform, depName = targetPlatformName, dep
if (depPlatform, depName) in targets:
yield from self.order_dependencies((depPlatform, depName), targets)
yield ('source', targetName)
yield targetDef
def instanciate_steps(self):
for stepDef in list(target_steps()):
stepPlatform, stepName = stepDef
stepClass = Dependency.all_deps[stepName]
if stepPlatform == 'source':
source = get_target_step(stepDef)(stepClass)
add_target_step(stepDef, source)
else:
source = get_target_step(stepName, 'source')
env = PlatformInfo.get_platform(stepPlatform).buildEnv
builder = get_target_step(stepDef)(stepClass, source, env)
add_target_step(stepDef, builder)
def prepare_sources(self):
if self.options.skip_source_prepare:
if option('skip_source_prepare'):
print("SKIP")
return
toolchain_sources = (tlc.source for tlc in self.buildEnv.toolchains if tlc.source)
for toolchain_source in toolchain_sources:
print("prepare sources for toolchain {} :".format(toolchain_source.name))
toolchain_source.prepare()
sources = (dep.source for dep in self.targets.values() if not dep.skip)
sources = remove_duplicates(sources, lambda s: s.__class__)
for source in sources:
print("prepare sources {} :".format(source.name))
sourceDefs = remove_duplicates(tDef for tDef in target_steps() if tDef[0]=='source')
for sourceDef in sourceDefs:
print("prepare sources {} :".format(sourceDef[1]))
source = get_target_step(sourceDef)
source.prepare()
def build(self):
toolchain_builders = (tlc.builder for tlc in self.buildEnv.toolchains if tlc.builder)
for toolchain_builder in toolchain_builders:
print("build toolchain {} :".format(toolchain_builder.name))
toolchain_builder.build()
builders = (dep.builder for dep in self.targets.values() if (dep.builder and not dep.skip))
for builder in builders:
if self.options.make_dist and builder.name == self.options.targets:
builderDefs = (tDef for tDef in target_steps() if tDef[0] != 'source')
for builderDef in builderDefs:
builder = get_target_step(builderDef)
if option('make_dist') and builderName == option('target'):
print("make dist {} ({}):".format(builder.name, builderDef[0]))
builder.make_dist()
continue
print("build {} :".format(builder.name))
print("build {} ({}):".format(builder.name, builderDef[0]))
add_target_step(builderDef, builder)
builder.build()
if self.options.make_dist:
dep = self.targets[self.options.targets]
builder = dep.builder
print("make dist {}:".format(builder.name))
builder.make_dist()
def _get_packages(self):
packages_list = []
for platform in PlatformInfo.all_running_platforms.values():
mapper_name = "{host}_{target}".format(
host=neutralEnv('distname'),
target=platform)
package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {})
packages_list += package_name_mapper.get('COMMON', [])
to_drop = []
for builderDef in self._targets:
platformName, builderName = builderDef
mapper_name = "{host}_{target}".format(
host=neutralEnv('distname'),
target=platformName)
package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {})
packages = package_name_mapper.get(builderName)
if packages:
packages_list += packages
to_drop.append(builderDef)
for dep in to_drop:
del self._targets[dep]
return packages_list
def install_packages(self):
packages_to_have = self._get_packages()
packages_to_have = remove_duplicates(packages_to_have)
distname = neutralEnv('distname')
if distname in ('fedora', 'redhat', 'centos'):
package_installer = 'sudo dnf install {}'
package_checker = 'rpm -q --quiet {}'
elif distname in ('debian', 'Ubuntu'):
package_installer = 'sudo apt-get install {}'
package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1'
elif distname == 'Darwin':
package_installer = 'brew install {}'
package_checker = 'brew list -1 | grep -q {}'
packages_to_install = []
for package in packages_to_have:
print(" - {} : ".format(package), end="")
command = package_checker.format(package)
try:
subprocess.check_call(command, shell=True)
except subprocess.CalledProcessError:
print("NEEDED")
packages_to_install.append(package)
else:
print("SKIP")
if packages_to_install:
command = package_installer.format(" ".join(packages_to_install))
print(command)
subprocess.check_call(command, shell=True)
else:
print("SKIP, No package to install.")
def run(self):
try:
print("[INSTALL PACKAGES]")
self.buildEnv.install_packages()
self.buildEnv.finalize_setup()
if option('dont_install_packages'):
print("SKIP")
else:
self.install_packages()
self.finalize_target_steps()
print("[SETUP PLATFORMS]")
for platform in PlatformInfo.all_running_platforms.values():
platform.finalize_setup()
print("[PREPARE]")
self.prepare_sources()
print("[BUILD]")
self.build()
# No error, clean intermediate file at end of build if needed.
print("[CLEAN]")
if self.buildEnv.options.clean_at_end:
self.buildEnv.clean_intermediate_directories()
if option('clean_at_end'):
for platform in PlatformInfo.all_running_platforms.values():
platform.clean_intermediate_directories()
else:
print("SKIP")
except StopBuild:

4
kiwixbuild/dependencies/__init__.py
View File

@ -2,10 +2,14 @@
from .base import *
from . import (
all_dependencies,
android_ndk,
android_sdk,
armhf,
ctpp2,
gradle,
gumbo,
icu4c,
ios_fat_lib,
kiwix_android,
kiwix_custom_app,
kiwix_lib,

34
kiwixbuild/dependencies/all_dependencies.py
View File

@ -3,26 +3,22 @@ from .base import (
NoopSource,
NoopBuilder)
from kiwixbuild._global import neutralEnv
class AllBaseDependencies(Dependency):
name = "alldependencies"
@property
def dependencies(self):
base_deps = ['zlib', 'lzma', 'xapian-core', 'gumbo', 'pugixml', 'libmicrohttpd', 'libaria2']
if self.buildEnv.platform_info.build != 'native':
base_deps += ["icu4c_cross-compile"]
if self.buildEnv.platform_info.build != 'win32':
base_deps += ["libmagic_cross-compile"]
else:
base_deps += ["icu4c", "libmagic"]
if ( self.buildEnv.platform_info.build != 'android'
and self.buildEnv.distname != 'Darwin'):
base_deps += ['ctpp2c', 'ctpp2']
if self.buildEnv.platform_info.build == 'android':
base_deps += ['Gradle']
return base_deps
Source = NoopSource
Builder = NoopBuilder
class Builder(NoopBuilder):
@classmethod
def get_dependencies(cls, platformInfo, allDeps):
base_deps = ['zlib', 'lzma', 'xapian-core', 'pugixml', 'libaria2', 'icu4c']
# zimwriterfs
if platformInfo.build not in ('android', 'win32'):
base_deps += ['libmagic', 'gumbo']
# kiwix-tools
if (platformInfo.build != 'android' and
neutralEnv('distname') != 'Darwin'):
base_deps += ['libmicrohttpd', 'ctpp2c', 'ctpp2']
return base_deps

77
kiwixbuild/dependencies/android_ndk.py Normal file
View File

@ -0,0 +1,77 @@
import os
from .base import Dependency, ReleaseDownload, Builder
from kiwixbuild.utils import Remotefile, add_execution_right, run_command
pj = os.path.join
class android_ndk(Dependency):
neutral = False
name = 'android-ndk'
gccver = '4.9.x'
class Source(ReleaseDownload):
archive = Remotefile('android-ndk-r13b-linux-x86_64.zip',
'3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c',
'https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip')
@property
def source_dir(self):
return self.target.full_name()
class Builder(Builder):
@property
def install_path(self):
return self.build_path
@property
def api(self):
return '21' if self.arch in ('arm64', 'mips64', 'x86_64') else '14'
@property
def platform(self):
return 'android-'+self.api
@property
def arch(self):
return self.buildEnv.platformInfo.arch
@property
def arch_full(self):
return self.buildEnv.platformInfo.arch_full
def _build_platform(self, context):
context.try_skip(self.build_path)
script = pj(self.source_path, 'build/tools/make_standalone_toolchain.py')
add_execution_right(script)
command = '{script} --arch={arch} --api={api} --install-dir={install_dir} --force'
command = command.format(
script=script,
arch=self.arch,
api=self.api,
install_dir=self.install_path
)
context.force_native_build = True
run_command(command, self.build_path, context, buildEnv=self.buildEnv)
def _fix_permission_right(self, context):
context.try_skip(self.build_path)
bin_dirs = [pj(self.install_path, 'bin'),
pj(self.install_path, self.arch_full, 'bin'),
pj(self.install_path, 'libexec', 'gcc', self.arch_full, self.target.gccver)
]
for root, dirs, files in os.walk(self.install_path):
if not root in bin_dirs:
continue
for file_ in files:
file_path = pj(root, file_)
if os.path.islink(file_path):
continue
add_execution_right(file_path)
def build(self):
self.command('build_platform', self._build_platform)
self.command('fix_permission_right', self._fix_permission_right)

16
kiwixbuild/toolchains/android_sdk.py → kiwixbuild/dependencies/android_sdk.py
View File

@ -1,15 +1,14 @@
import os
import shutil
from .base_toolchain import Toolchain
from kiwixbuild.dependencies import ReleaseDownload, Builder
from kiwixbuild.utils import Remotefile
from .base import Dependency, ReleaseDownload, Builder
from kiwixbuild.utils import Remotefile, run_command
pj = os.path.join
class android_sdk(Toolchain):
class android_sdk(Dependency):
neutral = True
name = 'android-sdk'
version = 'r25.2.3'
class Source(ReleaseDownload):
archive = Remotefile('tools_r25.2.3-linux.zip',
@ -20,7 +19,7 @@ class android_sdk(Toolchain):
@property
def install_path(self):
return pj(self.buildEnv.toolchain_dir, self.target.full_name)
return pj(self.buildEnv.toolchain_dir, self.target.full_name())
def _build_platform(self, context):
context.try_skip(self.install_path)
@ -38,7 +37,7 @@ class android_sdk(Toolchain):
# - 8 : Android SDK Build-tools, revision 24.0.1
# - 34 : SDK Platform Android 7.0, API 24, revision 2
# - 162 : Android Support Repository, revision 44
self.buildEnv.run_command(command, self.install_path, context, input="y\n")
run_command(command, self.install_path, context, input="y\n")
def _fix_licenses(self, context):
context.try_skip(self.install_path)
@ -49,6 +48,3 @@ class android_sdk(Toolchain):
def build(self):
self.command('build_platform', self._build_platform)
self.command('fix_licenses', self._fix_licenses)
def set_env(self, env):
env['ANDROID_HOME'] = self.builder.install_path

11
kiwixbuild/dependencies/armhf.py Normal file
View File

@ -0,0 +1,11 @@
from .base import Dependency, GitClone, NoopBuilder
class armhf_toolchain(Dependency):
neutral = True
name = 'armhf'
class Source(GitClone):
git_remote = "https://github.com/raspberrypi/tools"
git_dir = "raspberrypi-tools"
Builder = NoopBuilder

206
kiwixbuild/dependencies/base.py
View File

@ -2,8 +2,9 @@ import subprocess
import os
import shutil
from kiwixbuild.utils import pj, Context, SkipCommand, extract_archive, Defaultdict, StopBuild
from kiwixbuild.utils import pj, Context, SkipCommand, extract_archive, Defaultdict, StopBuild, run_command
from kiwixbuild.versions import main_project_versions, base_deps_versions
from kiwixbuild._global import neutralEnv, option
SCRIPT_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
@ -19,38 +20,53 @@ class _MetaDependency(type):
class Dependency(metaclass=_MetaDependency):
all_deps = {}
dependencies = []
force_native_build = False
def __init__(self, neutralEnv, buildEnv):
self.neutralEnv = neutralEnv
self.buildEnv = buildEnv
self.source = self.Source(self)
self.builder = self.Builder(self)
self.skip = False
@classmethod
def version(cls):
return base_deps_versions.get(cls.name, None)
@classmethod
def full_name(cls):
if cls.version():
return "{}-{}".format(cls.name, cls.version())
return cls.name
class Source:
"""Base Class to the real preparator
A source preparator must install source in the self.source_dir attribute
inside the neutralEnv.source_dir."""
def __init__(self, target):
self.target = target
@property
def version(self):
return base_deps_versions.get(self.name, None)
def name(self):
return self.target.name
@property
def full_name(self):
if self.version:
return "{}-{}".format(self.name, self.version)
return self.name
def source_dir(self):
return self.target.full_name()
@property
def source_path(self):
return pj(self.neutralEnv.source_dir, self.source.source_dir)
return pj(neutralEnv('source_dir'), self.source_dir)
@property
def _log_dir(self):
return self.buildEnv.log_dir
return neutralEnv('log_dir')
def _patch(self, context):
context.try_skip(self.source_path)
context.force_native_build = True
for p in self.patches:
with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input:
run_command("patch -p1", self.source_path, context, input=patch_input.read())
def command(self, name, function, *args):
print(" {} {} : ".format(name, self.name), end="", flush=True)
log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
context = Context(name, log, self.force_native_build)
context = Context(name, log, True)
try:
ret = function(*args, context=context)
context._finalise()
@ -71,34 +87,6 @@ class Dependency(metaclass=_MetaDependency):
raise
class Source:
"""Base Class to the real preparator
A source preparator must install source in the self.source_dir attribute
inside the neutralEnv.source_dir."""
def __init__(self, target):
self.target = target
self.neutralEnv = target.neutralEnv
@property
def name(self):
return self.target.name
@property
def source_dir(self):
return self.target.full_name
def _patch(self, context):
source_path = pj(self.neutralEnv.source_dir, self.source_dir)
context.try_skip(source_path)
context.force_native_build = True
for p in self.patches:
with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input:
self.neutralEnv.run_command("patch -p1", source_path, context, input=patch_input.read())
def command(self, *args, **kwargs):
return self.target.command(*args, **kwargs)
class NoopSource(Source):
def prepare(self):
pass
@ -109,18 +97,18 @@ class ReleaseDownload(Source):
@property
def extract_path(self):
return pj(self.neutralEnv.source_dir, self.source_dir)
return pj(neutralEnv('source_dir'), self.source_dir)
def _download(self, context):
context.try_skip(self.neutralEnv.archive_dir, self.name)
self.neutralEnv.download(self.archive)
context.try_skip(neutralEnv('archive_dir'), self.name)
neutralEnv('download')(self.archive)
def _extract(self, context):
context.try_skip(self.extract_path)
if os.path.exists(self.extract_path):
shutil.rmtree(self.extract_path)
extract_archive(pj(self.neutralEnv.archive_dir, self.archive.name),
self.neutralEnv.source_dir,
extract_archive(pj(neutralEnv('archive_dir'), self.archive.name),
neutralEnv('source_dir'),
topdir=self.archive_top_dir,
name=self.source_dir)
@ -142,18 +130,18 @@ class GitClone(Source):
@property
def source_dir(self):
if self.neutralEnv.make_release:
if option('make_release'):
return "{}_release".format(self.git_dir)
else:
return self.git_dir
@property
def git_path(self):
return pj(self.neutralEnv.source_dir, self.source_dir)
return pj(neutralEnv('source_dir'), self.source_dir)
@property
def git_ref(self):
if self.neutralEnv.make_release:
if option('make_release'):
return self.release_git_ref
else:
return self.base_git_ref
@ -163,13 +151,13 @@ class GitClone(Source):
raise SkipCommand()
command = "git clone --depth=1 --branch {} {} {}".format(
self.git_ref, self.git_remote, self.source_dir)
self.neutralEnv.run_command(command, self.neutralEnv.source_dir, context)
run_command(command, neutralEnv('source_dir'), context)
def _git_update(self, context):
command = "git fetch origin {}".format(
self.git_ref)
self.neutralEnv.run_command(command, self.git_path, context)
self.neutralEnv.run_command("git checkout "+self.git_ref, self.git_path, context)
run_command(command, self.git_path, context)
run_command("git checkout "+self.git_ref, self.git_path, context)
def prepare(self):
self.command('gitclone', self._git_clone)
@ -185,17 +173,17 @@ class SvnClone(Source):
@property
def svn_path(self):
return pj(self.neutralEnv.source_dir, self.svn_dir)
return pj(neutralEnv('source_dir'), self.svn_dir)
def _svn_checkout(self, context):
if os.path.exists(self.svn_path):
raise SkipCommand()
command = "svn checkout {} {}".format(self.svn_remote, self.svn_dir)
self.neutralEnv.run_command(command, self.neutralEnv.source_dir, context)
run_command(command, neutralEnv('source_dir'), context)
def _svn_update(self, context):
context.try_skip(self.svn_path)
self.neutralEnv.run_command("svn update", self.svn_path, context)
run_command("svn update", self.svn_path, context)
def prepare(self):
self.command('svncheckout', self._svn_checkout)
@ -206,10 +194,16 @@ class SvnClone(Source):
class Builder:
subsource_dir = None
dependencies = []
def __init__(self, target):
def __init__(self, target, source, buildEnv):
self.target = target
self.buildEnv = target.buildEnv
self.source = source
self.buildEnv = buildEnv
@classmethod
def get_dependencies(cls, platformInfo, allDeps):
return cls.dependencies
@property
def name(self):
@ -217,17 +211,41 @@ class Builder:
@property
def source_path(self):
base_source_path = self.target.source_path
base_source_path = self.source.source_path
if self.subsource_dir:
return pj(base_source_path, self.subsource_dir)
return base_source_path
@property
def build_path(self):
return pj(self.buildEnv.build_dir, self.target.full_name)
return pj(self.buildEnv.build_dir, self.target.full_name())
def command(self, *args, **kwargs):
return self.target.command(*args, **kwargs)
@property
def _log_dir(self):
return self.buildEnv.log_dir
def command(self, name, function, *args):
print(" {} {} : ".format(name, self.name), end="", flush=True)
log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
context = Context(name, log, self.target.force_native_build)
try:
ret = function(*args, context=context)
context._finalise()
print("OK")
return ret
except SkipCommand:
print("SKIP")
except subprocess.CalledProcessError:
print("ERROR")
try:
with open(log, 'r') as f:
print(f.read())
except:
pass
raise StopBuild()
except:
print("ERROR")
raise
def build(self):
if hasattr(self, '_pre_build_script'):
@ -271,8 +289,8 @@ class MakeBuilder(Builder):
def all_configure_option(self):
option = self.configure_option_template.format(
dep_options=self.configure_option,
static_option=self.static_configure_option if self.buildEnv.platform_info.static else self.dynamic_configure_option,
env_option=self.buildEnv.configure_option if not self.target.force_native_build else "",
static_option=self.static_configure_option if self.buildEnv.platformInfo.static else self.dynamic_configure_option,
env_option=self.buildEnv.platformInfo.configure_option if not self.target.force_native_build else "",
install_dir=self.buildEnv.install_dir,
libdir=pj(self.buildEnv.install_dir, self.buildEnv.libprefix)
)
@ -286,7 +304,7 @@ class MakeBuilder(Builder):
configure_option=self.all_configure_option
)
env = Defaultdict(str, os.environ)
if self.buildEnv.platform_info.static:
if self.buildEnv.platformInfo.static:
env['CFLAGS'] = env['CFLAGS'] + ' -fPIC'
if self.configure_env:
for k in self.configure_env:
@ -295,7 +313,7 @@ class MakeBuilder(Builder):
v = v.format(buildEnv=self.buildEnv, env=env)
self.configure_env[k[8:]] = v
env.update(self.configure_env)
self.buildEnv.run_command(command, self.build_path, context, env=env)
run_command(command, self.build_path, context, buildEnv=self.buildEnv, env=env)
def _compile(self, context):
context.try_skip(self.build_path)
@ -303,7 +321,7 @@ class MakeBuilder(Builder):
make_target=self.make_target,
make_option=self.make_option
)
self.buildEnv.run_command(command, self.build_path, context)
run_command(command, self.build_path, context, buildEnv=self.buildEnv)
def _install(self, context):
context.try_skip(self.build_path)
@ -311,12 +329,12 @@ class MakeBuilder(Builder):
make_install_target=self.make_install_target,
make_option=self.make_option
)
self.buildEnv.run_command(command, self.build_path, context)
run_command(command, self.build_path, context, buildEnv=self.buildEnv)
def _make_dist(self, context):
context.try_skip(self.build_path)
command = "make dist"
self.buildEnv.run_command(command, self.build_path, context)
run_command(command, self.build_path, context, buildEnv=self.buildEnv)
class CMakeBuilder(MakeBuilder):
@ -332,14 +350,14 @@ class CMakeBuilder(MakeBuilder):
" {source_path}"
" {cross_option}")
command = command.format(
configure_option="{} {}".format(self.buildEnv.cmake_option, self.configure_option),
configure_option=self.configure_option,
install_dir=self.buildEnv.install_dir,
libdir=self.buildEnv.libprefix,
source_path=self.source_path,
cross_option=cross_option
)
env = Defaultdict(str, os.environ)
if self.buildEnv.platform_info.static:
if self.buildEnv.platformInfo.static:
env['CFLAGS'] = env['CFLAGS'] + ' -fPIC'
if self.configure_env:
for k in self.configure_env:
@ -348,22 +366,16 @@ class CMakeBuilder(MakeBuilder):
v = v.format(buildEnv=self.buildEnv, env=env)
self.configure_env[k[8:]] = v
env.update(self.configure_env)
self.buildEnv.run_command(command, self.build_path, context, env=env, cross_env_only=True)
run_command(command, self.build_path, context, env=env, buildEnv=self.buildEnv, cross_env_only=True)
class MesonBuilder(Builder):
configure_option = ""
test_option = ""
def __init__(self, target):
super().__init__(target)
self.meson_command = self.buildEnv.neutralEnv.meson_command
self.mesontest_command = self.buildEnv.neutralEnv.mesontest_command
self.ninja_command = self.buildEnv.neutralEnv.ninja_command
@property
def library_type(self):
return 'static' if self.buildEnv.platform_info.static else 'shared'
return 'static' if self.buildEnv.platformInfo.static else 'shared'
def _configure(self, context):
context.try_skip(self.build_path)
@ -382,40 +394,40 @@ class MesonBuilder(Builder):
" --libdir={buildEnv.libprefix}"
" {cross_option}")
command = command.format(
command=self.meson_command,
command=neutralEnv('meson_command'),
library_type=self.library_type,
configure_option=configure_option,
build_path=self.build_path,
buildEnv=self.buildEnv,
cross_option=cross_option
)
self.buildEnv.run_command(command, self.source_path, context, cross_env_only=True)
run_command(command, self.source_path, context, buildEnv=self.buildEnv, cross_env_only=True)
def _compile(self, context):
command = "{} -v".format(self.ninja_command)
self.buildEnv.run_command(command, self.build_path, context)
command = "{} -v".format(neutralEnv('ninja_command'))
run_command(command, self.build_path, context, buildEnv=self.buildEnv)
def _test(self, context):
if ( self.buildEnv.platform_info.build == 'android'
or (self.buildEnv.platform_info.build != 'native'
and not self.buildEnv.platform_info.static)
if ( self.buildEnv.platformInfo.build == 'android'
or (self.buildEnv.platformInfo.build != 'native'
and not self.buildEnv.platformInfo.static)
):
raise SkipCommand()
command = "{} --verbose {}".format(self.mesontest_command, self.test_option)
self.buildEnv.run_command(command, self.build_path, context)
command = "{} --verbose {}".format(neutralEnv('mesontest_command'), self.test_option)
run_command(command, self.build_path, context, buildEnv=self.buildEnv)
def _install(self, context):
command = "{} -v install".format(self.ninja_command)
self.buildEnv.run_command(command, self.build_path, context)
command = "{} -v install".format(neutralEnv('ninja_command'))
run_command(command, self.build_path, context, buildEnv=self.buildEnv)
def _make_dist(self, context):
command = "{} -v dist".format(self.ninja_command)
self.buildEnv.run_command(command, self.build_path, context)
command = "{} -v dist".format(neutralEnv('ninja_command'))
run_command(command, self.build_path, context, buildEnv=self.buildEnv)
class GradleBuilder(Builder):
gradle_target = "build"
gradle_option = "-i"
gradle_option = "-i --no-daemon --build-cache"
def build(self):
self.command('configure', self._configure)
@ -435,4 +447,4 @@ class GradleBuilder(Builder):
command = command.format(
gradle_target=self.gradle_target,
gradle_option=self.gradle_option)
self.buildEnv.run_command(command, self.build_path, context)
run_command(command, self.build_path, context, buildEnv=self.buildEnv)

4
kiwixbuild/dependencies/ctpp2.py
View File

@ -3,7 +3,7 @@ from .base import (
ReleaseDownload,
CMakeBuilder)
from kiwixbuild.utils import Remotefile, pj
from kiwixbuild.utils import Remotefile, pj, run_command
class CTPP2(Dependency):
name = "ctpp2"
@ -51,4 +51,4 @@ class CTPP2C(CTPP2):
ctpp2c=pj(self.build_path, 'ctpp2c'),
install_dir=pj(self.buildEnv.install_dir, 'bin')
)
self.buildEnv.run_command(command, self.build_path, context)
run_command(command, self.build_path, context, buildEnv=self.buildEnv)

11
kiwixbuild/dependencies/gradle.py
View File

@ -6,7 +6,8 @@ from .base import (
from kiwixbuild.utils import Remotefile, pj, copy_tree, add_execution_right
class Gradle(Dependency):
name = "Gradle"
neutral = True
name = "gradle"
class Source(ReleaseDownload):
archive = Remotefile('gradle-4.6-bin.zip',
@ -14,14 +15,18 @@ class Gradle(Dependency):
'https://services.gradle.org/distributions/gradle-4.6-bin.zip')
class Builder(BaseBuilder):
@property
def install_path(self):
return self.buildEnv.install_dir
def build(self):
self.command('install', self._install)
def _install(self, context):
copy_tree(
pj(self.source_path, "bin"),
pj(self.buildEnv.install_dir, "bin"),
pj(self.install_path, "bin"),
post_copy_function = add_execution_right)
copy_tree(
pj(self.source_path, "lib"),
pj(self.buildEnv.install_dir, "lib"))
pj(self.install_path, "lib"))

4
kiwixbuild/dependencies/gumbo.py
View File

@ -4,7 +4,7 @@ from .base import (
MakeBuilder
)
from kiwixbuild.utils import Remotefile
from kiwixbuild.utils import Remotefile, run_command
class Gumbo(Dependency):
@ -18,6 +18,6 @@ class Gumbo(Dependency):
def _post_prepare_script(self, context):
context.try_skip(self.extract_path)
command = "./autogen.sh"
self.buildEnv.run_command(command, self.extract_path, context)
run_command(command, self.extract_path, context)
Builder = MakeBuilder

40
kiwixbuild/dependencies/icu4c.py
View File

@ -5,7 +5,7 @@ from .base import (
)
from kiwixbuild.utils import SkipCommand
from kiwixbuild._global import get_target_step
class Icu(Dependency):
name = "icu4c"
@ -25,33 +25,21 @@ class Icu(Dependency):
class Builder(MakeBuilder):
subsource_dir = "source"
@classmethod
def get_dependencies(cls, platformInfo, allDeps):
plt = 'native_static' if platformInfo.static else 'native_dyn'
return [(plt, 'icu4c')]
@property
def configure_option(self):
options = "--disable-samples --disable-tests --disable-extras --disable-dyload --enable-rpath"
if self.buildEnv.platform_info.build == 'android':
platformInfo = self.buildEnv.platformInfo
if platformInfo.build != 'native':
icu_native_builder = get_target_step(
'icu4c',
'native_static' if platformInfo.static else 'native_dyn')
options += " --with-cross-build={} --disable-tools".format(
icu_native_builder.build_path)
if platformInfo.build == 'android':
options += " --with-data-packaging=archive"
return options
class Icu_native(Icu):
name = "icu4c_native"
force_native_build = True
class Builder(Icu.Builder):
@property
def build_path(self):
return super().build_path+"_native"
def _install(self, context):
raise SkipCommand()
class Icu_cross_compile(Icu):
name = "icu4c_cross-compile"
dependencies = ['icu4c_native']
class Builder(Icu.Builder):
@property
def configure_option(self):
icu_native_dep = self.buildEnv.targetsDict['icu4c_native']
return super().configure_option + " --with-cross-build={} --disable-tools".format(icu_native_dep.builder.build_path)

52
kiwixbuild/dependencies/ios_fat_lib.py Normal file
View File

@ -0,0 +1,52 @@
import os
from kiwixbuild.platforms import PlatformInfo
from kiwixbuild.utils import pj, copy_tree, run_command
from kiwixbuild._global import option
from .base import (
Dependency,
NoopSource,
Builder as BaseBuilder)
class IOSFatLib(Dependency):
name = "_ios_fat_lib"
Source = NoopSource
class Builder(BaseBuilder):
@classmethod
def get_dependencies(self, platfomInfo, alldeps):
base_target = option('target')
return [('iOS_{}'.format(arch), base_target) for arch in option('ios_arch')]
def _copy_headers(self, context):
plt = PlatformInfo.get_platform('iOS_{}'.format(option('ios_arch')[0]))
include_src = pj(plt.buildEnv.install_dir, 'include')
include_dst = pj(self.buildEnv.install_dir, 'include')
copy_tree(include_src, include_dst)
def _merge_libs(self, context):
lib_dirs = []
for arch in option('ios_arch'):
plt = PlatformInfo.get_platform('iOS_{}'.format(arch))
lib_dirs.append(pj(plt.buildEnv.install_dir, 'lib'))
libs = []
for f in os.listdir(lib_dirs[0]):
if os.path.islink(pj(lib_dirs[0], f)):
continue
if f.endswith('.a') or f.endswith('.dylib'):
libs.append(f)
os.makedirs(pj(self.buildEnv.install_dir, 'lib'), exist_ok=True)
command_tmp = "lipo -create {input} -output {output}"
for l in libs:
command = command_tmp.format(
input=" ".join(pj(d, l) for d in lib_dirs),
output=pj(self.buildEnv.install_dir, 'lib', l))
run_command(command, self.buildEnv.install_dir, context)
def build(self):
self.command('copy_headers', self._copy_headers)
self.command('merge_libs', self._merge_libs)

42
kiwixbuild/dependencies/kiwix_android.py
View File

@ -5,19 +5,30 @@ from .base import (
GitClone,
GradleBuilder)
from kiwixbuild.utils import pj
from kiwixbuild.utils import pj, copy_tree
from kiwixbuild._global import option, get_target_step
class KiwixAndroid(Dependency):
name = "kiwix-android"
dependencies = ["Gradle", "kiwix-lib"]
class Source(GitClone):
git_remote = "https://github.com/kiwix/kiwix-android"
git_dir = "kiwix-android"
class Builder(GradleBuilder):
dependencies = ["kiwix-lib"]
@classmethod
def get_dependencies(cls, platformInfo, allDeps):
if not allDeps:
return super().get_dependencies(platformInfo, allDeps)
else:
deps = [('android_{}'.format(arch), 'kiwix-lib')
for arch in option('android_arch')]
return deps
def build(self):
if self.buildEnv.options.targets == 'kiwix-android-custom':
if option('target') == 'kiwix-android-custom':
print("SKIP")
else:
super().build()
@ -29,12 +40,25 @@ class KiwixAndroid(Dependency):
shutil.rmtree(pj(self.build_path, 'kiwixlib', 'src', 'main'))
except FileNotFoundError:
pass
shutil.copytree(pj(self.buildEnv.install_dir, 'kiwix-lib'),
pj(self.build_path, 'kiwixlib', 'src', 'main'))
for arch in option('android_arch'):
try:
kiwix_builder = get_target_step('kiwix-lib', 'android_{}'.format(arch))
except KeyError:
pass
else:
copy_tree(pj(kiwix_builder.buildEnv.install_dir, 'kiwix-lib'),
pj(self.build_path, 'kiwixlib', 'src', 'main'))
os.makedirs(
pj(self.build_path, 'app', 'src', 'main', 'assets', 'icu'),
exist_ok=True)
shutil.copy2(pj(self.buildEnv.install_dir, 'share', 'icu', '58.2',
'icudt58l.dat'),
pj(self.build_path, 'app', 'src', 'main', 'assets',
'icu', 'icudt58l.dat'))
for arch in option('android_arch'):
try:
kiwix_builder = get_target_step('kiwix-lib', 'android_{}'.format(arch))
except KeyError:
pass
else:
shutil.copy2(pj(kiwix_builder.buildEnv.install_dir, 'share', 'icu', '58.2',
'icudt58l.dat'),
pj(self.build_path, 'app', 'src', 'main', 'assets',
'icu', 'icudt58l.dat'))
break

14
kiwixbuild/dependencies/kiwix_custom_app.py
View File

@ -7,20 +7,22 @@ from .base import (
GradleBuilder)
from kiwixbuild.utils import Remotefile, pj, SkipCommand
from kiwixbuild._global import option, get_target_step
class KiwixCustomApp(Dependency):
name = "kiwix-android-custom"
dependencies = ["kiwix-android", "kiwix-lib"]
def __init__(self, buildEnv):
super().__init__(buildEnv)
self.custom_name = buildEnv.options.android_custom_app
self.custom_name = option('android_custom_app')
class Source(GitClone):
git_remote = "https://github.com/kiwix/kiwix-android-custom"
git_dir = "kiwix-android-custom"
class Builder(GradleBuilder):
dependencies = ["kiwix-android", "kiwix-lib"]
@property
def gradle_target(self):
return "assemble{}".format(self.target.custom_name)
@ -49,7 +51,7 @@ class KiwixCustomApp(Dependency):
def _get_zim_size(self):
try:
zim_size = self.buildEnv.options.zim_file_size
zim_size = option('zim_file_size')
except AttributeError:
with open(pj(self.source_path, self.target.custom_name, 'info.json')) as f:
app_info = json.load(f)
@ -62,7 +64,7 @@ class KiwixCustomApp(Dependency):
self.command('compile', self._compile)
def _download_zim(self, context):
zim_url = self.buildEnv.options.zim_file_url
zim_url = option('zim_file_url')
if zim_url is None:
raise SkipCommand()
with open(pj(self.source_path, self.target.custom_name, 'info.json')) as f:
@ -77,9 +79,9 @@ class KiwixCustomApp(Dependency):
def _configure(self, context):
# Copy kiwix-android in build dir.
kiwix_android_dep = self.buildEnv.targetsDict['kiwix-android']
kiwix_android_source = get_target_step('kiwix-android', 'source')
if not os.path.exists(self.build_path):
shutil.copytree(kiwix_android_dep.source_path, self.build_path)
shutil.copytree(kiwix_android_source.source_path, self.build_path)
# Copy kiwix-lib application in build dir
try:

25
kiwixbuild/dependencies/kiwix_lib.py
View File

@ -2,35 +2,34 @@ from .base import (
Dependency,
GitClone,
MesonBuilder)
from kiwixbuild._global import neutralEnv
class Kiwixlib(Dependency):
name = "kiwix-lib"
@property
def dependencies(self):
base_dependencies = ["pugixml", "libzim", "zlib", "lzma", "libaria2"]
if ( self.buildEnv.platform_info.build != 'android'
and self.buildEnv.distname != 'Darwin'):
base_dependencies += ['ctpp2c', 'ctpp2']
if self.buildEnv.platform_info.build != 'native':
return base_dependencies + ["icu4c_cross-compile"]
else:
return base_dependencies + ["icu4c"]
class Source(GitClone):
git_remote = "https://github.com/kiwix/kiwix-lib.git"
git_dir = "kiwix-lib"
class Builder(MesonBuilder):
@classmethod
def get_dependencies(cls, platformInfo, allDeps):
base_dependencies = ["pugixml", "libzim", "zlib", "lzma", "libaria2", "icu4c"]
if (platformInfo.build != 'android' and
neutralEnv('distname') != 'Darwin'):
base_dependencies += ['ctpp2c', 'ctpp2']
return base_dependencies
@property
def configure_option(self):
base_option = "-Dctpp2-install-prefix={buildEnv.install_dir}"
if self.buildEnv.platform_info.build == 'android':
if self.buildEnv.platformInfo.build == 'android':
base_option += ' -Dandroid=true'
return base_option
@property
def library_type(self):
if self.buildEnv.platform_info.build == 'android':
if self.buildEnv.platformInfo.build == 'android':
return 'shared'
return super().library_type

5
kiwixbuild/dependencies/kiwix_tools.py
View File

@ -5,15 +5,16 @@ from .base import (
class KiwixTools(Dependency):
name = "kiwix-tools"
dependencies = ["kiwix-lib", "libmicrohttpd", "zlib"]
class Source(GitClone):
git_remote = "https://github.com/kiwix/kiwix-tools.git"
git_dir = "kiwix-tools"
class Builder(MesonBuilder):
dependencies = ["kiwix-lib", "libmicrohttpd", "zlib"]
@property
def configure_option(self):
if self.buildEnv.platform_info.static:
if self.buildEnv.platformInfo.static:
return "-Dstatic-linkage=true"
return ""

6
kiwixbuild/dependencies/libaria2.py
View File

@ -4,11 +4,10 @@ from .base import (
MakeBuilder
)
from kiwixbuild.utils import Remotefile
from kiwixbuild.utils import Remotefile, run_command
class Aria2(Dependency):
name = "libaria2"
dependencies = ['zlib']
class Source(ReleaseDownload):
archive = Remotefile('libaria2-1.33.1.tar.gz',
@ -20,7 +19,8 @@ class Aria2(Dependency):
def _post_prepare_script(self, context):
context.try_skip(self.extract_path)
command = "autoreconf -i"
self.neutralEnv.run_command(command, self.extract_path, context)
run_command(command, self.extract_path, context)
class Builder(MakeBuilder):
dependencies = ['zlib']
configure_option = "--enable-libaria2 --disable-ssl --disable-bittorent --disable-metalink --without-sqlite3 --without-libxml2 --without-libexpat"

41
kiwixbuild/dependencies/libmagic.py
View File

@ -6,9 +6,10 @@ from .base import (
MakeBuilder,
)
from kiwixbuild.utils import Remotefile, pj, Defaultdict, SkipCommand
from kiwixbuild.utils import Remotefile, pj, Defaultdict, SkipCommand, run_command
from kiwixbuild._global import get_target_step
class LibMagicBase(Dependency):
class LibMagic(Dependency):
name = "libmagic"
class Source(ReleaseDownload):
@ -18,36 +19,24 @@ class LibMagicBase(Dependency):
'1c52c8c3d271cd898d5511c36a68059cda94036111ab293f01f83c3525b737c6',
'https://fossies.org/linux/misc/file-5.33.tar.gz')
Builder = MakeBuilder
class Builder(MakeBuilder):
@classmethod
def get_dependencies(cls, platformInfo, allDeps):
if platformInfo.build != 'native':
return [('native_static', 'libmagic')]
return []
class LibMagic_native(LibMagicBase):
name = "libmagic_native"
force_native_build = True
class Builder(LibMagicBase.Builder):
static_configure_option = dynamic_configure_option = "--disable-shared --enable-static"
@property
def build_path(self):
return super().build_path+"_native"
def _install(self, context):
raise SkipCommand()
class LibMagic_cross_compile(LibMagicBase):
name = "libmagic_cross-compile"
dependencies = ['libmagic_native']
class Builder(LibMagicBase.Builder):
def _compile(self, context):
platformInfo = self.buildEnv.platformInfo
if platformInfo.build == 'native':
return super()._compile(context)
context.try_skip(self.build_path)
command = "make -j4 {make_target} {make_option}".format(
make_target=self.make_target,
make_option=self.make_option
)
libmagic_native_dep = self.buildEnv.targetsDict['libmagic_native']
libmagic_native_builder = get_target_step('libmagic', 'native_static')
env = Defaultdict(str, os.environ)
env['PATH'] = ':'.join([pj(libmagic_native_dep.builder.build_path, 'src'), env['PATH']])
self.buildEnv.run_command(command, self.build_path, context, env=env)
env['PATH'] = ':'.join([pj(libmagic_native_builder.build_path, 'src'), env['PATH']])
run_command(command, self.build_path, context, buildEnv=self.buildEnv, env=env)

9
kiwixbuild/dependencies/libzim.py
View File

@ -6,17 +6,10 @@ from .base import (
class Libzim(Dependency):
name = "libzim"
@property
def dependencies(self):
base_dependencies = ['zlib', 'lzma', 'xapian-core']
if self.buildEnv.platform_info.build != 'native':
return base_dependencies + ["icu4c_cross-compile"]
else:
return base_dependencies + ["icu4c"]
class Source(GitClone):
git_remote = "https://github.com/openzim/libzim.git"
git_dir = "libzim"
class Builder(MesonBuilder):
test_option = "-t 8"
dependencies = ['zlib', 'lzma', 'xapian-core', 'icu4c']

15
kiwixbuild/dependencies/xapian.py
View File

@ -5,6 +5,7 @@ from .base import (
)
from kiwixbuild.utils import Remotefile
from kiwixbuild._global import neutralEnv
class Xapian(Dependency):
@ -19,10 +20,10 @@ class Xapian(Dependency):
configure_env = {'_format_LDFLAGS': "-L{buildEnv.install_dir}/{buildEnv.libprefix}",
'_format_CXXFLAGS': "-I{buildEnv.install_dir}/include"}
@property
def dependencies(self):
deps = ['zlib', 'lzma']
if (self.buildEnv.platform_info.build == 'win32'
or self.buildEnv.distname == 'Darwin'):
return deps
return deps + ['uuid']
@classmethod
def get_dependencies(cls, platformInfo, allDeps):
deps = ['zlib', 'lzma']
if (platformInfo.build == 'win32'
or neutralEnv('distname') == 'Darwin'):
return deps
return deps + ['uuid']

5
kiwixbuild/dependencies/zim_tools.py
View File

@ -5,15 +5,16 @@ from .base import (
class ZimTools(Dependency):
name = "zim-tools"
dependencies = ['libzim']
class Source(GitClone):
git_remote = "https://github.com/openzim/zim-tools.git"
git_dir = "zim-tools"
class Builder(MesonBuilder):
dependencies = ['libzim']
@property
def configure_option(self):
if self.buildEnv.platform_info.static:
if self.buildEnv.platformInfo.static:
return "-Dstatic-linkage=true"
return ""

12
kiwixbuild/dependencies/zimwriterfs.py
View File

@ -6,23 +6,17 @@ from .base import (
class Zimwriterfs(Dependency):
name = "zimwriterfs"
@property
def dependencies(self):
base_dependencies = ['libzim', 'zlib', 'xapian-core', 'gumbo']
if self.buildEnv.platform_info.build != 'native':
return base_dependencies + ["icu4c_cross-compile", "libmagic_cross-compile"]
else:
return base_dependencies + ["icu4c", "libmagic"]
class Source(GitClone):
git_remote = "https://github.com/openzim/zimwriterfs.git"
git_dir = "zimwriterfs"
release_git_ref = "1.1"
class Builder(MesonBuilder):
dependencies = ['libzim', 'zlib', 'xapian-core', 'gumbo', 'icu4c', 'libmagic']
@property
def configure_option(self):
base_option = "-Dmagic-install-prefix={buildEnv.install_dir}"
if self.buildEnv.platform_info.static:
if self.buildEnv.platformInfo.static:
base_option += " -Dstatic-linkage=true"
return base_option

6
kiwixbuild/dependencies/zlib.py
View File

@ -28,16 +28,16 @@ class zlib(Dependency):
def _configure(self, context):
if self.buildEnv.platform_info.build == 'win32':
if self.buildEnv.platformInfo.build == 'win32':
raise SkipCommand()
return super()._configure(context)
@property
def make_option(self):
if self.buildEnv.platform_info.build == 'win32':
if self.buildEnv.platformInfo.build == 'win32':
return "--makefile win32/Makefile.gcc PREFIX={host}- SHARED_MODE={static} INCLUDE_PATH={include_path} LIBRARY_PATH={library_path} BINARY_PATH={binary_path}".format(
host='i686-w64-mingw32',
static="0" if self.buildEnv.platform_info.static else "1",
static="0" if self.buildEnv.platformInfo.static else "1",
include_path=pj(self.buildEnv.install_dir, 'include'),
library_path=pj(self.buildEnv.install_dir, self.buildEnv.libprefix),
binary_path=pj(self.buildEnv.install_dir, 'bin'),

1
kiwixbuild/platforms/__init__.py
View File

@ -7,5 +7,6 @@ from . import (
i586,
ios,
native,
neutral,
win32
)

144
kiwixbuild/platforms/android.py
View File

@ -1,27 +1,41 @@
from .base import PlatformInfo
from .base import PlatformInfo, MetaPlatformInfo
from kiwixbuild.utils import pj
from kiwixbuild._global import get_target_step, option
class AndroidPlatformInfo(PlatformInfo):
__arch_infos = {
'arm' : ('arm-linux-androideabi', 'arm', 'armeabi'),
'arm64': ('aarch64-linux-android', 'aarch64', 'arm64-v8a'),
'mips': ('mipsel-linux-android', 'mipsel', 'mips'),
'mips64': ('mips64el-linux-android', 'mips64el', 'mips64'),
'x86': ('i686-linux-android', 'i686', 'x86'),
'x86_64': ('x86_64-linux-android', 'x86_64', 'x86_64'),
}
def __init__(self, name, arch):
super().__init__(name, 'android', True, ['android_ndk', 'android_sdk'],
hosts=['fedora', 'debian'])
self.arch = arch
self.arch_full, self.cpu, self.abi = self.__arch_infos[arch]
build = 'android'
static = True
toolchain_names = ['android-ndk']
compatible_hosts = ['fedora', 'debian']
def __str__(self):
return "android"
def binaries(self, install_path):
binaries = ((k,'{}-{}'.format(self.arch_full, v))
for k, v in (('CC', 'gcc'),
('CXX', 'g++'),
('AR', 'ar'),
('STRIP', 'strip'),
('WINDRES', 'windres'),
('RANLIB', 'ranlib'),
('LD', 'ld'))
)
return {k:pj(install_path, 'bin', v)
for k,v in binaries}
@property
def ndk_builder(self):
return get_target_step('android-ndk', self.name)
def get_cross_config(self):
install_path = self.ndk_builder.install_path
return {
'exec_wrapper_def': '',
'install_path': install_path,
'binaries': self.binaries(install_path),
'root_path': pj(install_path, 'sysroot'),
'extra_libs': [],
'extra_cflags': [],
'host_machine': {
@ -34,11 +48,99 @@ class AndroidPlatformInfo(PlatformInfo):
},
}
def get_bin_dir(self):
return [pj(self.ndk_builder.install_path, 'bin')]
def set_env(self, env):
root_path = pj(self.ndk_builder.install_path, 'sysroot')
env['PKG_CONFIG_LIBDIR'] = pj(root_path, 'lib', 'pkgconfig')
env['CFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(root_path) + env['CFLAGS']
env['CXXFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(root_path) + env['CXXFLAGS']
env['LDFLAGS'] = '--sysroot={} '.format(root_path) + env['LDFLAGS']
#env['CFLAGS'] = ' -fPIC -D_FILE_OFFSET_BITS=64 -O3 '+env['CFLAGS']
#env['CXXFLAGS'] = (' -D__OPTIMIZE__ -fno-strict-aliasing '
# ' -DU_HAVE_NL_LANGINFO_CODESET=0 '
# '-DU_STATIC_IMPLEMENTATION -O3 '
# '-DU_HAVE_STD_STRING -DU_TIMEZONE=0 ')+env['CXXFLAGS']
env['NDK_DEBUG'] = '0'
def set_compiler(self, env):
binaries = self.binaries(self.ndk_builder.install_path)
env['CC'] = binaries['CC']
env['CXX'] = binaries['CXX']
@property
def configure_option(self):
return '--host={}'.format(self.arch_full)
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_android_cross_file.txt')
self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
AndroidPlatformInfo('android_arm', 'arm')
AndroidPlatformInfo('android_arm64', 'arm64')
AndroidPlatformInfo('android_mips', 'mips')
AndroidPlatformInfo('android_mips64', 'mips64')
AndroidPlatformInfo('android_x86', 'x86')
AndroidPlatformInfo('android_x86_64', 'x86_64')
class AndroidArm(AndroidPlatformInfo):
name = 'android_arm'
arch = cpu = 'arm'
arch_full = 'arm-linux-androideabi'
abi = 'armeabi'
class AndroidArm(AndroidPlatformInfo):
name = 'android_arm64'
arch = 'arm64'
arch_full = 'aarch64-linux-android'
cpu = 'aarch64'
abi = 'arm64-v8a'
class AndroidArm(AndroidPlatformInfo):
name = 'android_mips'
arch = abi = 'mips'
arch_full = 'mipsel-linux-android'
cpu = 'mipsel'
class AndroidArm(AndroidPlatformInfo):
name = 'android_mips64'
arch = abi = 'mips64'
arch_full = 'mips64el-linux-android'
cpu = 'mips64el'
class AndroidArm(AndroidPlatformInfo):
name = 'android_x86'
arch = abi = 'x86'
arch_full = 'i686-linux-android'
cpu = 'i686'
class AndroidArm(AndroidPlatformInfo):
name = 'android_x86_64'
arch = cpu = abi = 'x86_64'
arch_full = 'x86_64-linux-android'
class Android(MetaPlatformInfo):
name = "android"
toolchain_names = ['android-sdk', 'gradle']
compatible_hosts = ['fedora', 'debian']
@property
def subPlatformNames(self):
return ['android_{}'.format(arch) for arch in option('android_arch')]
def add_targets(self, targetName, targets):
if targetName != 'kiwix-android':
return super().add_targets(targetName, targets)
else:
return AndroidPlatformInfo.add_targets(self, targetName, targets)
def __str__(self):
return self.name
@property
def sdk_builder(self):
return get_target_step('android-sdk', 'neutral')
@property
def gradle_builder(self):
return get_target_step('gradle', 'neutral')
def set_env(self, env):
env['ANDROID_HOME'] = self.sdk_builder.install_path
env['PATH'] = ':'.join([pj(self.gradle_builder.install_path, 'bin'), env['PATH']])

80
kiwixbuild/platforms/armhf.py
View File

@ -1,12 +1,20 @@
from .base import PlatformInfo
from kiwixbuild.utils import pj
from kiwixbuild._global import get_target_step
class ArmhfPlatformInfo(PlatformInfo):
def __init__(self, name, static):
super().__init__(name, 'armhf', static, ['armhf_toolchain'], ['fedora', 'debian'])
build = 'armhf'
arch_full = 'arm-linux-gnueabihf'
toolchain_names = ['armhf']
compatible_hosts = ['fedora', 'debian']
def get_cross_config(self):
return {
'binaries': self.binaries,
'exec_wrapper_def': '',
'root_path': self.root_path,
'extra_libs': [],
'extra_cflags': [],
'host_machine': {
@ -19,6 +27,70 @@ class ArmhfPlatformInfo(PlatformInfo):
}
}
@property
def tlc_source(self):
return get_target_step('armhf', 'source')
ArmhfPlatformInfo('armhf_dyn', False)
ArmhfPlatformInfo('armhf_static', True)
@property
def root_path(self):
return pj(self.tlc_source.source_path,
'arm-bcm2708',
'gcc-linaro-{}-raspbian-x64'.format(self.arch_full))
@property
def binaries(self):
binaries = ((k,'{}-{}'.format(self.arch_full, v))
for k, v in (('CC', 'gcc'),
('CXX', 'g++'),
('AR', 'ar'),
('STRIP', 'strip'),
('WINDRES', 'windres'),
('RANLIB', 'ranlib'),
('LD', 'ld'))
)
return {k:pj(self.root_path, 'bin', v)
for k,v in binaries}
@property
def exec_wrapper_def(self):
try:
which('qemu-arm')
except subprocess.CalledProcessError:
return ""
else:
return "exec_wrapper = 'qemu-arm'"
@property
def configure_option(self):
return '--host={}'.format(self.arch_full)
def get_bin_dir(self):
return [pj(self.root_path, 'bin')]
def set_env(self, env):
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
env['CFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS']
env['CXXFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
env['QEMU_LD_PREFIX'] = pj(self.root_path, "arm-linux-gnueabihf", "libc")
env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format(
':'.join([
pj(self.root_path, self.arch_full, "lib"),
env['LD_LIBRARY_PATH']
]))
def set_compiler(self, env):
env['CC'] = self.binaries['CC']
env['CXX'] = self.binaries['CXX']
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
class ArmhfDyn(ArmhfPlatformInfo):
name = 'armhf_dyn'
static = False
class ArmhfStatic(ArmhfPlatformInfo):
name = 'armhf_static'
static = True

106
kiwixbuild/platforms/base.py
View File

@ -1,21 +1,111 @@
import os, sys
import subprocess
from kiwixbuild.dependencies import Dependency
from kiwixbuild.utils import pj, remove_duplicates
from kiwixbuild.buildenv import BuildEnv
from kiwixbuild._global import neutralEnv, option, target_steps
_SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
TEMPLATES_DIR = pj(os.path.dirname(_SCRIPT_DIR), 'templates')
class _MetaPlatform(type):
def __new__(cls, name, bases, dct):
_class = type.__new__(cls, name, bases, dct)
if name not in ('PlatformInfo', 'MetaPlatformInfo') and 'name' in dct:
dep_name = dct['name']
PlatformInfo.all_platforms[dep_name] = _class
return _class
class PlatformInfo:
class PlatformInfo(metaclass=_MetaPlatform):
all_platforms = {}
all_running_platforms = {}
toolchain_names = []
configure_option = ""
def __init__(self, name, build, static, toolchains, hosts=None):
self.all_platforms[name] = self
self.build = build
self.static = static
self.toolchains = toolchains
self.compatible_hosts = hosts
@classmethod
def get_platform(cls, name, targets=None):
if name not in cls.all_running_platforms:
if targets is None:
print("Should not got there.")
print(cls.all_running_platforms)
raise KeyError(name)
cls.all_running_platforms[name] = cls.all_platforms[name](targets)
return cls.all_running_platforms[name]
def __init__(self, targets):
self.all_running_platforms[self.name] = self
self.buildEnv = BuildEnv(self)
self.setup_toolchains(targets)
def __str__(self):
return "{}_{}".format(self.build, 'static' if self.static else 'dyn')
def setup_toolchains(self, targets):
for tlc_name in self.toolchain_names:
ToolchainClass = Dependency.all_deps[tlc_name]
targets[('source', tlc_name)] = ToolchainClass.Source
plt_name = 'neutral' if ToolchainClass.neutral else self.name
targets[(plt_name, tlc_name)] = ToolchainClass.Builder
def add_targets(self, targetName, targets):
if (self.name, targetName) in targets:
return []
targetClass = Dependency.all_deps[targetName]
targets[('source', targetName)] = targetClass.Source
targets[(self.name, targetName)] = targetClass.Builder
for dep in targetClass.Builder.get_dependencies(self, False):
try:
depPlatformName, depName = dep
except ValueError:
depPlatformName, depName = self.name, dep
depPlatform = self.get_platform(depPlatformName, targets)
depPlatform.add_targets(depName, targets)
return [(self.name, targetName)]
def get_cross_config(self):
return {}
def set_env(self, env):
pass
def get_bind_dir(self):
return []
def set_compiler(self, env):
pass
def _gen_crossfile(self, name):
crossfile = pj(self.buildEnv.build_dir, name)
template_file = pj(TEMPLATES_DIR, name)
with open(template_file, 'r') as f:
template = f.read()
content = template.format(
**self.get_cross_config()
)
with open(crossfile, 'w') as outfile:
outfile.write(content)
return crossfile
def finalize_setup(self):
self.buildEnv.cross_config = self.get_cross_config()
self.buildEnv.meson_crossfile = None
self.buildEnv.cmake_crossfile = None
def clean_intermediate_directories(self):
self.buildEnv.clean_intermediate_directories()
class MetaPlatformInfo(PlatformInfo):
subPlatformNames = []
def add_targets(self, targetName, targets):
targetDefs = []
for platformName in self.subPlatformNames:
print("radd {}".format(platformName))
platform = self.get_platform(platformName, targets)
targetDefs += platform.add_targets(targetName, targets)
return targetDefs

46
kiwixbuild/platforms/i586.py
View File

@ -1,13 +1,18 @@
import os
from .base import PlatformInfo
from kiwixbuild.utils import which
class I586PlatformInfo(PlatformInfo):
def __init__(self, name, static):
super().__init__(name, 'i586', static, ['linux_i586_toolchain'], ['fedora', 'debian'])
build = 'i586'
arch_full = 'i586-linux-gnu'
compatible_hosts = ['fedora', 'debian']
def get_cross_config(self):
return {
'binaries': self.binaries,
'exec_wrapper_def': '',
'extra_libs': ['-m32', '-march=i586', '-mno-sse'],
'extra_cflags': ['-m32', '-march=i586', '-mno-sse'],
'host_machine': {
@ -20,6 +25,39 @@ class I586PlatformInfo(PlatformInfo):
}
}
@property
def configure_option(self):
return '--host={}'.format(self.arch_full)
I586PlatformInfo('i586_dyn', False)
I586PlatformInfo('i586_static', True)
@property
def binaries(self):
return {k:which(v)
for k, v in (('CC', os.environ.get('CC', 'gcc')),
('CXX', os.environ.get('CXX', 'g++')),
('AR', 'ar'),
('STRIP', 'strip'),
('RANLIB', 'ranlib'),
('LD', 'ld'))
}
def set_env(self, env):
env['CFLAGS'] = "-m32 -march=i586 -mno-sse "+env['CFLAGS']
env['CXXFLAGS'] = "-m32 -march=i586 -mno-sse "+env['CXXFLAGS']
env['LDFLAGS'] = "-m32 -march=i586 -mno-sse "+env['LDFLAGS']
def get_bin_dir(self):
return []
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_i586_cross_file.txt')
self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
class I586Dyn(I586PlatformInfo):
name = 'i586_dyn'
static = False
class I586Static(I586PlatformInfo):
name = 'i586_static'
static = True

90
kiwixbuild/platforms/ios.py
View File

@ -1,22 +1,17 @@
from .base import PlatformInfo
import subprocess
from .base import PlatformInfo, MetaPlatformInfo
from kiwixbuild.utils import pj, xrun_find
from kiwixbuild._global import option
class iOSPlatformInfo(PlatformInfo):
__arch_infos = {
'armv7': ('arm-apple-darwin', 'armv7', 'iphoneos'),
'arm64': ('arm-apple-darwin', 'arm64', 'iphoneos'),
'i386': ('', 'i386', 'iphonesimulator'),
'x86_64': ('', 'x86_64', 'iphonesimulator'),
}
build = 'iOS'
static = True
compatible_hosts = ['Darwin']
def __init__(self, name, arch):
super().__init__(name, 'iOS', True, ['iOS_sdk'],
hosts=['Darwin'])
self.arch = arch
self.arch_full, self.cpu, self.sdk_name = self.__arch_infos[arch]
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._root_path = None
@property
@ -29,9 +24,16 @@ class iOSPlatformInfo(PlatformInfo):
def __str__(self):
return "iOS"
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_ios_cross_file.txt')
self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def get_cross_config(self):
return {
'root_path': self.root_path,
'binaries': self.binaries,
'exec_wrapper_def': '',
'extra_libs': ['-fembed-bitcode', '-isysroot', self.root_path, '-arch', self.arch, '-miphoneos-version-min=9.0', '-stdlib=libc++'],
'extra_cflags': ['-fembed-bitcode', '-isysroot', self.root_path, '-arch', self.arch, '-miphoneos-version-min=9.0', '-stdlib=libc++'],
'host_machine': {
@ -53,7 +55,61 @@ class iOSPlatformInfo(PlatformInfo):
def get_bin_dir(self):
return [pj(self.root_path, 'bin')]
iOSPlatformInfo('iOS_armv7', 'armv7')
iOSPlatformInfo('iOS_arm64', 'arm64')
iOSPlatformInfo('iOS_i386', 'i386')
iOSPlatformInfo('iOS_x86_64', 'x86_64')
@property
def binaries(self):
return {
'CC': xrun_find('clang'),
'CXX': xrun_find('clang++'),
'AR': '/usr/bin/ar',
'STRIP': '/usr/bin/strip',
'RANLIB': '/usr/bin/ranlib',
'LD': '/usr/bin/ld',
}
@property
def configure_option(self):
return '--host={}'.format(self.arch_full)
def set_compiler(self, env):
env['CC'] = self.binaries['CC']
env['CXX'] = self.binaries['CXX']
class iOSArmv7(iOSPlatformInfo):
name = 'iOS_armv7'
arch = cpu = 'armv7'
arch_full = 'arm-apple-darwin'
sdk_name = 'iphoneos'
class iOSArm64(iOSPlatformInfo):
name = 'iOS_arm64'
arch = cpu = 'arm64'
arch_full = 'aarch64-apple-darwin'
sdk_name = 'iphoneos'
class iOSi386(iOSPlatformInfo):
name = 'iOS_i386'
arch = cpu = 'i386'
arch_full = 'i386-apple-darwin'
sdk_name = 'iphonesimulator'
class iOSx64(iOSPlatformInfo):
name = 'iOS_x86_64'
arch = cpu = 'x86_64'
arch_full = 'x86_64-apple-darwin'
sdk_name = 'iphonesimulator'
class IOS(MetaPlatformInfo):
name = "iOS_multi"
compatible_hosts = ['Darwin']
@property
def subPlatformNames(self):
return ['iOS_{}'.format(arch) for arch in option('ios_arch')]
def add_targets(self, targetName, targets):
super().add_targets(targetName, targets)
return PlatformInfo.add_targets(self, '_ios_fat_lib', targets)
def __str__(self):
return self.name

17
kiwixbuild/platforms/native.py
View File

@ -2,12 +2,15 @@ from .base import PlatformInfo
class NativePlatformInfo(PlatformInfo):
def __init__(self, name, static, hosts):
super().__init__(name, 'native', static, [], hosts)
def get_cross_config(self):
return {}
build = 'native'
NativePlatformInfo('native_dyn', False, ['fedora', 'debian', 'Darwin'])
NativePlatformInfo('native_static', True, ['fedora', 'debian'])
class NativeDyn(NativePlatformInfo):
name = 'native_dyn'
static = False
compatible_hosts = ['fedora', 'debian', 'Darwin']
class NativeStatic(NativePlatformInfo):
name = 'native_static'
static = True
compatible_hosts = ['fedora', 'debian']

9
kiwixbuild/platforms/neutral.py Normal file
View File

@ -0,0 +1,9 @@
from .base import PlatformInfo
class NeutralPlatformInfo(PlatformInfo):
name = 'neutral'
static = ''
compatible_hosts = ['fedora', 'debian', 'Darwin']
def __str__(self):
return "neutral"

69
kiwixbuild/platforms/win32.py
View File

@ -1,13 +1,21 @@
import subprocess
from .base import PlatformInfo
from kiwixbuild.utils import which, pj
from kiwixbuild._global import neutralEnv
class Win32PlatformInfo(PlatformInfo):
extra_libs = ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4', '-lmsvcr90', '-liphlpapi']
def __init__(self, name, static):
super().__init__(name, 'win32', static, ['mingw32_toolchain'], ['fedora', 'debian'])
build = 'win32'
compatible_hosts = ['fedora', 'debian']
arch_full = 'i686-w64-mingw32'
def get_cross_config(self):
return {
'exec_wrapper_def': self.exec_wrapper_def,
'binaries': self.binaries,
'root_path': self.root_path,
'extra_libs': self.extra_libs,
'extra_cflags': ['-DWIN32'],
'host_machine': {
@ -20,5 +28,58 @@ class Win32PlatformInfo(PlatformInfo):
}
}
Win32PlatformInfo('win32_dyn', False)
Win32PlatformInfo('win32_static', True)
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
@property
def root_path(self):
root_paths = {
'fedora': '/usr/i686-w64-mingw32/sys-root/mingw',
'debian': '/usr/i686-w64-mingw32'
}
return root_paths[neutralEnv('distname')]
@property
def binaries(self):
return {k:which('{}-{}'.format(self.arch_full, v))
for k, v in (('CC', 'gcc'),
('CXX', 'g++'),
('AR', 'ar'),
('STRIP', 'strip'),
('WINDRES', 'windres'),
('RANLIB', 'ranlib'))
}
@property
def exec_wrapper_def(self):
try:
which('wine')
except subprocess.CalledProcessError:
return ""
else:
return "exec_wrapper = 'wine'"
@property
def configure_option(self):
return '--host={}'.format(self.arch_full)
def set_compiler(self, env):
for k, v in self.binaries.items():
env[k] = v
def get_bin_dir(self):
return [pj(self.root_path, 'bin')]
def set_env(self, env):
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
env['LIBS'] = " ".join(self.extra_libs) + " " +env['LIBS']
class Win32Dyn(Win32PlatformInfo):
name = 'win32_dyn'
static = False
class Win32Static(Win32PlatformInfo):
name = 'win32_static'
static = True

9
kiwixbuild/templates/cmake_android_cross_file.txt
View File

@ -1,7 +1,8 @@
SET(CMAKE_SYSTEM_NAME {host_machine[system]})
SET(CMAKE_ANDROID_STANDALONE_TOOLCHAIN {toolchain.builder.install_path})
SET(CMAKE_C_COMPILER "{toolchain.binaries[CC]}")
SET(CMAKE_CXX_COMPILER "{toolchain.binaries[CXX]}")
SET(CMAKE_ANDROID_STANDALONE_TOOLCHAIN {install_path})
SET(CMAKE_FIND_ROOT_PATH {toolchain.root_path})
SET(CMAKE_C_COMPILER "{binaries[CC]}")
SET(CMAKE_CXX_COMPILER "{binaries[CXX]}")
SET(CMAKE_FIND_ROOT_PATH {root_path})

12
kiwixbuild/templates/cmake_cross_file.txt
View File

@ -2,11 +2,11 @@ SET(CMAKE_SYSTEM_NAME {host_machine[system]})
SET(CMAKE_SYSTEM_PROCESSOR {host_machine[cpu_family]})
# specify the cross compiler
SET(CMAKE_C_COMPILER "{toolchain.binaries[CC]}")
SET(CMAKE_CXX_COMPILER "{toolchain.binaries[CXX]}")
SET(CMAKE_RC_COMPILER {toolchain.binaries[WINDRES]})
SET(CMAKE_AR:FILEPATH {toolchain.binaries[AR]})
SET(CMAKE_RANLIB:FILEPATH {toolchain.binaries[RANLIB]})
SET(CMAKE_C_COMPILER "{binaries[CC]}")
SET(CMAKE_CXX_COMPILER "{binaries[CXX]}")
SET(CMAKE_RC_COMPILER {binaries[WINDRES]})
SET(CMAKE_AR:FILEPATH {binaries[AR]})
SET(CMAKE_RANLIB:FILEPATH {binaries[RANLIB]})
find_program(CCACHE_FOUND ccache)
if(CCACHE_FOUND)
@ -15,7 +15,7 @@ if(CCACHE_FOUND)
endif(CCACHE_FOUND)
# where is the target environment
SET(CMAKE_FIND_ROOT_PATH {toolchain.root_path})
SET(CMAKE_FIND_ROOT_PATH {root_path})
# search for programs in the build host directories
SET(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)

8
kiwixbuild/templates/cmake_i586_cross_file.txt
View File

@ -2,13 +2,13 @@ SET(CMAKE_SYSTEM_NAME {host_machine[system]})
SET(CMAKE_SYSTEM_PROCESSOR {host_machine[cpu_family]})
# specify the cross compiler
SET(CMAKE_C_COMPILER "{toolchain.binaries[CC]}")
SET(CMAKE_CXX_COMPILER "{toolchain.binaries[CXX]}")
SET(CMAKE_C_COMPILER "{binaries[CC]}")
SET(CMAKE_CXX_COMPILER "{binaries[CXX]}")
SET(C_FLAGS "-m32 -march=i586")
SET(CXX_FLAGS "-m32 -march=i586")
SET(CMAKE_LD_FLAGS "-m32 -march=i586")
SET(CMAKE_AR:FILEPATH {toolchain.binaries[AR]})
SET(CMAKE_RANLIB:FILEPATH {toolchain.binaries[RANLIB]})
SET(CMAKE_AR:FILEPATH {binaries[AR]})
SET(CMAKE_RANLIB:FILEPATH {binaries[RANLIB]})
find_program(CCACHE_FOUND ccache)
if(CCACHE_FOUND)

4
kiwixbuild/templates/cmake_ios_cross_file.txt
View File

@ -1,7 +1,7 @@
SET(CMAKE_SYSTEM_NAME {host_machine[system]})
SET(CMAKE_C_COMPILER "{toolchain.binaries[CC]}")
SET(CMAKE_CXX_COMPILER "{toolchain.binaries[CXX]}")
SET(CMAKE_C_COMPILER "{binaries[CC]}")
SET(CMAKE_CXX_COMPILER "{binaries[CXX]}")
SET(CMAKE_FIND_ROOT_PATH {root_path})

8
kiwixbuild/templates/meson_android_cross_file.txt
View File

@ -1,9 +1,9 @@
[binaries]
pkgconfig = 'pkg-config'
c = '{toolchain.binaries[CC]}'
ar = '{toolchain.binaries[AR]}'
cpp = '{toolchain.binaries[CXX]}'
strip = '{toolchain.binaries[STRIP]}'
c = '{binaries[CC]}'
ar = '{binaries[AR]}'
cpp = '{binaries[CXX]}'
strip = '{binaries[STRIP]}'
[properties]
c_link_args = {extra_libs!r}

10
kiwixbuild/templates/meson_cross_file.txt
View File

@ -1,10 +1,10 @@
[binaries]
pkgconfig = 'pkg-config'
c = '{toolchain.binaries[CC]}'
ar = '{toolchain.binaries[AR]}'
cpp = '{toolchain.binaries[CXX]}'
strip = '{toolchain.binaries[STRIP]}'
{toolchain.exec_wrapper_def}
c = '{binaries[CC]}'
ar = '{binaries[AR]}'
cpp = '{binaries[CXX]}'
strip = '{binaries[STRIP]}'
{exec_wrapper_def}
[properties]
c_link_args = {extra_libs!r}

4
kiwixbuild/toolchains/__init__.py
View File

@ -1,4 +0,0 @@
from . import android_ndk, android_sdk, armhf, ios, mingw32, linux_i586
from .base_toolchain import Toolchain

134
kiwixbuild/toolchains/android_ndk.py
View File

@ -1,134 +0,0 @@
import os
from .base_toolchain import Toolchain
from kiwixbuild.dependencies import ReleaseDownload, Builder
from kiwixbuild.utils import Remotefile, add_execution_right
pj = os.path.join
class android_ndk(Toolchain):
neutral = False
name = 'android-ndk'
version = 'r13b'
gccver = '4.9.x'
@property
def api(self):
return '21' if self.arch in ('arm64', 'mips64', 'x86_64') else '14'
@property
def platform(self):
return 'android-'+self.api
@property
def arch(self):
return self.buildEnv.platform_info.arch
@property
def arch_full(self):
return self.buildEnv.platform_info.arch_full
@property
def toolchain(self):
return self.arch_full+"-4.9"
@property
def root_path(self):
return pj(self.builder.install_path, 'sysroot')
@property
def binaries(self):
binaries = ((k,'{}-{}'.format(self.arch_full, v))
for k, v in (('CC', 'gcc'),
('CXX', 'g++'),
('AR', 'ar'),
('STRIP', 'strip'),
('WINDRES', 'windres'),
('RANLIB', 'ranlib'),
('LD', 'ld'))
)
return {k:pj(self.builder.install_path, 'bin', v)
for k,v in binaries}
@property
def configure_option(self):
return '--host={}'.format(self.arch_full)
@property
def full_name(self):
return "{name}-{version}-{arch}-{api}".format(
name = self.name,
version = self.version,
arch = self.arch,
api = self.api)
class Source(ReleaseDownload):
archive = Remotefile('android-ndk-r13b-linux-x86_64.zip',
'3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c',
'https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip')
@property
def source_dir(self):
return "{}-{}".format(
self.target.name,
self.target.version)
class Builder(Builder):
@property
def install_path(self):
return self.build_path
def _build_platform(self, context):
context.try_skip(self.build_path)
script = pj(self.source_path, 'build/tools/make_standalone_toolchain.py')
add_execution_right(script)
command = '{script} --arch={arch} --api={api} --install-dir={install_dir} --force'
command = command.format(
script=script,
arch=self.target.arch,
api=self.target.api,
install_dir=self.install_path
)
self.buildEnv.run_command(command, self.build_path, context)
def _fix_permission_right(self, context):
context.try_skip(self.build_path)
bin_dirs = [pj(self.install_path, 'bin'),
pj(self.install_path, self.target.arch_full, 'bin'),
pj(self.install_path, 'libexec', 'gcc', self.target.arch_full, self.target.gccver)
]
for root, dirs, files in os.walk(self.install_path):
if not root in bin_dirs:
continue
for file_ in files:
file_path = pj(root, file_)
if os.path.islink(file_path):
continue
add_execution_right(file_path)
def build(self):
self.command('build_platform', self._build_platform)
self.command('fix_permission_right', self._fix_permission_right)
def get_bin_dir(self):
return [pj(self.builder.install_path, 'bin')]
def set_env(self, env):
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
env['CFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(self.root_path) + env['CFLAGS']
env['CXXFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(self.root_path) + env['CXXFLAGS']
env['LDFLAGS'] = '--sysroot={} '.format(self.root_path) + env['LDFLAGS']
#env['CFLAGS'] = ' -fPIC -D_FILE_OFFSET_BITS=64 -O3 '+env['CFLAGS']
#env['CXXFLAGS'] = (' -D__OPTIMIZE__ -fno-strict-aliasing '
# ' -DU_HAVE_NL_LANGINFO_CODESET=0 '
# '-DU_STATIC_IMPLEMENTATION -O3 '
# '-DU_HAVE_STD_STRING -DU_TIMEZONE=0 ')+env['CXXFLAGS']
env['NDK_DEBUG'] = '0'
def set_compiler(self, env):
env['CC'] = self.binaries['CC']
env['CXX'] = self.binaries['CXX']

64
kiwixbuild/toolchains/armhf.py
View File

@ -1,64 +0,0 @@
import os
import subprocess
from .base_toolchain import Toolchain
from kiwixbuild.dependencies import GitClone
from kiwixbuild.utils import which
pj = os.path.join
class armhf_toolchain(Toolchain):
name = 'armhf'
arch_full = 'arm-linux-gnueabihf'
class Source(GitClone):
git_remote = "https://github.com/raspberrypi/tools"
git_dir = "raspberrypi-tools"
@property
def root_path(self):
return pj(self.source_path, 'arm-bcm2708', 'gcc-linaro-arm-linux-gnueabihf-raspbian-x64')
@property
def binaries(self):
binaries = ((k,'{}-{}'.format(self.arch_full, v))
for k, v in (('CC', 'gcc'),
('CXX', 'g++'),
('AR', 'ar'),
('STRIP', 'strip'),
('WINDRES', 'windres'),
('RANLIB', 'ranlib'),
('LD', 'ld'))
)
return {k:pj(self.root_path, 'bin', v)
for k,v in binaries}
@property
def exec_wrapper_def(self):
try:
which('qemu-arm')
except subprocess.CalledProcessError:
return ""
else:
return "exec_wrapper = 'qemu-arm'"
@property
def configure_option(self):
return '--host={}'.format(self.arch_full)
def get_bin_dir(self):
return [pj(self.root_path, 'bin')]
def set_env(self, env):
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
env['CFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS']
env['CXXFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
env['QEMU_LD_PREFIX'] = pj(self.root_path, "arm-linux-gnueabihf", "libc")
env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format(
':'.join([
pj(self.root_path, "arm-linux-gnueabihf", "lib"),
env['LD_LIBRARY_PATH']
]))
def set_compiler(self, env):
env['CC'] = self.binaries['CC']
env['CXX'] = self.binaries['CXX']

75
kiwixbuild/toolchains/base_toolchain.py
View File

@ -1,75 +0,0 @@
import os
import subprocess
pj = os.path.join
from kiwixbuild.utils import Context, SkipCommand, StopBuild
class _MetaToolchain(type):
def __new__(cls, name, bases, dct):
_class = type.__new__(cls, name, bases, dct)
if name != 'Toolchain':
Toolchain.all_toolchains[name] = _class
return _class
class Toolchain(metaclass=_MetaToolchain):
neutral = True
all_toolchains = {}
configure_option = ""
cmake_option = ""
exec_wrapper_def = ""
Builder = None
Source = None
def __init__(self, neutralEnv):
self.neutralEnv = neutralEnv
self.buildEnv = neutralEnv
self.source = self.Source(self) if self.Source else None
self.builder = self.Builder(self) if self.Builder else None
@property
def full_name(self):
return "{name}-{version}".format(
name = self.name,
version = self.version)
@property
def source_path(self):
return pj(self.neutralEnv.source_dir, self.source.source_dir)
@property
def _log_dir(self):
return self.neutralEnv.log_dir
def set_env(self, env):
pass
def set_compiler(self, env):
pass
def get_bin_dir(self):
return []
def command(self, name, function, *args):
print(" {} {} : ".format(name, self.name), end="", flush=True)
log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
context = Context(name, log, True)
try:
ret = function(*args, context=context)
context._finalise()
print("OK")
return ret
except SkipCommand:
print("SKIP")
except subprocess.CalledProcessError:
print("ERROR")
try:
with open(log, 'r') as f:
print(f.read())
except:
pass
raise StopBuild()
except:
print("ERROR")
raise

23
kiwixbuild/toolchains/ios.py
View File

@ -1,23 +0,0 @@
from .base_toolchain import Toolchain
from kiwixbuild.utils import pj, xrun_find
class iOS_sdk(Toolchain):
@property
def binaries(self):
return {
'CC': xrun_find('clang'),
'CXX': xrun_find('clang++'),
'AR': '/usr/bin/ar',
'STRIP': '/usr/bin/strip',
'RANLIB': '/usr/bin/ranlib',
'LD': '/usr/bin/ld',
}
@property
def configure_option(self):
return '--host=arm-apple-darwin'
def set_compiler(self, env):
env['CC'] = self.binaries['CC']
env['CXX'] = self.binaries['CXX']

37
kiwixbuild/toolchains/linux_i586.py
View File

@ -1,37 +0,0 @@
import os
from .base_toolchain import Toolchain
from kiwixbuild.dependencies import GitClone
from kiwixbuild.utils import which
pj = os.path.join
class linux_i586_toolchain(Toolchain):
name = 'linux_i586'
arch_full = 'i586-linux-gnu'
@property
def configure_option(self):
return '--host={}'.format(self.arch_full)
@property
def binaries(self):
return {k:which(v)
for k, v in (('CC', os.environ.get('CC', 'gcc')),
('CXX', os.environ.get('CXX', 'g++')),
('AR', 'ar'),
('STRIP', 'strip'),
('RANLIB', 'ranlib'),
('LD', 'ld'))
}
@property
def configure_option(self):
return '--host={}'.format(self.arch_full)
def set_env(self, env):
env['CFLAGS'] = "-m32 -march=i586 -mno-sse "+env['CFLAGS']
env['CXXFLAGS'] = "-m32 -march=i586 -mno-sse "+env['CXXFLAGS']
env['LDFLAGS'] = "-m32 -march=i586 -mno-sse "+env['LDFLAGS']
def get_bin_dir(self):
return []

55
kiwixbuild/toolchains/mingw32.py
View File

@ -1,55 +0,0 @@
import os
import subprocess
from .base_toolchain import Toolchain
from kiwixbuild.utils import which
pj = os.path.join
class mingw32_toolchain(Toolchain):
name = 'mingw32'
arch_full = 'i686-w64-mingw32'
extra_libs = ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4', '-lmsvcr90', '-liphlpapi']
@property
def root_path(self):
root_paths = {
'fedora': '/usr/i686-w64-mingw32/sys-root/mingw',
'debian': '/usr/i686-w64-mingw32'
}
return root_paths[self.neutralEnv.distname]
@property
def binaries(self):
return {k:which('{}-{}'.format(self.arch_full, v))
for k, v in (('CC', 'gcc'),
('CXX', 'g++'),
('AR', 'ar'),
('STRIP', 'strip'),
('WINDRES', 'windres'),
('RANLIB', 'ranlib'))
}
@property
def exec_wrapper_def(self):
try:
which('wine')
except subprocess.CalledProcessError:
return ""
else:
return "exec_wrapper = 'wine'"
@property
def configure_option(self):
return '--host={}'.format(self.arch_full)
def set_compiler(self, env):
for k, v in self.binaries.items():
env[k] = v
def get_bin_dir(self):
return [pj(self.root_path, 'bin')]
def set_env(self, env):
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
env['LIBS'] = " ".join(self.extra_libs) + " " +env['LIBS']

52
kiwixbuild/utils.py
View File

@ -10,9 +10,10 @@ import ssl
import subprocess
from collections import namedtuple, defaultdict
from kiwixbuild._global import neutralEnv, option
pj = os.path.join
g_print_progress = True
REMOTE_PREFIX = 'http://download.kiwix.org/dev/'
@ -28,11 +29,6 @@ def xrun_find(name):
return output[:-1].decode()
def setup_print_progress(print_progress):
global g_print_progress
g_print_progress = print_progress
class Defaultdict(defaultdict):
def __getattr__(self, name):
return self[name]
@ -67,7 +63,7 @@ def get_sha256(path):
def print_progress(progress):
if g_print_progress:
if option('show_progress'):
text = "{}\033[{}D".format(progress, len(progress))
print(text, end="")
@ -90,7 +86,7 @@ def copy_tree(src, dst, post_copy_function=None):
post_copy_function(dstfile)
def download_remote(what, where, check_certificate=True):
def download_remote(what, where):
file_path = pj(where, what.name)
file_url = what.url or (REMOTE_PREFIX + what.name)
if os.path.exists(file_path):
@ -98,7 +94,7 @@ def download_remote(what, where, check_certificate=True):
raise SkipCommand()
os.remove(file_path)
if not check_certificate:
if option('no_cert_check'):
context = ssl.create_default_context()
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
@ -221,3 +217,41 @@ def extract_archive(archive_path, dest_dir, topdir=None, name=None):
if archive is not None:
archive.close()
def run_command(command, cwd, context, buildEnv=None, env=None, input=None, cross_env_only=False):
os.makedirs(cwd, exist_ok=True)
if env is None:
env = Defaultdict(str, os.environ)
if buildEnv is not None:
cross_compile_env = True
cross_compile_compiler = True
cross_compile_path = True
if context.force_native_build:
cross_compile_env = False
cross_compile_compiler = False
cross_compile_path = False
if cross_env_only:
cross_compile_compiler = False
env = buildEnv._set_env(env, cross_compile_env, cross_compile_compiler, cross_compile_path)
log = None
try:
if not option('verbose'):
log = open(context.log_file, 'w')
print("run command '{}'".format(command), file=log)
print("current directory is '{}'".format(cwd), file=log)
print("env is :", file=log)
for k, v in env.items():
print(" {} : {!r}".format(k, v), file=log)
kwargs = dict()
if input:
kwargs['stdin'] = subprocess.PIPE
process = subprocess.Popen(command, shell=True, cwd=cwd, env=env, stdout=log or sys.stdout, stderr=subprocess.STDOUT, **kwargs)
if input:
process.communicate(input.encode())
retcode = process.wait()
if retcode:
raise subprocess.CalledProcessError(retcode, command)
finally:
if log:
log.close()

8
kiwixbuild/versions.py
View File

@ -10,7 +10,7 @@ main_project_versions = {
# This is the "version" of the whole base_deps_versions dict.
# Change this when you change base_deps_versions.
base_deps_meta_version = '3'
base_deps_meta_version = '5'
base_deps_versions = {
@ -23,7 +23,9 @@ base_deps_versions = {
'libmicrohttpd' : '0.9.46',
'gumbo' : '0.10.1',
'icu4c' : '58.2',
'Gradle' : '4.6',
'gradle' : '4.6',
'libaria2' : '1.33.1',
'libmagic' : '5.33'
'libmagic' : '5.33',
'android-sdk' : 'r25.2.3',
'android-ndk' : 'r13b'
}

33
travis/compile_all.py
View File

@ -66,8 +66,14 @@ def run_kiwix_build(target, platform, build_deps_only=False, make_release=False,
command = ['kiwix-build']
command.append(target)
command.append('--hide-progress')
command.append('--force-install-packages')
command.extend(['--target-platform', platform])
if target == 'kiwix-android' and platform.startswith('android_'):
command.extend(['--target-platform', 'android', '--android-arch', platform[8:]])
elif platform == 'android':
command.extend(['--target-platform', 'android'])
for arch in ('arm', 'arm64', 'x86', 'x86_64'):
command.extend(['--android-arch', arch])
else:
command.extend(['--target-platform', platform])
if build_deps_only:
command.append('--build-deps-only')
if make_release:
@ -121,12 +127,11 @@ def make_archive(project, platform):
def make_deps_archive(target, full=False):
(BASE_DIR/'.install_packages_ok').unlink()
archive_name = "deps_{}_{}_{}.tar.gz".format(
TRAVIS_OS_NAME, PLATFORM, target)
files_to_archive = [BASE_DIR/'INSTALL']
files_to_archive += BASE_DIR.glob('**/android-ndk*')
files_to_archive += HOME.glob('BUILD_*/android-ndk*')
files_to_archive += HOME.glob('BUILD_*/android-sdk*')
if (BASE_DIR/'meson_cross_file.txt').exists():
files_to_archive.append(BASE_DIR/'meson_cross_file.txt')
@ -138,17 +143,19 @@ def make_deps_archive(target, full=False):
if full:
files_to_archive += ARCHIVE_DIR.glob(".*_ok")
files_to_archive += BASE_DIR.glob('*/.*_ok')
files_to_archive += (HOME/"BUILD_native_dyn").glob('*/.*_ok')
files_to_archive += (HOME/"BUILD_native_static").glob('*/.*_ok')
files_to_archive += HOME.glob('BUILD_android*/.*_ok')
files_to_archive += SOURCE_DIR.glob('*/.*_ok')
files_to_archive += [SOURCE_DIR/'pugixml-{}'.format(
base_deps_versions['pugixml'])]
files_to_archive += [BASE_DIR/'pugixml-{}'.format(
base_deps_versions['pugixml'])]
if (TOOLCHAINS_DIR).exists():
files_to_archive.append(TOOLCHAINS_DIR)
files_to_archive += HOME.glob('BUILD_*/pugixml-{}'.format(
base_deps_versions['pugixml']))
files_to_archive += HOME.glob('**/TOOLCHAINS')
relative_path = HOME
with tarfile.open(str(relative_path/archive_name), 'w:gz') as tar:
for name in files_to_archive:
for name in set(files_to_archive):
tar.add(str(name), arcname=str(name.relative_to(relative_path)))
return relative_path/archive_name
@ -281,10 +288,10 @@ elif PLATFORM == 'armhf_static':
make_archive('kiwix-tools', 'linux-armhf')
elif PLATFORM == 'i586_static':
make_archive('kiwix-tools', 'linux-i586')
elif PLATFORM.startswith('android_') and 'kiwix-android' in TARGETS:
elif PLATFORM.startswith('android') and 'kiwix-android' in TARGETS:
APK_NAME = "kiwix-{}".format(PLATFORM)
source_debug_dir = BASE_DIR/'kiwix-android'/'app'/'build'/'outputs'/'apk'/'kiwix'/'debug'
source_release_dir = BASE_DIR/'kiwix-android'/'app'/'build'/'outputs'/'apk'/'kiwix'/'release'
source_debug_dir = HOME/'BUILD_android'/'kiwix-android'/'app'/'build'/'outputs'/'apk'/'kiwix'/'debug'
source_release_dir = HOME/'BUILD_android'/'kiwix-android'/'app'/'build'/'outputs'/'apk'/'kiwix'/'release'
shutil.copy(str(source_debug_dir/'app-kiwix-debug.apk'),
str(NIGHTLY_KIWIX_ARCHIVES_DIR/"{}-debug.apk".format(APK_NAME)))
shutil.copy(str(source_release_dir/'app-kiwix-release-unsigned.apk'),