smallsolar
/
kiwix-build
mirror of https://github.com/kiwix/kiwix-build.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #174 from kiwix/split_files 

Split files
This commit is contained in:
Matthieu Gautier 2018-05-22 19:14:27 +02:00 committed by GitHub
parent b27d1f0e17 a413c5f064
commit f44042f910
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
46 changed files with 1629 additions and 1422 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

730
kiwixbuild/__init__.py
View File

@ -1,732 +1,12 @@
#!/usr/bin/env python3
import os, sys, shutil
import os, sys
import argparse
import ssl
import urllib.request
import subprocess
import platform
from collections import OrderedDict
from .toolchains import Toolchain
from .dependencies import Dependency
from .utils import (
pj,
remove_duplicates,
add_execution_right,
get_sha256,
print_progress,
setup_print_progress,
download_remote,
StopBuild,
SkipCommand,
Defaultdict,
Remotefile,
Context)
REMOTE_PREFIX = 'http://download.kiwix.org/dev/'
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
_fedora_common = ['automake', 'libtool', 'cmake', 'git', 'subversion', 'ccache', 'pkgconfig', 'gcc-c++', 'gettext-devel']
_debian_common = ['automake', 'libtool', 'cmake', 'git', 'subversion', 'ccache', 'pkg-config', 'gcc', 'autopoint']
PACKAGE_NAME_MAPPERS = {
'fedora_native_dyn': {
'COMMON': _fedora_common,
'uuid': ['libuuid-devel'],
'xapian-core': None, # Not the right version on fedora 25
'ctpp2': None,
'pugixml': None, # ['pugixml-devel'] but package doesn't provide pkg-config file
'libmicrohttpd': ['libmicrohttpd-devel'],
'zlib': ['zlib-devel'],
'lzma': ['xz-devel'],
'icu4c': None,
'zimlib': None,
'file' : ['file-devel'],
'gumbo' : ['gumbo-parser-devel'],
},
'fedora_native_static': {
'COMMON': _fedora_common + ['glibc-static', 'libstdc++-static'],
'zlib': ['zlib-devel', 'zlib-static'],
'lzma': ['xz-devel', 'xz-static']
# Either there is no packages, or no static or too old
},
'fedora_i586_dyn': {
'COMMON': _fedora_common + ['glibc-devel.i686', 'libstdc++-devel.i686'],
},
'fedora_i586_static': {
'COMMON': _fedora_common + ['glibc-devel.i686'],
},
'fedora_win32_dyn': {
'COMMON': _fedora_common + ['mingw32-gcc-c++', 'mingw32-bzip2', 'mingw32-win-iconv', 'mingw32-winpthreads', 'wine'],
'zlib': ['mingw32-zlib'],
'lzma': ['mingw32-xz-libs'],
'libmicrohttpd': ['mingw32-libmicrohttpd'],
},
'fedora_win32_static': {
'COMMON': _fedora_common + ['mingw32-gcc-c++', 'mingw32-bzip2-static', 'mingw32-win-iconv-static', 'mingw32-winpthreads-static', 'wine'],
'zlib': ['mingw32-zlib-static'],
'lzma': ['mingw32-xz-libs-static'],
'libmicrohttpd': None, # ['mingw32-libmicrohttpd-static'] packaging dependecy seems buggy, and some static lib are name libfoo.dll.a and
# gcc cannot found them.
},
'fedora_armhf_static': {
'COMMON': _fedora_common
},
'fedora_armhf_dyn': {
'COMMON': _fedora_common
},
'fedora_android': {
'COMMON': _fedora_common + ['java-1.8.0-openjdk-devel']
},
'debian_native_dyn': {
'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'],
'zlib': ['zlib1g-dev'],
'uuid': ['uuid-dev'],
'ctpp2': ['libctpp2-dev'],
'ctpp2c': ['ctpp2-utils'],
'libmicrohttpd': ['libmicrohttpd-dev', 'ccache']
},
'debian_native_static': {
'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'],
'zlib': ['zlib1g-dev'],
'uuid': ['uuid-dev'],
'ctpp2': ['libctpp2-dev'],
'ctpp2c': ['ctpp2-utils'],
},
'debian_i586_dyn': {
'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib'],
},
'debian_i586_static': {
'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib'],
},
'debian_win32_dyn': {
'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'],
'ctpp2c': ['ctpp2-utils'],
},
'debian_win32_static': {
'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'],
'ctpp2c': ['ctpp2-utils'],
},
'debian_armhf_static': {
'COMMON': _debian_common,
'ctpp2c': ['ctpp2-utils'],
},
'debian_armhf_dyn': {
'COMMON': _debian_common,
'ctpp2c': ['ctpp2-utils'],
},
'debian_android': {
'COMMON': _debian_common + ['default-jdk'],
'ctpp2c': ['ctpp2-utils'],
},
'Darwin_native_dyn': {
'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'],
'file': ['libmagic']
},
'Darwin_iOS': {
'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'],
'file': ['libmagic']
},
}
def xrun_find(name):
command = "xcrun -find {}".format(name)
output = subprocess.check_output(command, shell=True)
return output[:-1].decode()
class TargetInfo:
def __init__(self, build, static, toolchains, hosts=None):
self.build = build
self.static = static
self.toolchains = toolchains
self.compatible_hosts = hosts
def __str__(self):
return "{}_{}".format(self.build, 'static' if self.static else 'dyn')
def get_cross_config(self, host):
if self.build == 'native':
return {}
elif self.build == 'win32':
root_paths = {
'fedora': '/usr/i686-w64-mingw32/sys-root/mingw',
'debian': '/usr/i686-w64-mingw32'
}
return {
'root_path': root_paths[host],
'extra_libs': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4', '-lmsvcr90', '-liphlpapi'],
'extra_cflags': ['-DWIN32'],
'host_machine': {
'system': 'Windows',
'lsystem': 'windows',
'cpu_family': 'x86',
'cpu': 'i686',
'endian': 'little',
'abi': ''
}
}
elif self.build == 'armhf':
return {
'extra_libs': [],
'extra_cflags': [],
'host_machine': {
'system': 'linux',
'lsystem': 'linux',
'cpu_family': 'arm',
'cpu': 'armhf',
'endian': 'little',
'abi': ''
}
}
elif self.build == 'i586':
return {
'extra_libs': ['-m32', '-march=i586', '-mno-sse'],
'extra_cflags': ['-m32', '-march=i586', '-mno-sse'],
'host_machine': {
'system': 'linux',
'lsystem': 'linux',
'cpu_family': 'x86',
'cpu': 'i586',
'endian': 'little',
'abi': ''
}
}
class AndroidTargetInfo(TargetInfo):
__arch_infos = {
'arm' : ('arm-linux-androideabi', 'arm', 'armeabi'),
'arm64': ('aarch64-linux-android', 'aarch64', 'arm64-v8a'),
'mips': ('mipsel-linux-android', 'mipsel', 'mips'),
'mips64': ('mips64el-linux-android', 'mips64el', 'mips64'),
'x86': ('i686-linux-android', 'i686', 'x86'),
'x86_64': ('x86_64-linux-android', 'x86_64', 'x86_64'),
}
def __init__(self, arch):
super().__init__('android', True, ['android_ndk', 'android_sdk'],
hosts=['fedora', 'debian'])
self.arch = arch
self.arch_full, self.cpu, self.abi = self.__arch_infos[arch]
def __str__(self):
return "android"
def get_cross_config(self, host):
return {
'extra_libs': [],
'extra_cflags': [],
'host_machine': {
'system': 'Android',
'lsystem': 'android',
'cpu_family': self.arch,
'cpu': self.cpu,
'endian': 'little',
'abi': self.abi
},
}
class iOSTargetInfo(TargetInfo):
__arch_infos = {
'armv7': ('arm-apple-darwin', 'armv7', 'iphoneos'),
'arm64': ('arm-apple-darwin', 'arm64', 'iphoneos'),
'i386': ('', 'i386', 'iphonesimulator'),
'x86_64': ('', 'x86_64', 'iphonesimulator'),
}
def __init__(self, arch):
super().__init__('iOS', True, ['iOS_sdk'],
hosts=['Darwin'])
self.arch = arch
self.arch_full, self.cpu, self.sdk_name = self.__arch_infos[arch]
self._root_path = None
@property
def root_path(self):
if self._root_path is None:
command = "xcodebuild -version -sdk {} | grep -E '^Path' | sed 's/Path: //'".format(self.sdk_name)
self._root_path = subprocess.check_output(command, shell=True)[:-1].decode()
return self._root_path
def __str__(self):
return "iOS"
def get_cross_config(self, host):
return {
'extra_libs': ['-fembed-bitcode', '-isysroot', self.root_path, '-arch', self.arch, '-miphoneos-version-min=9.0', '-stdlib=libc++'],
'extra_cflags': ['-fembed-bitcode', '-isysroot', self.root_path, '-arch', self.arch, '-miphoneos-version-min=9.0', '-stdlib=libc++'],
'host_machine': {
'system': 'Darwin',
'lsystem': 'darwin',
'cpu_family': self.arch,
'cpu': self.cpu,
'endian': '',
'abi': ''
},
}
class BuildEnv:
target_platforms = {
'native_dyn': TargetInfo('native', False, [],
hosts=['fedora', 'debian', 'Darwin']),
'native_static': TargetInfo('native', True, [],
hosts=['fedora', 'debian']),
'i586_dyn': TargetInfo('i586', False, ['linux_i586_toolchain'],
hosts=['fedora', 'debian']),
'i586_static': TargetInfo('i586', True, ['linux_i586_toolchain'],
hosts=['fedora', 'debian']),
'win32_dyn': TargetInfo('win32', False, ['mingw32_toolchain'],
hosts=['fedora', 'debian']),
'win32_static': TargetInfo('win32', True, ['mingw32_toolchain'],
hosts=['fedora', 'debian']),
'armhf_dyn': TargetInfo('armhf', False, ['armhf_toolchain'],
hosts=['fedora', 'debian']),
'armhf_static': TargetInfo('armhf', True, ['armhf_toolchain'],
hosts=['fedora', 'debian']),
'android_arm': AndroidTargetInfo('arm'),
'android_arm64': AndroidTargetInfo('arm64'),
'android_mips': AndroidTargetInfo('mips'),
'android_mips64': AndroidTargetInfo('mips64'),
'android_x86': AndroidTargetInfo('x86'),
'android_x86_64': AndroidTargetInfo('x86_64'),
'iOS_armv7': iOSTargetInfo('armv7'),
'iOS_arm64': iOSTargetInfo('arm64'),
'iOS_i386': iOSTargetInfo('i386'),
'iOS_x86_64': iOSTargetInfo('x86_64'),
}
def __init__(self, options, targetsDict):
self.source_dir = pj(options.working_dir, "SOURCE")
build_dir = "BUILD_{}".format(options.target_platform)
self.build_dir = pj(options.working_dir, build_dir)
self.archive_dir = pj(options.working_dir, "ARCHIVE")
self.toolchain_dir = pj(options.working_dir, "TOOLCHAINS")
self.log_dir = pj(self.build_dir, 'LOGS')
self.install_dir = pj(self.build_dir, "INSTALL")
for d in (self.source_dir,
self.build_dir,
self.archive_dir,
self.toolchain_dir,
self.log_dir,
self.install_dir):
os.makedirs(d, exist_ok=True)
self.detect_platform()
self.ninja_command = self._detect_ninja()
if not self.ninja_command:
sys.exit("ERROR: ninja command not found")
self.meson_command = self._detect_meson()
if not self.meson_command:
sys.exit("ERROR: meson command not fount")
self.mesontest_command = "meson test"
self.setup_build(options.target_platform)
self.setup_toolchains()
self.options = options
self.libprefix = options.libprefix or self._detect_libdir()
self.targetsDict = targetsDict
def clean_intermediate_directories(self):
for subdir in os.listdir(self.build_dir):
subpath = pj(self.build_dir, subdir)
if subpath == self.install_dir:
continue
if os.path.isdir(subpath):
shutil.rmtree(subpath)
else:
os.remove(subpath)
def detect_platform(self):
_platform = platform.system()
self.distname = _platform
if _platform == 'Windows':
print('ERROR: kiwix-build is not intented to run on Windows platform.\n'
'It should probably not work, but well, you still can have a try.')
cont = input('Do you want to continue ? [y/N]')
if cont.lower() != 'y':
sys.exit(0)
if _platform == 'Linux':
self.distname, _, _ = platform.linux_distribution()
self.distname = self.distname.lower()
if self.distname == 'ubuntu':
self.distname = 'debian'
def setup_build(self, target_platform):
self.platform_info = self.target_platforms[target_platform]
if self.distname not in self.platform_info.compatible_hosts:
print(('ERROR: The target {} cannot be build on host {}.\n'
'Select another target platform, or change your host system.'
).format(target_platform, self.distname))
sys.exit(-1)
self.cross_config = self.platform_info.get_cross_config(self.distname)
def setup_toolchains(self):
toolchain_names = self.platform_info.toolchains
self.toolchains =[Toolchain.all_toolchains[toolchain_name](self)
for toolchain_name in toolchain_names]
def finalize_setup(self):
getattr(self, 'setup_{}'.format(self.platform_info.build))()
def setup_native(self):
self.cmake_crossfile = None
self.meson_crossfile = None
def _gen_crossfile(self, name):
crossfile = pj(self.build_dir, name)
template_file = pj(SCRIPT_DIR, 'templates', name)
with open(template_file, 'r') as f:
template = f.read()
content = template.format(
toolchain=self.toolchains[0],
**self.cross_config
)
with open(crossfile, 'w') as outfile:
outfile.write(content)
return crossfile
def setup_win32(self):
self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def setup_android(self):
self.cmake_crossfile = self._gen_crossfile('cmake_android_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def setup_armhf(self):
self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def setup_iOS(self):
self.cmake_crossfile = self._gen_crossfile('cmake_ios_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def setup_i586(self):
self.cmake_crossfile = self._gen_crossfile('cmake_i586_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def __getattr__(self, name):
return getattr(self.options, name)
def _is_debianlike(self):
return os.path.isfile('/etc/debian_version')
def _detect_libdir(self):
if self._is_debianlike():
try:
pc = subprocess.Popen(['dpkg-architecture', '-qDEB_HOST_MULTIARCH'],
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL)
(stdo, _) = pc.communicate()
if pc.returncode == 0:
archpath = stdo.decode().strip()
return 'lib/' + archpath
except Exception:
pass
if os.path.isdir('/usr/lib64') and not os.path.islink('/usr/lib64'):
return 'lib64'
return 'lib'
def _detect_ninja(self):
for n in ['ninja', 'ninja-build']:
try:
retcode = subprocess.check_call([n, '--version'],
stdout=subprocess.DEVNULL)
except (FileNotFoundError, PermissionError):
# Doesn't exist in PATH or isn't executable
continue
if retcode == 0:
return n
def _detect_meson(self):
for n in ['meson.py', 'meson']:
try:
retcode = subprocess.check_call([n, '--version'],
stdout=subprocess.DEVNULL)
except (FileNotFoundError, PermissionError):
# Doesn't exist in PATH or isn't executable
continue
if retcode == 0:
return n
@property
def configure_option(self):
configure_options = [tlc.configure_option for tlc in self.toolchains]
return " ".join(configure_options)
@property
def cmake_option(self):
cmake_options = [tlc.cmake_option for tlc in self.toolchains]
return " ".join(cmake_options)
def _set_env(self, env, cross_compile_env, cross_compile_compiler, cross_compile_path):
if env is None:
env = Defaultdict(str, os.environ)
bin_dirs = []
if cross_compile_env:
for k, v in self.cross_config.get('env', {}).items():
if k.startswith('_format_'):
v = v.format(**self.cross_config)
k = k[8:]
env[k] = v
for toolchain in self.toolchains:
toolchain.set_env(env)
if cross_compile_compiler:
for toolchain in self.toolchains:
toolchain.set_compiler(env)
if cross_compile_path:
for tlc in self.toolchains:
bin_dirs += tlc.get_bin_dir()
pkgconfig_path = pj(self.install_dir, self.libprefix, 'pkgconfig')
env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path])
# Add ccache path
for p in ('/usr/lib/ccache', '/usr/lib64/ccache'):
if os.path.isdir(p):
ccache_path = [p]
break
else:
ccache_path = []
env['PATH'] = ':'.join(bin_dirs +
[pj(self.install_dir, 'bin')] +
ccache_path +
[env['PATH']])
env['LD_LIBRARY_PATH'] = ':'.join([env['LD_LIBRARY_PATH'],
pj(self.install_dir, 'lib'),
pj(self.install_dir, self.libprefix)
])
env['CPPFLAGS'] = " ".join(['-I'+pj(self.install_dir, 'include'), env['CPPFLAGS']])
env['LDFLAGS'] = " ".join(['-L'+pj(self.install_dir, 'lib'),
'-L'+pj(self.install_dir, self.libprefix),
env['LDFLAGS']])
return env
def run_command(self, command, cwd, context, env=None, input=None, cross_env_only=False):
os.makedirs(cwd, exist_ok=True)
cross_compile_env = True
cross_compile_compiler = True
cross_compile_path = True
if context.force_native_build:
cross_compile_env = False
cross_compile_compiler = False
cross_compile_path = False
if cross_env_only:
cross_compile_compiler = False
env = self._set_env(env, cross_compile_env, cross_compile_compiler, cross_compile_path)
log = None
try:
if not self.options.verbose:
log = open(context.log_file, 'w')
print("run command '{}'".format(command), file=log)
print("current directory is '{}'".format(cwd), file=log)
print("env is :", file=log)
for k, v in env.items():
print(" {} : {!r}".format(k, v), file=log)
kwargs = dict()
if input:
kwargs['stdin'] = subprocess.PIPE
process = subprocess.Popen(command, shell=True, cwd=cwd, env=env, stdout=log or sys.stdout, stderr=subprocess.STDOUT, **kwargs)
if input:
process.communicate(input.encode())
retcode = process.wait()
if retcode:
raise subprocess.CalledProcessError(retcode, command)
finally:
if log:
log.close()
def download(self, what, where=None):
where = where or self.archive_dir
download_remote(what, where, not self.options.no_cert_check)
def install_packages(self):
autoskip_file = pj(self.build_dir, ".install_packages_ok")
if self.distname in ('fedora', 'redhat', 'centos'):
package_installer = 'sudo dnf install {}'
package_checker = 'rpm -q --quiet {}'
elif self.distname in ('debian', 'Ubuntu'):
package_installer = 'sudo apt-get install {}'
package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1'
elif self.distname == 'Darwin':
package_installer = 'brew install {}'
package_checker = 'brew list -1 | grep -q {}'
mapper_name = "{host}_{target}".format(
host=self.distname,
target=self.platform_info)
try:
package_name_mapper = PACKAGE_NAME_MAPPERS[mapper_name]
except KeyError:
print("SKIP : We don't know which packages we must install to compile"
" a {target} {build_type} version on a {host} host.".format(
target=self.platform_info,
host=self.distname))
return
packages_list = package_name_mapper.get('COMMON', [])
for dep in self.targetsDict.values():
packages = package_name_mapper.get(dep.name)
if packages:
packages_list += packages
dep.skip = True
for dep in self.targetsDict.values():
packages = getattr(dep, 'extra_packages', [])
for package in packages:
packages_list += package_name_mapper.get(package, [])
if not self.options.force_install_packages and os.path.exists(autoskip_file):
print("SKIP")
return
packages_to_install = []
for package in packages_list:
print(" - {} : ".format(package), end="")
command = package_checker.format(package)
try:
subprocess.check_call(command, shell=True)
except subprocess.CalledProcessError:
print("NEEDED")
packages_to_install.append(package)
else:
print("SKIP")
if packages_to_install:
command = package_installer.format(" ".join(packages_to_install))
print(command)
subprocess.check_call(command, shell=True)
else:
print("SKIP, No package to install.")
with open(autoskip_file, 'w'):
pass
class iOS_sdk(Toolchain):
@property
def root_path(self):
return self.buildEnv.platform_info.root_path
@property
def binaries(self):
return {
'CC': xrun_find('clang'),
'CXX': xrun_find('clang++'),
'AR': '/usr/bin/ar',
'STRIP': '/usr/bin/strip',
'RANLIB': '/usr/bin/ranlib',
'LD': '/usr/bin/ld',
}
@property
def configure_option(self):
return '--host=arm-apple-darwin'
def get_bin_dir(self):
return [pj(self.root_path, 'bin')]
def set_env(self, env):
arch = self.buildEnv.platform_info.arch
env['CFLAGS'] = " -fembed-bitcode -isysroot {SDKROOT} -arch {arch} -miphoneos-version-min=9.0 ".format(SDKROOT=self.root_path, arch=arch) + env['CFLAGS']
env['CXXFLAGS'] = env['CFLAGS'] + " -stdlib=libc++ -std=c++11 "+env['CXXFLAGS']
env['LDFLAGS'] = " -arch {arch} -isysroot {SDKROOT} ".format(SDKROOT=self.root_path, arch=arch)
env['MACOSX_DEPLOYMENT_TARGET'] = "10.7"
def set_compiler(self, env):
env['CC'] = self.binaries['CC']
env['CXX'] = self.binaries['CXX']
class Builder:
def __init__(self, options):
self.options = options
self.targets = OrderedDict()
self.buildEnv = buildEnv = BuildEnv(options, self.targets)
_targets = {}
targetDef = options.targets
self.add_targets(targetDef, _targets)
dependencies = self.order_dependencies(_targets, targetDef)
dependencies = list(remove_duplicates(dependencies))
if options.build_nodeps:
self.targets[targetDef] = _targets[targetDef]
else:
for dep in dependencies:
if self.options.build_deps_only and dep == targetDef:
continue
self.targets[dep] = _targets[dep]
def add_targets(self, targetName, targets):
if targetName in targets:
return
targetClass = Dependency.all_deps[targetName]
target = targetClass(self.buildEnv)
targets[targetName] = target
for dep in target.dependencies:
self.add_targets(dep, targets)
def order_dependencies(self, _targets, targetName):
target = _targets[targetName]
for depName in target.dependencies:
yield from self.order_dependencies(_targets, depName)
yield targetName
def prepare_sources(self):
if self.options.skip_source_prepare:
print("SKIP")
return
toolchain_sources = (tlc.source for tlc in self.buildEnv.toolchains if tlc.source)
for toolchain_source in toolchain_sources:
print("prepare sources for toolchain {} :".format(toolchain_source.name))
toolchain_source.prepare()
sources = (dep.source for dep in self.targets.values() if not dep.skip)
sources = remove_duplicates(sources, lambda s: s.__class__)
for source in sources:
print("prepare sources {} :".format(source.name))
source.prepare()
def build(self):
toolchain_builders = (tlc.builder for tlc in self.buildEnv.toolchains if tlc.builder)
for toolchain_builder in toolchain_builders:
print("build toolchain {} :".format(toolchain_builder.name))
toolchain_builder.build()
builders = (dep.builder for dep in self.targets.values() if (dep.builder and not dep.skip))
for builder in builders:
if self.options.make_dist and builder.name == self.options.targets:
continue
print("build {} :".format(builder.name))
builder.build()
if self.options.make_dist:
dep = self.targets[self.options.targets]
builder = dep.builder
print("make dist {}:".format(builder.name))
builder.make_dist()
def run(self):
try:
print("[INSTALL PACKAGES]")
self.buildEnv.install_packages()
self.buildEnv.finalize_setup()
print("[PREPARE]")
self.prepare_sources()
print("[BUILD]")
self.build()
# No error, clean intermediate file at end of build if needed.
print("[CLEAN]")
if self.buildEnv.options.clean_at_end:
self.buildEnv.clean_intermediate_directories()
else:
print("SKIP")
except StopBuild:
sys.exit("Stopping build due to errors")
from .platforms import PlatformInfo
from .builder import Builder
from .utils import setup_print_progress
def parse_args():
parser = argparse.ArgumentParser()
@ -734,7 +14,7 @@ def parse_args():
choices=Dependency.all_deps.keys())
parser.add_argument('--working-dir', default=".")
parser.add_argument('--libprefix', default=None)
parser.add_argument('--target-platform', default="native_dyn", choices=BuildEnv.target_platforms)
parser.add_argument('--target-platform', default="native_dyn", choices=PlatformInfo.all_platforms)
parser.add_argument('--verbose', '-v', action="store_true",
help=("Print all logs on stdout instead of in specific"
" log files per commands"))

376
kiwixbuild/buildenv.py Normal file
View File

@ -0,0 +1,376 @@
import os, sys, shutil
import subprocess
import platform
from .platforms import PlatformInfo
from .toolchains import Toolchain
from .packages import PACKAGE_NAME_MAPPERS
from .utils import (
pj,
download_remote,
Defaultdict)
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
class PlatformNeutralEnv:
def __init__(self, options):
self.options = options
self.source_dir = pj(options.working_dir, "SOURCE")
self.archive_dir = pj(options.working_dir, "ARCHIVE")
self.toolchain_dir = pj(options.working_dir, "TOOLCHAINS")
self.log_dir = pj(self.working_dir, 'LOGS')
for d in (self.source_dir,
self.archive_dir,
self.toolchain_dir,
self.log_dir):
os.makedirs(d, exist_ok=True)
self.toolchains = {}
self.detect_platform()
self.ninja_command = self._detect_ninja()
if not self.ninja_command:
sys.exit("ERROR: ninja command not found")
self.meson_command = self._detect_meson()
if not self.meson_command:
sys.exit("ERROR: meson command not fount")
self.mesontest_command = "{} test".format(self.meson_command)
def run_command(self, command, cwd, context, env=None, input=None):
os.makedirs(cwd, exist_ok=True)
if env is None:
env = Defaultdict(str, os.environ)
log = None
try:
if not self.options.verbose:
log = open(context.log_file, 'w')
print("run command '{}'".format(command), file=log)
print("current directory is '{}'".format(cwd), file=log)
print("env is :", file=log)
for k, v in env.items():
print(" {} : {!r}".format(k, v), file=log)
kwargs = dict()
if input:
kwargs['stdin'] = subprocess.PIPE
process = subprocess.Popen(command, shell=True, cwd=cwd, env=env, stdout=log or sys.stdout, stderr=subprocess.STDOUT, **kwargs)
if input:
process.communicate(input.encode())
retcode = process.wait()
if retcode:
raise subprocess.CalledProcessError(retcode, command)
finally:
if log:
log.close()
def detect_platform(self):
_platform = platform.system()
self.distname = _platform
if _platform == 'Windows':
print('ERROR: kiwix-build is not intented to run on Windows platform.\n'
'It should probably not work, but well, you still can have a try.')
cont = input('Do you want to continue ? [y/N]')
if cont.lower() != 'y':
sys.exit(0)
if _platform == 'Linux':
self.distname, _, _ = platform.linux_distribution()
self.distname = self.distname.lower()
if self.distname == 'ubuntu':
self.distname = 'debian'
def download(self, what, where=None):
where = where or self.archive_dir
download_remote(what, where, not self.options.no_cert_check)
def _detect_ninja(self):
for n in ['ninja', 'ninja-build']:
try:
retcode = subprocess.check_call([n, '--version'],
stdout=subprocess.DEVNULL)
except (FileNotFoundError, PermissionError):
# Doesn't exist in PATH or isn't executable
continue
if retcode == 0:
return n
def add_toolchain(self, toolchain_name):
if toolchain_name not in self.toolchains:
ToolchainClass = Toolchain.all_toolchains[toolchain_name]
self.toolchains[toolchain_name] = ToolchainClass(self)
return self.toolchains[toolchain_name]
def _detect_meson(self):
for n in ['meson.py', 'meson']:
try:
retcode = subprocess.check_call([n, '--version'],
stdout=subprocess.DEVNULL)
except (FileNotFoundError, PermissionError):
# Doesn't exist in PATH or isn't executable
continue
if retcode == 0:
return n
def __getattr__(self, name):
return getattr(self.options, name)
class BuildEnv:
def __init__(self, options, neutralEnv, targetsDict):
build_dir = "BUILD_{}".format(options.target_platform)
self.neutralEnv = neutralEnv
self.build_dir = pj(options.working_dir, build_dir)
self.install_dir = pj(self.build_dir, "INSTALL")
for d in (self.build_dir,
self.install_dir):
os.makedirs(d, exist_ok=True)
self.setup_build(options.target_platform)
self.setup_toolchains()
self.options = options
self.libprefix = options.libprefix or self._detect_libdir()
self.targetsDict = targetsDict
def clean_intermediate_directories(self):
for subdir in os.listdir(self.build_dir):
subpath = pj(self.build_dir, subdir)
if subpath == self.install_dir:
continue
if os.path.isdir(subpath):
shutil.rmtree(subpath)
else:
os.remove(subpath)
def setup_build(self, target_platform):
self.platform_info = PlatformInfo.all_platforms[target_platform]
if self.distname not in self.platform_info.compatible_hosts:
print(('ERROR: The target {} cannot be build on host {}.\n'
'Select another target platform, or change your host system.'
).format(target_platform, self.distname))
sys.exit(-1)
self.cross_config = self.platform_info.get_cross_config()
def setup_toolchains(self):
toolchain_names = self.platform_info.toolchains
self.toolchains = []
for toolchain_name in toolchain_names:
ToolchainClass = Toolchain.all_toolchains[toolchain_name]
if ToolchainClass.neutral:
self.toolchains.append(
self.neutralEnv.add_toolchain(toolchain_name)
)
else:
self.toolchains.append(ToolchainClass(self))
def finalize_setup(self):
getattr(self, 'setup_{}'.format(self.platform_info.build))()
def setup_native(self):
self.cmake_crossfile = None
self.meson_crossfile = None
def _gen_crossfile(self, name):
crossfile = pj(self.build_dir, name)
template_file = pj(SCRIPT_DIR, 'templates', name)
with open(template_file, 'r') as f:
template = f.read()
content = template.format(
toolchain=self.toolchains[0],
**self.cross_config
)
with open(crossfile, 'w') as outfile:
outfile.write(content)
return crossfile
def setup_win32(self):
self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def setup_android(self):
self.cmake_crossfile = self._gen_crossfile('cmake_android_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def setup_armhf(self):
self.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def setup_iOS(self):
self.cmake_crossfile = self._gen_crossfile('cmake_ios_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def setup_i586(self):
self.cmake_crossfile = self._gen_crossfile('cmake_i586_cross_file.txt')
self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
def __getattr__(self, name):
return getattr(self.neutralEnv, name)
def _is_debianlike(self):
return os.path.isfile('/etc/debian_version')
def _detect_libdir(self):
if self._is_debianlike():
try:
pc = subprocess.Popen(['dpkg-architecture', '-qDEB_HOST_MULTIARCH'],
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL)
(stdo, _) = pc.communicate()
if pc.returncode == 0:
archpath = stdo.decode().strip()
return 'lib/' + archpath
except Exception:
pass
if os.path.isdir('/usr/lib64') and not os.path.islink('/usr/lib64'):
return 'lib64'
return 'lib'
@property
def configure_option(self):
configure_options = [tlc.configure_option for tlc in self.toolchains]
return " ".join(configure_options)
@property
def cmake_option(self):
cmake_options = [tlc.cmake_option for tlc in self.toolchains]
return " ".join(cmake_options)
def _set_env(self, env, cross_compile_env, cross_compile_compiler, cross_compile_path):
if env is None:
env = Defaultdict(str, os.environ)
pkgconfig_path = pj(self.install_dir, self.libprefix, 'pkgconfig')
env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path])
# Add ccache path
for p in ('/usr/lib/ccache', '/usr/lib64/ccache'):
if os.path.isdir(p):
ccache_path = [p]
break
else:
ccache_path = []
env['PATH'] = ':'.join([pj(self.install_dir, 'bin')] +
ccache_path +
[env['PATH']])
env['LD_LIBRARY_PATH'] = ':'.join([env['LD_LIBRARY_PATH'],
pj(self.install_dir, 'lib'),
pj(self.install_dir, self.libprefix)
])
env['CPPFLAGS'] = " ".join(['-I'+pj(self.install_dir, 'include'), env['CPPFLAGS']])
env['LDFLAGS'] = " ".join(['-L'+pj(self.install_dir, 'lib'),
'-L'+pj(self.install_dir, self.libprefix),
env['LDFLAGS']])
if cross_compile_env:
for k, v in self.cross_config.get('env', {}).items():
if k.startswith('_format_'):
v = v.format(**self.cross_config)
k = k[8:]
env[k] = v
for toolchain in self.toolchains:
toolchain.set_env(env)
self.platform_info.set_env(env)
if cross_compile_compiler:
for toolchain in self.toolchains:
toolchain.set_compiler(env)
if cross_compile_path:
bin_dirs = []
for tlc in self.toolchains:
bin_dirs += tlc.get_bin_dir()
bin_dirs += self.platform_info.get_bind_dir()
env['PATH'] = ':'.join(bin_dirs + [env['PATH']])
return env
def run_command(self, command, cwd, context, env=None, input=None, cross_env_only=False):
os.makedirs(cwd, exist_ok=True)
cross_compile_env = True
cross_compile_compiler = True
cross_compile_path = True
if context.force_native_build:
cross_compile_env = False
cross_compile_compiler = False
cross_compile_path = False
if cross_env_only:
cross_compile_compiler = False
env = self._set_env(env, cross_compile_env, cross_compile_compiler, cross_compile_path)
log = None
try:
if not self.options.verbose:
log = open(context.log_file, 'w')
print("run command '{}'".format(command), file=log)
print("current directory is '{}'".format(cwd), file=log)
print("env is :", file=log)
for k, v in env.items():
print(" {} : {!r}".format(k, v), file=log)
kwargs = dict()
if input:
kwargs['stdin'] = subprocess.PIPE
process = subprocess.Popen(command, shell=True, cwd=cwd, env=env, stdout=log or sys.stdout, stderr=subprocess.STDOUT, **kwargs)
if input:
process.communicate(input.encode())
retcode = process.wait()
if retcode:
raise subprocess.CalledProcessError(retcode, command)
finally:
if log:
log.close()
def install_packages(self):
autoskip_file = pj(self.build_dir, ".install_packages_ok")
if self.distname in ('fedora', 'redhat', 'centos'):
package_installer = 'sudo dnf install {}'
package_checker = 'rpm -q --quiet {}'
elif self.distname in ('debian', 'Ubuntu'):
package_installer = 'sudo apt-get install {}'
package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1'
elif self.distname == 'Darwin':
package_installer = 'brew install {}'
package_checker = 'brew list -1 | grep -q {}'
mapper_name = "{host}_{target}".format(
host=self.distname,
target=self.platform_info)
try:
package_name_mapper = PACKAGE_NAME_MAPPERS[mapper_name]
except KeyError:
print("SKIP : We don't know which packages we must install to compile"
" a {target} {build_type} version on a {host} host.".format(
target=self.platform_info,
host=self.distname))
return
packages_list = package_name_mapper.get('COMMON', [])
for dep in self.targetsDict.values():
packages = package_name_mapper.get(dep.name)
if packages:
packages_list += packages
dep.skip = True
for dep in self.targetsDict.values():
packages = getattr(dep, 'extra_packages', [])
for package in packages:
packages_list += package_name_mapper.get(package, [])
if not self.options.force_install_packages and os.path.exists(autoskip_file):
print("SKIP")
return
packages_to_install = []
for package in packages_list:
print(" - {} : ".format(package), end="")
command = package_checker.format(package)
try:
subprocess.check_call(command, shell=True)
except subprocess.CalledProcessError:
print("NEEDED")
packages_to_install.append(package)
else:
print("SKIP")
if packages_to_install:
command = package_installer.format(" ".join(packages_to_install))
print(command)
subprocess.check_call(command, shell=True)
else:
print("SKIP, No package to install.")
with open(autoskip_file, 'w'):
pass

97
kiwixbuild/builder.py Normal file
View File

@ -0,0 +1,97 @@
import sys
from collections import OrderedDict
from .buildenv import *
from .utils import remove_duplicates, StopBuild
from .dependencies import Dependency
class Builder:
def __init__(self, options):
self.options = options
self.targets = OrderedDict()
self.neutralEnv = PlatformNeutralEnv(options)
self.buildEnv = BuildEnv(options, self.neutralEnv, self.targets)
_targets = {}
targetDef = options.targets
self.add_targets(targetDef, _targets)
dependencies = self.order_dependencies(_targets, targetDef)
dependencies = list(remove_duplicates(dependencies))
if options.build_nodeps:
self.targets[targetDef] = _targets[targetDef]
else:
for dep in dependencies:
if self.options.build_deps_only and dep == targetDef:
continue
self.targets[dep] = _targets[dep]
def add_targets(self, targetName, targets):
if targetName in targets:
return
targetClass = Dependency.all_deps[targetName]
target = targetClass(self.neutralEnv, self.buildEnv)
targets[targetName] = target
for dep in target.dependencies:
self.add_targets(dep, targets)
def order_dependencies(self, _targets, targetName):
target = _targets[targetName]
for depName in target.dependencies:
yield from self.order_dependencies(_targets, depName)
yield targetName
def prepare_sources(self):
if self.options.skip_source_prepare:
print("SKIP")
return
toolchain_sources = (tlc.source for tlc in self.buildEnv.toolchains if tlc.source)
for toolchain_source in toolchain_sources:
print("prepare sources for toolchain {} :".format(toolchain_source.name))
toolchain_source.prepare()
sources = (dep.source for dep in self.targets.values() if not dep.skip)
sources = remove_duplicates(sources, lambda s: s.__class__)
for source in sources:
print("prepare sources {} :".format(source.name))
source.prepare()
def build(self):
toolchain_builders = (tlc.builder for tlc in self.buildEnv.toolchains if tlc.builder)
for toolchain_builder in toolchain_builders:
print("build toolchain {} :".format(toolchain_builder.name))
toolchain_builder.build()
builders = (dep.builder for dep in self.targets.values() if (dep.builder and not dep.skip))
for builder in builders:
if self.options.make_dist and builder.name == self.options.targets:
continue
print("build {} :".format(builder.name))
builder.build()
if self.options.make_dist:
dep = self.targets[self.options.targets]
builder = dep.builder
print("make dist {}:".format(builder.name))
builder.make_dist()
def run(self):
try:
print("[INSTALL PACKAGES]")
self.buildEnv.install_packages()
self.buildEnv.finalize_setup()
print("[PREPARE]")
self.prepare_sources()
print("[BUILD]")
self.build()
# No error, clean intermediate file at end of build if needed.
print("[CLEAN]")
if self.buildEnv.options.clean_at_end:
self.buildEnv.clean_intermediate_directories()
else:
print("SKIP")
except StopBuild:
sys.exit("Stopping build due to errors")

623
kiwixbuild/dependencies.py
View File

@ -1,623 +0,0 @@
import shutil, os, json
from urllib.parse import urlparse
from kiwixbuild.dependency_utils import (
Dependency,
ReleaseDownload,
GitClone,
SvnClone,
NoopSource,
MakeBuilder,
CMakeBuilder,
MesonBuilder,
GradleBuilder,
NoopBuilder,
Builder as BaseBuilder)
from kiwixbuild.utils import Remotefile, pj, SkipCommand, copy_tree, add_execution_right, Defaultdict
# *************************************
# Missing dependencies
# Is this ok to assume that those libs
# exist in your "distri" (linux/mac) ?
# If not, we need to compile them here
# *************************************
# Argtable
# MSVirtual
# Android
# libiconv
# gettext
# *************************************
class zlib(Dependency):
name = 'zlib'
class Source(ReleaseDownload):
archive = Remotefile('zlib-1.2.8.tar.gz',
'36658cb768a54c1d4dec43c3116c27ed893e88b02ecfcb44f2166f9c0b7f2a0d')
patches = ['zlib_std_libname.patch']
class Builder(MakeBuilder):
dynamic_configure_option = "--shared"
static_configure_option = "--static"
def _pre_build_script(self, context):
context.try_skip(self.build_path)
shutil.copytree(self.source_path, self.build_path)
@property
def all_configure_option(self):
return '--static' if self.buildEnv.platform_info.static else '--shared'
@property
def configure_option(self):
options = "-DINSTALL_PKGCONFIG_DIR={}".format(pj(self.buildEnv.install_dir, self.buildEnv.libprefix, 'pkgconfig'))
if self.buildEnv.platform_info.static:
options += " -DBUILD_SHARED_LIBS=false"
else:
options += " -DBUILD_SHARED_LIBS=true"
return options
def _configure(self, context):
if self.buildEnv.platform_info.build == 'win32':
raise SkipCommand()
return super()._configure(context)
@property
def make_option(self):
if self.buildEnv.platform_info.build == 'win32':
return "--makefile win32/Makefile.gcc PREFIX={host}- SHARED_MODE={static} INCLUDE_PATH={include_path} LIBRARY_PATH={library_path} BINARY_PATH={binary_path}".format(
host='i686-w64-mingw32',
static="0" if self.buildEnv.platform_info.static else "1",
include_path=pj(self.buildEnv.install_dir, 'include'),
library_path=pj(self.buildEnv.install_dir, self.buildEnv.libprefix),
binary_path=pj(self.buildEnv.install_dir, 'bin'),
)
return ""
class lzma(Dependency):
name = 'lzma'
class Source(ReleaseDownload):
archive = Remotefile('xz-5.2.3.tar.bz2',
'fd9ca16de1052aac899ad3495ad20dfa906c27b4a5070102a2ec35ca3a4740c1',
'https://tukaani.org/xz/xz-5.2.3.tar.bz2')
class Builder(MakeBuilder):
@property
def configure_option(self):
return "--disable-assembler --disable-xz --disable-xzdec"
class UUID(Dependency):
name = 'uuid'
class Source(ReleaseDownload):
archive = Remotefile('e2fsprogs-libs-1.43.4.tar.gz',
'eed4516325768255c9745e7b82c9d7d0393abce302520a5b2cde693204b0e419',
'https://www.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v1.43.4/e2fsprogs-libs-1.43.4.tar.gz')
extract_dir = 'e2fsprogs-libs-1.43.4'
class Builder(MakeBuilder):
configure_option = ("--enable-libuuid --disable-debugfs --disable-imager --disable-resizer --disable-defrag --enable-fsck"
" --disable-uuidd")
configure_env = {'_format_CFLAGS': "{env.CFLAGS} -fPIC"}
static_configure_option = dynamic_configure_option = ""
make_target = 'libs'
make_install_target = 'install-libs'
class Xapian(Dependency):
name = "xapian-core"
class Source(ReleaseDownload):
archive = Remotefile('xapian-core-1.4.5.tar.xz',
'85b5f952de9df925fd13e00f6e82484162fd506d38745613a50b0a2064c6b02b')
class Builder(MakeBuilder):
configure_option = "--disable-sse --disable-backend-inmemory --disable-documentation"
configure_env = {'_format_LDFLAGS': "-L{buildEnv.install_dir}/{buildEnv.libprefix}",
'_format_CXXFLAGS': "-I{buildEnv.install_dir}/include"}
@property
def dependencies(self):
deps = ['zlib', 'lzma']
if (self.buildEnv.platform_info.build == 'win32'
or self.buildEnv.distname == 'Darwin'):
return deps
return deps + ['uuid']
class CTPP2(Dependency):
name = "ctpp2"
class Source(ReleaseDownload):
name = "ctpp2"
source_dir = "ctpp2-2.8.3"
archive = Remotefile('ctpp2-2.8.3.tar.gz',
'a83ffd07817adb575295ef40fbf759892512e5a63059c520f9062d9ab8fb42fc')
patches = ["ctpp2_include.patch",
"ctpp2_no_src_modification.patch",
"ctpp2_fix-static-libname.patch",
"ctpp2_mingw32.patch",
"ctpp2_dll_export_VMExecutable.patch",
"ctpp2_win_install_lib_in_lib_dir.patch",
"ctpp2_iconv_support.patch",
"ctpp2_compile_ctpp2c_static.patch",
]
class Builder(CMakeBuilder):
@property
def configure_option(self):
libprefix = self.buildEnv.libprefix
options = "-DMD5_SUPPORT=OFF -DICONV_SUPPORT=OFF"
if libprefix.startswith('lib'):
libprefix = libprefix[3:]
options += " -DLIB_SUFFIX={}".format(libprefix)
return options
class CTPP2C(CTPP2):
name = "ctpp2c"
force_native_build = True
class Builder(CTPP2.Builder):
make_target = "ctpp2c"
@property
def build_path(self):
return super().build_path+"_native"
def _install(self, context):
context.try_skip(self.build_path)
command = "cp {ctpp2c}* {install_dir}".format(
ctpp2c=pj(self.build_path, 'ctpp2c'),
install_dir=pj(self.buildEnv.install_dir, 'bin')
)
self.buildEnv.run_command(command, self.build_path, context)
class Pugixml(Dependency):
name = "pugixml"
class Source(ReleaseDownload):
archive = Remotefile('pugixml-1.2.tar.gz',
'0f422dad86da0a2e56a37fb2a88376aae6e931f22cc8b956978460c9db06136b')
patches = ["pugixml_meson.patch"]
Builder = MesonBuilder
class MicroHttpd(Dependency):
name = "libmicrohttpd"
class Source(ReleaseDownload):
archive = Remotefile('libmicrohttpd-0.9.46.tar.gz',
'06dbd2654f390fa1e8196fe063fc1449a6c2ed65a38199a49bf29ad8a93b8979',
'http://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.46.tar.gz')
class Builder(MakeBuilder):
configure_option = "--disable-https --without-libgcrypt --without-libcurl"
class Aria2(Dependency):
name = "libaria2"
dependencies = ['zlib']
class Source(ReleaseDownload):
archive = Remotefile('libaria2-1.33.1.tar.gz',
'0616f11ef3ddd0c74be74ea2536be62ce168b99e3d6a35dea9d166b9cab9fbd1',
'https://github.com/aria2/aria2/archive/release-1.33.1.tar.gz')
patches = ["libaria2_android.patch"]
def _post_prepare_script(self, context):
context.try_skip(self.extract_path)
command = "autoreconf -i"
self.buildEnv.run_command(command, self.extract_path, context)
class Builder(MakeBuilder):
configure_option = "--enable-libaria2 --disable-ssl --disable-bittorent --disable-metalink --without-sqlite3 --without-libxml2 --without-libexpat"
class Gumbo(Dependency):
name = "gumbo"
class Source(ReleaseDownload):
archive = Remotefile('gumbo-0.10.1.tar.gz',
'28463053d44a5dfbc4b77bcf49c8cee119338ffa636cc17fc3378421d714efad',
'https://github.com/google/gumbo-parser/archive/v0.10.1.tar.gz')
def _post_prepare_script(self, context):
context.try_skip(self.extract_path)
command = "./autogen.sh"
self.buildEnv.run_command(command, self.extract_path, context)
Builder = MakeBuilder
class Icu(Dependency):
name = "icu4c"
class Source(SvnClone):
name = "icu4c"
svn_remote = "http://source.icu-project.org/repos/icu/tags/release-58-2/icu4c"
svn_dir = "icu4c"
patches = ["icu4c_fix_static_lib_name_mingw.patch",
"icu4c_android_elf64_st_info.patch",
"icu4c_custom_data.patch",
"icu4c_noxlocale.patch",
"icu4c_rpath.patch"]
class Builder(MakeBuilder):
subsource_dir = "source"
@property
def configure_option(self):
options = "--disable-samples --disable-tests --disable-extras --disable-dyload --enable-rpath"
if self.buildEnv.platform_info.build == 'android':
options += " --with-data-packaging=archive"
return options
class Icu_native(Icu):
name = "icu4c_native"
force_native_build = True
class Builder(Icu.Builder):
@property
def build_path(self):
return super().build_path+"_native"
def _install(self, context):
raise SkipCommand()
class Icu_cross_compile(Icu):
name = "icu4c_cross-compile"
dependencies = ['icu4c_native']
class Builder(Icu.Builder):
@property
def configure_option(self):
icu_native_dep = self.buildEnv.targetsDict['icu4c_native']
return super().configure_option + " --with-cross-build={} --disable-tools".format(icu_native_dep.builder.build_path)
class Libzim(Dependency):
name = "libzim"
@property
def dependencies(self):
base_dependencies = ['zlib', 'lzma', 'xapian-core']
if self.buildEnv.platform_info.build != 'native':
return base_dependencies + ["icu4c_cross-compile"]
else:
return base_dependencies + ["icu4c"]
class Source(GitClone):
git_remote = "https://github.com/openzim/libzim.git"
git_dir = "libzim"
class Builder(MesonBuilder):
test_option = "-t 8"
class ZimTools(Dependency):
name = "zim-tools"
dependencies = ['libzim']
class Source(GitClone):
git_remote = "https://github.com/openzim/zim-tools.git"
git_dir = "zim-tools"
class Builder(MesonBuilder):
@property
def configure_option(self):
if self.buildEnv.platform_info.static:
return "-Dstatic-linkage=true"
return ""
class Zimwriterfs(Dependency):
name = "zimwriterfs"
@property
def dependencies(self):
base_dependencies = ['libzim', 'zlib', 'xapian-core', 'gumbo']
if self.buildEnv.platform_info.build != 'native':
return base_dependencies + ["icu4c_cross-compile", "libmagic_cross-compile"]
else:
return base_dependencies + ["icu4c", "libmagic"]
class Source(GitClone):
git_remote = "https://github.com/openzim/zimwriterfs.git"
git_dir = "zimwriterfs"
release_git_ref = "1.1"
class Builder(MesonBuilder):
@property
def configure_option(self):
base_option = "-Dmagic-install-prefix={buildEnv.install_dir}"
if self.buildEnv.platform_info.static:
base_option += " -Dstatic-linkage=true"
return base_option
class Kiwixlib(Dependency):
name = "kiwix-lib"
@property
def dependencies(self):
base_dependencies = ["pugixml", "libzim", "zlib", "lzma", "libaria2"]
if ( self.buildEnv.platform_info.build != 'android'
and self.buildEnv.distname != 'Darwin'):
base_dependencies += ['ctpp2c', 'ctpp2']
if self.buildEnv.platform_info.build != 'native':
return base_dependencies + ["icu4c_cross-compile"]
else:
return base_dependencies + ["icu4c"]
class Source(GitClone):
git_remote = "https://github.com/kiwix/kiwix-lib.git"
git_dir = "kiwix-lib"
class Builder(MesonBuilder):
@property
def configure_option(self):
base_option = "-Dctpp2-install-prefix={buildEnv.install_dir}"
if self.buildEnv.platform_info.build == 'android':
base_option += ' -Dandroid=true'
return base_option
@property
def library_type(self):
if self.buildEnv.platform_info.build == 'android':
return 'shared'
return super().library_type
class KiwixTools(Dependency):
name = "kiwix-tools"
dependencies = ["kiwix-lib", "libmicrohttpd", "zlib"]
class Source(GitClone):
git_remote = "https://github.com/kiwix/kiwix-tools.git"
git_dir = "kiwix-tools"
class Builder(MesonBuilder):
@property
def configure_option(self):
if self.buildEnv.platform_info.static:
return "-Dstatic-linkage=true"
return ""
class Gradle(Dependency):
name = "Gradle"
class Source(ReleaseDownload):
archive = Remotefile('gradle-4.6-bin.zip',
'98bd5fd2b30e070517e03c51cbb32beee3e2ee1a84003a5a5d748996d4b1b915',
'https://services.gradle.org/distributions/gradle-4.6-bin.zip')
class Builder(BaseBuilder):
def build(self):
self.command('install', self._install)
def _install(self, context):
copy_tree(
pj(self.source_path, "bin"),
pj(self.buildEnv.install_dir, "bin"),
post_copy_function = add_execution_right)
copy_tree(
pj(self.source_path, "lib"),
pj(self.buildEnv.install_dir, "lib"))
class LibMagicBase(Dependency):
name = "libmagic"
class Source(ReleaseDownload):
name = "libmagic"
source_dir = "libmagic"
archive = Remotefile('file-5.33.tar.gz',
'1c52c8c3d271cd898d5511c36a68059cda94036111ab293f01f83c3525b737c6',
'https://fossies.org/linux/misc/file-5.33.tar.gz')
Builder = MakeBuilder
class LibMagic_native(LibMagicBase):
name = "libmagic_native"
force_native_build = True
class Builder(LibMagicBase.Builder):
static_configure_option = dynamic_configure_option = "--disable-shared --enable-static"
@property
def build_path(self):
return super().build_path+"_native"
def _install(self, context):
raise SkipCommand()
class LibMagic_cross_compile(LibMagicBase):
name = "libmagic_cross-compile"
dependencies = ['libmagic_native']
class Builder(LibMagicBase.Builder):
def _compile(self, context):
context.try_skip(self.build_path)
command = "make -j4 {make_target} {make_option}".format(
make_target=self.make_target,
make_option=self.make_option
)
libmagic_native_dep = self.buildEnv.targetsDict['libmagic_native']
env = Defaultdict(str, os.environ)
env['PATH'] = ':'.join([pj(libmagic_native_dep.builder.build_path, 'src'), env['PATH']])
self.buildEnv.run_command(command, self.build_path, context, env=env)
class AllBaseDependencies(Dependency):
name = "alldependencies"
@property
def dependencies(self):
base_deps = ['zlib', 'lzma', 'xapian-core', 'gumbo', 'pugixml', 'libmicrohttpd', 'libaria2']
if self.buildEnv.platform_info.build != 'native':
base_deps += ["icu4c_cross-compile"]
if self.buildEnv.platform_info.build != 'win32':
base_deps += ["libmagic_cross-compile"]
else:
base_deps += ["icu4c", "libmagic"]
if ( self.buildEnv.platform_info.build != 'android'
and self.buildEnv.distname != 'Darwin'):
base_deps += ['ctpp2c', 'ctpp2']
if self.buildEnv.platform_info.build == 'android':
base_deps += ['Gradle']
return base_deps
Source = NoopSource
Builder = NoopBuilder
class KiwixAndroid(Dependency):
name = "kiwix-android"
dependencies = ["Gradle", "kiwix-lib"]
class Source(GitClone):
git_remote = "https://github.com/kiwix/kiwix-android"
git_dir = "kiwix-android"
class Builder(GradleBuilder):
def build(self):
if self.buildEnv.options.targets == 'kiwix-android-custom':
print("SKIP")
else:
super().build()
def _configure(self, context):
if not os.path.exists(self.build_path):
shutil.copytree(self.source_path, self.build_path, symlinks=True)
try:
shutil.rmtree(pj(self.build_path, 'kiwixlib', 'src', 'main'))
except FileNotFoundError:
pass
shutil.copytree(pj(self.buildEnv.install_dir, 'kiwix-lib'),
pj(self.build_path, 'kiwixlib', 'src', 'main'))
os.makedirs(
pj(self.build_path, 'app', 'src', 'main', 'assets', 'icu'),
exist_ok=True)
shutil.copy2(pj(self.buildEnv.install_dir, 'share', 'icu', '58.2',
'icudt58l.dat'),
pj(self.build_path, 'app', 'src', 'main', 'assets',
'icu', 'icudt58l.dat'))
class KiwixCustomApp(Dependency):
name = "kiwix-android-custom"
dependencies = ["kiwix-android", "kiwix-lib"]
def __init__(self, buildEnv):
super().__init__(buildEnv)
self.custom_name = buildEnv.options.android_custom_app
class Source(GitClone):
git_remote = "https://github.com/kiwix/kiwix-android-custom"
git_dir = "kiwix-android-custom"
class Builder(GradleBuilder):
@property
def gradle_target(self):
return "assemble{}".format(self.target.custom_name)
@property
def gradle_option(self):
template = ("-i -P customDir={customDir}"
" -P zim_file_size={zim_size}"
" -P version_code={version_code}"
" -P version_name={version_name}"
" -P content_version_code={content_version_code}")
return template.format(
customDir=pj(self.build_path, 'custom'),
zim_size=self._get_zim_size(),
version_code=os.environ['VERSION_CODE'],
version_name=os.environ['VERSION_NAME'],
content_version_code=os.environ['CONTENT_VERSION_CODE'])
@property
def build_path(self):
return pj(self.buildEnv.build_dir, "{}_{}".format(self.target.full_name, self.target.custom_name))
@property
def custom_build_path(self):
return pj(self.build_path, 'custom', self.target.custom_name)
def _get_zim_size(self):
try:
zim_size = self.buildEnv.options.zim_file_size
except AttributeError:
with open(pj(self.source_path, self.target.custom_name, 'info.json')) as f:
app_info = json.load(f)
zim_size = os.path.getsize(pj(self.custom_build_path, app_info['zim_file']))
return zim_size
def build(self):
self.command('configure', self._configure)
self.command('download_zim', self._download_zim)
self.command('compile', self._compile)
def _download_zim(self, context):
zim_url = self.buildEnv.options.zim_file_url
if zim_url is None:
raise SkipCommand()
with open(pj(self.source_path, self.target.custom_name, 'info.json')) as f:
app_info = json.load(f)
zim_url = app_info.get('zim_url', zim_url)
out_filename = urlparse(zim_url).path
out_filename = os.path.basename(out_filename)
zim_file = Remotefile(out_filename, '', zim_url)
self.buildEnv.download(zim_file)
shutil.copy(pj(self.buildEnv.archive_dir, out_filename),
pj(self.custom_build_path, app_info['zim_file']))
def _configure(self, context):
# Copy kiwix-android in build dir.
kiwix_android_dep = self.buildEnv.targetsDict['kiwix-android']
if not os.path.exists(self.build_path):
shutil.copytree(kiwix_android_dep.source_path, self.build_path)
# Copy kiwix-lib application in build dir
try:
shutil.rmtree(pj(self.build_path, 'kiwixlib', 'src', 'main'))
except FileNotFoundError:
pass
shutil.copytree(pj(self.buildEnv.install_dir, 'kiwix-lib'),
pj(self.build_path, 'kiwixlib', 'src', 'main'))
os.makedirs(
pj(self.build_path, 'app', 'src', 'main', 'assets', 'icu'),
exist_ok=True)
shutil.copy2(pj(self.buildEnv.install_dir, 'share', 'icu', '58.2',
'icudt58l.dat'),
pj(self.build_path, 'app', 'src', 'main', 'assets',
'icu', 'icudt58l.dat'))
# Generate custom directory
try:
shutil.rmtree(pj(self.build_path, 'custom'))
except FileNotFoundError:
pass
os.makedirs(pj(self.build_path, 'custom'))
command = "./gen-custom-android-directory.py {custom_name} --output-dir {custom_dir}"
command = command.format(
custom_name=self.target.custom_name,
custom_dir=pj(self.build_path, 'custom', self.target.custom_name)
)
self.buildEnv.run_command(command, self.source_path, context)

24
kiwixbuild/dependencies/__init__.py Normal file
View File

@ -0,0 +1,24 @@
from .base import *
from . import (
all_dependencies,
ctpp2,
gradle,
gumbo,
icu4c,
kiwix_android,
kiwix_custom_app,
kiwix_lib,
kiwix_tools,
libaria2,
libmagic,
libmicrohttpd,
libzim,
lzma,
pugixml,
uuid,
xapian,
zim_tools,
zimwriterfs,
zlib
)

28
kiwixbuild/dependencies/all_dependencies.py Normal file
View File

@ -0,0 +1,28 @@
from .base import (
Dependency,
NoopSource,
NoopBuilder)
class AllBaseDependencies(Dependency):
name = "alldependencies"
@property
def dependencies(self):
base_deps = ['zlib', 'lzma', 'xapian-core', 'gumbo', 'pugixml', 'libmicrohttpd', 'libaria2']
if self.buildEnv.platform_info.build != 'native':
base_deps += ["icu4c_cross-compile"]
if self.buildEnv.platform_info.build != 'win32':
base_deps += ["libmagic_cross-compile"]
else:
base_deps += ["icu4c", "libmagic"]
if ( self.buildEnv.platform_info.build != 'android'
and self.buildEnv.distname != 'Darwin'):
base_deps += ['ctpp2c', 'ctpp2']
if self.buildEnv.platform_info.build == 'android':
base_deps += ['Gradle']
return base_deps
Source = NoopSource
Builder = NoopBuilder

84
kiwixbuild/dependency_utils.py → kiwixbuild/dependencies/base.py
View File

@ -3,9 +3,9 @@ import os
import shutil
from kiwixbuild.utils import pj, Context, SkipCommand, extract_archive, Defaultdict, StopBuild
from kiwixbuild.dependency_versions import main_project_versions, base_deps_versions
from kiwixbuild.versions import main_project_versions, base_deps_versions
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
SCRIPT_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
class _MetaDependency(type):
@ -22,7 +22,8 @@ class Dependency(metaclass=_MetaDependency):
dependencies = []
force_native_build = False
def __init__(self, buildEnv):
def __init__(self, neutralEnv, buildEnv):
self.neutralEnv = neutralEnv
self.buildEnv = buildEnv
self.source = self.Source(self)
self.builder = self.Builder(self)
@ -40,7 +41,7 @@ class Dependency(metaclass=_MetaDependency):
@property
def source_path(self):
return pj(self.buildEnv.source_dir, self.source.source_dir)
return pj(self.neutralEnv.source_dir, self.source.source_dir)
@property
def _log_dir(self):
@ -73,10 +74,10 @@ class Dependency(metaclass=_MetaDependency):
class Source:
"""Base Class to the real preparator
A source preparator must install source in the self.source_dir attribute
inside the buildEnv.source_dir."""
inside the neutralEnv.source_dir."""
def __init__(self, target):
self.target = target
self.buildEnv = target.buildEnv
self.neutralEnv = target.neutralEnv
@property
def name(self):
@ -87,12 +88,12 @@ class Source:
return self.target.full_name
def _patch(self, context):
source_path = pj(self.buildEnv.source_dir, self.source_dir)
source_path = pj(self.neutralEnv.source_dir, self.source_dir)
context.try_skip(source_path)
context.force_native_build = True
for p in self.patches:
with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input:
self.buildEnv.run_command("patch -p1", source_path, context, input=patch_input.read())
self.neutralEnv.run_command("patch -p1", source_path, context, input=patch_input.read())
def command(self, *args, **kwargs):
return self.target.command(*args, **kwargs)
@ -108,18 +109,18 @@ class ReleaseDownload(Source):
@property
def extract_path(self):
return pj(self.buildEnv.source_dir, self.source_dir)
return pj(self.neutralEnv.source_dir, self.source_dir)
def _download(self, context):
context.try_skip(self.buildEnv.archive_dir, self.name)
self.buildEnv.download(self.archive)
context.try_skip(self.neutralEnv.archive_dir, self.name)
self.neutralEnv.download(self.archive)
def _extract(self, context):
context.try_skip(self.extract_path)
if os.path.exists(self.extract_path):
shutil.rmtree(self.extract_path)
extract_archive(pj(self.buildEnv.archive_dir, self.archive.name),
self.buildEnv.source_dir,
extract_archive(pj(self.neutralEnv.archive_dir, self.archive.name),
self.neutralEnv.source_dir,
topdir=self.archive_top_dir,
name=self.source_dir)
@ -141,36 +142,34 @@ class GitClone(Source):
@property
def source_dir(self):
if self.buildEnv.make_release:
if self.neutralEnv.make_release:
return "{}_release".format(self.git_dir)
else:
return self.git_dir
@property
def git_path(self):
return pj(self.buildEnv.source_dir, self.source_dir)
return pj(self.neutralEnv.source_dir, self.source_dir)
@property
def git_ref(self):
if self.buildEnv.make_release:
if self.neutralEnv.make_release:
return self.release_git_ref
else:
return self.base_git_ref
def _git_clone(self, context):
context.force_native_build = True
if os.path.exists(self.git_path):
raise SkipCommand()
command = "git clone --depth=1 --branch {} {} {}".format(
self.git_ref, self.git_remote, self.source_dir)
self.buildEnv.run_command(command, self.buildEnv.source_dir, context)
self.neutralEnv.run_command(command, self.neutralEnv.source_dir, context)
def _git_update(self, context):
context.force_native_build = True
command = "git fetch origin {}".format(
self.git_ref)
self.buildEnv.run_command(command, self.git_path, context)
self.buildEnv.run_command("git checkout "+self.git_ref, self.git_path, context)
self.neutralEnv.run_command(command, self.git_path, context)
self.neutralEnv.run_command("git checkout "+self.git_ref, self.git_path, context)
def prepare(self):
self.command('gitclone', self._git_clone)
@ -186,19 +185,17 @@ class SvnClone(Source):
@property
def svn_path(self):
return pj(self.buildEnv.source_dir, self.svn_dir)
return pj(self.neutralEnv.source_dir, self.svn_dir)
def _svn_checkout(self, context):
context.force_native_build = True
if os.path.exists(self.svn_path):
raise SkipCommand()
command = "svn checkout {} {}".format(self.svn_remote, self.svn_dir)
self.buildEnv.run_command(command, self.buildEnv.source_dir, context)
self.neutralEnv.run_command(command, self.neutralEnv.source_dir, context)
def _svn_update(self, context):
context.try_skip(self.svn_path)
context.force_native_build = True
self.buildEnv.run_command("svn update", self.svn_path, context)
self.neutralEnv.run_command("svn update", self.svn_path, context)
def prepare(self):
self.command('svncheckout', self._svn_checkout)
@ -259,6 +256,7 @@ class NoopBuilder(Builder):
class MakeBuilder(Builder):
configure_option_template = "{dep_options} {static_option} {env_option} --prefix {install_dir} --libdir {libdir}"
configure_option = ""
dynamic_configure_option = "--enable-shared --disable-static"
static_configure_option = "--enable-static --disable-shared"
@ -271,19 +269,21 @@ class MakeBuilder(Builder):
@property
def all_configure_option(self):
return "{} {} {}".format(
self.configure_option,
self.static_configure_option if self.buildEnv.platform_info.static else self.dynamic_configure_option,
self.buildEnv.configure_option if not self.target.force_native_build else "")
option = self.configure_option_template.format(
dep_options=self.configure_option,
static_option=self.static_configure_option if self.buildEnv.platform_info.static else self.dynamic_configure_option,
env_option=self.buildEnv.configure_option if not self.target.force_native_build else "",
install_dir=self.buildEnv.install_dir,
libdir=pj(self.buildEnv.install_dir, self.buildEnv.libprefix)
)
return option
def _configure(self, context):
context.try_skip(self.build_path)
command = "{configure_script} {configure_option} --prefix {install_dir} --libdir {libdir}"
command = "{configure_script} {configure_option}"
command = command.format(
configure_script=pj(self.source_path, self.configure_script),
configure_option=self.all_configure_option,
install_dir=self.buildEnv.install_dir,
libdir=pj(self.buildEnv.install_dir, self.buildEnv.libprefix)
configure_option=self.all_configure_option
)
env = Defaultdict(str, os.environ)
if self.buildEnv.platform_info.static:
@ -355,6 +355,12 @@ class MesonBuilder(Builder):
configure_option = ""
test_option = ""
def __init__(self, target):
super().__init__(target)
self.meson_command = self.buildEnv.neutralEnv.meson_command
self.mesontest_command = self.buildEnv.neutralEnv.mesontest_command
self.ninja_command = self.buildEnv.neutralEnv.ninja_command
@property
def library_type(self):
return 'static' if self.buildEnv.platform_info.static else 'shared'
@ -376,7 +382,7 @@ class MesonBuilder(Builder):
" --libdir={buildEnv.libprefix}"
" {cross_option}")
command = command.format(
command=self.buildEnv.meson_command,
command=self.meson_command,
library_type=self.library_type,
configure_option=configure_option,
build_path=self.build_path,
@ -386,7 +392,7 @@ class MesonBuilder(Builder):
self.buildEnv.run_command(command, self.source_path, context, cross_env_only=True)
def _compile(self, context):
command = "{} -v".format(self.buildEnv.ninja_command)
command = "{} -v".format(self.ninja_command)
self.buildEnv.run_command(command, self.build_path, context)
def _test(self, context):
@ -395,15 +401,15 @@ class MesonBuilder(Builder):
and not self.buildEnv.platform_info.static)
):
raise SkipCommand()
command = "{} --verbose {}".format(self.buildEnv.mesontest_command, self.test_option)
command = "{} --verbose {}".format(self.mesontest_command, self.test_option)
self.buildEnv.run_command(command, self.build_path, context)
def _install(self, context):
command = "{} -v install".format(self.buildEnv.ninja_command)
command = "{} -v install".format(self.ninja_command)
self.buildEnv.run_command(command, self.build_path, context)
def _make_dist(self, context):
command = "{} -v dist".format(self.buildEnv.ninja_command)
command = "{} -v dist".format(self.ninja_command)
self.buildEnv.run_command(command, self.build_path, context)

54
kiwixbuild/dependencies/ctpp2.py Normal file
View File

@ -0,0 +1,54 @@
from .base import (
Dependency,
ReleaseDownload,
CMakeBuilder)
from kiwixbuild.utils import Remotefile, pj
class CTPP2(Dependency):
name = "ctpp2"
class Source(ReleaseDownload):
name = "ctpp2"
source_dir = "ctpp2-2.8.3"
archive = Remotefile('ctpp2-2.8.3.tar.gz',
'a83ffd07817adb575295ef40fbf759892512e5a63059c520f9062d9ab8fb42fc')
patches = ["ctpp2_include.patch",
"ctpp2_no_src_modification.patch",
"ctpp2_fix-static-libname.patch",
"ctpp2_mingw32.patch",
"ctpp2_dll_export_VMExecutable.patch",
"ctpp2_win_install_lib_in_lib_dir.patch",
"ctpp2_iconv_support.patch",
"ctpp2_compile_ctpp2c_static.patch",
]
class Builder(CMakeBuilder):
@property
def configure_option(self):
libprefix = self.buildEnv.libprefix
options = "-DMD5_SUPPORT=OFF -DICONV_SUPPORT=OFF"
if libprefix.startswith('lib'):
libprefix = libprefix[3:]
options += " -DLIB_SUFFIX={}".format(libprefix)
return options
class CTPP2C(CTPP2):
name = "ctpp2c"
force_native_build = True
class Builder(CTPP2.Builder):
make_target = "ctpp2c"
@property
def build_path(self):
return super().build_path+"_native"
def _install(self, context):
context.try_skip(self.build_path)
command = "cp {ctpp2c}* {install_dir}".format(
ctpp2c=pj(self.build_path, 'ctpp2c'),
install_dir=pj(self.buildEnv.install_dir, 'bin')
)
self.buildEnv.run_command(command, self.build_path, context)

27
kiwixbuild/dependencies/gradle.py Normal file
View File

@ -0,0 +1,27 @@
from .base import (
Dependency,
ReleaseDownload,
Builder as BaseBuilder)
from kiwixbuild.utils import Remotefile, pj, copy_tree, add_execution_right
class Gradle(Dependency):
name = "Gradle"
class Source(ReleaseDownload):
archive = Remotefile('gradle-4.6-bin.zip',
'98bd5fd2b30e070517e03c51cbb32beee3e2ee1a84003a5a5d748996d4b1b915',
'https://services.gradle.org/distributions/gradle-4.6-bin.zip')
class Builder(BaseBuilder):
def build(self):
self.command('install', self._install)
def _install(self, context):
copy_tree(
pj(self.source_path, "bin"),
pj(self.buildEnv.install_dir, "bin"),
post_copy_function = add_execution_right)
copy_tree(
pj(self.source_path, "lib"),
pj(self.buildEnv.install_dir, "lib"))

23
kiwixbuild/dependencies/gumbo.py Normal file
View File

@ -0,0 +1,23 @@
from .base import (
Dependency,
ReleaseDownload,
MakeBuilder
)
from kiwixbuild.utils import Remotefile
class Gumbo(Dependency):
name = "gumbo"
class Source(ReleaseDownload):
archive = Remotefile('gumbo-0.10.1.tar.gz',
'28463053d44a5dfbc4b77bcf49c8cee119338ffa636cc17fc3378421d714efad',
'https://github.com/google/gumbo-parser/archive/v0.10.1.tar.gz')
def _post_prepare_script(self, context):
context.try_skip(self.extract_path)
command = "./autogen.sh"
self.buildEnv.run_command(command, self.extract_path, context)
Builder = MakeBuilder

57
kiwixbuild/dependencies/icu4c.py Normal file
View File

@ -0,0 +1,57 @@
from .base import (
Dependency,
SvnClone,
MakeBuilder
)
from kiwixbuild.utils import SkipCommand
class Icu(Dependency):
name = "icu4c"
class Source(SvnClone):
name = "icu4c"
svn_remote = "http://source.icu-project.org/repos/icu/tags/release-58-2/icu4c"
svn_dir = "icu4c"
patches = ["icu4c_fix_static_lib_name_mingw.patch",
"icu4c_android_elf64_st_info.patch",
"icu4c_custom_data.patch",
"icu4c_noxlocale.patch",
"icu4c_rpath.patch"]
class Builder(MakeBuilder):
subsource_dir = "source"
@property
def configure_option(self):
options = "--disable-samples --disable-tests --disable-extras --disable-dyload --enable-rpath"
if self.buildEnv.platform_info.build == 'android':
options += " --with-data-packaging=archive"
return options
class Icu_native(Icu):
name = "icu4c_native"
force_native_build = True
class Builder(Icu.Builder):
@property
def build_path(self):
return super().build_path+"_native"
def _install(self, context):
raise SkipCommand()
class Icu_cross_compile(Icu):
name = "icu4c_cross-compile"
dependencies = ['icu4c_native']
class Builder(Icu.Builder):
@property
def configure_option(self):
icu_native_dep = self.buildEnv.targetsDict['icu4c_native']
return super().configure_option + " --with-cross-build={} --disable-tools".format(icu_native_dep.builder.build_path)

40
kiwixbuild/dependencies/kiwix_android.py Normal file
View File

@ -0,0 +1,40 @@
import shutil, os
from .base import (
Dependency,
GitClone,
GradleBuilder)
from kiwixbuild.utils import pj
class KiwixAndroid(Dependency):
name = "kiwix-android"
dependencies = ["Gradle", "kiwix-lib"]
class Source(GitClone):
git_remote = "https://github.com/kiwix/kiwix-android"
git_dir = "kiwix-android"
class Builder(GradleBuilder):
def build(self):
if self.buildEnv.options.targets == 'kiwix-android-custom':
print("SKIP")
else:
super().build()
def _configure(self, context):
if not os.path.exists(self.build_path):
shutil.copytree(self.source_path, self.build_path, symlinks=True)
try:
shutil.rmtree(pj(self.build_path, 'kiwixlib', 'src', 'main'))
except FileNotFoundError:
pass
shutil.copytree(pj(self.buildEnv.install_dir, 'kiwix-lib'),
pj(self.build_path, 'kiwixlib', 'src', 'main'))
os.makedirs(
pj(self.build_path, 'app', 'src', 'main', 'assets', 'icu'),
exist_ok=True)
shutil.copy2(pj(self.buildEnv.install_dir, 'share', 'icu', '58.2',
'icudt58l.dat'),
pj(self.build_path, 'app', 'src', 'main', 'assets',
'icu', 'icudt58l.dat'))

110
kiwixbuild/dependencies/kiwix_custom_app.py Normal file
View File

@ -0,0 +1,110 @@
import shutil, os, json
from urllib.parse import urlparse
from .base import (
Dependency,
GitClone,
GradleBuilder)
from kiwixbuild.utils import Remotefile, pj, SkipCommand
class KiwixCustomApp(Dependency):
name = "kiwix-android-custom"
dependencies = ["kiwix-android", "kiwix-lib"]
def __init__(self, buildEnv):
super().__init__(buildEnv)
self.custom_name = buildEnv.options.android_custom_app
class Source(GitClone):
git_remote = "https://github.com/kiwix/kiwix-android-custom"
git_dir = "kiwix-android-custom"
class Builder(GradleBuilder):
@property
def gradle_target(self):
return "assemble{}".format(self.target.custom_name)
@property
def gradle_option(self):
template = ("-i -P customDir={customDir}"
" -P zim_file_size={zim_size}"
" -P version_code={version_code}"
" -P version_name={version_name}"
" -P content_version_code={content_version_code}")
return template.format(
customDir=pj(self.build_path, 'custom'),
zim_size=self._get_zim_size(),
version_code=os.environ['VERSION_CODE'],
version_name=os.environ['VERSION_NAME'],
content_version_code=os.environ['CONTENT_VERSION_CODE'])
@property
def build_path(self):
return pj(self.buildEnv.build_dir, "{}_{}".format(self.target.full_name, self.target.custom_name))
@property
def custom_build_path(self):
return pj(self.build_path, 'custom', self.target.custom_name)
def _get_zim_size(self):
try:
zim_size = self.buildEnv.options.zim_file_size
except AttributeError:
with open(pj(self.source_path, self.target.custom_name, 'info.json')) as f:
app_info = json.load(f)
zim_size = os.path.getsize(pj(self.custom_build_path, app_info['zim_file']))
return zim_size
def build(self):
self.command('configure', self._configure)
self.command('download_zim', self._download_zim)
self.command('compile', self._compile)
def _download_zim(self, context):
zim_url = self.buildEnv.options.zim_file_url
if zim_url is None:
raise SkipCommand()
with open(pj(self.source_path, self.target.custom_name, 'info.json')) as f:
app_info = json.load(f)
zim_url = app_info.get('zim_url', zim_url)
out_filename = urlparse(zim_url).path
out_filename = os.path.basename(out_filename)
zim_file = Remotefile(out_filename, '', zim_url)
self.buildEnv.download(zim_file)
shutil.copy(pj(self.buildEnv.archive_dir, out_filename),
pj(self.custom_build_path, app_info['zim_file']))
def _configure(self, context):
# Copy kiwix-android in build dir.
kiwix_android_dep = self.buildEnv.targetsDict['kiwix-android']
if not os.path.exists(self.build_path):
shutil.copytree(kiwix_android_dep.source_path, self.build_path)
# Copy kiwix-lib application in build dir
try:
shutil.rmtree(pj(self.build_path, 'kiwixlib', 'src', 'main'))
except FileNotFoundError:
pass
shutil.copytree(pj(self.buildEnv.install_dir, 'kiwix-lib'),
pj(self.build_path, 'kiwixlib', 'src', 'main'))
os.makedirs(
pj(self.build_path, 'app', 'src', 'main', 'assets', 'icu'),
exist_ok=True)
shutil.copy2(pj(self.buildEnv.install_dir, 'share', 'icu', '58.2',
'icudt58l.dat'),
pj(self.build_path, 'app', 'src', 'main', 'assets',
'icu', 'icudt58l.dat'))
# Generate custom directory
try:
shutil.rmtree(pj(self.build_path, 'custom'))
except FileNotFoundError:
pass
os.makedirs(pj(self.build_path, 'custom'))
command = "./gen-custom-android-directory.py {custom_name} --output-dir {custom_dir}"
command = command.format(
custom_name=self.target.custom_name,
custom_dir=pj(self.build_path, 'custom', self.target.custom_name)
)
self.buildEnv.run_command(command, self.source_path, context)

36
kiwixbuild/dependencies/kiwix_lib.py Normal file
View File

@ -0,0 +1,36 @@
from .base import (
Dependency,
GitClone,
MesonBuilder)
class Kiwixlib(Dependency):
name = "kiwix-lib"
@property
def dependencies(self):
base_dependencies = ["pugixml", "libzim", "zlib", "lzma", "libaria2"]
if ( self.buildEnv.platform_info.build != 'android'
and self.buildEnv.distname != 'Darwin'):
base_dependencies += ['ctpp2c', 'ctpp2']
if self.buildEnv.platform_info.build != 'native':
return base_dependencies + ["icu4c_cross-compile"]
else:
return base_dependencies + ["icu4c"]
class Source(GitClone):
git_remote = "https://github.com/kiwix/kiwix-lib.git"
git_dir = "kiwix-lib"
class Builder(MesonBuilder):
@property
def configure_option(self):
base_option = "-Dctpp2-install-prefix={buildEnv.install_dir}"
if self.buildEnv.platform_info.build == 'android':
base_option += ' -Dandroid=true'
return base_option
@property
def library_type(self):
if self.buildEnv.platform_info.build == 'android':
return 'shared'
return super().library_type

19
kiwixbuild/dependencies/kiwix_tools.py Normal file
View File

@ -0,0 +1,19 @@
from .base import (
Dependency,
GitClone,
MesonBuilder)
class KiwixTools(Dependency):
name = "kiwix-tools"
dependencies = ["kiwix-lib", "libmicrohttpd", "zlib"]
class Source(GitClone):
git_remote = "https://github.com/kiwix/kiwix-tools.git"
git_dir = "kiwix-tools"
class Builder(MesonBuilder):
@property
def configure_option(self):
if self.buildEnv.platform_info.static:
return "-Dstatic-linkage=true"
return ""

26
kiwixbuild/dependencies/libaria2.py Normal file
View File

@ -0,0 +1,26 @@
from .base import (
Dependency,
ReleaseDownload,
MakeBuilder
)
from kiwixbuild.utils import Remotefile
class Aria2(Dependency):
name = "libaria2"
dependencies = ['zlib']
class Source(ReleaseDownload):
archive = Remotefile('libaria2-1.33.1.tar.gz',
'0616f11ef3ddd0c74be74ea2536be62ce168b99e3d6a35dea9d166b9cab9fbd1',
'https://github.com/aria2/aria2/archive/release-1.33.1.tar.gz')
patches = ["libaria2_android.patch"]
def _post_prepare_script(self, context):
context.try_skip(self.extract_path)
command = "autoreconf -i"
self.neutralEnv.run_command(command, self.extract_path, context)
class Builder(MakeBuilder):
configure_option = "--enable-libaria2 --disable-ssl --disable-bittorent --disable-metalink --without-sqlite3 --without-libxml2 --without-libexpat"

53
kiwixbuild/dependencies/libmagic.py Normal file
View File

@ -0,0 +1,53 @@
import os
from .base import (
Dependency,
ReleaseDownload,
MakeBuilder,
)
from kiwixbuild.utils import Remotefile, pj, Defaultdict, SkipCommand
class LibMagicBase(Dependency):
name = "libmagic"
class Source(ReleaseDownload):
name = "libmagic"
source_dir = "libmagic"
archive = Remotefile('file-5.33.tar.gz',
'1c52c8c3d271cd898d5511c36a68059cda94036111ab293f01f83c3525b737c6',
'https://fossies.org/linux/misc/file-5.33.tar.gz')
Builder = MakeBuilder
class LibMagic_native(LibMagicBase):
name = "libmagic_native"
force_native_build = True
class Builder(LibMagicBase.Builder):
static_configure_option = dynamic_configure_option = "--disable-shared --enable-static"
@property
def build_path(self):
return super().build_path+"_native"
def _install(self, context):
raise SkipCommand()
class LibMagic_cross_compile(LibMagicBase):
name = "libmagic_cross-compile"
dependencies = ['libmagic_native']
class Builder(LibMagicBase.Builder):
def _compile(self, context):
context.try_skip(self.build_path)
command = "make -j4 {make_target} {make_option}".format(
make_target=self.make_target,
make_option=self.make_option
)
libmagic_native_dep = self.buildEnv.targetsDict['libmagic_native']
env = Defaultdict(str, os.environ)
env['PATH'] = ':'.join([pj(libmagic_native_dep.builder.build_path, 'src'), env['PATH']])
self.buildEnv.run_command(command, self.build_path, context, env=env)

17
kiwixbuild/dependencies/libmicrohttpd.py Normal file
View File

@ -0,0 +1,17 @@
from .base import (
Dependency,
ReleaseDownload,
MakeBuilder)
from kiwixbuild.utils import Remotefile
class MicroHttpd(Dependency):
name = "libmicrohttpd"
class Source(ReleaseDownload):
archive = Remotefile('libmicrohttpd-0.9.46.tar.gz',
'06dbd2654f390fa1e8196fe063fc1449a6c2ed65a38199a49bf29ad8a93b8979',
'http://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.46.tar.gz')
class Builder(MakeBuilder):
configure_option = "--disable-https --without-libgcrypt --without-libcurl"

22
kiwixbuild/dependencies/libzim.py Normal file
View File

@ -0,0 +1,22 @@
from .base import (
Dependency,
GitClone,
MesonBuilder)
class Libzim(Dependency):
name = "libzim"
@property
def dependencies(self):
base_dependencies = ['zlib', 'lzma', 'xapian-core']
if self.buildEnv.platform_info.build != 'native':
return base_dependencies + ["icu4c_cross-compile"]
else:
return base_dependencies + ["icu4c"]
class Source(GitClone):
git_remote = "https://github.com/openzim/libzim.git"
git_dir = "libzim"
class Builder(MesonBuilder):
test_option = "-t 8"

19
kiwixbuild/dependencies/lzma.py Normal file
View File

@ -0,0 +1,19 @@
from .base import (
Dependency,
ReleaseDownload,
MakeBuilder)
from kiwixbuild.utils import Remotefile
class lzma(Dependency):
name = 'lzma'
class Source(ReleaseDownload):
archive = Remotefile('xz-5.2.3.tar.bz2',
'fd9ca16de1052aac899ad3495ad20dfa906c27b4a5070102a2ec35ca3a4740c1',
'https://tukaani.org/xz/xz-5.2.3.tar.bz2')
class Builder(MakeBuilder):
@property
def configure_option(self):
return "--disable-assembler --disable-xz --disable-xzdec"

16
kiwixbuild/dependencies/pugixml.py Normal file
View File

@ -0,0 +1,16 @@
from .base import (
Dependency,
ReleaseDownload,
MesonBuilder)
from kiwixbuild.utils import Remotefile
class Pugixml(Dependency):
name = "pugixml"
class Source(ReleaseDownload):
archive = Remotefile('pugixml-1.2.tar.gz',
'0f422dad86da0a2e56a37fb2a88376aae6e931f22cc8b956978460c9db06136b')
patches = ["pugixml_meson.patch"]
Builder = MesonBuilder

25
kiwixbuild/dependencies/uuid.py Normal file
View File

@ -0,0 +1,25 @@
from .base import (
Dependency,
ReleaseDownload,
MakeBuilder
)
from kiwixbuild.utils import Remotefile
class UUID(Dependency):
name = 'uuid'
class Source(ReleaseDownload):
archive = Remotefile('e2fsprogs-libs-1.43.4.tar.gz',
'eed4516325768255c9745e7b82c9d7d0393abce302520a5b2cde693204b0e419',
'https://www.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v1.43.4/e2fsprogs-libs-1.43.4.tar.gz')
extract_dir = 'e2fsprogs-libs-1.43.4'
class Builder(MakeBuilder):
configure_option = ("--enable-libuuid --disable-debugfs --disable-imager --disable-resizer --disable-defrag --enable-fsck"
" --disable-uuidd")
configure_env = {'_format_CFLAGS': "{env.CFLAGS} -fPIC"}
static_configure_option = dynamic_configure_option = ""
make_target = 'libs'
make_install_target = 'install-libs'

28
kiwixbuild/dependencies/xapian.py Normal file
View File

@ -0,0 +1,28 @@
from .base import (
Dependency,
ReleaseDownload,
MakeBuilder
)
from kiwixbuild.utils import Remotefile
class Xapian(Dependency):
name = "xapian-core"
class Source(ReleaseDownload):
archive = Remotefile('xapian-core-1.4.5.tar.xz',
'85b5f952de9df925fd13e00f6e82484162fd506d38745613a50b0a2064c6b02b')
class Builder(MakeBuilder):
configure_option = "--disable-sse --disable-backend-inmemory --disable-documentation"
configure_env = {'_format_LDFLAGS': "-L{buildEnv.install_dir}/{buildEnv.libprefix}",
'_format_CXXFLAGS': "-I{buildEnv.install_dir}/include"}
@property
def dependencies(self):
deps = ['zlib', 'lzma']
if (self.buildEnv.platform_info.build == 'win32'
or self.buildEnv.distname == 'Darwin'):
return deps
return deps + ['uuid']

19
kiwixbuild/dependencies/zim_tools.py Normal file
View File

@ -0,0 +1,19 @@
from .base import (
Dependency,
GitClone,
MesonBuilder)
class ZimTools(Dependency):
name = "zim-tools"
dependencies = ['libzim']
class Source(GitClone):
git_remote = "https://github.com/openzim/zim-tools.git"
git_dir = "zim-tools"
class Builder(MesonBuilder):
@property
def configure_option(self):
if self.buildEnv.platform_info.static:
return "-Dstatic-linkage=true"
return ""

28
kiwixbuild/dependencies/zimwriterfs.py Normal file
View File

@ -0,0 +1,28 @@
from .base import (
Dependency,
GitClone,
MesonBuilder)
class Zimwriterfs(Dependency):
name = "zimwriterfs"
@property
def dependencies(self):
base_dependencies = ['libzim', 'zlib', 'xapian-core', 'gumbo']
if self.buildEnv.platform_info.build != 'native':
return base_dependencies + ["icu4c_cross-compile", "libmagic_cross-compile"]
else:
return base_dependencies + ["icu4c", "libmagic"]
class Source(GitClone):
git_remote = "https://github.com/openzim/zimwriterfs.git"
git_dir = "zimwriterfs"
release_git_ref = "1.1"
class Builder(MesonBuilder):
@property
def configure_option(self):
base_option = "-Dmagic-install-prefix={buildEnv.install_dir}"
if self.buildEnv.platform_info.static:
base_option += " -Dstatic-linkage=true"
return base_option

45
kiwixbuild/dependencies/zlib.py Normal file
View File

@ -0,0 +1,45 @@
import shutil
from .base import (
Dependency,
ReleaseDownload,
MakeBuilder)
from kiwixbuild.utils import Remotefile, pj, SkipCommand
class zlib(Dependency):
name = 'zlib'
class Source(ReleaseDownload):
archive = Remotefile('zlib-1.2.8.tar.gz',
'36658cb768a54c1d4dec43c3116c27ed893e88b02ecfcb44f2166f9c0b7f2a0d')
patches = ['zlib_std_libname.patch']
class Builder(MakeBuilder):
dynamic_configure_option = "--shared"
static_configure_option = "--static"
configure_option_template = "{dep_options} {static_option} --prefix {install_dir} --libdir {libdir}"
def _pre_build_script(self, context):
context.try_skip(self.build_path)
shutil.copytree(self.source_path, self.build_path)
def _configure(self, context):
if self.buildEnv.platform_info.build == 'win32':
raise SkipCommand()
return super()._configure(context)
@property
def make_option(self):
if self.buildEnv.platform_info.build == 'win32':
return "--makefile win32/Makefile.gcc PREFIX={host}- SHARED_MODE={static} INCLUDE_PATH={include_path} LIBRARY_PATH={library_path} BINARY_PATH={binary_path}".format(
host='i686-w64-mingw32',
static="0" if self.buildEnv.platform_info.static else "1",
include_path=pj(self.buildEnv.install_dir, 'include'),
library_path=pj(self.buildEnv.install_dir, self.buildEnv.libprefix),
binary_path=pj(self.buildEnv.install_dir, 'bin'),
)
return ""

103
kiwixbuild/packages.py Normal file
View File

@ -0,0 +1,103 @@
_fedora_common = ['automake', 'libtool', 'cmake', 'git', 'subversion', 'ccache', 'pkgconfig', 'gcc-c++', 'gettext-devel']
_debian_common = ['automake', 'libtool', 'cmake', 'git', 'subversion', 'ccache', 'pkg-config', 'gcc', 'autopoint']
PACKAGE_NAME_MAPPERS = {
'fedora_native_dyn': {
'COMMON': _fedora_common,
'uuid': ['libuuid-devel'],
'xapian-core': None, # Not the right version on fedora 25
'ctpp2': None,
'pugixml': None, # ['pugixml-devel'] but package doesn't provide pkg-config file
'libmicrohttpd': ['libmicrohttpd-devel'],
'zlib': ['zlib-devel'],
'lzma': ['xz-devel'],
'icu4c': None,
'zimlib': None,
'file' : ['file-devel'],
'gumbo' : ['gumbo-parser-devel'],
},
'fedora_native_static': {
'COMMON': _fedora_common + ['glibc-static', 'libstdc++-static'],
'zlib': ['zlib-devel', 'zlib-static'],
'lzma': ['xz-devel', 'xz-static']
# Either there is no packages, or no static or too old
},
'fedora_i586_dyn': {
'COMMON': _fedora_common + ['glibc-devel.i686', 'libstdc++-devel.i686'],
},
'fedora_i586_static': {
'COMMON': _fedora_common + ['glibc-devel.i686'],
},
'fedora_win32_dyn': {
'COMMON': _fedora_common + ['mingw32-gcc-c++', 'mingw32-bzip2', 'mingw32-win-iconv', 'mingw32-winpthreads', 'wine'],
'zlib': ['mingw32-zlib'],
'lzma': ['mingw32-xz-libs'],
'libmicrohttpd': ['mingw32-libmicrohttpd'],
},
'fedora_win32_static': {
'COMMON': _fedora_common + ['mingw32-gcc-c++', 'mingw32-bzip2-static', 'mingw32-win-iconv-static', 'mingw32-winpthreads-static', 'wine'],
'zlib': ['mingw32-zlib-static'],
'lzma': ['mingw32-xz-libs-static'],
'libmicrohttpd': None, # ['mingw32-libmicrohttpd-static'] packaging dependecy seems buggy, and some static lib are name libfoo.dll.a and
# gcc cannot found them.
},
'fedora_armhf_static': {
'COMMON': _fedora_common
},
'fedora_armhf_dyn': {
'COMMON': _fedora_common
},
'fedora_android': {
'COMMON': _fedora_common + ['java-1.8.0-openjdk-devel']
},
'debian_native_dyn': {
'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'],
'zlib': ['zlib1g-dev'],
'uuid': ['uuid-dev'],
'ctpp2': ['libctpp2-dev'],
'ctpp2c': ['ctpp2-utils'],
'libmicrohttpd': ['libmicrohttpd-dev', 'ccache']
},
'debian_native_static': {
'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'],
'zlib': ['zlib1g-dev'],
'uuid': ['uuid-dev'],
'ctpp2': ['libctpp2-dev'],
'ctpp2c': ['ctpp2-utils'],
},
'debian_i586_dyn': {
'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib'],
},
'debian_i586_static': {
'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib'],
},
'debian_win32_dyn': {
'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'],
'ctpp2c': ['ctpp2-utils'],
},
'debian_win32_static': {
'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'],
'ctpp2c': ['ctpp2-utils'],
},
'debian_armhf_static': {
'COMMON': _debian_common,
'ctpp2c': ['ctpp2-utils'],
},
'debian_armhf_dyn': {
'COMMON': _debian_common,
'ctpp2c': ['ctpp2-utils'],
},
'debian_android': {
'COMMON': _debian_common + ['default-jdk'],
'ctpp2c': ['ctpp2-utils'],
},
'Darwin_native_dyn': {
'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'],
'file': ['libmagic']
},
'Darwin_iOS': {
'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'],
'file': ['libmagic']
},
}

11
kiwixbuild/platforms/__init__.py Normal file
View File

@ -0,0 +1,11 @@
from .base import *
from . import (
android,
armhf,
i586,
ios,
native,
win32
)

44
kiwixbuild/platforms/android.py Normal file
View File

@ -0,0 +1,44 @@
from .base import PlatformInfo
class AndroidPlatformInfo(PlatformInfo):
__arch_infos = {
'arm' : ('arm-linux-androideabi', 'arm', 'armeabi'),
'arm64': ('aarch64-linux-android', 'aarch64', 'arm64-v8a'),
'mips': ('mipsel-linux-android', 'mipsel', 'mips'),
'mips64': ('mips64el-linux-android', 'mips64el', 'mips64'),
'x86': ('i686-linux-android', 'i686', 'x86'),
'x86_64': ('x86_64-linux-android', 'x86_64', 'x86_64'),
}
def __init__(self, name, arch):
super().__init__(name, 'android', True, ['android_ndk', 'android_sdk'],
hosts=['fedora', 'debian'])
self.arch = arch
self.arch_full, self.cpu, self.abi = self.__arch_infos[arch]
def __str__(self):
return "android"
def get_cross_config(self):
return {
'extra_libs': [],
'extra_cflags': [],
'host_machine': {
'system': 'Android',
'lsystem': 'android',
'cpu_family': self.arch,
'cpu': self.cpu,
'endian': 'little',
'abi': self.abi
},
}
AndroidPlatformInfo('android_arm', 'arm')
AndroidPlatformInfo('android_arm64', 'arm64')
AndroidPlatformInfo('android_mips', 'mips')
AndroidPlatformInfo('android_mips64', 'mips64')
AndroidPlatformInfo('android_x86', 'x86')
AndroidPlatformInfo('android_x86_64', 'x86_64')

24
kiwixbuild/platforms/armhf.py Normal file
View File

@ -0,0 +1,24 @@
from .base import PlatformInfo
class ArmhfPlatformInfo(PlatformInfo):
def __init__(self, name, static):
super().__init__(name, 'armhf', static, ['armhf_toolchain'], ['fedora', 'debian'])
def get_cross_config(self):
return {
'extra_libs': [],
'extra_cflags': [],
'host_machine': {
'system': 'linux',
'lsystem': 'linux',
'cpu_family': 'arm',
'cpu': 'armhf',
'endian': 'little',
'abi': ''
}
}
ArmhfPlatformInfo('armhf_dyn', False)
ArmhfPlatformInfo('armhf_static', True)

21
kiwixbuild/platforms/base.py Normal file
View File

@ -0,0 +1,21 @@
class PlatformInfo:
all_platforms = {}
def __init__(self, name, build, static, toolchains, hosts=None):
self.all_platforms[name] = self
self.build = build
self.static = static
self.toolchains = toolchains
self.compatible_hosts = hosts
def __str__(self):
return "{}_{}".format(self.build, 'static' if self.static else 'dyn')
def set_env(self, env):
pass
def get_bind_dir(self):
return []

25
kiwixbuild/platforms/i586.py Normal file
View File

@ -0,0 +1,25 @@
from .base import PlatformInfo
class I586PlatformInfo(PlatformInfo):
def __init__(self, name, static):
super().__init__(name, 'i586', static, ['linux_i586_toolchain'], ['fedora', 'debian'])
def get_cross_config(self):
return {
'extra_libs': ['-m32', '-march=i586', '-mno-sse'],
'extra_cflags': ['-m32', '-march=i586', '-mno-sse'],
'host_machine': {
'system': 'linux',
'lsystem': 'linux',
'cpu_family': 'x86',
'cpu': 'i586',
'endian': 'little',
'abi': ''
}
}
I586PlatformInfo('i586_dyn', False)
I586PlatformInfo('i586_static', True)

59
kiwixbuild/platforms/ios.py Normal file
View File

@ -0,0 +1,59 @@
from .base import PlatformInfo
import subprocess
class iOSPlatformInfo(PlatformInfo):
__arch_infos = {
'armv7': ('arm-apple-darwin', 'armv7', 'iphoneos'),
'arm64': ('arm-apple-darwin', 'arm64', 'iphoneos'),
'i386': ('', 'i386', 'iphonesimulator'),
'x86_64': ('', 'x86_64', 'iphonesimulator'),
}
def __init__(self, name, arch):
super().__init__(name, 'iOS', True, ['iOS_sdk'],
hosts=['Darwin'])
self.arch = arch
self.arch_full, self.cpu, self.sdk_name = self.__arch_infos[arch]
self._root_path = None
@property
def root_path(self):
if self._root_path is None:
command = "xcodebuild -version -sdk {} | grep -E '^Path' | sed 's/Path: //'".format(self.sdk_name)
self._root_path = subprocess.check_output(command, shell=True)[:-1].decode()
return self._root_path
def __str__(self):
return "iOS"
def get_cross_config(self):
return {
'root_path': self.root_path,
'extra_libs': ['-fembed-bitcode', '-isysroot', self.root_path, '-arch', self.arch, '-miphoneos-version-min=9.0', '-stdlib=libc++'],
'extra_cflags': ['-fembed-bitcode', '-isysroot', self.root_path, '-arch', self.arch, '-miphoneos-version-min=9.0', '-stdlib=libc++'],
'host_machine': {
'system': 'Darwin',
'lsystem': 'darwin',
'cpu_family': self.arch,
'cpu': self.cpu,
'endian': '',
'abi': ''
},
}
def set_env(self, env):
env['CFLAGS'] = " -fembed-bitcode -isysroot {SDKROOT} -arch {arch} -miphoneos-version-min=9.0 ".format(SDKROOT=self.root_path, arch=self.arch) + env['CFLAGS']
env['CXXFLAGS'] = env['CFLAGS'] + " -stdlib=libc++ -std=c++11 "+env['CXXFLAGS']
env['LDFLAGS'] = " -arch {arch} -isysroot {SDKROOT} ".format(SDKROOT=self.root_path, arch=self.arch)
env['MACOSX_DEPLOYMENT_TARGET'] = "10.7"
def get_bin_dir(self):
return [pj(self.root_path, 'bin')]
iOSPlatformInfo('iOS_armv7', 'armv7')
iOSPlatformInfo('iOS_arm64', 'arm64')
iOSPlatformInfo('iOS_i386', 'i386')
iOSPlatformInfo('iOS_x86_64', 'x86_64')

13
kiwixbuild/platforms/native.py Normal file
View File

@ -0,0 +1,13 @@
from .base import PlatformInfo
class NativePlatformInfo(PlatformInfo):
def __init__(self, name, static, hosts):
super().__init__(name, 'native', static, [], hosts)
def get_cross_config(self):
return {}
NativePlatformInfo('native_dyn', False, ['fedora', 'debian', 'Darwin'])
NativePlatformInfo('native_static', True, ['fedora', 'debian'])

24
kiwixbuild/platforms/win32.py Normal file
View File

@ -0,0 +1,24 @@
from .base import PlatformInfo
class Win32PlatformInfo(PlatformInfo):
extra_libs = ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4', '-lmsvcr90', '-liphlpapi']
def __init__(self, name, static):
super().__init__(name, 'win32', static, ['mingw32_toolchain'], ['fedora', 'debian'])
def get_cross_config(self):
return {
'extra_libs': self.extra_libs,
'extra_cflags': ['-DWIN32'],
'host_machine': {
'system': 'Windows',
'lsystem': 'windows',
'cpu_family': 'x86',
'cpu': 'i686',
'endian': 'little',
'abi': ''
}
}
Win32PlatformInfo('win32_dyn', False)
Win32PlatformInfo('win32_static', True)

2
kiwixbuild/templates/cmake_ios_cross_file.txt
View File

@ -3,5 +3,5 @@ SET(CMAKE_SYSTEM_NAME {host_machine[system]})
SET(CMAKE_C_COMPILER "{toolchain.binaries[CC]}")
SET(CMAKE_CXX_COMPILER "{toolchain.binaries[CXX]}")
SET(CMAKE_FIND_ROOT_PATH {toolchain.root_path})
SET(CMAKE_FIND_ROOT_PATH {root_path})

3
kiwixbuild/toolchains/android_ndk.py
View File

@ -1,12 +1,13 @@
import os
from .base_toolchain import Toolchain
from kiwixbuild.dependency_utils import ReleaseDownload, Builder
from kiwixbuild.dependencies import ReleaseDownload, Builder
from kiwixbuild.utils import Remotefile, add_execution_right
pj = os.path.join
class android_ndk(Toolchain):
neutral = False
name = 'android-ndk'
version = 'r13b'
gccver = '4.9.x'

5
kiwixbuild/toolchains/android_sdk.py
View File

@ -2,7 +2,7 @@ import os
import shutil
from .base_toolchain import Toolchain
from kiwixbuild.dependency_utils import ReleaseDownload, Builder
from kiwixbuild.dependencies import ReleaseDownload, Builder
from kiwixbuild.utils import Remotefile
pj = os.path.join
@ -50,8 +50,5 @@ class android_sdk(Toolchain):
self.command('build_platform', self._build_platform)
self.command('fix_licenses', self._fix_licenses)
def get_bin_dir(self):
return []
def set_env(self, env):
env['ANDROID_HOME'] = self.builder.install_path

8
kiwixbuild/toolchains/armhf.py
View File

@ -2,8 +2,8 @@ import os
import subprocess
from .base_toolchain import Toolchain
from kiwixbuild.dependency_utils import GitClone
from kiwixbuild.utils import Remotefile, which
from kiwixbuild.dependencies import GitClone
from kiwixbuild.utils import which
pj = os.path.join
class armhf_toolchain(Toolchain):
@ -52,13 +52,11 @@ class armhf_toolchain(Toolchain):
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
env['CFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS']
env['CXXFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
env['LIBS'] = " ".join(self.buildEnv.cross_config['extra_libs']) + " " +env['LIBS']
env['QEMU_LD_PREFIX'] = pj(self.root_path, "arm-linux-gnueabihf", "libc")
env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format(
':'.join([
pj(self.root_path, "arm-linux-gnueabihf", "lib"),
pj(self.buildEnv.install_dir, 'lib'),
pj(self.buildEnv.install_dir, self.buildEnv.libprefix)
env['LD_LIBRARY_PATH']
]))
def set_compiler(self, env):

13
kiwixbuild/toolchains/base_toolchain.py
View File

@ -14,6 +14,7 @@ class _MetaToolchain(type):
class Toolchain(metaclass=_MetaToolchain):
neutral = True
all_toolchains = {}
configure_option = ""
cmake_option = ""
@ -21,8 +22,9 @@ class Toolchain(metaclass=_MetaToolchain):
Builder = None
Source = None
def __init__(self, buildEnv):
self.buildEnv = buildEnv
def __init__(self, neutralEnv):
self.neutralEnv = neutralEnv
self.buildEnv = neutralEnv
self.source = self.Source(self) if self.Source else None
self.builder = self.Builder(self) if self.Builder else None
@ -34,11 +36,11 @@ class Toolchain(metaclass=_MetaToolchain):
@property
def source_path(self):
return pj(self.buildEnv.source_dir, self.source.source_dir)
return pj(self.neutralEnv.source_dir, self.source.source_dir)
@property
def _log_dir(self):
return self.buildEnv.log_dir
return self.neutralEnv.log_dir
def set_env(self, env):
pass
@ -46,6 +48,9 @@ class Toolchain(metaclass=_MetaToolchain):
def set_compiler(self, env):
pass
def get_bin_dir(self):
return []
def command(self, name, function, *args):
print(" {} {} : ".format(name, self.name), end="", flush=True)
log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))

21
kiwixbuild/toolchains/ios.py
View File

@ -1,4 +1,23 @@
from .base_toolchain import Toolchain
from kiwixbuild.utils import pj, xrun_find
class iOS_sdk(Toolchain):
pass
@property
def binaries(self):
return {
'CC': xrun_find('clang'),
'CXX': xrun_find('clang++'),
'AR': '/usr/bin/ar',
'STRIP': '/usr/bin/strip',
'RANLIB': '/usr/bin/ranlib',
'LD': '/usr/bin/ld',
}
@property
def configure_option(self):
return '--host=arm-apple-darwin'
def set_compiler(self, env):
env['CC'] = self.binaries['CC']
env['CXX'] = self.binaries['CXX']

5
kiwixbuild/toolchains/linux_i586.py
View File

@ -1,9 +1,8 @@
import os
import subprocess
from .base_toolchain import Toolchain
from kiwixbuild.dependency_utils import GitClone
from kiwixbuild.utils import Remotefile, which
from kiwixbuild.dependencies import GitClone
from kiwixbuild.utils import which
pj = os.path.join
class linux_i586_toolchain(Toolchain):

18
kiwixbuild/toolchains/mingw32.py
View File

@ -9,10 +9,15 @@ pj = os.path.join
class mingw32_toolchain(Toolchain):
name = 'mingw32'
arch_full = 'i686-w64-mingw32'
extra_libs = ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4', '-lmsvcr90', '-liphlpapi']
@property
def root_path(self):
return self.buildEnv.cross_config['root_path']
root_paths = {
'fedora': '/usr/i686-w64-mingw32/sys-root/mingw',
'debian': '/usr/i686-w64-mingw32'
}
return root_paths[self.neutralEnv.distname]
@property
def binaries(self):
@ -34,18 +39,17 @@ class mingw32_toolchain(Toolchain):
else:
return "exec_wrapper = 'wine'"
@property
def configure_option(self):
return '--host={}'.format(self.arch_full)
def set_compiler(self, env):
for k, v in self.binaries.items():
env[k] = v
def get_bin_dir(self):
return [pj(self.root_path, 'bin')]
def set_env(self, env):
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
env['LIBS'] = " ".join(self.buildEnv.cross_config['extra_libs']) + " " +env['LIBS']
def set_compiler(self, env):
for k, v in self.binaries.items():
env[k] = v
env['LIBS'] = " ".join(self.extra_libs) + " " +env['LIBS']

8
kiwixbuild/utils.py
View File

@ -4,7 +4,8 @@ import tarfile, zipfile
import tempfile
import shutil
import os, stat, sys
import urllib
import urllib.request
import urllib.error
import ssl
import subprocess
from collections import namedtuple, defaultdict
@ -21,6 +22,11 @@ def which(name):
output = subprocess.check_output(command, shell=True)
return output[:-1].decode()
def xrun_find(name):
command = "xcrun -find {}".format(name)
output = subprocess.check_output(command, shell=True)
return output[:-1].decode()
def setup_print_progress(print_progress):
global g_print_progress

0
kiwixbuild/dependency_versions.py → kiwixbuild/versions.py
View File

18
travis/compile_all.py
View File

@ -11,7 +11,10 @@ import re
from urllib.request import urlretrieve
from urllib.error import URLError
from kiwixbuild import dependency_versions
from kiwixbuild.versions import (
main_project_versions,
base_deps_versions,
base_deps_meta_version)
PLATFORM = environ['PLATFORM']
TRAVIS_OS_NAME = environ['TRAVIS_OS_NAME']
@ -81,7 +84,7 @@ def make_archive(project, platform):
base_bin_dir = BASE_DIR/'INSTALL'/'bin'
if make_release:
postfix = dependency_versions.main_project_versions[project]
postfix = main_project_versions[project]
if project in ('kiwix-lib', 'kiwix-tools'):
archive_dir = RELEASE_KIWIX_ARCHIVES_DIR/project
else:
@ -137,9 +140,9 @@ def make_deps_archive(target, full=False):
files_to_archive += BASE_DIR.glob('*/.*_ok')
files_to_archive += SOURCE_DIR.glob('*/.*_ok')
files_to_archive += [SOURCE_DIR/'pugixml-{}'.format(
dependency_versions.base_deps_versions['pugixml'])]
base_deps_versions['pugixml'])]
files_to_archive += [BASE_DIR/'pugixml-{}'.format(
dependency_versions.base_deps_versions['pugixml'])]
base_deps_versions['pugixml'])]
if (TOOLCHAINS_DIR).exists():
files_to_archive.append(TOOLCHAINS_DIR)
relative_path = HOME
@ -170,9 +173,8 @@ for p in (NIGHTLY_KIWIX_ARCHIVES_DIR,
make_release = re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", environ.get('TRAVIS_TAG', '')) is not None
# The first thing we need to do is to (potentially) download already compiled base dependencies.
BASE_DEP_VERSION = dependency_versions.base_deps_meta_version
base_dep_archive_name = "base_deps_{}_{}_{}.tar.gz".format(
TRAVIS_OS_NAME, PLATFORM, BASE_DEP_VERSION)
TRAVIS_OS_NAME, PLATFORM, base_deps_meta_version)
print_message("Getting archive {}", base_dep_archive_name)
try:
@ -263,11 +265,11 @@ if make_release and PLATFORM == 'native_dyn':
if target == 'zimwriterfs':
in_file = BASE_DIR/target/'{}-{}.tar.gz'.format(
target,
dependency_versions.main_project_versions[target])
main_project_versions[target])
else:
in_file = BASE_DIR/target/'meson-dist'/'{}-{}.tar.xz'.format(
target,
dependency_versions.main_project_versions[target])
main_project_versions[target])
shutil.copy(str(in_file), str(out_dir/target))
elif PLATFORM == 'native_static':