Create a real module kiwixbuild and make it installable. Split files.
Make kiwix-build installable using pip. Module is now called `kiwixbuild` because `kiwix-build` is not a valid python identifier. Also split toolchains in separated directory.
This commit is contained in:
Matthieu Gautier
parent
1513204f42
commit
97666b0fc9
|
|
@ -0,0 +1,4 @@
|
||||||
|
include kiwixbuild/templates/cmake_cross_file.txt
|
||||||
|
include kiwixbuild/templates/meson_cross_file.txt
|
||||||
|
include kiwixbuild/templates/cmake_android_cross_file.txt
|
||||||
|
recursise-include kiwixbuild/patches *.patch
|
||||||
|
|
@ -8,9 +8,9 @@ import subprocess
|
||||||
import platform
|
import platform
|
||||||
from collections import OrderedDict
|
from collections import OrderedDict
|
||||||
|
|
||||||
from dependencies import Dependency
|
from .toolchains import Toolchain
|
||||||
from dependency_utils import ReleaseDownload, Builder, GitClone
|
from .dependencies import Dependency
|
||||||
from utils import (
|
from .utils import (
|
||||||
pj,
|
pj,
|
||||||
remove_duplicates,
|
remove_duplicates,
|
||||||
add_execution_right,
|
add_execution_right,
|
||||||
|
|
@ -47,7 +47,7 @@ PACKAGE_NAME_MAPPERS = {
|
||||||
'gumbo' : ['gumbo-parser-devel'],
|
'gumbo' : ['gumbo-parser-devel'],
|
||||||
},
|
},
|
||||||
'fedora_native_static': {
|
'fedora_native_static': {
|
||||||
'COMMON': _fedora_common + ['glibc-static', 'libstdc++-static'],
|
'COMMON': _fedora_common,
|
||||||
'zlib': ['zlib-devel', 'zlib-static'],
|
'zlib': ['zlib-devel', 'zlib-static'],
|
||||||
'lzma': ['xz-devel', 'xz-static']
|
'lzma': ['xz-devel', 'xz-static']
|
||||||
# Either there is no packages, or no static or too old
|
# Either there is no packages, or no static or too old
|
||||||
|
|
@ -118,12 +118,6 @@ PACKAGE_NAME_MAPPERS = {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def which(name):
|
|
||||||
command = "which {}".format(name)
|
|
||||||
output = subprocess.check_output(command, shell=True)
|
|
||||||
return output[:-1].decode()
|
|
||||||
|
|
||||||
|
|
||||||
class TargetInfo:
|
class TargetInfo:
|
||||||
def __init__(self, build, static, toolchains, hosts=None):
|
def __init__(self, build, static, toolchains, hosts=None):
|
||||||
self.build = build
|
self.build = build
|
||||||
|
|
@ -536,355 +530,6 @@ class BuildEnv:
|
||||||
with open(autoskip_file, 'w'):
|
with open(autoskip_file, 'w'):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
class _MetaToolchain(type):
|
|
||||||
def __new__(cls, name, bases, dct):
|
|
||||||
_class = type.__new__(cls, name, bases, dct)
|
|
||||||
if name != 'Toolchain':
|
|
||||||
Toolchain.all_toolchains[name] = _class
|
|
||||||
return _class
|
|
||||||
|
|
||||||
|
|
||||||
class Toolchain(metaclass=_MetaToolchain):
|
|
||||||
all_toolchains = {}
|
|
||||||
configure_option = ""
|
|
||||||
cmake_option = ""
|
|
||||||
exec_wrapper_def = ""
|
|
||||||
Builder = None
|
|
||||||
Source = None
|
|
||||||
|
|
||||||
def __init__(self, buildEnv):
|
|
||||||
self.buildEnv = buildEnv
|
|
||||||
self.source = self.Source(self) if self.Source else None
|
|
||||||
self.builder = self.Builder(self) if self.Builder else None
|
|
||||||
|
|
||||||
@property
|
|
||||||
def full_name(self):
|
|
||||||
return "{name}-{version}".format(
|
|
||||||
name = self.name,
|
|
||||||
version = self.version)
|
|
||||||
|
|
||||||
@property
|
|
||||||
def source_path(self):
|
|
||||||
return pj(self.buildEnv.source_dir, self.source.source_dir)
|
|
||||||
|
|
||||||
@property
|
|
||||||
def _log_dir(self):
|
|
||||||
return self.buildEnv.log_dir
|
|
||||||
|
|
||||||
def set_env(self, env):
|
|
||||||
pass
|
|
||||||
|
|
||||||
def set_compiler(self, env):
|
|
||||||
pass
|
|
||||||
|
|
||||||
def command(self, name, function, *args):
|
|
||||||
print(" {} {} : ".format(name, self.name), end="", flush=True)
|
|
||||||
log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
|
|
||||||
context = Context(name, log, True)
|
|
||||||
try:
|
|
||||||
ret = function(*args, context=context)
|
|
||||||
context._finalise()
|
|
||||||
print("OK")
|
|
||||||
return ret
|
|
||||||
except SkipCommand:
|
|
||||||
print("SKIP")
|
|
||||||
except subprocess.CalledProcessError:
|
|
||||||
print("ERROR")
|
|
||||||
try:
|
|
||||||
with open(log, 'r') as f:
|
|
||||||
print(f.read())
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
raise StopBuild()
|
|
||||||
except:
|
|
||||||
print("ERROR")
|
|
||||||
raise
|
|
||||||
|
|
||||||
|
|
||||||
class mingw32_toolchain(Toolchain):
|
|
||||||
name = 'mingw32'
|
|
||||||
arch_full = 'i686-w64-mingw32'
|
|
||||||
|
|
||||||
@property
|
|
||||||
def root_path(self):
|
|
||||||
return self.buildEnv.cross_config['root_path']
|
|
||||||
|
|
||||||
@property
|
|
||||||
def binaries(self):
|
|
||||||
return {k:which('{}-{}'.format(self.arch_full, v))
|
|
||||||
for k, v in (('CC', 'gcc'),
|
|
||||||
('CXX', 'g++'),
|
|
||||||
('AR', 'ar'),
|
|
||||||
('STRIP', 'strip'),
|
|
||||||
('WINDRES', 'windres'),
|
|
||||||
('RANLIB', 'ranlib'))
|
|
||||||
}
|
|
||||||
|
|
||||||
@property
|
|
||||||
def exec_wrapper_def(self):
|
|
||||||
try:
|
|
||||||
which('wine')
|
|
||||||
except subprocess.CalledProcessError:
|
|
||||||
return ""
|
|
||||||
else:
|
|
||||||
return "exec_wrapper = 'wine'"
|
|
||||||
|
|
||||||
|
|
||||||
@property
|
|
||||||
def configure_option(self):
|
|
||||||
return '--host={}'.format(self.arch_full)
|
|
||||||
|
|
||||||
def get_bin_dir(self):
|
|
||||||
return [pj(self.root_path, 'bin')]
|
|
||||||
|
|
||||||
def set_env(self, env):
|
|
||||||
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
|
|
||||||
env['LIBS'] = " ".join(self.buildEnv.cross_config['extra_libs']) + " " +env['LIBS']
|
|
||||||
|
|
||||||
def set_compiler(self, env):
|
|
||||||
for k, v in self.binaries.items():
|
|
||||||
env[k] = v
|
|
||||||
|
|
||||||
|
|
||||||
class android_ndk(Toolchain):
|
|
||||||
name = 'android-ndk'
|
|
||||||
version = 'r13b'
|
|
||||||
gccver = '4.9.x'
|
|
||||||
|
|
||||||
@property
|
|
||||||
def api(self):
|
|
||||||
return '21' if self.arch in ('arm64', 'mips64', 'x86_64') else '14'
|
|
||||||
|
|
||||||
@property
|
|
||||||
def platform(self):
|
|
||||||
return 'android-'+self.api
|
|
||||||
|
|
||||||
@property
|
|
||||||
def arch(self):
|
|
||||||
return self.buildEnv.platform_info.arch
|
|
||||||
|
|
||||||
@property
|
|
||||||
def arch_full(self):
|
|
||||||
return self.buildEnv.platform_info.arch_full
|
|
||||||
|
|
||||||
@property
|
|
||||||
def toolchain(self):
|
|
||||||
return self.arch_full+"-4.9"
|
|
||||||
|
|
||||||
@property
|
|
||||||
def root_path(self):
|
|
||||||
return pj(self.builder.install_path, 'sysroot')
|
|
||||||
|
|
||||||
@property
|
|
||||||
def binaries(self):
|
|
||||||
binaries = ((k,'{}-{}'.format(self.arch_full, v))
|
|
||||||
for k, v in (('CC', 'gcc'),
|
|
||||||
('CXX', 'g++'),
|
|
||||||
('AR', 'ar'),
|
|
||||||
('STRIP', 'strip'),
|
|
||||||
('WINDRES', 'windres'),
|
|
||||||
('RANLIB', 'ranlib'),
|
|
||||||
('LD', 'ld'))
|
|
||||||
)
|
|
||||||
return {k:pj(self.builder.install_path, 'bin', v)
|
|
||||||
for k,v in binaries}
|
|
||||||
|
|
||||||
@property
|
|
||||||
def configure_option(self):
|
|
||||||
return '--host={}'.format(self.arch_full)
|
|
||||||
|
|
||||||
@property
|
|
||||||
def full_name(self):
|
|
||||||
return "{name}-{version}-{arch}-{api}".format(
|
|
||||||
name = self.name,
|
|
||||||
version = self.version,
|
|
||||||
arch = self.arch,
|
|
||||||
api = self.api)
|
|
||||||
|
|
||||||
class Source(ReleaseDownload):
|
|
||||||
archive = Remotefile('android-ndk-r13b-linux-x86_64.zip',
|
|
||||||
'3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c',
|
|
||||||
'https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip')
|
|
||||||
|
|
||||||
@property
|
|
||||||
def source_dir(self):
|
|
||||||
return "{}-{}".format(
|
|
||||||
self.target.name,
|
|
||||||
self.target.version)
|
|
||||||
|
|
||||||
|
|
||||||
class Builder(Builder):
|
|
||||||
|
|
||||||
@property
|
|
||||||
def install_path(self):
|
|
||||||
return self.build_path
|
|
||||||
|
|
||||||
def _build_platform(self, context):
|
|
||||||
context.try_skip(self.build_path)
|
|
||||||
script = pj(self.source_path, 'build/tools/make_standalone_toolchain.py')
|
|
||||||
add_execution_right(script)
|
|
||||||
command = '{script} --arch={arch} --api={api} --install-dir={install_dir} --force'
|
|
||||||
command = command.format(
|
|
||||||
script=script,
|
|
||||||
arch=self.target.arch,
|
|
||||||
api=self.target.api,
|
|
||||||
install_dir=self.install_path
|
|
||||||
)
|
|
||||||
self.buildEnv.run_command(command, self.build_path, context)
|
|
||||||
|
|
||||||
def _fix_permission_right(self, context):
|
|
||||||
context.try_skip(self.build_path)
|
|
||||||
bin_dirs = [pj(self.install_path, 'bin'),
|
|
||||||
pj(self.install_path, self.target.arch_full, 'bin'),
|
|
||||||
pj(self.install_path, 'libexec', 'gcc', self.target.arch_full, self.target.gccver)
|
|
||||||
]
|
|
||||||
for root, dirs, files in os.walk(self.install_path):
|
|
||||||
if not root in bin_dirs:
|
|
||||||
continue
|
|
||||||
|
|
||||||
for file_ in files:
|
|
||||||
file_path = pj(root, file_)
|
|
||||||
if os.path.islink(file_path):
|
|
||||||
continue
|
|
||||||
add_execution_right(file_path)
|
|
||||||
|
|
||||||
def build(self):
|
|
||||||
self.command('build_platform', self._build_platform)
|
|
||||||
self.command('fix_permission_right', self._fix_permission_right)
|
|
||||||
|
|
||||||
def get_bin_dir(self):
|
|
||||||
return [pj(self.builder.install_path, 'bin')]
|
|
||||||
|
|
||||||
def set_env(self, env):
|
|
||||||
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
|
|
||||||
env['CFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(self.root_path) + env['CFLAGS']
|
|
||||||
env['CXXFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(self.root_path) + env['CXXFLAGS']
|
|
||||||
env['LDFLAGS'] = '--sysroot={} '.format(self.root_path) + env['LDFLAGS']
|
|
||||||
#env['CFLAGS'] = ' -fPIC -D_FILE_OFFSET_BITS=64 -O3 '+env['CFLAGS']
|
|
||||||
#env['CXXFLAGS'] = (' -D__OPTIMIZE__ -fno-strict-aliasing '
|
|
||||||
# ' -DU_HAVE_NL_LANGINFO_CODESET=0 '
|
|
||||||
# '-DU_STATIC_IMPLEMENTATION -O3 '
|
|
||||||
# '-DU_HAVE_STD_STRING -DU_TIMEZONE=0 ')+env['CXXFLAGS']
|
|
||||||
env['NDK_DEBUG'] = '0'
|
|
||||||
|
|
||||||
def set_compiler(self, env):
|
|
||||||
env['CC'] = self.binaries['CC']
|
|
||||||
env['CXX'] = self.binaries['CXX']
|
|
||||||
|
|
||||||
|
|
||||||
class android_sdk(Toolchain):
|
|
||||||
name = 'android-sdk'
|
|
||||||
version = 'r25.2.3'
|
|
||||||
|
|
||||||
class Source(ReleaseDownload):
|
|
||||||
archive = Remotefile('tools_r25.2.3-linux.zip',
|
|
||||||
'1b35bcb94e9a686dff6460c8bca903aa0281c6696001067f34ec00093145b560',
|
|
||||||
'https://dl.google.com/android/repository/tools_r25.2.3-linux.zip')
|
|
||||||
|
|
||||||
class Builder(Builder):
|
|
||||||
|
|
||||||
@property
|
|
||||||
def install_path(self):
|
|
||||||
return pj(self.buildEnv.toolchain_dir, self.target.full_name)
|
|
||||||
|
|
||||||
def _build_platform(self, context):
|
|
||||||
context.try_skip(self.install_path)
|
|
||||||
tools_dir = pj(self.install_path, 'tools')
|
|
||||||
shutil.copytree(self.source_path, tools_dir)
|
|
||||||
script = pj(tools_dir, 'android')
|
|
||||||
command = '{script} --verbose update sdk -a --no-ui --filter {packages}'
|
|
||||||
command = command.format(
|
|
||||||
script=script,
|
|
||||||
packages = ','.join(str(i) for i in [1,2,8,34,162])
|
|
||||||
)
|
|
||||||
# packages correspond to :
|
|
||||||
# - 1 : Android SDK Tools, revision 25.2.5
|
|
||||||
# - 2 : Android SDK Platform-tools, revision 25.0.3
|
|
||||||
# - 8 : Android SDK Build-tools, revision 24.0.1
|
|
||||||
# - 34 : SDK Platform Android 7.0, API 24, revision 2
|
|
||||||
# - 162 : Android Support Repository, revision 44
|
|
||||||
self.buildEnv.run_command(command, self.install_path, context, input="y\n")
|
|
||||||
|
|
||||||
def _fix_licenses(self, context):
|
|
||||||
context.try_skip(self.install_path)
|
|
||||||
os.makedirs(pj(self.install_path, 'licenses'), exist_ok=True)
|
|
||||||
with open(pj(self.install_path, 'licenses', 'android-sdk-license'), 'w') as f:
|
|
||||||
f.write("\n8933bad161af4178b1185d1a37fbf41ea5269c55\nd56f5187479451eabf01fb78af6dfcb131a6481e")
|
|
||||||
|
|
||||||
def build(self):
|
|
||||||
self.command('build_platform', self._build_platform)
|
|
||||||
self.command('fix_licenses', self._fix_licenses)
|
|
||||||
|
|
||||||
def get_bin_dir(self):
|
|
||||||
return []
|
|
||||||
|
|
||||||
def set_env(self, env):
|
|
||||||
env['ANDROID_HOME'] = self.builder.install_path
|
|
||||||
|
|
||||||
|
|
||||||
class armhf_toolchain(Toolchain):
|
|
||||||
name = 'armhf'
|
|
||||||
arch_full = 'arm-linux-gnueabihf'
|
|
||||||
|
|
||||||
class Source(GitClone):
|
|
||||||
git_remote = "https://github.com/raspberrypi/tools"
|
|
||||||
git_dir = "raspberrypi-tools"
|
|
||||||
|
|
||||||
@property
|
|
||||||
def root_path(self):
|
|
||||||
return pj(self.source_path, 'arm-bcm2708', 'gcc-linaro-arm-linux-gnueabihf-raspbian-x64')
|
|
||||||
|
|
||||||
@property
|
|
||||||
def binaries(self):
|
|
||||||
binaries = ((k,'{}-{}'.format(self.arch_full, v))
|
|
||||||
for k, v in (('CC', 'gcc'),
|
|
||||||
('CXX', 'g++'),
|
|
||||||
('AR', 'ar'),
|
|
||||||
('STRIP', 'strip'),
|
|
||||||
('WINDRES', 'windres'),
|
|
||||||
('RANLIB', 'ranlib'),
|
|
||||||
('LD', 'ld'))
|
|
||||||
)
|
|
||||||
return {k:pj(self.root_path, 'bin', v)
|
|
||||||
for k,v in binaries}
|
|
||||||
|
|
||||||
@property
|
|
||||||
def exec_wrapper_def(self):
|
|
||||||
try:
|
|
||||||
which('qemu-arm')
|
|
||||||
except subprocess.CalledProcessError:
|
|
||||||
return ""
|
|
||||||
else:
|
|
||||||
return "exec_wrapper = 'qemu-arm'"
|
|
||||||
|
|
||||||
@property
|
|
||||||
def configure_option(self):
|
|
||||||
return '--host={}'.format(self.arch_full)
|
|
||||||
|
|
||||||
def get_bin_dir(self):
|
|
||||||
return [pj(self.root_path, 'bin')]
|
|
||||||
|
|
||||||
def set_env(self, env):
|
|
||||||
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
|
|
||||||
env['CFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS']
|
|
||||||
env['CXXFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
|
|
||||||
env['LIBS'] = " ".join(self.buildEnv.cross_config['extra_libs']) + " " +env['LIBS']
|
|
||||||
env['QEMU_LD_PREFIX'] = pj(self.root_path, "arm-linux-gnueabihf", "libc")
|
|
||||||
env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format(
|
|
||||||
':'.join([
|
|
||||||
pj(self.root_path, "arm-linux-gnueabihf", "lib"),
|
|
||||||
pj(self.buildEnv.install_dir, 'lib'),
|
|
||||||
pj(self.buildEnv.install_dir, self.buildEnv.libprefix)
|
|
||||||
]))
|
|
||||||
|
|
||||||
def set_compiler(self, env):
|
|
||||||
env['CC'] = self.binaries['CC']
|
|
||||||
env['CXX'] = self.binaries['CXX']
|
|
||||||
|
|
||||||
|
|
||||||
class iOS_sdk(Toolchain):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class Builder:
|
class Builder:
|
||||||
def __init__(self, options):
|
def __init__(self, options):
|
||||||
|
|
@ -1021,10 +666,10 @@ def parse_args():
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
return options
|
return options
|
||||||
|
|
||||||
|
def main():
|
||||||
if __name__ == "__main__":
|
|
||||||
options = parse_args()
|
options = parse_args()
|
||||||
options.working_dir = os.path.abspath(options.working_dir)
|
options.working_dir = os.path.abspath(options.working_dir)
|
||||||
setup_print_progress(options.show_progress)
|
setup_print_progress(options.show_progress)
|
||||||
builder = Builder(options)
|
builder = Builder(options)
|
||||||
builder.run()
|
builder.run()
|
||||||
|
|
||||||
|
|
@ -0,0 +1,7 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
from . import main
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
import shutil, os, json
|
import shutil, os, json
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
from dependency_utils import (
|
from kiwixbuild.dependency_utils import (
|
||||||
Dependency,
|
Dependency,
|
||||||
ReleaseDownload,
|
ReleaseDownload,
|
||||||
GitClone,
|
GitClone,
|
||||||
|
|
@ -14,7 +14,7 @@ from dependency_utils import (
|
||||||
NoopBuilder,
|
NoopBuilder,
|
||||||
Builder as BaseBuilder)
|
Builder as BaseBuilder)
|
||||||
|
|
||||||
from utils import Remotefile, pj, SkipCommand, copy_tree, add_execution_right
|
from kiwixbuild.utils import Remotefile, pj, SkipCommand, copy_tree, add_execution_right
|
||||||
|
|
||||||
# *************************************
|
# *************************************
|
||||||
# Missing dependencies
|
# Missing dependencies
|
||||||
|
|
@ -22,7 +22,6 @@ from utils import Remotefile, pj, SkipCommand, copy_tree, add_execution_right
|
||||||
# exist in your "distri" (linux/mac) ?
|
# exist in your "distri" (linux/mac) ?
|
||||||
# If not, we need to compile them here
|
# If not, we need to compile them here
|
||||||
# *************************************
|
# *************************************
|
||||||
# aria2
|
|
||||||
# Argtable
|
# Argtable
|
||||||
# MSVirtual
|
# MSVirtual
|
||||||
# Android
|
# Android
|
||||||
|
|
@ -415,7 +414,7 @@ class AllBaseDependencies(Dependency):
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def dependencies(self):
|
def dependencies(self):
|
||||||
base_deps = ['zlib', 'lzma', 'xapian-core', 'gumbo', 'pugixml', 'libmicrohttpd', 'libaria2']
|
base_deps = ['zlib', 'lzma', 'xapian-core', 'gumbo', 'pugixml', 'libmicrohttpd']
|
||||||
if self.buildEnv.platform_info.build != 'native':
|
if self.buildEnv.platform_info.build != 'native':
|
||||||
base_deps += ["icu4c_cross-compile"]
|
base_deps += ["icu4c_cross-compile"]
|
||||||
else:
|
else:
|
||||||
|
|
@ -2,8 +2,8 @@ import subprocess
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
|
|
||||||
from utils import pj, Context, SkipCommand, extract_archive, Defaultdict, StopBuild
|
from kiwixbuild.utils import pj, Context, SkipCommand, extract_archive, Defaultdict, StopBuild
|
||||||
from dependency_versions import main_project_versions, base_deps_versions
|
from kiwixbuild.dependency_versions import main_project_versions, base_deps_versions
|
||||||
|
|
||||||
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
|
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
|
||||||
|
|
||||||
|
|
@ -0,0 +1,4 @@
|
||||||
|
|
||||||
|
from . import android_ndk, android_sdk, armhf, ios, mingw32
|
||||||
|
|
||||||
|
from .base_toolchain import Toolchain
|
||||||
|
|
@ -0,0 +1,133 @@
|
||||||
|
import os
|
||||||
|
|
||||||
|
from .base_toolchain import Toolchain
|
||||||
|
from kiwixbuild.dependency_utils import ReleaseDownload, Builder
|
||||||
|
from kiwixbuild.utils import Remotefile, add_execution_right
|
||||||
|
|
||||||
|
pj = os.path.join
|
||||||
|
|
||||||
|
class android_ndk(Toolchain):
|
||||||
|
name = 'android-ndk'
|
||||||
|
version = 'r13b'
|
||||||
|
gccver = '4.9.x'
|
||||||
|
|
||||||
|
@property
|
||||||
|
def api(self):
|
||||||
|
return '21' if self.arch in ('arm64', 'mips64', 'x86_64') else '14'
|
||||||
|
|
||||||
|
@property
|
||||||
|
def platform(self):
|
||||||
|
return 'android-'+self.api
|
||||||
|
|
||||||
|
@property
|
||||||
|
def arch(self):
|
||||||
|
return self.buildEnv.platform_info.arch
|
||||||
|
|
||||||
|
@property
|
||||||
|
def arch_full(self):
|
||||||
|
return self.buildEnv.platform_info.arch_full
|
||||||
|
|
||||||
|
@property
|
||||||
|
def toolchain(self):
|
||||||
|
return self.arch_full+"-4.9"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def root_path(self):
|
||||||
|
return pj(self.builder.install_path, 'sysroot')
|
||||||
|
|
||||||
|
@property
|
||||||
|
def binaries(self):
|
||||||
|
binaries = ((k,'{}-{}'.format(self.arch_full, v))
|
||||||
|
for k, v in (('CC', 'gcc'),
|
||||||
|
('CXX', 'g++'),
|
||||||
|
('AR', 'ar'),
|
||||||
|
('STRIP', 'strip'),
|
||||||
|
('WINDRES', 'windres'),
|
||||||
|
('RANLIB', 'ranlib'),
|
||||||
|
('LD', 'ld'))
|
||||||
|
)
|
||||||
|
return {k:pj(self.builder.install_path, 'bin', v)
|
||||||
|
for k,v in binaries}
|
||||||
|
|
||||||
|
@property
|
||||||
|
def configure_option(self):
|
||||||
|
return '--host={}'.format(self.arch_full)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def full_name(self):
|
||||||
|
return "{name}-{version}-{arch}-{api}".format(
|
||||||
|
name = self.name,
|
||||||
|
version = self.version,
|
||||||
|
arch = self.arch,
|
||||||
|
api = self.api)
|
||||||
|
|
||||||
|
class Source(ReleaseDownload):
|
||||||
|
archive = Remotefile('android-ndk-r13b-linux-x86_64.zip',
|
||||||
|
'3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c',
|
||||||
|
'https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip')
|
||||||
|
|
||||||
|
@property
|
||||||
|
def source_dir(self):
|
||||||
|
return "{}-{}".format(
|
||||||
|
self.target.name,
|
||||||
|
self.target.version)
|
||||||
|
|
||||||
|
|
||||||
|
class Builder(Builder):
|
||||||
|
|
||||||
|
@property
|
||||||
|
def install_path(self):
|
||||||
|
return self.build_path
|
||||||
|
|
||||||
|
def _build_platform(self, context):
|
||||||
|
context.try_skip(self.build_path)
|
||||||
|
script = pj(self.source_path, 'build/tools/make_standalone_toolchain.py')
|
||||||
|
add_execution_right(script)
|
||||||
|
command = '{script} --arch={arch} --api={api} --install-dir={install_dir} --force'
|
||||||
|
command = command.format(
|
||||||
|
script=script,
|
||||||
|
arch=self.target.arch,
|
||||||
|
api=self.target.api,
|
||||||
|
install_dir=self.install_path
|
||||||
|
)
|
||||||
|
self.buildEnv.run_command(command, self.build_path, context)
|
||||||
|
|
||||||
|
def _fix_permission_right(self, context):
|
||||||
|
context.try_skip(self.build_path)
|
||||||
|
bin_dirs = [pj(self.install_path, 'bin'),
|
||||||
|
pj(self.install_path, self.target.arch_full, 'bin'),
|
||||||
|
pj(self.install_path, 'libexec', 'gcc', self.target.arch_full, self.target.gccver)
|
||||||
|
]
|
||||||
|
for root, dirs, files in os.walk(self.install_path):
|
||||||
|
if not root in bin_dirs:
|
||||||
|
continue
|
||||||
|
|
||||||
|
for file_ in files:
|
||||||
|
file_path = pj(root, file_)
|
||||||
|
if os.path.islink(file_path):
|
||||||
|
continue
|
||||||
|
add_execution_right(file_path)
|
||||||
|
|
||||||
|
def build(self):
|
||||||
|
self.command('build_platform', self._build_platform)
|
||||||
|
self.command('fix_permission_right', self._fix_permission_right)
|
||||||
|
|
||||||
|
def get_bin_dir(self):
|
||||||
|
return [pj(self.builder.install_path, 'bin')]
|
||||||
|
|
||||||
|
def set_env(self, env):
|
||||||
|
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
|
||||||
|
env['CFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(self.root_path) + env['CFLAGS']
|
||||||
|
env['CXXFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(self.root_path) + env['CXXFLAGS']
|
||||||
|
env['LDFLAGS'] = '--sysroot={} '.format(self.root_path) + env['LDFLAGS']
|
||||||
|
#env['CFLAGS'] = ' -fPIC -D_FILE_OFFSET_BITS=64 -O3 '+env['CFLAGS']
|
||||||
|
#env['CXXFLAGS'] = (' -D__OPTIMIZE__ -fno-strict-aliasing '
|
||||||
|
# ' -DU_HAVE_NL_LANGINFO_CODESET=0 '
|
||||||
|
# '-DU_STATIC_IMPLEMENTATION -O3 '
|
||||||
|
# '-DU_HAVE_STD_STRING -DU_TIMEZONE=0 ')+env['CXXFLAGS']
|
||||||
|
env['NDK_DEBUG'] = '0'
|
||||||
|
|
||||||
|
def set_compiler(self, env):
|
||||||
|
env['CC'] = self.binaries['CC']
|
||||||
|
env['CXX'] = self.binaries['CXX']
|
||||||
|
|
||||||
|
|
@ -0,0 +1,57 @@
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
|
||||||
|
from .base_toolchain import Toolchain
|
||||||
|
from kiwixbuild.dependency_utils import ReleaseDownload, Builder
|
||||||
|
from kiwixbuild.utils import Remotefile
|
||||||
|
|
||||||
|
pj = os.path.join
|
||||||
|
|
||||||
|
class android_sdk(Toolchain):
|
||||||
|
name = 'android-sdk'
|
||||||
|
version = 'r25.2.3'
|
||||||
|
|
||||||
|
class Source(ReleaseDownload):
|
||||||
|
archive = Remotefile('tools_r25.2.3-linux.zip',
|
||||||
|
'1b35bcb94e9a686dff6460c8bca903aa0281c6696001067f34ec00093145b560',
|
||||||
|
'https://dl.google.com/android/repository/tools_r25.2.3-linux.zip')
|
||||||
|
|
||||||
|
class Builder(Builder):
|
||||||
|
|
||||||
|
@property
|
||||||
|
def install_path(self):
|
||||||
|
return pj(self.buildEnv.toolchain_dir, self.target.full_name)
|
||||||
|
|
||||||
|
def _build_platform(self, context):
|
||||||
|
context.try_skip(self.install_path)
|
||||||
|
tools_dir = pj(self.install_path, 'tools')
|
||||||
|
shutil.copytree(self.source_path, tools_dir)
|
||||||
|
script = pj(tools_dir, 'android')
|
||||||
|
command = '{script} --verbose update sdk -a --no-ui --filter {packages}'
|
||||||
|
command = command.format(
|
||||||
|
script=script,
|
||||||
|
packages = ','.join(str(i) for i in [1,2,8,34,162])
|
||||||
|
)
|
||||||
|
# packages correspond to :
|
||||||
|
# - 1 : Android SDK Tools, revision 25.2.5
|
||||||
|
# - 2 : Android SDK Platform-tools, revision 25.0.3
|
||||||
|
# - 8 : Android SDK Build-tools, revision 24.0.1
|
||||||
|
# - 34 : SDK Platform Android 7.0, API 24, revision 2
|
||||||
|
# - 162 : Android Support Repository, revision 44
|
||||||
|
self.buildEnv.run_command(command, self.install_path, context, input="y\n")
|
||||||
|
|
||||||
|
def _fix_licenses(self, context):
|
||||||
|
context.try_skip(self.install_path)
|
||||||
|
os.makedirs(pj(self.install_path, 'licenses'), exist_ok=True)
|
||||||
|
with open(pj(self.install_path, 'licenses', 'android-sdk-license'), 'w') as f:
|
||||||
|
f.write("\n8933bad161af4178b1185d1a37fbf41ea5269c55\nd56f5187479451eabf01fb78af6dfcb131a6481e")
|
||||||
|
|
||||||
|
def build(self):
|
||||||
|
self.command('build_platform', self._build_platform)
|
||||||
|
self.command('fix_licenses', self._fix_licenses)
|
||||||
|
|
||||||
|
def get_bin_dir(self):
|
||||||
|
return []
|
||||||
|
|
||||||
|
def set_env(self, env):
|
||||||
|
env['ANDROID_HOME'] = self.builder.install_path
|
||||||
|
|
@ -0,0 +1,66 @@
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
from .base_toolchain import Toolchain
|
||||||
|
from kiwixbuild.dependency_utils import GitClone
|
||||||
|
from kiwixbuild.utils import Remotefile, which
|
||||||
|
pj = os.path.join
|
||||||
|
|
||||||
|
class armhf_toolchain(Toolchain):
|
||||||
|
name = 'armhf'
|
||||||
|
arch_full = 'arm-linux-gnueabihf'
|
||||||
|
|
||||||
|
class Source(GitClone):
|
||||||
|
git_remote = "https://github.com/raspberrypi/tools"
|
||||||
|
git_dir = "raspberrypi-tools"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def root_path(self):
|
||||||
|
return pj(self.source_path, 'arm-bcm2708', 'gcc-linaro-arm-linux-gnueabihf-raspbian-x64')
|
||||||
|
|
||||||
|
@property
|
||||||
|
def binaries(self):
|
||||||
|
binaries = ((k,'{}-{}'.format(self.arch_full, v))
|
||||||
|
for k, v in (('CC', 'gcc'),
|
||||||
|
('CXX', 'g++'),
|
||||||
|
('AR', 'ar'),
|
||||||
|
('STRIP', 'strip'),
|
||||||
|
('WINDRES', 'windres'),
|
||||||
|
('RANLIB', 'ranlib'),
|
||||||
|
('LD', 'ld'))
|
||||||
|
)
|
||||||
|
return {k:pj(self.root_path, 'bin', v)
|
||||||
|
for k,v in binaries}
|
||||||
|
|
||||||
|
@property
|
||||||
|
def exec_wrapper_def(self):
|
||||||
|
try:
|
||||||
|
which('qemu-arm')
|
||||||
|
except subprocess.CalledProcessError:
|
||||||
|
return ""
|
||||||
|
else:
|
||||||
|
return "exec_wrapper = 'qemu-arm'"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def configure_option(self):
|
||||||
|
return '--host={}'.format(self.arch_full)
|
||||||
|
|
||||||
|
def get_bin_dir(self):
|
||||||
|
return [pj(self.root_path, 'bin')]
|
||||||
|
|
||||||
|
def set_env(self, env):
|
||||||
|
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
|
||||||
|
env['CFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS']
|
||||||
|
env['CXXFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
|
||||||
|
env['LIBS'] = " ".join(self.buildEnv.cross_config['extra_libs']) + " " +env['LIBS']
|
||||||
|
env['QEMU_LD_PREFIX'] = pj(self.root_path, "arm-linux-gnueabihf", "libc")
|
||||||
|
env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format(
|
||||||
|
':'.join([
|
||||||
|
pj(self.root_path, "arm-linux-gnueabihf", "lib"),
|
||||||
|
pj(self.buildEnv.install_dir, 'lib'),
|
||||||
|
pj(self.buildEnv.install_dir, self.buildEnv.libprefix)
|
||||||
|
]))
|
||||||
|
|
||||||
|
def set_compiler(self, env):
|
||||||
|
env['CC'] = self.binaries['CC']
|
||||||
|
env['CXX'] = self.binaries['CXX']
|
||||||
|
|
@ -0,0 +1,70 @@
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
pj = os.path.join
|
||||||
|
|
||||||
|
from kiwixbuild.utils import Context, SkipCommand, StopBuild
|
||||||
|
|
||||||
|
class _MetaToolchain(type):
|
||||||
|
def __new__(cls, name, bases, dct):
|
||||||
|
_class = type.__new__(cls, name, bases, dct)
|
||||||
|
if name != 'Toolchain':
|
||||||
|
Toolchain.all_toolchains[name] = _class
|
||||||
|
return _class
|
||||||
|
|
||||||
|
|
||||||
|
class Toolchain(metaclass=_MetaToolchain):
|
||||||
|
all_toolchains = {}
|
||||||
|
configure_option = ""
|
||||||
|
cmake_option = ""
|
||||||
|
exec_wrapper_def = ""
|
||||||
|
Builder = None
|
||||||
|
Source = None
|
||||||
|
|
||||||
|
def __init__(self, buildEnv):
|
||||||
|
self.buildEnv = buildEnv
|
||||||
|
self.source = self.Source(self) if self.Source else None
|
||||||
|
self.builder = self.Builder(self) if self.Builder else None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def full_name(self):
|
||||||
|
return "{name}-{version}".format(
|
||||||
|
name = self.name,
|
||||||
|
version = self.version)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def source_path(self):
|
||||||
|
return pj(self.buildEnv.source_dir, self.source.source_dir)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def _log_dir(self):
|
||||||
|
return self.buildEnv.log_dir
|
||||||
|
|
||||||
|
def set_env(self, env):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def set_compiler(self, env):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def command(self, name, function, *args):
|
||||||
|
print(" {} {} : ".format(name, self.name), end="", flush=True)
|
||||||
|
log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
|
||||||
|
context = Context(name, log, True)
|
||||||
|
try:
|
||||||
|
ret = function(*args, context=context)
|
||||||
|
context._finalise()
|
||||||
|
print("OK")
|
||||||
|
return ret
|
||||||
|
except SkipCommand:
|
||||||
|
print("SKIP")
|
||||||
|
except subprocess.CalledProcessError:
|
||||||
|
print("ERROR")
|
||||||
|
try:
|
||||||
|
with open(log, 'r') as f:
|
||||||
|
print(f.read())
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
raise StopBuild()
|
||||||
|
except:
|
||||||
|
print("ERROR")
|
||||||
|
raise
|
||||||
|
|
@ -0,0 +1,4 @@
|
||||||
|
from .base_toolchain import Toolchain
|
||||||
|
|
||||||
|
class iOS_sdk(Toolchain):
|
||||||
|
pass
|
||||||
|
|
@ -0,0 +1,51 @@
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
from .base_toolchain import Toolchain
|
||||||
|
from kiwixbuild.utils import which
|
||||||
|
|
||||||
|
pj = os.path.join
|
||||||
|
|
||||||
|
class mingw32_toolchain(Toolchain):
|
||||||
|
name = 'mingw32'
|
||||||
|
arch_full = 'i686-w64-mingw32'
|
||||||
|
|
||||||
|
@property
|
||||||
|
def root_path(self):
|
||||||
|
return self.buildEnv.cross_config['root_path']
|
||||||
|
|
||||||
|
@property
|
||||||
|
def binaries(self):
|
||||||
|
return {k:which('{}-{}'.format(self.arch_full, v))
|
||||||
|
for k, v in (('CC', 'gcc'),
|
||||||
|
('CXX', 'g++'),
|
||||||
|
('AR', 'ar'),
|
||||||
|
('STRIP', 'strip'),
|
||||||
|
('WINDRES', 'windres'),
|
||||||
|
('RANLIB', 'ranlib'))
|
||||||
|
}
|
||||||
|
|
||||||
|
@property
|
||||||
|
def exec_wrapper_def(self):
|
||||||
|
try:
|
||||||
|
which('wine')
|
||||||
|
except subprocess.CalledProcessError:
|
||||||
|
return ""
|
||||||
|
else:
|
||||||
|
return "exec_wrapper = 'wine'"
|
||||||
|
|
||||||
|
|
||||||
|
@property
|
||||||
|
def configure_option(self):
|
||||||
|
return '--host={}'.format(self.arch_full)
|
||||||
|
|
||||||
|
def get_bin_dir(self):
|
||||||
|
return [pj(self.root_path, 'bin')]
|
||||||
|
|
||||||
|
def set_env(self, env):
|
||||||
|
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
|
||||||
|
env['LIBS'] = " ".join(self.buildEnv.cross_config['extra_libs']) + " " +env['LIBS']
|
||||||
|
|
||||||
|
def set_compiler(self, env):
|
||||||
|
for k, v in self.binaries.items():
|
||||||
|
env[k] = v
|
||||||
|
|
@ -6,6 +6,7 @@ import shutil
|
||||||
import os, stat, sys
|
import os, stat, sys
|
||||||
import urllib
|
import urllib
|
||||||
import ssl
|
import ssl
|
||||||
|
import subprocess
|
||||||
from collections import namedtuple, defaultdict
|
from collections import namedtuple, defaultdict
|
||||||
|
|
||||||
pj = os.path.join
|
pj = os.path.join
|
||||||
|
|
@ -15,6 +16,12 @@ g_print_progress = True
|
||||||
REMOTE_PREFIX = 'http://download.kiwix.org/dev/'
|
REMOTE_PREFIX = 'http://download.kiwix.org/dev/'
|
||||||
|
|
||||||
|
|
||||||
|
def which(name):
|
||||||
|
command = "which {}".format(name)
|
||||||
|
output = subprocess.check_output(command, shell=True)
|
||||||
|
return output[:-1].decode()
|
||||||
|
|
||||||
|
|
||||||
def setup_print_progress(print_progress):
|
def setup_print_progress(print_progress):
|
||||||
global g_print_progress
|
global g_print_progress
|
||||||
g_print_progress = print_progress
|
g_print_progress = print_progress
|
||||||
|
|
@ -63,6 +70,7 @@ def add_execution_right(file_path):
|
||||||
current_permissions = stat.S_IMODE(os.lstat(file_path).st_mode)
|
current_permissions = stat.S_IMODE(os.lstat(file_path).st_mode)
|
||||||
os.chmod(file_path, current_permissions | stat.S_IXUSR)
|
os.chmod(file_path, current_permissions | stat.S_IXUSR)
|
||||||
|
|
||||||
|
|
||||||
def copy_tree(src, dst, post_copy_function=None):
|
def copy_tree(src, dst, post_copy_function=None):
|
||||||
os.makedirs(dst, exist_ok=True)
|
os.makedirs(dst, exist_ok=True)
|
||||||
for root, dirs, files in os.walk(src):
|
for root, dirs, files in os.walk(src):
|
||||||
|
|
@ -152,6 +160,7 @@ class Context:
|
||||||
|
|
||||||
def extract_archive(archive_path, dest_dir, topdir=None, name=None):
|
def extract_archive(archive_path, dest_dir, topdir=None, name=None):
|
||||||
is_zip_archive = archive_path.endswith('.zip')
|
is_zip_archive = archive_path.endswith('.zip')
|
||||||
|
archive = None
|
||||||
try:
|
try:
|
||||||
if is_zip_archive:
|
if is_zip_archive:
|
||||||
archive = zipfile.ZipFile(archive_path)
|
archive = zipfile.ZipFile(archive_path)
|
||||||
|
|
@ -0,0 +1,42 @@
|
||||||
|
from setuptools import setup, find_packages
|
||||||
|
from codecs import open # To use a consistent encoding
|
||||||
|
from os import path
|
||||||
|
|
||||||
|
here = path.abspath(path.dirname(__file__))
|
||||||
|
|
||||||
|
# Get the long description from the relevant file
|
||||||
|
with open(path.join(here, 'README.md'), encoding='utf-8') as f:
|
||||||
|
long_description = f.read()
|
||||||
|
|
||||||
|
# Doc: http://pythonhosted.org/setuptools/setuptools.html#including-data-files.
|
||||||
|
setup(
|
||||||
|
name='kiwix-build',
|
||||||
|
version="0.1",
|
||||||
|
description=('A tool to build kiwix and openzim projects.'),
|
||||||
|
long_description=long_description,
|
||||||
|
url='https://github.com/kiwix/kiwix-build/',
|
||||||
|
author='Matthieu Gautier',
|
||||||
|
author_email='mgautier@kymeria.fr',
|
||||||
|
license='GPLv3',
|
||||||
|
|
||||||
|
# See https://pypi.python.org/pypi?%3Aaction=list_classifiers
|
||||||
|
classifiers=[
|
||||||
|
'Development Status :: 3 - Alpha',
|
||||||
|
|
||||||
|
# Indicate who your project is intended for
|
||||||
|
'Intended Audience :: Developers',
|
||||||
|
'Topic :: Software Development :: Build Tools',
|
||||||
|
|
||||||
|
# Specify the Python versions you support here. In particular, ensure
|
||||||
|
# that you indicate whether you support Python 2, Python 3 or both.
|
||||||
|
'Programming Language :: Python :: 3',
|
||||||
|
'Programming Language :: Python :: 3.4',
|
||||||
|
],
|
||||||
|
packages=find_packages(),
|
||||||
|
include_package_data=True,
|
||||||
|
entry_points={
|
||||||
|
'console_scripts': [
|
||||||
|
'kiwix-build = kiwixbuild:main'
|
||||||
|
]
|
||||||
|
}
|
||||||
|
)
|
||||||
Loading…
Reference in New Issue