Create a real module kiwixbuild and make it installable. Split files.
Make kiwix-build installable using pip. Module is now called `kiwixbuild` because `kiwix-build` is not a valid python identifier. Also split toolchains in separated directory.
This commit is contained in:
Matthieu Gautier
parent
1513204f42
commit
97666b0fc9
|
|
@ -0,0 +1,4 @@
|
|||
include kiwixbuild/templates/cmake_cross_file.txt
|
||||
include kiwixbuild/templates/meson_cross_file.txt
|
||||
include kiwixbuild/templates/cmake_android_cross_file.txt
|
||||
recursise-include kiwixbuild/patches *.patch
|
||||
|
|
@ -8,9 +8,9 @@ import subprocess
|
|||
import platform
|
||||
from collections import OrderedDict
|
||||
|
||||
from dependencies import Dependency
|
||||
from dependency_utils import ReleaseDownload, Builder, GitClone
|
||||
from utils import (
|
||||
from .toolchains import Toolchain
|
||||
from .dependencies import Dependency
|
||||
from .utils import (
|
||||
pj,
|
||||
remove_duplicates,
|
||||
add_execution_right,
|
||||
|
|
@ -47,7 +47,7 @@ PACKAGE_NAME_MAPPERS = {
|
|||
'gumbo' : ['gumbo-parser-devel'],
|
||||
},
|
||||
'fedora_native_static': {
|
||||
'COMMON': _fedora_common + ['glibc-static', 'libstdc++-static'],
|
||||
'COMMON': _fedora_common,
|
||||
'zlib': ['zlib-devel', 'zlib-static'],
|
||||
'lzma': ['xz-devel', 'xz-static']
|
||||
# Either there is no packages, or no static or too old
|
||||
|
|
@ -118,12 +118,6 @@ PACKAGE_NAME_MAPPERS = {
|
|||
}
|
||||
|
||||
|
||||
def which(name):
|
||||
command = "which {}".format(name)
|
||||
output = subprocess.check_output(command, shell=True)
|
||||
return output[:-1].decode()
|
||||
|
||||
|
||||
class TargetInfo:
|
||||
def __init__(self, build, static, toolchains, hosts=None):
|
||||
self.build = build
|
||||
|
|
@ -536,355 +530,6 @@ class BuildEnv:
|
|||
with open(autoskip_file, 'w'):
|
||||
pass
|
||||
|
||||
class _MetaToolchain(type):
|
||||
def __new__(cls, name, bases, dct):
|
||||
_class = type.__new__(cls, name, bases, dct)
|
||||
if name != 'Toolchain':
|
||||
Toolchain.all_toolchains[name] = _class
|
||||
return _class
|
||||
|
||||
|
||||
class Toolchain(metaclass=_MetaToolchain):
|
||||
all_toolchains = {}
|
||||
configure_option = ""
|
||||
cmake_option = ""
|
||||
exec_wrapper_def = ""
|
||||
Builder = None
|
||||
Source = None
|
||||
|
||||
def __init__(self, buildEnv):
|
||||
self.buildEnv = buildEnv
|
||||
self.source = self.Source(self) if self.Source else None
|
||||
self.builder = self.Builder(self) if self.Builder else None
|
||||
|
||||
@property
|
||||
def full_name(self):
|
||||
return "{name}-{version}".format(
|
||||
name = self.name,
|
||||
version = self.version)
|
||||
|
||||
@property
|
||||
def source_path(self):
|
||||
return pj(self.buildEnv.source_dir, self.source.source_dir)
|
||||
|
||||
@property
|
||||
def _log_dir(self):
|
||||
return self.buildEnv.log_dir
|
||||
|
||||
def set_env(self, env):
|
||||
pass
|
||||
|
||||
def set_compiler(self, env):
|
||||
pass
|
||||
|
||||
def command(self, name, function, *args):
|
||||
print(" {} {} : ".format(name, self.name), end="", flush=True)
|
||||
log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
|
||||
context = Context(name, log, True)
|
||||
try:
|
||||
ret = function(*args, context=context)
|
||||
context._finalise()
|
||||
print("OK")
|
||||
return ret
|
||||
except SkipCommand:
|
||||
print("SKIP")
|
||||
except subprocess.CalledProcessError:
|
||||
print("ERROR")
|
||||
try:
|
||||
with open(log, 'r') as f:
|
||||
print(f.read())
|
||||
except:
|
||||
pass
|
||||
raise StopBuild()
|
||||
except:
|
||||
print("ERROR")
|
||||
raise
|
||||
|
||||
|
||||
class mingw32_toolchain(Toolchain):
|
||||
name = 'mingw32'
|
||||
arch_full = 'i686-w64-mingw32'
|
||||
|
||||
@property
|
||||
def root_path(self):
|
||||
return self.buildEnv.cross_config['root_path']
|
||||
|
||||
@property
|
||||
def binaries(self):
|
||||
return {k:which('{}-{}'.format(self.arch_full, v))
|
||||
for k, v in (('CC', 'gcc'),
|
||||
('CXX', 'g++'),
|
||||
('AR', 'ar'),
|
||||
('STRIP', 'strip'),
|
||||
('WINDRES', 'windres'),
|
||||
('RANLIB', 'ranlib'))
|
||||
}
|
||||
|
||||
@property
|
||||
def exec_wrapper_def(self):
|
||||
try:
|
||||
which('wine')
|
||||
except subprocess.CalledProcessError:
|
||||
return ""
|
||||
else:
|
||||
return "exec_wrapper = 'wine'"
|
||||
|
||||
|
||||
@property
|
||||
def configure_option(self):
|
||||
return '--host={}'.format(self.arch_full)
|
||||
|
||||
def get_bin_dir(self):
|
||||
return [pj(self.root_path, 'bin')]
|
||||
|
||||
def set_env(self, env):
|
||||
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
|
||||
env['LIBS'] = " ".join(self.buildEnv.cross_config['extra_libs']) + " " +env['LIBS']
|
||||
|
||||
def set_compiler(self, env):
|
||||
for k, v in self.binaries.items():
|
||||
env[k] = v
|
||||
|
||||
|
||||
class android_ndk(Toolchain):
|
||||
name = 'android-ndk'
|
||||
version = 'r13b'
|
||||
gccver = '4.9.x'
|
||||
|
||||
@property
|
||||
def api(self):
|
||||
return '21' if self.arch in ('arm64', 'mips64', 'x86_64') else '14'
|
||||
|
||||
@property
|
||||
def platform(self):
|
||||
return 'android-'+self.api
|
||||
|
||||
@property
|
||||
def arch(self):
|
||||
return self.buildEnv.platform_info.arch
|
||||
|
||||
@property
|
||||
def arch_full(self):
|
||||
return self.buildEnv.platform_info.arch_full
|
||||
|
||||
@property
|
||||
def toolchain(self):
|
||||
return self.arch_full+"-4.9"
|
||||
|
||||
@property
|
||||
def root_path(self):
|
||||
return pj(self.builder.install_path, 'sysroot')
|
||||
|
||||
@property
|
||||
def binaries(self):
|
||||
binaries = ((k,'{}-{}'.format(self.arch_full, v))
|
||||
for k, v in (('CC', 'gcc'),
|
||||
('CXX', 'g++'),
|
||||
('AR', 'ar'),
|
||||
('STRIP', 'strip'),
|
||||
('WINDRES', 'windres'),
|
||||
('RANLIB', 'ranlib'),
|
||||
('LD', 'ld'))
|
||||
)
|
||||
return {k:pj(self.builder.install_path, 'bin', v)
|
||||
for k,v in binaries}
|
||||
|
||||
@property
|
||||
def configure_option(self):
|
||||
return '--host={}'.format(self.arch_full)
|
||||
|
||||
@property
|
||||
def full_name(self):
|
||||
return "{name}-{version}-{arch}-{api}".format(
|
||||
name = self.name,
|
||||
version = self.version,
|
||||
arch = self.arch,
|
||||
api = self.api)
|
||||
|
||||
class Source(ReleaseDownload):
|
||||
archive = Remotefile('android-ndk-r13b-linux-x86_64.zip',
|
||||
'3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c',
|
||||
'https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip')
|
||||
|
||||
@property
|
||||
def source_dir(self):
|
||||
return "{}-{}".format(
|
||||
self.target.name,
|
||||
self.target.version)
|
||||
|
||||
|
||||
class Builder(Builder):
|
||||
|
||||
@property
|
||||
def install_path(self):
|
||||
return self.build_path
|
||||
|
||||
def _build_platform(self, context):
|
||||
context.try_skip(self.build_path)
|
||||
script = pj(self.source_path, 'build/tools/make_standalone_toolchain.py')
|
||||
add_execution_right(script)
|
||||
command = '{script} --arch={arch} --api={api} --install-dir={install_dir} --force'
|
||||
command = command.format(
|
||||
script=script,
|
||||
arch=self.target.arch,
|
||||
api=self.target.api,
|
||||
install_dir=self.install_path
|
||||
)
|
||||
self.buildEnv.run_command(command, self.build_path, context)
|
||||
|
||||
def _fix_permission_right(self, context):
|
||||
context.try_skip(self.build_path)
|
||||
bin_dirs = [pj(self.install_path, 'bin'),
|
||||
pj(self.install_path, self.target.arch_full, 'bin'),
|
||||
pj(self.install_path, 'libexec', 'gcc', self.target.arch_full, self.target.gccver)
|
||||
]
|
||||
for root, dirs, files in os.walk(self.install_path):
|
||||
if not root in bin_dirs:
|
||||
continue
|
||||
|
||||
for file_ in files:
|
||||
file_path = pj(root, file_)
|
||||
if os.path.islink(file_path):
|
||||
continue
|
||||
add_execution_right(file_path)
|
||||
|
||||
def build(self):
|
||||
self.command('build_platform', self._build_platform)
|
||||
self.command('fix_permission_right', self._fix_permission_right)
|
||||
|
||||
def get_bin_dir(self):
|
||||
return [pj(self.builder.install_path, 'bin')]
|
||||
|
||||
def set_env(self, env):
|
||||
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
|
||||
env['CFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(self.root_path) + env['CFLAGS']
|
||||
env['CXXFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(self.root_path) + env['CXXFLAGS']
|
||||
env['LDFLAGS'] = '--sysroot={} '.format(self.root_path) + env['LDFLAGS']
|
||||
#env['CFLAGS'] = ' -fPIC -D_FILE_OFFSET_BITS=64 -O3 '+env['CFLAGS']
|
||||
#env['CXXFLAGS'] = (' -D__OPTIMIZE__ -fno-strict-aliasing '
|
||||
# ' -DU_HAVE_NL_LANGINFO_CODESET=0 '
|
||||
# '-DU_STATIC_IMPLEMENTATION -O3 '
|
||||
# '-DU_HAVE_STD_STRING -DU_TIMEZONE=0 ')+env['CXXFLAGS']
|
||||
env['NDK_DEBUG'] = '0'
|
||||
|
||||
def set_compiler(self, env):
|
||||
env['CC'] = self.binaries['CC']
|
||||
env['CXX'] = self.binaries['CXX']
|
||||
|
||||
|
||||
class android_sdk(Toolchain):
|
||||
name = 'android-sdk'
|
||||
version = 'r25.2.3'
|
||||
|
||||
class Source(ReleaseDownload):
|
||||
archive = Remotefile('tools_r25.2.3-linux.zip',
|
||||
'1b35bcb94e9a686dff6460c8bca903aa0281c6696001067f34ec00093145b560',
|
||||
'https://dl.google.com/android/repository/tools_r25.2.3-linux.zip')
|
||||
|
||||
class Builder(Builder):
|
||||
|
||||
@property
|
||||
def install_path(self):
|
||||
return pj(self.buildEnv.toolchain_dir, self.target.full_name)
|
||||
|
||||
def _build_platform(self, context):
|
||||
context.try_skip(self.install_path)
|
||||
tools_dir = pj(self.install_path, 'tools')
|
||||
shutil.copytree(self.source_path, tools_dir)
|
||||
script = pj(tools_dir, 'android')
|
||||
command = '{script} --verbose update sdk -a --no-ui --filter {packages}'
|
||||
command = command.format(
|
||||
script=script,
|
||||
packages = ','.join(str(i) for i in [1,2,8,34,162])
|
||||
)
|
||||
# packages correspond to :
|
||||
# - 1 : Android SDK Tools, revision 25.2.5
|
||||
# - 2 : Android SDK Platform-tools, revision 25.0.3
|
||||
# - 8 : Android SDK Build-tools, revision 24.0.1
|
||||
# - 34 : SDK Platform Android 7.0, API 24, revision 2
|
||||
# - 162 : Android Support Repository, revision 44
|
||||
self.buildEnv.run_command(command, self.install_path, context, input="y\n")
|
||||
|
||||
def _fix_licenses(self, context):
|
||||
context.try_skip(self.install_path)
|
||||
os.makedirs(pj(self.install_path, 'licenses'), exist_ok=True)
|
||||
with open(pj(self.install_path, 'licenses', 'android-sdk-license'), 'w') as f:
|
||||
f.write("\n8933bad161af4178b1185d1a37fbf41ea5269c55\nd56f5187479451eabf01fb78af6dfcb131a6481e")
|
||||
|
||||
def build(self):
|
||||
self.command('build_platform', self._build_platform)
|
||||
self.command('fix_licenses', self._fix_licenses)
|
||||
|
||||
def get_bin_dir(self):
|
||||
return []
|
||||
|
||||
def set_env(self, env):
|
||||
env['ANDROID_HOME'] = self.builder.install_path
|
||||
|
||||
|
||||
class armhf_toolchain(Toolchain):
|
||||
name = 'armhf'
|
||||
arch_full = 'arm-linux-gnueabihf'
|
||||
|
||||
class Source(GitClone):
|
||||
git_remote = "https://github.com/raspberrypi/tools"
|
||||
git_dir = "raspberrypi-tools"
|
||||
|
||||
@property
|
||||
def root_path(self):
|
||||
return pj(self.source_path, 'arm-bcm2708', 'gcc-linaro-arm-linux-gnueabihf-raspbian-x64')
|
||||
|
||||
@property
|
||||
def binaries(self):
|
||||
binaries = ((k,'{}-{}'.format(self.arch_full, v))
|
||||
for k, v in (('CC', 'gcc'),
|
||||
('CXX', 'g++'),
|
||||
('AR', 'ar'),
|
||||
('STRIP', 'strip'),
|
||||
('WINDRES', 'windres'),
|
||||
('RANLIB', 'ranlib'),
|
||||
('LD', 'ld'))
|
||||
)
|
||||
return {k:pj(self.root_path, 'bin', v)
|
||||
for k,v in binaries}
|
||||
|
||||
@property
|
||||
def exec_wrapper_def(self):
|
||||
try:
|
||||
which('qemu-arm')
|
||||
except subprocess.CalledProcessError:
|
||||
return ""
|
||||
else:
|
||||
return "exec_wrapper = 'qemu-arm'"
|
||||
|
||||
@property
|
||||
def configure_option(self):
|
||||
return '--host={}'.format(self.arch_full)
|
||||
|
||||
def get_bin_dir(self):
|
||||
return [pj(self.root_path, 'bin')]
|
||||
|
||||
def set_env(self, env):
|
||||
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
|
||||
env['CFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS']
|
||||
env['CXXFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
|
||||
env['LIBS'] = " ".join(self.buildEnv.cross_config['extra_libs']) + " " +env['LIBS']
|
||||
env['QEMU_LD_PREFIX'] = pj(self.root_path, "arm-linux-gnueabihf", "libc")
|
||||
env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format(
|
||||
':'.join([
|
||||
pj(self.root_path, "arm-linux-gnueabihf", "lib"),
|
||||
pj(self.buildEnv.install_dir, 'lib'),
|
||||
pj(self.buildEnv.install_dir, self.buildEnv.libprefix)
|
||||
]))
|
||||
|
||||
def set_compiler(self, env):
|
||||
env['CC'] = self.binaries['CC']
|
||||
env['CXX'] = self.binaries['CXX']
|
||||
|
||||
|
||||
class iOS_sdk(Toolchain):
|
||||
pass
|
||||
|
||||
|
||||
class Builder:
|
||||
def __init__(self, options):
|
||||
|
|
@ -897,7 +542,7 @@ class Builder:
|
|||
self.add_targets(targetDef, _targets)
|
||||
dependencies = self.order_dependencies(_targets, targetDef)
|
||||
dependencies = list(remove_duplicates(dependencies))
|
||||
|
||||
|
||||
for dep in dependencies:
|
||||
if self.options.build_deps_only and dep == targetDef:
|
||||
continue
|
||||
|
|
@ -1021,10 +666,10 @@ def parse_args():
|
|||
sys.exit(1)
|
||||
return options
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
def main():
|
||||
options = parse_args()
|
||||
options.working_dir = os.path.abspath(options.working_dir)
|
||||
setup_print_progress(options.show_progress)
|
||||
builder = Builder(options)
|
||||
builder.run()
|
||||
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
from . import main
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
import shutil, os, json
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from dependency_utils import (
|
||||
from kiwixbuild.dependency_utils import (
|
||||
Dependency,
|
||||
ReleaseDownload,
|
||||
GitClone,
|
||||
|
|
@ -14,7 +14,7 @@ from dependency_utils import (
|
|||
NoopBuilder,
|
||||
Builder as BaseBuilder)
|
||||
|
||||
from utils import Remotefile, pj, SkipCommand, copy_tree, add_execution_right
|
||||
from kiwixbuild.utils import Remotefile, pj, SkipCommand, copy_tree, add_execution_right
|
||||
|
||||
# *************************************
|
||||
# Missing dependencies
|
||||
|
|
@ -22,7 +22,6 @@ from utils import Remotefile, pj, SkipCommand, copy_tree, add_execution_right
|
|||
# exist in your "distri" (linux/mac) ?
|
||||
# If not, we need to compile them here
|
||||
# *************************************
|
||||
# aria2
|
||||
# Argtable
|
||||
# MSVirtual
|
||||
# Android
|
||||
|
|
@ -415,7 +414,7 @@ class AllBaseDependencies(Dependency):
|
|||
|
||||
@property
|
||||
def dependencies(self):
|
||||
base_deps = ['zlib', 'lzma', 'xapian-core', 'gumbo', 'pugixml', 'libmicrohttpd', 'libaria2']
|
||||
base_deps = ['zlib', 'lzma', 'xapian-core', 'gumbo', 'pugixml', 'libmicrohttpd']
|
||||
if self.buildEnv.platform_info.build != 'native':
|
||||
base_deps += ["icu4c_cross-compile"]
|
||||
else:
|
||||
|
|
@ -2,8 +2,8 @@ import subprocess
|
|||
import os
|
||||
import shutil
|
||||
|
||||
from utils import pj, Context, SkipCommand, extract_archive, Defaultdict, StopBuild
|
||||
from dependency_versions import main_project_versions, base_deps_versions
|
||||
from kiwixbuild.utils import pj, Context, SkipCommand, extract_archive, Defaultdict, StopBuild
|
||||
from kiwixbuild.dependency_versions import main_project_versions, base_deps_versions
|
||||
|
||||
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
|
||||
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
|
||||
from . import android_ndk, android_sdk, armhf, ios, mingw32
|
||||
|
||||
from .base_toolchain import Toolchain
|
||||
|
|
@ -0,0 +1,133 @@
|
|||
import os
|
||||
|
||||
from .base_toolchain import Toolchain
|
||||
from kiwixbuild.dependency_utils import ReleaseDownload, Builder
|
||||
from kiwixbuild.utils import Remotefile, add_execution_right
|
||||
|
||||
pj = os.path.join
|
||||
|
||||
class android_ndk(Toolchain):
|
||||
name = 'android-ndk'
|
||||
version = 'r13b'
|
||||
gccver = '4.9.x'
|
||||
|
||||
@property
|
||||
def api(self):
|
||||
return '21' if self.arch in ('arm64', 'mips64', 'x86_64') else '14'
|
||||
|
||||
@property
|
||||
def platform(self):
|
||||
return 'android-'+self.api
|
||||
|
||||
@property
|
||||
def arch(self):
|
||||
return self.buildEnv.platform_info.arch
|
||||
|
||||
@property
|
||||
def arch_full(self):
|
||||
return self.buildEnv.platform_info.arch_full
|
||||
|
||||
@property
|
||||
def toolchain(self):
|
||||
return self.arch_full+"-4.9"
|
||||
|
||||
@property
|
||||
def root_path(self):
|
||||
return pj(self.builder.install_path, 'sysroot')
|
||||
|
||||
@property
|
||||
def binaries(self):
|
||||
binaries = ((k,'{}-{}'.format(self.arch_full, v))
|
||||
for k, v in (('CC', 'gcc'),
|
||||
('CXX', 'g++'),
|
||||
('AR', 'ar'),
|
||||
('STRIP', 'strip'),
|
||||
('WINDRES', 'windres'),
|
||||
('RANLIB', 'ranlib'),
|
||||
('LD', 'ld'))
|
||||
)
|
||||
return {k:pj(self.builder.install_path, 'bin', v)
|
||||
for k,v in binaries}
|
||||
|
||||
@property
|
||||
def configure_option(self):
|
||||
return '--host={}'.format(self.arch_full)
|
||||
|
||||
@property
|
||||
def full_name(self):
|
||||
return "{name}-{version}-{arch}-{api}".format(
|
||||
name = self.name,
|
||||
version = self.version,
|
||||
arch = self.arch,
|
||||
api = self.api)
|
||||
|
||||
class Source(ReleaseDownload):
|
||||
archive = Remotefile('android-ndk-r13b-linux-x86_64.zip',
|
||||
'3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c',
|
||||
'https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip')
|
||||
|
||||
@property
|
||||
def source_dir(self):
|
||||
return "{}-{}".format(
|
||||
self.target.name,
|
||||
self.target.version)
|
||||
|
||||
|
||||
class Builder(Builder):
|
||||
|
||||
@property
|
||||
def install_path(self):
|
||||
return self.build_path
|
||||
|
||||
def _build_platform(self, context):
|
||||
context.try_skip(self.build_path)
|
||||
script = pj(self.source_path, 'build/tools/make_standalone_toolchain.py')
|
||||
add_execution_right(script)
|
||||
command = '{script} --arch={arch} --api={api} --install-dir={install_dir} --force'
|
||||
command = command.format(
|
||||
script=script,
|
||||
arch=self.target.arch,
|
||||
api=self.target.api,
|
||||
install_dir=self.install_path
|
||||
)
|
||||
self.buildEnv.run_command(command, self.build_path, context)
|
||||
|
||||
def _fix_permission_right(self, context):
|
||||
context.try_skip(self.build_path)
|
||||
bin_dirs = [pj(self.install_path, 'bin'),
|
||||
pj(self.install_path, self.target.arch_full, 'bin'),
|
||||
pj(self.install_path, 'libexec', 'gcc', self.target.arch_full, self.target.gccver)
|
||||
]
|
||||
for root, dirs, files in os.walk(self.install_path):
|
||||
if not root in bin_dirs:
|
||||
continue
|
||||
|
||||
for file_ in files:
|
||||
file_path = pj(root, file_)
|
||||
if os.path.islink(file_path):
|
||||
continue
|
||||
add_execution_right(file_path)
|
||||
|
||||
def build(self):
|
||||
self.command('build_platform', self._build_platform)
|
||||
self.command('fix_permission_right', self._fix_permission_right)
|
||||
|
||||
def get_bin_dir(self):
|
||||
return [pj(self.builder.install_path, 'bin')]
|
||||
|
||||
def set_env(self, env):
|
||||
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
|
||||
env['CFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(self.root_path) + env['CFLAGS']
|
||||
env['CXXFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(self.root_path) + env['CXXFLAGS']
|
||||
env['LDFLAGS'] = '--sysroot={} '.format(self.root_path) + env['LDFLAGS']
|
||||
#env['CFLAGS'] = ' -fPIC -D_FILE_OFFSET_BITS=64 -O3 '+env['CFLAGS']
|
||||
#env['CXXFLAGS'] = (' -D__OPTIMIZE__ -fno-strict-aliasing '
|
||||
# ' -DU_HAVE_NL_LANGINFO_CODESET=0 '
|
||||
# '-DU_STATIC_IMPLEMENTATION -O3 '
|
||||
# '-DU_HAVE_STD_STRING -DU_TIMEZONE=0 ')+env['CXXFLAGS']
|
||||
env['NDK_DEBUG'] = '0'
|
||||
|
||||
def set_compiler(self, env):
|
||||
env['CC'] = self.binaries['CC']
|
||||
env['CXX'] = self.binaries['CXX']
|
||||
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
import os
|
||||
import shutil
|
||||
|
||||
from .base_toolchain import Toolchain
|
||||
from kiwixbuild.dependency_utils import ReleaseDownload, Builder
|
||||
from kiwixbuild.utils import Remotefile
|
||||
|
||||
pj = os.path.join
|
||||
|
||||
class android_sdk(Toolchain):
|
||||
name = 'android-sdk'
|
||||
version = 'r25.2.3'
|
||||
|
||||
class Source(ReleaseDownload):
|
||||
archive = Remotefile('tools_r25.2.3-linux.zip',
|
||||
'1b35bcb94e9a686dff6460c8bca903aa0281c6696001067f34ec00093145b560',
|
||||
'https://dl.google.com/android/repository/tools_r25.2.3-linux.zip')
|
||||
|
||||
class Builder(Builder):
|
||||
|
||||
@property
|
||||
def install_path(self):
|
||||
return pj(self.buildEnv.toolchain_dir, self.target.full_name)
|
||||
|
||||
def _build_platform(self, context):
|
||||
context.try_skip(self.install_path)
|
||||
tools_dir = pj(self.install_path, 'tools')
|
||||
shutil.copytree(self.source_path, tools_dir)
|
||||
script = pj(tools_dir, 'android')
|
||||
command = '{script} --verbose update sdk -a --no-ui --filter {packages}'
|
||||
command = command.format(
|
||||
script=script,
|
||||
packages = ','.join(str(i) for i in [1,2,8,34,162])
|
||||
)
|
||||
# packages correspond to :
|
||||
# - 1 : Android SDK Tools, revision 25.2.5
|
||||
# - 2 : Android SDK Platform-tools, revision 25.0.3
|
||||
# - 8 : Android SDK Build-tools, revision 24.0.1
|
||||
# - 34 : SDK Platform Android 7.0, API 24, revision 2
|
||||
# - 162 : Android Support Repository, revision 44
|
||||
self.buildEnv.run_command(command, self.install_path, context, input="y\n")
|
||||
|
||||
def _fix_licenses(self, context):
|
||||
context.try_skip(self.install_path)
|
||||
os.makedirs(pj(self.install_path, 'licenses'), exist_ok=True)
|
||||
with open(pj(self.install_path, 'licenses', 'android-sdk-license'), 'w') as f:
|
||||
f.write("\n8933bad161af4178b1185d1a37fbf41ea5269c55\nd56f5187479451eabf01fb78af6dfcb131a6481e")
|
||||
|
||||
def build(self):
|
||||
self.command('build_platform', self._build_platform)
|
||||
self.command('fix_licenses', self._fix_licenses)
|
||||
|
||||
def get_bin_dir(self):
|
||||
return []
|
||||
|
||||
def set_env(self, env):
|
||||
env['ANDROID_HOME'] = self.builder.install_path
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
import os
|
||||
import subprocess
|
||||
|
||||
from .base_toolchain import Toolchain
|
||||
from kiwixbuild.dependency_utils import GitClone
|
||||
from kiwixbuild.utils import Remotefile, which
|
||||
pj = os.path.join
|
||||
|
||||
class armhf_toolchain(Toolchain):
|
||||
name = 'armhf'
|
||||
arch_full = 'arm-linux-gnueabihf'
|
||||
|
||||
class Source(GitClone):
|
||||
git_remote = "https://github.com/raspberrypi/tools"
|
||||
git_dir = "raspberrypi-tools"
|
||||
|
||||
@property
|
||||
def root_path(self):
|
||||
return pj(self.source_path, 'arm-bcm2708', 'gcc-linaro-arm-linux-gnueabihf-raspbian-x64')
|
||||
|
||||
@property
|
||||
def binaries(self):
|
||||
binaries = ((k,'{}-{}'.format(self.arch_full, v))
|
||||
for k, v in (('CC', 'gcc'),
|
||||
('CXX', 'g++'),
|
||||
('AR', 'ar'),
|
||||
('STRIP', 'strip'),
|
||||
('WINDRES', 'windres'),
|
||||
('RANLIB', 'ranlib'),
|
||||
('LD', 'ld'))
|
||||
)
|
||||
return {k:pj(self.root_path, 'bin', v)
|
||||
for k,v in binaries}
|
||||
|
||||
@property
|
||||
def exec_wrapper_def(self):
|
||||
try:
|
||||
which('qemu-arm')
|
||||
except subprocess.CalledProcessError:
|
||||
return ""
|
||||
else:
|
||||
return "exec_wrapper = 'qemu-arm'"
|
||||
|
||||
@property
|
||||
def configure_option(self):
|
||||
return '--host={}'.format(self.arch_full)
|
||||
|
||||
def get_bin_dir(self):
|
||||
return [pj(self.root_path, 'bin')]
|
||||
|
||||
def set_env(self, env):
|
||||
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
|
||||
env['CFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS']
|
||||
env['CXXFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
|
||||
env['LIBS'] = " ".join(self.buildEnv.cross_config['extra_libs']) + " " +env['LIBS']
|
||||
env['QEMU_LD_PREFIX'] = pj(self.root_path, "arm-linux-gnueabihf", "libc")
|
||||
env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format(
|
||||
':'.join([
|
||||
pj(self.root_path, "arm-linux-gnueabihf", "lib"),
|
||||
pj(self.buildEnv.install_dir, 'lib'),
|
||||
pj(self.buildEnv.install_dir, self.buildEnv.libprefix)
|
||||
]))
|
||||
|
||||
def set_compiler(self, env):
|
||||
env['CC'] = self.binaries['CC']
|
||||
env['CXX'] = self.binaries['CXX']
|
||||
|
|
@ -0,0 +1,70 @@
|
|||
import os
|
||||
import subprocess
|
||||
|
||||
pj = os.path.join
|
||||
|
||||
from kiwixbuild.utils import Context, SkipCommand, StopBuild
|
||||
|
||||
class _MetaToolchain(type):
|
||||
def __new__(cls, name, bases, dct):
|
||||
_class = type.__new__(cls, name, bases, dct)
|
||||
if name != 'Toolchain':
|
||||
Toolchain.all_toolchains[name] = _class
|
||||
return _class
|
||||
|
||||
|
||||
class Toolchain(metaclass=_MetaToolchain):
|
||||
all_toolchains = {}
|
||||
configure_option = ""
|
||||
cmake_option = ""
|
||||
exec_wrapper_def = ""
|
||||
Builder = None
|
||||
Source = None
|
||||
|
||||
def __init__(self, buildEnv):
|
||||
self.buildEnv = buildEnv
|
||||
self.source = self.Source(self) if self.Source else None
|
||||
self.builder = self.Builder(self) if self.Builder else None
|
||||
|
||||
@property
|
||||
def full_name(self):
|
||||
return "{name}-{version}".format(
|
||||
name = self.name,
|
||||
version = self.version)
|
||||
|
||||
@property
|
||||
def source_path(self):
|
||||
return pj(self.buildEnv.source_dir, self.source.source_dir)
|
||||
|
||||
@property
|
||||
def _log_dir(self):
|
||||
return self.buildEnv.log_dir
|
||||
|
||||
def set_env(self, env):
|
||||
pass
|
||||
|
||||
def set_compiler(self, env):
|
||||
pass
|
||||
|
||||
def command(self, name, function, *args):
|
||||
print(" {} {} : ".format(name, self.name), end="", flush=True)
|
||||
log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
|
||||
context = Context(name, log, True)
|
||||
try:
|
||||
ret = function(*args, context=context)
|
||||
context._finalise()
|
||||
print("OK")
|
||||
return ret
|
||||
except SkipCommand:
|
||||
print("SKIP")
|
||||
except subprocess.CalledProcessError:
|
||||
print("ERROR")
|
||||
try:
|
||||
with open(log, 'r') as f:
|
||||
print(f.read())
|
||||
except:
|
||||
pass
|
||||
raise StopBuild()
|
||||
except:
|
||||
print("ERROR")
|
||||
raise
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
from .base_toolchain import Toolchain
|
||||
|
||||
class iOS_sdk(Toolchain):
|
||||
pass
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
import os
|
||||
import subprocess
|
||||
|
||||
from .base_toolchain import Toolchain
|
||||
from kiwixbuild.utils import which
|
||||
|
||||
pj = os.path.join
|
||||
|
||||
class mingw32_toolchain(Toolchain):
|
||||
name = 'mingw32'
|
||||
arch_full = 'i686-w64-mingw32'
|
||||
|
||||
@property
|
||||
def root_path(self):
|
||||
return self.buildEnv.cross_config['root_path']
|
||||
|
||||
@property
|
||||
def binaries(self):
|
||||
return {k:which('{}-{}'.format(self.arch_full, v))
|
||||
for k, v in (('CC', 'gcc'),
|
||||
('CXX', 'g++'),
|
||||
('AR', 'ar'),
|
||||
('STRIP', 'strip'),
|
||||
('WINDRES', 'windres'),
|
||||
('RANLIB', 'ranlib'))
|
||||
}
|
||||
|
||||
@property
|
||||
def exec_wrapper_def(self):
|
||||
try:
|
||||
which('wine')
|
||||
except subprocess.CalledProcessError:
|
||||
return ""
|
||||
else:
|
||||
return "exec_wrapper = 'wine'"
|
||||
|
||||
|
||||
@property
|
||||
def configure_option(self):
|
||||
return '--host={}'.format(self.arch_full)
|
||||
|
||||
def get_bin_dir(self):
|
||||
return [pj(self.root_path, 'bin')]
|
||||
|
||||
def set_env(self, env):
|
||||
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
|
||||
env['LIBS'] = " ".join(self.buildEnv.cross_config['extra_libs']) + " " +env['LIBS']
|
||||
|
||||
def set_compiler(self, env):
|
||||
for k, v in self.binaries.items():
|
||||
env[k] = v
|
||||
|
|
@ -6,6 +6,7 @@ import shutil
|
|||
import os, stat, sys
|
||||
import urllib
|
||||
import ssl
|
||||
import subprocess
|
||||
from collections import namedtuple, defaultdict
|
||||
|
||||
pj = os.path.join
|
||||
|
|
@ -15,6 +16,12 @@ g_print_progress = True
|
|||
REMOTE_PREFIX = 'http://download.kiwix.org/dev/'
|
||||
|
||||
|
||||
def which(name):
|
||||
command = "which {}".format(name)
|
||||
output = subprocess.check_output(command, shell=True)
|
||||
return output[:-1].decode()
|
||||
|
||||
|
||||
def setup_print_progress(print_progress):
|
||||
global g_print_progress
|
||||
g_print_progress = print_progress
|
||||
|
|
@ -63,6 +70,7 @@ def add_execution_right(file_path):
|
|||
current_permissions = stat.S_IMODE(os.lstat(file_path).st_mode)
|
||||
os.chmod(file_path, current_permissions | stat.S_IXUSR)
|
||||
|
||||
|
||||
def copy_tree(src, dst, post_copy_function=None):
|
||||
os.makedirs(dst, exist_ok=True)
|
||||
for root, dirs, files in os.walk(src):
|
||||
|
|
@ -152,6 +160,7 @@ class Context:
|
|||
|
||||
def extract_archive(archive_path, dest_dir, topdir=None, name=None):
|
||||
is_zip_archive = archive_path.endswith('.zip')
|
||||
archive = None
|
||||
try:
|
||||
if is_zip_archive:
|
||||
archive = zipfile.ZipFile(archive_path)
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
from setuptools import setup, find_packages
|
||||
from codecs import open # To use a consistent encoding
|
||||
from os import path
|
||||
|
||||
here = path.abspath(path.dirname(__file__))
|
||||
|
||||
# Get the long description from the relevant file
|
||||
with open(path.join(here, 'README.md'), encoding='utf-8') as f:
|
||||
long_description = f.read()
|
||||
|
||||
# Doc: http://pythonhosted.org/setuptools/setuptools.html#including-data-files.
|
||||
setup(
|
||||
name='kiwix-build',
|
||||
version="0.1",
|
||||
description=('A tool to build kiwix and openzim projects.'),
|
||||
long_description=long_description,
|
||||
url='https://github.com/kiwix/kiwix-build/',
|
||||
author='Matthieu Gautier',
|
||||
author_email='mgautier@kymeria.fr',
|
||||
license='GPLv3',
|
||||
|
||||
# See https://pypi.python.org/pypi?%3Aaction=list_classifiers
|
||||
classifiers=[
|
||||
'Development Status :: 3 - Alpha',
|
||||
|
||||
# Indicate who your project is intended for
|
||||
'Intended Audience :: Developers',
|
||||
'Topic :: Software Development :: Build Tools',
|
||||
|
||||
# Specify the Python versions you support here. In particular, ensure
|
||||
# that you indicate whether you support Python 2, Python 3 or both.
|
||||
'Programming Language :: Python :: 3',
|
||||
'Programming Language :: Python :: 3.4',
|
||||
],
|
||||
packages=find_packages(),
|
||||
include_package_data=True,
|
||||
entry_points={
|
||||
'console_scripts': [
|
||||
'kiwix-build = kiwixbuild:main'
|
||||
]
|
||||
}
|
||||
)
|
||||
Loading…
Reference in New Issue