From 8f0c6a321beb019d0398a2827200ac891d53c972 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 4 Apr 2018 18:15:10 +0200 Subject: [PATCH 1/6] Add an option to not build all dependencies but only the specified target. --- kiwixbuild/__init__.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/kiwixbuild/__init__.py b/kiwixbuild/__init__.py index 32bbf51..3718c1c 100644 --- a/kiwixbuild/__init__.py +++ b/kiwixbuild/__init__.py @@ -543,10 +543,13 @@ class Builder: dependencies = self.order_dependencies(_targets, targetDef) dependencies = list(remove_duplicates(dependencies)) - for dep in dependencies: - if self.options.build_deps_only and dep == targetDef: - continue - self.targets[dep] = _targets[dep] + if options.build_nodeps: + self.targets[targetDef] = _targets[targetDef] + else: + for dep in dependencies: + if self.options.build_deps_only and dep == targetDef: + continue + self.targets[dep] = _targets[dep] def add_targets(self, targetName, targets): if targetName in targets: @@ -633,6 +636,8 @@ def parse_args(): help="Skip the source download part") parser.add_argument('--build-deps-only', action='store_true', help="Build only the dependencies of the specified targets.") + parser.add_argument('--build-nodeps', action='store_true', + help="Build only the target, not its dependencies.") parser.add_argument('--make-dist', action='store_true', help="Build distrubution (dist) source archive") parser.add_argument('--make-release', action='store_true', From 5d08673a52144dbda3a93174ff6ada2a595995e9 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 4 Apr 2018 18:16:20 +0200 Subject: [PATCH 2/6] Correctly stop the build if there is an error during the downloading. --- kiwixbuild/utils.py | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/kiwixbuild/utils.py b/kiwixbuild/utils.py index 6c7e6eb..858c495 100644 --- a/kiwixbuild/utils.py +++ b/kiwixbuild/utils.py @@ -101,22 +101,26 @@ def download_remote(what, where, check_certificate=True): batch_size = 1024 * 8 extra_args = {'context':context} if sys.version_info >= (3, 4, 3) else {} progress_chars = "/-\|" - with urllib.request.urlopen(file_url, **extra_args) as resource, open(file_path, 'wb') as file: - tsize = resource.getheader('Content-Length', None) - if tsize is not None: - tsize = int(tsize) - current = 0 - while True: - batch = resource.read(batch_size) - if not batch: - break - if tsize: - current += batch_size - print_progress("{:.2%}".format(current/tsize)) - else: - print_progress(progress_chars[current]) - current = (current+1)%4 - file.write(batch) + try: + with urllib.request.urlopen(file_url, **extra_args) as resource, open(file_path, 'wb') as file: + tsize = resource.getheader('Content-Length', None) + if tsize is not None: + tsize = int(tsize) + current = 0 + while True: + batch = resource.read(batch_size) + if not batch: + break + if tsize: + current += batch_size + print_progress("{:.2%}".format(current/tsize)) + else: + print_progress(progress_chars[current]) + current = (current+1)%4 + file.write(batch) + except urllib.error.HTTPError: + print("Cannot download url {}".format(file_url)) + raise StopBuild() if not what.sha256: print('Sha256 for {} not set, do no verify download'.format(what.name)) From 468c3edff940d7c6e64bfba45d49704213724a5a Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 5 Apr 2018 11:23:29 +0200 Subject: [PATCH 3/6] Add a i586 toolchain. This will allow to build kiwix-tools for i586 architectures. --- MANIFEST.in | 1 + kiwixbuild/__init__.py | 37 +++++++++++++++--- .../templates/cmake_i586_cross_file.txt | 18 +++++++++ kiwixbuild/toolchains/__init__.py | 2 +- kiwixbuild/toolchains/linux_i586.py | 38 +++++++++++++++++++ 5 files changed, 90 insertions(+), 6 deletions(-) create mode 100644 kiwixbuild/templates/cmake_i586_cross_file.txt create mode 100644 kiwixbuild/toolchains/linux_i586.py diff --git a/MANIFEST.in b/MANIFEST.in index 44ac497..ea3afa9 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,5 @@ include kiwixbuild/templates/cmake_cross_file.txt include kiwixbuild/templates/meson_cross_file.txt include kiwixbuild/templates/cmake_android_cross_file.txt +include kiwixbuild/templates/cmake_i586_cross_file.txt include kiwixbuild/patches/*.patch diff --git a/kiwixbuild/__init__.py b/kiwixbuild/__init__.py index 3718c1c..a7a6c49 100644 --- a/kiwixbuild/__init__.py +++ b/kiwixbuild/__init__.py @@ -52,6 +52,12 @@ PACKAGE_NAME_MAPPERS = { 'lzma': ['xz-devel', 'xz-static'] # Either there is no packages, or no static or too old }, + 'fedora_i586_dyn': { + 'COMMON': _fedora_common + ['glibc-devel.i686', 'libstdc++-devel.i686'], + }, + 'fedora_i586_static': { + 'COMMON': _fedora_common + ['glibc-devel.i686'], + }, 'fedora_win32_dyn': { 'COMMON': _fedora_common + ['mingw32-gcc-c++', 'mingw32-bzip2', 'mingw32-win-iconv', 'mingw32-winpthreads', 'wine'], 'zlib': ['mingw32-zlib'], @@ -162,6 +168,19 @@ class TargetInfo: 'abi': '' } } + elif self.build == 'i586': + return { + 'extra_libs': ['-m32', '-march=i586'], + 'extra_cflags': ['-m32', '-march=i586'], + 'host_machine': { + 'system': 'linux', + 'lsystem': 'linux', + 'cpu_family': 'x86', + 'cpu': 'i586', + 'endian': 'little', + 'abi': '' + } + } class AndroidTargetInfo(TargetInfo): __arch_infos = { @@ -208,15 +227,19 @@ class BuildEnv: 'native_dyn': TargetInfo('native', False, [], hosts=['fedora', 'debian', 'Darwin']), 'native_static': TargetInfo('native', True, [], - hosts=['fedora', 'debian']), + hosts=['fedora', 'debian']), + 'i586_dyn': TargetInfo('i586', False, ['linux_i586_toolchain'], + hosts=['fedora', 'debian']), + 'i586_static': TargetInfo('i586', True, ['linux_i586_toolchain'], + hosts=['fedora', 'debian']), 'win32_dyn': TargetInfo('win32', False, ['mingw32_toolchain'], - hosts=['fedora', 'debian']), + hosts=['fedora', 'debian']), 'win32_static': TargetInfo('win32', True, ['mingw32_toolchain'], - hosts=['fedora', 'debian']), + hosts=['fedora', 'debian']), 'armhf_dyn': TargetInfo('armhf', False, ['armhf_toolchain'], - hosts=['fedora', 'debian']), + hosts=['fedora', 'debian']), 'armhf_static': TargetInfo('armhf', True, ['armhf_toolchain'], - hosts=['fedora', 'debian']), + hosts=['fedora', 'debian']), 'android_arm': AndroidTargetInfo('arm'), 'android_arm64': AndroidTargetInfo('arm64'), 'android_mips': AndroidTargetInfo('mips'), @@ -332,6 +355,10 @@ class BuildEnv: def setup_iOS(self): pass + def setup_i586(self): + self.cmake_crossfile = self._gen_crossfile('cmake_i586_cross_file.txt') + self.meson_crossfile = self._gen_crossfile('meson_cross_file.txt') + def __getattr__(self, name): return getattr(self.options, name) diff --git a/kiwixbuild/templates/cmake_i586_cross_file.txt b/kiwixbuild/templates/cmake_i586_cross_file.txt new file mode 100644 index 0000000..03d186a --- /dev/null +++ b/kiwixbuild/templates/cmake_i586_cross_file.txt @@ -0,0 +1,18 @@ +SET(CMAKE_SYSTEM_NAME {host_machine[system]}) +SET(CMAKE_SYSTEM_PROCESSOR {host_machine[cpu_family]}) + +# specify the cross compiler +SET(CMAKE_C_COMPILER "{toolchain.binaries[CC]}") +SET(CMAKE_CXX_COMPILER "{toolchain.binaries[CXX]}") +SET(C_FLAGS "-m32 -march=i586") +SET(CXX_FLAGS "-m32 -march=i586") +SET(CMAKE_LD_FLAGS "-m32 -march=i586") +SET(CMAKE_AR:FILEPATH {toolchain.binaries[AR]}) +SET(CMAKE_RANLIB:FILEPATH {toolchain.binaries[RANLIB]}) + +find_program(CCACHE_FOUND ccache) +if(CCACHE_FOUND) + set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache) + set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache) +endif(CCACHE_FOUND) + diff --git a/kiwixbuild/toolchains/__init__.py b/kiwixbuild/toolchains/__init__.py index fcfee2f..d0ca8b1 100644 --- a/kiwixbuild/toolchains/__init__.py +++ b/kiwixbuild/toolchains/__init__.py @@ -1,4 +1,4 @@ -from . import android_ndk, android_sdk, armhf, ios, mingw32 +from . import android_ndk, android_sdk, armhf, ios, mingw32, linux_i586 from .base_toolchain import Toolchain diff --git a/kiwixbuild/toolchains/linux_i586.py b/kiwixbuild/toolchains/linux_i586.py new file mode 100644 index 0000000..b4908fe --- /dev/null +++ b/kiwixbuild/toolchains/linux_i586.py @@ -0,0 +1,38 @@ +import os +import subprocess + +from .base_toolchain import Toolchain +from kiwixbuild.dependency_utils import GitClone +from kiwixbuild.utils import Remotefile, which +pj = os.path.join + +class linux_i586_toolchain(Toolchain): + name = 'linux_i586' + arch_full = 'i586-linux-gnu' + + @property + def configure_option(self): + return '--host={}'.format(self.arch_full) + + @property + def binaries(self): + return {k:which(v) + for k, v in (('CC', os.environ.get('CC', 'gcc')), + ('CXX', os.environ.get('CXX', 'g++')), + ('AR', 'ar'), + ('STRIP', 'strip'), + ('RANLIB', 'ranlib'), + ('LD', 'ld')) + } + + @property + def configure_option(self): + return '--host={}'.format(self.arch_full) + + def set_env(self, env): + env['CFLAGS'] = "-m32 -march=i586 "+env['CFLAGS'] + env['CXXFLAGS'] = "-m32 -march=i586 "+env['CXXFLAGS'] + env['LDFLAGS'] = "-m32 -march=i586 "+env['LDFLAGS'] + + def get_bin_dir(self): + return [] From fd39245c40bc72c70f3ae4891767c28d95bbe62d Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 5 Apr 2018 11:28:01 +0200 Subject: [PATCH 4/6] Update travis to also compile i586 variants. --- .travis.yml | 2 ++ travis/compile_all.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 8695ca3..fb18dd4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,6 +46,8 @@ env: - PLATFORM="win32_static" - PLATFORM="armhf_dyn" - PLATFORM="armhf_static" + - PLATFORM="i586_dyn" + - PLATFORM="i586_static" - PLATFORM="android_arm" - PLATFORM="android_arm64" - PLATFORM="android_mips" diff --git a/travis/compile_all.py b/travis/compile_all.py index 047bfe4..3f43196 100755 --- a/travis/compile_all.py +++ b/travis/compile_all.py @@ -255,6 +255,8 @@ elif PLATFORM == 'win32_static': make_archive('kiwix-tools', 'win32') elif PLATFORM == 'armhf_static': make_archive('kiwix-tools', 'armhf') +elif PLATFORM == 'i586_static': + make_archive('kiwix-tools', 'i586') elif PLATFORM.startswith('android_') and 'kiwix-android' in TARGETS: APK_NAME = "kiwix-{}".format(PLATFORM) source_debug_dir = BASE_DIR/'kiwix-android'/'app'/'build'/'outputs'/'apk'/'kiwix'/'debug' From 27bce3cd99ef80e3110fa30d8cc262a7381d30b2 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Fri, 6 Apr 2018 15:03:12 +0200 Subject: [PATCH 5/6] Add missing package for i586 ubuntu in travis. --- .travis.yml | 4 ++++ kiwixbuild/__init__.py | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/.travis.yml b/.travis.yml index fb18dd4..741df91 100644 --- a/.travis.yml +++ b/.travis.yml @@ -92,3 +92,7 @@ addons: - mingw-w64-tools - default-jdk - autopoint + - libc6-dev:i386 + - libstdc++-6-dev:i386 + - gcc-5-multilib + - g++-5-multilib diff --git a/kiwixbuild/__init__.py b/kiwixbuild/__init__.py index a7a6c49..77848ee 100644 --- a/kiwixbuild/__init__.py +++ b/kiwixbuild/__init__.py @@ -95,6 +95,12 @@ PACKAGE_NAME_MAPPERS = { 'ctpp2': ['libctpp2-dev'], 'ctpp2c': ['ctpp2-utils'], }, + 'debian_i586_dyn': { + 'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib', 'libxml2-dev:i386', 'libsqlite3-dev:i386'], + }, + 'debian_i586_static': { + 'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib', 'libxml2-dev:i386', 'libsqlite3-dev:i386'], + }, 'debian_win32_dyn': { 'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'], 'ctpp2c': ['ctpp2-utils'], From 83a4841ea2b7ab5349f3357211be6928d6e8b440 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Fri, 6 Apr 2018 17:47:46 +0200 Subject: [PATCH 6/6] Do not build ctpp2 with iconv support. --- kiwixbuild/dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kiwixbuild/dependencies.py b/kiwixbuild/dependencies.py index 5b35c3b..b395414 100644 --- a/kiwixbuild/dependencies.py +++ b/kiwixbuild/dependencies.py @@ -147,7 +147,7 @@ class CTPP2(Dependency): ] class Builder(CMakeBuilder): - configure_option = "-DMD5_SUPPORT=OFF" + configure_option = "-DMD5_SUPPORT=OFF -DICONV_SUPPORT=OFF" class CTPP2C(CTPP2):