Split the too long kiwix-build.py file into several smaller ones.
This commit is contained in:
Matthieu Gautier
parent
38a14d8af6
commit
ffee068fd0
|
|
@ -0,0 +1,229 @@
|
|||
import shutil
|
||||
|
||||
from dependency_utils import (
|
||||
Dependency,
|
||||
ReleaseDownload,
|
||||
GitClone,
|
||||
MakeBuilder,
|
||||
CMakeBuilder,
|
||||
MesonBuilder)
|
||||
|
||||
from utils import Remotefile, pj, SkipCommand
|
||||
|
||||
# *************************************
|
||||
# Missing dependencies
|
||||
# Is this ok to assume that those libs
|
||||
# exist in your "distri" (linux/mac) ?
|
||||
# If not, we need to compile them here
|
||||
# *************************************
|
||||
# Zlib
|
||||
# LZMA
|
||||
# aria2
|
||||
# Argtable
|
||||
# MSVirtual
|
||||
# Android
|
||||
# libiconv
|
||||
# gettext
|
||||
# *************************************
|
||||
|
||||
|
||||
class zlib(Dependency):
|
||||
name = 'zlib'
|
||||
version = '1.2.8'
|
||||
|
||||
class Source(ReleaseDownload):
|
||||
archive = Remotefile('zlib-1.2.8.tar.gz',
|
||||
'36658cb768a54c1d4dec43c3116c27ed893e88b02ecfcb44f2166f9c0b7f2a0d')
|
||||
patches = ['zlib_std_libname.patch']
|
||||
|
||||
class Builder(CMakeBuilder):
|
||||
@property
|
||||
def configure_option(self):
|
||||
return "-DINSTALL_PKGCONFIG_DIR={}".format(pj(self.buildEnv.install_dir, self.buildEnv.libprefix, 'pkgconfig'))
|
||||
|
||||
|
||||
class UUID(Dependency):
|
||||
name = 'uuid'
|
||||
version = "1.43.4"
|
||||
|
||||
class Source(ReleaseDownload):
|
||||
archive = Remotefile('e2fsprogs-libs-1.43.4.tar.gz',
|
||||
'eed4516325768255c9745e7b82c9d7d0393abce302520a5b2cde693204b0e419',
|
||||
'https://www.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v1.43.4/e2fsprogs-libs-1.43.4.tar.gz')
|
||||
extract_dir = 'e2fsprogs-libs-1.43.4'
|
||||
|
||||
class Builder(MakeBuilder):
|
||||
configure_option = "--enable-libuuid"
|
||||
configure_env = {'_format_CFLAGS': "{env.CFLAGS} -fPIC"}
|
||||
make_target = 'libs'
|
||||
make_install_target = 'install-libs'
|
||||
|
||||
|
||||
class Xapian(Dependency):
|
||||
name = "xapian-core"
|
||||
version = "1.4.2"
|
||||
|
||||
class Source(ReleaseDownload):
|
||||
archive = Remotefile('xapian-core-1.4.2.tar.xz',
|
||||
'aec2c4352998127a2f2316218bf70f48cef0a466a87af3939f5f547c5246e1ce')
|
||||
patches = ["xapian_pkgconfig.patch"]
|
||||
|
||||
class Builder(MakeBuilder):
|
||||
configure_option = "--disable-sse --disable-backend-inmemory --disable-documentation"
|
||||
dynamic_configure_option = "--enable-shared --disable-static"
|
||||
static_configure_option = "--enable-static --disable-shared"
|
||||
configure_env = {'_format_LDFLAGS': "-L{buildEnv.install_dir}/{buildEnv.libprefix}",
|
||||
'_format_CXXFLAGS': "-I{buildEnv.install_dir}/include"}
|
||||
|
||||
@property
|
||||
def dependencies(self):
|
||||
deps = ['zlib']
|
||||
if self.buildEnv.build_target == 'win32':
|
||||
return deps
|
||||
return deps + ['UUID']
|
||||
|
||||
|
||||
class CTPP2(Dependency):
|
||||
name = "ctpp2"
|
||||
version = "2.8.3"
|
||||
|
||||
class Source(ReleaseDownload):
|
||||
archive = Remotefile('ctpp2-2.8.3.tar.gz',
|
||||
'a83ffd07817adb575295ef40fbf759892512e5a63059c520f9062d9ab8fb42fc')
|
||||
patches = ["ctpp2_include.patch",
|
||||
"ctpp2_no_src_modification.patch",
|
||||
"ctpp2_fix-static-libname.patch",
|
||||
"ctpp2_mingw32.patch",
|
||||
"ctpp2_dll_export_VMExecutable.patch",
|
||||
"ctpp2_win_install_lib_in_lib_dir.patch",
|
||||
"ctpp2_iconv_support.patch"]
|
||||
|
||||
class Builder(CMakeBuilder):
|
||||
configure_option = "-DMD5_SUPPORT=OFF"
|
||||
|
||||
|
||||
class Pugixml(Dependency):
|
||||
name = "pugixml"
|
||||
version = "1.2"
|
||||
|
||||
class Source(ReleaseDownload):
|
||||
archive = Remotefile('pugixml-1.2.tar.gz',
|
||||
'0f422dad86da0a2e56a37fb2a88376aae6e931f22cc8b956978460c9db06136b')
|
||||
patches = ["pugixml_meson.patch"]
|
||||
|
||||
Builder = MesonBuilder
|
||||
|
||||
|
||||
class MicroHttpd(Dependency):
|
||||
name = "libmicrohttpd"
|
||||
version = "0.9.46"
|
||||
|
||||
class Source(ReleaseDownload):
|
||||
archive = Remotefile('libmicrohttpd-0.9.46.tar.gz',
|
||||
'06dbd2654f390fa1e8196fe063fc1449a6c2ed65a38199a49bf29ad8a93b8979',
|
||||
'http://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.46.tar.gz')
|
||||
|
||||
class Builder(MakeBuilder):
|
||||
configure_option = "--disable-https --without-libgcrypt --without-libcurl"
|
||||
dynamic_configure_option = "--enable-shared --disable-static"
|
||||
static_configure_option = "--enable-static --disable-shared"
|
||||
|
||||
|
||||
class Icu(Dependency):
|
||||
name = "icu4c"
|
||||
version = "56_1"
|
||||
|
||||
class Source(ReleaseDownload):
|
||||
archive = Remotefile('icu4c-56_1-src.tgz',
|
||||
'3a64e9105c734dcf631c0b3ed60404531bce6c0f5a64bfe1a6402a4cc2314816')
|
||||
patches = ["icu4c_fix_static_lib_name_mingw.patch"]
|
||||
data = Remotefile('icudt56l.dat',
|
||||
'e23d85eee008f335fc49e8ef37b1bc2b222db105476111e3d16f0007d371cbca')
|
||||
|
||||
def _download_data(self, context):
|
||||
self.buildEnv.download(self.data)
|
||||
|
||||
def _copy_data(self, context):
|
||||
context.try_skip(self.extract_path)
|
||||
shutil.copyfile(pj(self.buildEnv.archive_dir, self.data.name), pj(self.extract_path, 'source', 'data', 'in', self.data.name))
|
||||
|
||||
def prepare(self):
|
||||
super().prepare()
|
||||
self.command("download_data", self._download_data)
|
||||
self.command("copy_data", self._copy_data)
|
||||
|
||||
class Builder(MakeBuilder):
|
||||
subsource_dir = "source"
|
||||
configure_option = "--disable-samples --disable-tests --disable-extras --disable-dyload"
|
||||
dynamic_configure_option = "--enable-shared --disable-static"
|
||||
static_configure_option = "--enable-static --disable-shared"
|
||||
|
||||
|
||||
class Icu_native(Icu):
|
||||
force_native_build = True
|
||||
|
||||
class Builder(Icu.Builder):
|
||||
@property
|
||||
def build_path(self):
|
||||
return super().build_path+"_native"
|
||||
|
||||
def _install(self, context):
|
||||
raise SkipCommand()
|
||||
|
||||
|
||||
class Icu_cross_compile(Icu):
|
||||
dependencies = ['Icu_native']
|
||||
|
||||
class Builder(Icu.Builder):
|
||||
@property
|
||||
def configure_option(self):
|
||||
Icu_native = self.buildEnv.targetsDict['Icu_native']
|
||||
return super().configure_option + " --with-cross-build=" + Icu_native.builder.build_path
|
||||
|
||||
|
||||
class Zimlib(Dependency):
|
||||
name = "zimlib"
|
||||
|
||||
class Source(GitClone):
|
||||
#git_remote = "https://gerrit.wikimedia.org/r/p/openzim.git"
|
||||
git_remote = "https://github.com/mgautierfr/openzim"
|
||||
git_dir = "openzim"
|
||||
git_ref = "meson"
|
||||
|
||||
class Builder(MesonBuilder):
|
||||
subsource_dir = "zimlib"
|
||||
|
||||
|
||||
class Kiwixlib(Dependency):
|
||||
name = "kiwix-lib"
|
||||
dependencies = ['zlib']
|
||||
|
||||
@property
|
||||
def dependencies(self):
|
||||
if self.buildEnv.build_target == 'win32':
|
||||
return ["Xapian", "CTPP2", "Pugixml", "Icu_cross_compile", "Zimlib"]
|
||||
return ["Xapian", "CTPP2", "Pugixml", "Icu", "Zimlib"]
|
||||
|
||||
class Source(GitClone):
|
||||
git_remote = "https://github.com/kiwix/kiwix-lib.git"
|
||||
git_dir = "kiwix-lib"
|
||||
|
||||
class Builder(MesonBuilder):
|
||||
configure_option = "-Dctpp2-install-prefix={buildEnv.install_dir}"
|
||||
|
||||
|
||||
class KiwixTools(Dependency):
|
||||
name = "kiwix-tools"
|
||||
dependencies = ["Kiwixlib", "MicroHttpd", "zlib"]
|
||||
|
||||
class Source(GitClone):
|
||||
git_remote = "https://github.com/kiwix/kiwix-tools.git"
|
||||
git_dir = "kiwix-tools"
|
||||
|
||||
class Builder(MesonBuilder):
|
||||
@property
|
||||
def configure_option(self):
|
||||
base_options = "-Dctpp2-install-prefix={buildEnv.install_dir}"
|
||||
if self.buildEnv.build_static:
|
||||
base_options += " -Dstatic-linkage=true"
|
||||
return base_options
|
||||
|
|
@ -0,0 +1,292 @@
|
|||
import subprocess
|
||||
import os
|
||||
import shutil
|
||||
|
||||
from utils import pj, Context, SkipCommand, extract_archive, Defaultdict, StopBuild
|
||||
|
||||
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
|
||||
|
||||
|
||||
class _MetaDependency(type):
|
||||
def __new__(cls, name, bases, dct):
|
||||
_class = type.__new__(cls, name, bases, dct)
|
||||
if name != 'Dependency':
|
||||
Dependency.all_deps[name] = _class
|
||||
return _class
|
||||
|
||||
|
||||
class Dependency(metaclass=_MetaDependency):
|
||||
all_deps = {}
|
||||
dependencies = []
|
||||
force_native_build = False
|
||||
version = None
|
||||
|
||||
def __init__(self, buildEnv):
|
||||
self.buildEnv = buildEnv
|
||||
self.source = self.Source(self)
|
||||
self.builder = self.Builder(self)
|
||||
self.skip = False
|
||||
|
||||
@property
|
||||
def full_name(self):
|
||||
if self.version:
|
||||
return "{}-{}".format(self.name, self.version)
|
||||
return self.name
|
||||
|
||||
@property
|
||||
def source_path(self):
|
||||
return pj(self.buildEnv.source_dir, self.source.source_dir)
|
||||
|
||||
@property
|
||||
def _log_dir(self):
|
||||
return self.buildEnv.log_dir
|
||||
|
||||
def command(self, name, function, *args):
|
||||
print(" {} {} : ".format(name, self.name), end="", flush=True)
|
||||
log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
|
||||
context = Context(name, log)
|
||||
try:
|
||||
ret = function(*args, context=context)
|
||||
context._finalise()
|
||||
print("OK")
|
||||
return ret
|
||||
except SkipCommand:
|
||||
print("SKIP")
|
||||
except subprocess.CalledProcessError:
|
||||
print("ERROR")
|
||||
try:
|
||||
with open(log, 'r') as f:
|
||||
print(f.read())
|
||||
except:
|
||||
pass
|
||||
raise StopBuild()
|
||||
except:
|
||||
print("ERROR")
|
||||
raise
|
||||
|
||||
|
||||
class Source:
|
||||
"""Base Class to the real preparator
|
||||
A source preparator must install source in the self.source_dir attribute
|
||||
inside the buildEnv.source_dir."""
|
||||
def __init__(self, target):
|
||||
self.target = target
|
||||
self.buildEnv = target.buildEnv
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return self.target.name
|
||||
|
||||
@property
|
||||
def source_dir(self):
|
||||
return self.target.full_name
|
||||
|
||||
def command(self, *args, **kwargs):
|
||||
return self.target.command(*args, **kwargs)
|
||||
|
||||
|
||||
class ReleaseDownload(Source):
|
||||
archive_top_dir = None
|
||||
|
||||
@property
|
||||
def extract_path(self):
|
||||
return pj(self.buildEnv.source_dir, self.source_dir)
|
||||
|
||||
def _download(self, context):
|
||||
self.buildEnv.download(self.archive)
|
||||
|
||||
def _extract(self, context):
|
||||
context.try_skip(self.extract_path)
|
||||
if os.path.exists(self.extract_path):
|
||||
shutil.rmtree(self.extract_path)
|
||||
extract_archive(pj(self.buildEnv.archive_dir, self.archive.name),
|
||||
self.buildEnv.source_dir,
|
||||
topdir=self.archive_top_dir,
|
||||
name=self.source_dir)
|
||||
|
||||
def _patch(self, context):
|
||||
context.try_skip(self.extract_path)
|
||||
for p in self.patches:
|
||||
with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input:
|
||||
self.buildEnv.run_command("patch -p1", self.extract_path, context, input=patch_input, allow_wrapper=False)
|
||||
|
||||
def prepare(self):
|
||||
self.command('download', self._download)
|
||||
self.command('extract', self._extract)
|
||||
if hasattr(self, 'patches'):
|
||||
self.command('patch', self._patch)
|
||||
|
||||
|
||||
class GitClone(Source):
|
||||
git_ref = "master"
|
||||
|
||||
@property
|
||||
def source_dir(self):
|
||||
return self.git_dir
|
||||
|
||||
@property
|
||||
def git_path(self):
|
||||
return pj(self.buildEnv.source_dir, self.git_dir)
|
||||
|
||||
def _git_clone(self, context):
|
||||
if os.path.exists(self.git_path):
|
||||
raise SkipCommand()
|
||||
command = "git clone " + self.git_remote
|
||||
self.buildEnv.run_command(command, self.buildEnv.source_dir, context)
|
||||
|
||||
def _git_update(self, context):
|
||||
self.buildEnv.run_command("git fetch", self.git_path, context)
|
||||
self.buildEnv.run_command("git checkout "+self.git_ref, self.git_path, context)
|
||||
|
||||
def prepare(self):
|
||||
self.command('gitclone', self._git_clone)
|
||||
self.command('gitupdate', self._git_update)
|
||||
|
||||
|
||||
class Builder:
|
||||
subsource_dir = None
|
||||
|
||||
def __init__(self, target):
|
||||
self.target = target
|
||||
self.buildEnv = target.buildEnv
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return self.target.name
|
||||
|
||||
@property
|
||||
def source_path(self):
|
||||
base_source_path = self.target.source_path
|
||||
if self.subsource_dir:
|
||||
return pj(base_source_path, self.subsource_dir)
|
||||
return base_source_path
|
||||
|
||||
@property
|
||||
def build_path(self):
|
||||
return pj(self.buildEnv.build_dir, self.target.full_name)
|
||||
|
||||
def command(self, *args, **kwargs):
|
||||
return self.target.command(*args, **kwargs)
|
||||
|
||||
def build(self):
|
||||
self.command('configure', self._configure)
|
||||
self.command('compile', self._compile)
|
||||
self.command('install', self._install)
|
||||
|
||||
|
||||
class MakeBuilder(Builder):
|
||||
configure_option = static_configure_option = dynamic_configure_option = ""
|
||||
make_option = ""
|
||||
install_option = ""
|
||||
configure_script = "configure"
|
||||
configure_env = None
|
||||
make_target = ""
|
||||
make_install_target = "install"
|
||||
|
||||
def _configure(self, context):
|
||||
context.try_skip(self.build_path)
|
||||
configure_option = "{} {} {}".format(
|
||||
self.configure_option,
|
||||
self.static_configure_option if self.buildEnv.build_static else self.dynamic_configure_option,
|
||||
self.buildEnv.configure_option)
|
||||
command = "{configure_script} {configure_option} --prefix {install_dir} --libdir {libdir}"
|
||||
command = command.format(
|
||||
configure_script=pj(self.source_path, self.configure_script),
|
||||
configure_option=configure_option,
|
||||
install_dir=self.buildEnv.install_dir,
|
||||
libdir=pj(self.buildEnv.install_dir, self.buildEnv.libprefix)
|
||||
)
|
||||
env = Defaultdict(str, os.environ)
|
||||
if self.buildEnv.build_static:
|
||||
env['CFLAGS'] = env['CFLAGS'] + ' -fPIC'
|
||||
if self.configure_env:
|
||||
for k in self.configure_env:
|
||||
if k.startswith('_format_'):
|
||||
v = self.configure_env.pop(k)
|
||||
v = v.format(buildEnv=self.buildEnv, env=env)
|
||||
self.configure_env[k[8:]] = v
|
||||
env.update(self.configure_env)
|
||||
self.buildEnv.run_command(command, self.build_path, context, env=env)
|
||||
|
||||
def _compile(self, context):
|
||||
context.try_skip(self.build_path)
|
||||
command = "make -j4 {make_target} {make_option}".format(
|
||||
make_target=self.make_target,
|
||||
make_option=self.make_option
|
||||
)
|
||||
self.buildEnv.run_command(command, self.build_path, context)
|
||||
|
||||
def _install(self, context):
|
||||
context.try_skip(self.build_path)
|
||||
command = "make {make_install_target} {make_option}".format(
|
||||
make_install_target=self.make_install_target,
|
||||
make_option=self.make_option
|
||||
)
|
||||
self.buildEnv.run_command(command, self.build_path, context)
|
||||
|
||||
|
||||
class CMakeBuilder(MakeBuilder):
|
||||
def _configure(self, context):
|
||||
context.try_skip(self.build_path)
|
||||
command = ("cmake {configure_option}"
|
||||
" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON"
|
||||
" -DCMAKE_INSTALL_PREFIX={install_dir}"
|
||||
" -DCMAKE_INSTALL_LIBDIR={libdir}"
|
||||
" {source_path}"
|
||||
" {cross_option}")
|
||||
command = command.format(
|
||||
configure_option="{} {}".format(self.buildEnv.cmake_option, self.configure_option),
|
||||
install_dir=self.buildEnv.install_dir,
|
||||
libdir=self.buildEnv.libprefix,
|
||||
source_path=self.source_path,
|
||||
cross_option="-DCMAKE_TOOLCHAIN_FILE={}".format(self.buildEnv.cmake_crossfile) if self.buildEnv.cmake_crossfile else ""
|
||||
)
|
||||
env = Defaultdict(str, os.environ)
|
||||
if self.buildEnv.build_static:
|
||||
env['CFLAGS'] = env['CFLAGS'] + ' -fPIC'
|
||||
if self.configure_env:
|
||||
for k in self.configure_env:
|
||||
if k.startswith('_format_'):
|
||||
v = self.configure_env.pop(k)
|
||||
v = v.format(buildEnv=self.buildEnv, env=env)
|
||||
self.configure_env[k[8:]] = v
|
||||
env.update(self.configure_env)
|
||||
self.buildEnv.run_command(command, self.build_path, context, env=env, allow_wrapper=False)
|
||||
|
||||
|
||||
class MesonBuilder(Builder):
|
||||
configure_option = ""
|
||||
|
||||
def _configure(self, context):
|
||||
context.try_skip(self.build_path)
|
||||
if os.path.exists(self.build_path):
|
||||
shutil.rmtree(self.build_path)
|
||||
os.makedirs(self.build_path)
|
||||
if self.buildEnv.build_static:
|
||||
library_type = 'static'
|
||||
else:
|
||||
library_type = 'shared'
|
||||
configure_option = self.configure_option.format(buildEnv=self.buildEnv)
|
||||
command = ("{command} . {build_path}"
|
||||
" --default-library={library_type}"
|
||||
" {configure_option}"
|
||||
" --prefix={buildEnv.install_dir}"
|
||||
" --libdir={buildEnv.libprefix}"
|
||||
" {cross_option}")
|
||||
command = command.format(
|
||||
command=self.buildEnv.meson_command,
|
||||
library_type=library_type,
|
||||
configure_option=configure_option,
|
||||
build_path=self.build_path,
|
||||
buildEnv=self.buildEnv,
|
||||
cross_option="--cross-file {}".format(self.buildEnv.meson_crossfile) if self.buildEnv.meson_crossfile else ""
|
||||
)
|
||||
self.buildEnv.run_command(command, self.source_path, context, allow_wrapper=False)
|
||||
|
||||
def _compile(self, context):
|
||||
command = "{} -v".format(self.buildEnv.ninja_command)
|
||||
self.buildEnv.run_command(command, self.build_path, context, allow_wrapper=False)
|
||||
|
||||
def _install(self, context):
|
||||
command = "{} -v install".format(self.buildEnv.ninja_command)
|
||||
self.buildEnv.run_command(command, self.build_path, context, allow_wrapper=False)
|
||||
767
kiwix-build.py
767
kiwix-build.py
|
|
@ -4,71 +4,72 @@ import os, sys, stat
|
|||
import argparse
|
||||
import ssl
|
||||
import urllib.request
|
||||
import tarfile
|
||||
import subprocess
|
||||
import hashlib
|
||||
import shutil
|
||||
import tempfile
|
||||
import configparser
|
||||
import platform
|
||||
from collections import defaultdict, namedtuple, OrderedDict
|
||||
from collections import OrderedDict
|
||||
|
||||
|
||||
pj = os.path.join
|
||||
from dependencies import Dependency
|
||||
from utils import (
|
||||
pj,
|
||||
remove_duplicates,
|
||||
get_sha256,
|
||||
StopBuild,
|
||||
SkipCommand,
|
||||
Defaultdict)
|
||||
|
||||
REMOTE_PREFIX = 'http://download.kiwix.org/dev/'
|
||||
|
||||
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
|
||||
|
||||
CROSS_ENV = {
|
||||
'fedora_win32' : {
|
||||
'root_path' : '/usr/i686-w64-mingw32/sys-root/mingw',
|
||||
'binaries' : {
|
||||
'c' : 'i686-w64-mingw32-gcc',
|
||||
'cpp' : 'i686-w64-mingw32-g++',
|
||||
'ar' : 'i686-w64-mingw32-ar',
|
||||
'strip' : 'i686-w64-mingw32-strip',
|
||||
'windres' : 'i686-w64-mingw32-windres',
|
||||
'ranlib' : 'i686-w64-mingw32-ranlib',
|
||||
'exe_wrapper' : 'wine'
|
||||
'fedora_win32': {
|
||||
'root_path': '/usr/i686-w64-mingw32/sys-root/mingw',
|
||||
'binaries': {
|
||||
'c': 'i686-w64-mingw32-gcc',
|
||||
'cpp': 'i686-w64-mingw32-g++',
|
||||
'ar': 'i686-w64-mingw32-ar',
|
||||
'strip': 'i686-w64-mingw32-strip',
|
||||
'windres': 'i686-w64-mingw32-windres',
|
||||
'ranlib': 'i686-w64-mingw32-ranlib',
|
||||
'exe_wrapper': 'wine'
|
||||
},
|
||||
'properties' : {
|
||||
'properties': {
|
||||
'c_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4'],
|
||||
'cpp_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4']
|
||||
},
|
||||
'host_machine' : {
|
||||
'system' : 'windows',
|
||||
'cpu_family' : 'x86',
|
||||
'cpu' : 'i686',
|
||||
'endian' : 'little'
|
||||
'host_machine': {
|
||||
'system': 'windows',
|
||||
'cpu_family': 'x86',
|
||||
'cpu': 'i686',
|
||||
'endian': 'little'
|
||||
},
|
||||
'env': {
|
||||
'_format_PKG_CONFIG_LIBDIR' : '{root_path}/lib/pkgconfig'
|
||||
'_format_PKG_CONFIG_LIBDIR': '{root_path}/lib/pkgconfig'
|
||||
}
|
||||
},
|
||||
'debian_win32' : {
|
||||
'root_path' : '/usr/i686-w64-mingw32/',
|
||||
'binaries' : {
|
||||
'c' : 'i686-w64-mingw32-gcc',
|
||||
'cpp' : 'i686-w64-mingw32-g++',
|
||||
'ar' : 'i686-w64-mingw32-ar',
|
||||
'strip' : 'i686-w64-mingw32-strip',
|
||||
'windres' : 'i686-w64-mingw32-windres',
|
||||
'ranlib' : 'i686-w64-mingw32-ranlib',
|
||||
'exe_wrapper' : 'wine'
|
||||
'debian_win32': {
|
||||
'root_path': '/usr/i686-w64-mingw32/',
|
||||
'binaries': {
|
||||
'c': 'i686-w64-mingw32-gcc',
|
||||
'cpp': 'i686-w64-mingw32-g++',
|
||||
'ar': 'i686-w64-mingw32-ar',
|
||||
'strip': 'i686-w64-mingw32-strip',
|
||||
'windres': 'i686-w64-mingw32-windres',
|
||||
'ranlib': 'i686-w64-mingw32-ranlib',
|
||||
'exe_wrapper': 'wine'
|
||||
},
|
||||
'properties' : {
|
||||
'properties': {
|
||||
'c_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4'],
|
||||
'cpp_link_args': ['-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4']
|
||||
},
|
||||
'host_machine' : {
|
||||
'system' : 'windows',
|
||||
'cpu_family' : 'x86',
|
||||
'cpu' : 'i686',
|
||||
'endian' : 'little'
|
||||
'host_machine': {
|
||||
'system': 'windows',
|
||||
'cpu_family': 'x86',
|
||||
'cpu': 'i686',
|
||||
'endian': 'little'
|
||||
},
|
||||
'env': {
|
||||
'_format_PKG_CONFIG_LIBDIR' : '{root_path}/lib/pkgconfig'
|
||||
'_format_PKG_CONFIG_LIBDIR': '{root_path}/lib/pkgconfig'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -76,57 +77,53 @@ CROSS_ENV = {
|
|||
|
||||
PACKAGE_NAME_MAPPERS = {
|
||||
'fedora_native_dyn': {
|
||||
'COMMON' : ['gcc-c++', 'cmake', 'automake', 'ccache'],
|
||||
'COMMON': ['gcc-c++', 'cmake', 'automake', 'ccache'],
|
||||
'uuid': ['libuuid-devel'],
|
||||
'xapian-core' : None, # Not the right version on fedora 25
|
||||
'ctpp2' : None,
|
||||
'pugixml' : None, # ['pugixml-devel'] but package doesn't provide pkg-config file
|
||||
'libmicrohttpd' : ['libmicrohttpd-devel'],
|
||||
'zlib' : ['zlib-devel'],
|
||||
'xapian-core': None, # Not the right version on fedora 25
|
||||
'ctpp2': None,
|
||||
'pugixml': None, # ['pugixml-devel'] but package doesn't provide pkg-config file
|
||||
'libmicrohttpd': ['libmicrohttpd-devel'],
|
||||
'zlib': ['zlib-devel'],
|
||||
'icu4c': None,
|
||||
'zimlib': None,
|
||||
},
|
||||
'fedora_native_static' : {
|
||||
'COMMON' : ['gcc-c++', 'cmake', 'automake', 'glibc-static', 'libstdc++-static', 'ccache'],
|
||||
'zlib' : ['zlib-devel', 'zlib-static']
|
||||
'fedora_native_static': {
|
||||
'COMMON': ['gcc-c++', 'cmake', 'automake', 'glibc-static', 'libstdc++-static', 'ccache'],
|
||||
'zlib': ['zlib-devel', 'zlib-static']
|
||||
# Either there is no packages, or no static or too old
|
||||
},
|
||||
'fedora_win32_dyn' : {
|
||||
'COMMON' : ['mingw32-gcc-c++', 'mingw32-bzip2', 'mingw32-win-iconv', 'mingw32-winpthreads', 'wine', 'ccache'],
|
||||
'zlib' : ['mingw32-zlib'],
|
||||
'libmicrohttpd' : ['mingw32-libmicrohttpd'],
|
||||
'fedora_win32_dyn': {
|
||||
'COMMON': ['mingw32-gcc-c++', 'mingw32-bzip2', 'mingw32-win-iconv', 'mingw32-winpthreads', 'wine', 'ccache'],
|
||||
'zlib': ['mingw32-zlib'],
|
||||
'libmicrohttpd': ['mingw32-libmicrohttpd'],
|
||||
},
|
||||
'fedora_win32_static' : {
|
||||
'COMMON' : ['mingw32-gcc-c++', 'mingw32-bzip2-static', 'mingw32-win-iconv-static', 'mingw32-winpthreads-static', 'wine', 'ccache'],
|
||||
'zlib' : ['mingw32-zlib-static'],
|
||||
'libmicrohttpd' : None, # ['mingw32-libmicrohttpd-static'] packaging dependecy seems buggy, and some static lib are name libfoo.dll.a and
|
||||
# gcc cannot found them.
|
||||
'fedora_win32_static': {
|
||||
'COMMON': ['mingw32-gcc-c++', 'mingw32-bzip2-static', 'mingw32-win-iconv-static', 'mingw32-winpthreads-static', 'wine', 'ccache'],
|
||||
'zlib': ['mingw32-zlib-static'],
|
||||
'libmicrohttpd': None, # ['mingw32-libmicrohttpd-static'] packaging dependecy seems buggy, and some static lib are name libfoo.dll.a and
|
||||
# gcc cannot found them.
|
||||
},
|
||||
'debian_native_dyn' : {
|
||||
'COMMON' : ['gcc', 'cmake', 'libbz2-dev', 'ccache'],
|
||||
'zlib' : ['zlib1g-dev'],
|
||||
'uuid' : ['uuid-dev'],
|
||||
'debian_native_dyn': {
|
||||
'COMMON': ['gcc', 'cmake', 'libbz2-dev', 'ccache'],
|
||||
'zlib': ['zlib1g-dev'],
|
||||
'uuid': ['uuid-dev'],
|
||||
'ctpp2': ['libctpp2-dev'],
|
||||
'libmicrohttpd' : ['libmicrohttpd-dev', 'ccache']
|
||||
'libmicrohttpd': ['libmicrohttpd-dev', 'ccache']
|
||||
},
|
||||
'debian_native_static' : {
|
||||
'COMMON' : ['gcc', 'cmake', 'libbz2-dev', 'ccache'],
|
||||
'zlib' : ['zlib1g-dev'],
|
||||
'uuid' : ['uuid-dev'],
|
||||
'debian_native_static': {
|
||||
'COMMON': ['gcc', 'cmake', 'libbz2-dev', 'ccache'],
|
||||
'zlib': ['zlib1g-dev'],
|
||||
'uuid': ['uuid-dev'],
|
||||
'ctpp2': ['libctpp2-dev'],
|
||||
},
|
||||
'debian_win32_dyn' : {
|
||||
'COMMON' : ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools', 'ccache']
|
||||
'debian_win32_dyn': {
|
||||
'COMMON': ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools', 'ccache']
|
||||
},
|
||||
'debian_win32_static' : {
|
||||
'COMMON' : ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools', 'ccache']
|
||||
'debian_win32_static': {
|
||||
'COMMON': ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools', 'ccache']
|
||||
},
|
||||
}
|
||||
|
||||
class Defaultdict(defaultdict):
|
||||
def __getattr__(self, name):
|
||||
return self[name]
|
||||
|
||||
|
||||
class Which():
|
||||
def __getattr__(self, name):
|
||||
|
|
@ -137,77 +134,6 @@ class Which():
|
|||
def __format__(self, format_spec):
|
||||
return getattr(self, format_spec)
|
||||
|
||||
def remove_duplicates(iterable, key_function=None):
|
||||
seen = set()
|
||||
if key_function is None:
|
||||
key_function = lambda e:e
|
||||
for elem in iterable:
|
||||
key = key_function(elem)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
yield elem
|
||||
|
||||
def get_sha256(path):
|
||||
sha256 = hashlib.sha256()
|
||||
with open(path, 'br') as f:
|
||||
sha256.update(f.read())
|
||||
return sha256.hexdigest()
|
||||
|
||||
class SkipCommand(Exception):
|
||||
pass
|
||||
|
||||
class StopBuild(Exception):
|
||||
pass
|
||||
|
||||
class Remotefile(namedtuple('Remotefile', ('name', 'sha256', 'url'))):
|
||||
def __new__(cls, name, sha256, url=None):
|
||||
return super().__new__(cls, name, sha256, url)
|
||||
|
||||
class Context:
|
||||
def __init__(self, command_name, log_file):
|
||||
self.command_name = command_name
|
||||
self.log_file = log_file
|
||||
self.autoskip_file = None
|
||||
|
||||
def try_skip(self, path):
|
||||
self.autoskip_file = pj(path, ".{}_ok".format(self.command_name))
|
||||
if os.path.exists(self.autoskip_file):
|
||||
raise SkipCommand()
|
||||
|
||||
def _finalise(self):
|
||||
if self.autoskip_file is not None:
|
||||
with open(self.autoskip_file, 'w') as f: pass
|
||||
|
||||
|
||||
def extract_archive(archive_path, dest_dir, topdir=None, name=None):
|
||||
with tarfile.open(archive_path) as archive:
|
||||
members = archive.getmembers()
|
||||
if not topdir:
|
||||
for d in (m for m in members if m.isdir()):
|
||||
if not os.path.dirname(d.name):
|
||||
if topdir:
|
||||
# There is already a top dir.
|
||||
# Two topdirs in the same archive.
|
||||
# Extract all
|
||||
topdir = None
|
||||
break
|
||||
topdir = d
|
||||
else:
|
||||
topdir = archive.getmember(topdir)
|
||||
if topdir:
|
||||
members_to_extract = [m for m in members if m.name.startswith(topdir.name+'/')]
|
||||
os.makedirs(dest_dir, exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(prefix=os.path.basename(archive_path), dir=dest_dir) as tmpdir:
|
||||
archive.extractall(path=tmpdir, members=members_to_extract)
|
||||
name = name or topdir.name
|
||||
os.rename(pj(tmpdir, topdir.name), pj(dest_dir, name))
|
||||
else:
|
||||
if name:
|
||||
dest_dir = pj(dest_dir, name)
|
||||
os.makedirs(dest_dir)
|
||||
archive.extractall(path=dest_dir)
|
||||
|
||||
|
||||
class BuildEnv:
|
||||
build_targets = ['native', 'win32']
|
||||
|
|
@ -282,16 +208,16 @@ class BuildEnv:
|
|||
|
||||
def setup_win32(self):
|
||||
cross_name = "{host}_{target}".format(
|
||||
host = self.distname,
|
||||
target = self.build_target)
|
||||
host=self.distname,
|
||||
target=self.build_target)
|
||||
try:
|
||||
self.cross_env = CROSS_ENV[cross_name]
|
||||
except KeyError:
|
||||
sys.exit("ERROR : We don't know how to set env to compile"
|
||||
" a {target} version on a {host} host.".format(
|
||||
target = self.build_target,
|
||||
host = self.distname
|
||||
))
|
||||
target=self.build_target,
|
||||
host=self.distname
|
||||
))
|
||||
|
||||
self.wrapper = self._gen_crossfile('bash_wrapper.sh')
|
||||
current_permissions = stat.S_IMODE(os.lstat(self.wrapper).st_mode)
|
||||
|
|
@ -357,11 +283,11 @@ class BuildEnv:
|
|||
env['PKG_CONFIG_PATH'] = (env['PKG_CONFIG_PATH'] + ':' + pkgconfig_path
|
||||
if env['PKG_CONFIG_PATH']
|
||||
else pkgconfig_path
|
||||
)
|
||||
)
|
||||
# Add ccache path
|
||||
for p in ('/usr/lib/ccache', '/usr/lib64/ccache'):
|
||||
if os.path.isdir(p):
|
||||
ccache_path=[p]
|
||||
ccache_path = [p]
|
||||
break
|
||||
else:
|
||||
ccache_path = []
|
||||
|
|
@ -373,7 +299,7 @@ class BuildEnv:
|
|||
env['LD_LIBRARY_PATH'] = (env['LD_LIBRARY_PATH'] + ':' + ld_library_path
|
||||
if env['LD_LIBRARY_PATH']
|
||||
else ld_library_path
|
||||
)
|
||||
)
|
||||
env['CPPFLAGS'] = '-I'+pj(self.install_dir, 'include')
|
||||
env['LDFLAGS'] = '-L'+pj(self.install_dir, 'lib')
|
||||
return env
|
||||
|
|
@ -388,7 +314,7 @@ class BuildEnv:
|
|||
if not self.options.verbose:
|
||||
log = open(context.log_file, 'w')
|
||||
print("run command '{}'".format(command), file=log)
|
||||
print("env is :",file=log)
|
||||
print("env is :", file=log)
|
||||
for k, v in env.items():
|
||||
print(" {} : {!r}".format(k, v), file=log)
|
||||
|
||||
|
|
@ -436,18 +362,17 @@ class BuildEnv:
|
|||
elif self.distname in ('debian', 'Ubuntu'):
|
||||
package_installer = 'apt-get'
|
||||
mapper_name = "{host}_{target}_{build_type}".format(
|
||||
host = self.distname,
|
||||
target = self.build_target,
|
||||
build_type = 'static' if self.options.build_static else 'dyn')
|
||||
host=self.distname,
|
||||
target=self.build_target,
|
||||
build_type='static' if self.options.build_static else 'dyn')
|
||||
try:
|
||||
package_name_mapper = PACKAGE_NAME_MAPPERS[mapper_name]
|
||||
except KeyError:
|
||||
print("SKIP : We don't know which packages we must install to compile"
|
||||
" a {target} {build_type} version on a {host} host.".format(
|
||||
target = self.build_target,
|
||||
build_type = 'static' if self.options.build_static else 'dyn',
|
||||
host = self.distname
|
||||
))
|
||||
target=self.build_target,
|
||||
build_type='static' if self.options.build_static else 'dyn',
|
||||
host=self.distname))
|
||||
return
|
||||
|
||||
packages_list = package_name_mapper.get('COMMON', [])
|
||||
|
|
@ -461,516 +386,16 @@ class BuildEnv:
|
|||
return
|
||||
if packages_list:
|
||||
command = "sudo {package_installer} install {packages_list}".format(
|
||||
package_installer = package_installer,
|
||||
packages_list = " ".join(packages_list)
|
||||
package_installer=package_installer,
|
||||
packages_list=" ".join(packages_list)
|
||||
)
|
||||
print(command)
|
||||
subprocess.check_call(command, shell=True)
|
||||
else:
|
||||
print("SKIP, No package to install.")
|
||||
|
||||
with open(autoskip_file, 'w') as f: pass
|
||||
|
||||
################################################################################
|
||||
##### PROJECT
|
||||
################################################################################
|
||||
class _MetaDependency(type):
|
||||
def __new__(cls, name, bases, dct):
|
||||
_class = type.__new__(cls, name, bases, dct)
|
||||
if name != 'Dependency':
|
||||
Dependency.all_deps[name] = _class
|
||||
return _class
|
||||
|
||||
|
||||
class Dependency(metaclass=_MetaDependency):
|
||||
all_deps = {}
|
||||
dependencies = []
|
||||
force_native_build = False
|
||||
version = None
|
||||
def __init__(self, buildEnv):
|
||||
self.buildEnv = buildEnv
|
||||
self.source = self.Source(self)
|
||||
self.builder = self.Builder(self)
|
||||
self.skip = False
|
||||
|
||||
@property
|
||||
def full_name(self):
|
||||
if self.version:
|
||||
return "{}-{}".format(self.name, self.version)
|
||||
return self.name
|
||||
|
||||
@property
|
||||
def source_path(self):
|
||||
return pj(self.buildEnv.source_dir, self.source.source_dir)
|
||||
|
||||
@property
|
||||
def _log_dir(self):
|
||||
return self.buildEnv.log_dir
|
||||
|
||||
def command(self, name, function, *args):
|
||||
print(" {} {} : ".format(name, self.name), end="", flush=True)
|
||||
log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
|
||||
context = Context(name, log)
|
||||
try:
|
||||
ret = function(*args, context=context)
|
||||
context._finalise()
|
||||
print("OK")
|
||||
return ret
|
||||
except SkipCommand:
|
||||
print("SKIP")
|
||||
except subprocess.CalledProcessError:
|
||||
print("ERROR")
|
||||
try:
|
||||
with open(log, 'r') as f:
|
||||
print(f.read())
|
||||
except:
|
||||
pass
|
||||
raise StopBuild()
|
||||
except:
|
||||
print("ERROR")
|
||||
raise
|
||||
|
||||
|
||||
class Source:
|
||||
"""Base Class to the real preparator
|
||||
A source preparator must install source in the self.source_dir attribute
|
||||
inside the buildEnv.source_dir."""
|
||||
def __init__(self, target):
|
||||
self.target = target
|
||||
self.buildEnv = target.buildEnv
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return self.target.name
|
||||
|
||||
@property
|
||||
def source_dir(self):
|
||||
return self.target.full_name
|
||||
|
||||
def command(self, *args, **kwargs):
|
||||
return self.target.command(*args, **kwargs)
|
||||
|
||||
|
||||
class ReleaseDownload(Source):
|
||||
archive_top_dir = None
|
||||
@property
|
||||
def extract_path(self):
|
||||
return pj(self.buildEnv.source_dir, self.source_dir)
|
||||
|
||||
def _download(self, context):
|
||||
self.buildEnv.download(self.archive)
|
||||
|
||||
def _extract(self, context):
|
||||
context.try_skip(self.extract_path)
|
||||
if os.path.exists(self.extract_path):
|
||||
shutil.rmtree(self.extract_path)
|
||||
extract_archive(pj(self.buildEnv.archive_dir, self.archive.name),
|
||||
self.buildEnv.source_dir,
|
||||
topdir=self.archive_top_dir,
|
||||
name=self.source_dir)
|
||||
|
||||
def _patch(self, context):
|
||||
context.try_skip(self.extract_path)
|
||||
for p in self.patches:
|
||||
with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input:
|
||||
self.buildEnv.run_command("patch -p1", self.extract_path, context, input=patch_input, allow_wrapper=False)
|
||||
|
||||
def prepare(self):
|
||||
self.command('download', self._download)
|
||||
self.command('extract', self._extract)
|
||||
if hasattr(self, 'patches'):
|
||||
self.command('patch', self._patch)
|
||||
|
||||
|
||||
class GitClone(Source):
|
||||
git_ref = "master"
|
||||
@property
|
||||
def source_dir(self):
|
||||
return self.git_dir
|
||||
|
||||
@property
|
||||
def git_path(self):
|
||||
return pj(self.buildEnv.source_dir, self.git_dir)
|
||||
|
||||
def _git_clone(self, context):
|
||||
if os.path.exists(self.git_path):
|
||||
raise SkipCommand()
|
||||
command = "git clone " + self.git_remote
|
||||
self.buildEnv.run_command(command, self.buildEnv.source_dir, context)
|
||||
|
||||
def _git_update(self, context):
|
||||
self.buildEnv.run_command("git fetch", self.git_path, context)
|
||||
self.buildEnv.run_command("git checkout "+self.git_ref, self.git_path, context)
|
||||
|
||||
def prepare(self):
|
||||
self.command('gitclone', self._git_clone)
|
||||
self.command('gitupdate', self._git_update)
|
||||
|
||||
|
||||
class Builder:
|
||||
subsource_dir = None
|
||||
def __init__(self, target):
|
||||
self.target = target
|
||||
self.buildEnv = target.buildEnv
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return self.target.name
|
||||
|
||||
@property
|
||||
def source_path(self):
|
||||
base_source_path = self.target.source_path
|
||||
if self.subsource_dir:
|
||||
return pj(base_source_path, self.subsource_dir)
|
||||
return base_source_path
|
||||
|
||||
@property
|
||||
def build_path(self):
|
||||
return pj(self.buildEnv.build_dir, self.target.full_name)
|
||||
|
||||
def command(self, *args, **kwargs):
|
||||
return self.target.command(*args, **kwargs)
|
||||
|
||||
def build(self):
|
||||
self.command('configure', self._configure)
|
||||
self.command('compile', self._compile)
|
||||
self.command('install', self._install)
|
||||
|
||||
|
||||
class MakeBuilder(Builder):
|
||||
configure_option = static_configure_option = dynamic_configure_option = ""
|
||||
make_option = ""
|
||||
install_option = ""
|
||||
configure_script = "configure"
|
||||
configure_env = None
|
||||
make_target = ""
|
||||
make_install_target = "install"
|
||||
|
||||
def _configure(self, context):
|
||||
context.try_skip(self.build_path)
|
||||
configure_option = "{} {} {}".format(
|
||||
self.configure_option,
|
||||
self.static_configure_option if self.buildEnv.build_static else self.dynamic_configure_option,
|
||||
self.buildEnv.configure_option)
|
||||
command = "{configure_script} {configure_option} --prefix {install_dir} --libdir {libdir}"
|
||||
command = command.format(
|
||||
configure_script = pj(self.source_path, self.configure_script),
|
||||
configure_option = configure_option,
|
||||
install_dir = self.buildEnv.install_dir,
|
||||
libdir = pj(self.buildEnv.install_dir, self.buildEnv.libprefix)
|
||||
)
|
||||
env = Defaultdict(str, os.environ)
|
||||
if self.buildEnv.build_static:
|
||||
env['CFLAGS'] = env['CFLAGS'] + ' -fPIC'
|
||||
if self.configure_env:
|
||||
for k in self.configure_env:
|
||||
if k.startswith('_format_'):
|
||||
v = self.configure_env.pop(k)
|
||||
v = v.format(buildEnv=self.buildEnv, env=env)
|
||||
self.configure_env[k[8:]] = v
|
||||
env.update(self.configure_env)
|
||||
self.buildEnv.run_command(command, self.build_path, context, env=env)
|
||||
|
||||
def _compile(self, context):
|
||||
context.try_skip(self.build_path)
|
||||
command = "make -j4 {make_target} {make_option}".format(
|
||||
make_target = self.make_target,
|
||||
make_option = self.make_option
|
||||
)
|
||||
self.buildEnv.run_command(command, self.build_path, context)
|
||||
|
||||
def _install(self, context):
|
||||
context.try_skip(self.build_path)
|
||||
command = "make {make_install_target} {make_option}".format(
|
||||
make_install_target = self.make_install_target,
|
||||
make_option = self.make_option
|
||||
)
|
||||
self.buildEnv.run_command(command, self.build_path, context)
|
||||
|
||||
|
||||
class CMakeBuilder(MakeBuilder):
|
||||
def _configure(self, context):
|
||||
context.try_skip(self.build_path)
|
||||
command = ("cmake {configure_option}"
|
||||
" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON"
|
||||
" -DCMAKE_INSTALL_PREFIX={install_dir}"
|
||||
" -DCMAKE_INSTALL_LIBDIR={libdir}"
|
||||
" {source_path}"
|
||||
" {cross_option}")
|
||||
command = command.format(
|
||||
configure_option = "{} {}".format(self.buildEnv.cmake_option, self.configure_option),
|
||||
install_dir = self.buildEnv.install_dir,
|
||||
libdir = self.buildEnv.libprefix,
|
||||
source_path = self.source_path,
|
||||
cross_option = "-DCMAKE_TOOLCHAIN_FILE={}".format(self.buildEnv.cmake_crossfile) if self.buildEnv.cmake_crossfile else ""
|
||||
)
|
||||
env = Defaultdict(str, os.environ)
|
||||
if self.buildEnv.build_static:
|
||||
env['CFLAGS'] = env['CFLAGS'] + ' -fPIC'
|
||||
if self.configure_env:
|
||||
for k in self.configure_env:
|
||||
if k.startswith('_format_'):
|
||||
v = self.configure_env.pop(k)
|
||||
v = v.format(buildEnv=self.buildEnv, env=env)
|
||||
self.configure_env[k[8:]] = v
|
||||
env.update(self.configure_env)
|
||||
self.buildEnv.run_command(command, self.build_path, context, env=env, allow_wrapper=False)
|
||||
|
||||
|
||||
class MesonBuilder(Builder):
|
||||
configure_option = ""
|
||||
|
||||
def _configure(self, context):
|
||||
context.try_skip(self.build_path)
|
||||
if os.path.exists(self.build_path):
|
||||
shutil.rmtree(self.build_path)
|
||||
os.makedirs(self.build_path)
|
||||
if self.buildEnv.build_static:
|
||||
library_type = 'static'
|
||||
else:
|
||||
library_type = 'shared'
|
||||
configure_option = self.configure_option.format(buildEnv=self.buildEnv)
|
||||
command = ("{command} . {build_path}"
|
||||
" --default-library={library_type}"
|
||||
" {configure_option}"
|
||||
" --prefix={buildEnv.install_dir}"
|
||||
" --libdir={buildEnv.libprefix}"
|
||||
" {cross_option}")
|
||||
command = command.format(
|
||||
command = self.buildEnv.meson_command,
|
||||
library_type=library_type,
|
||||
configure_option=configure_option,
|
||||
build_path = self.build_path,
|
||||
buildEnv=self.buildEnv,
|
||||
cross_option = "--cross-file {}".format(self.buildEnv.meson_crossfile) if self.buildEnv.meson_crossfile else ""
|
||||
)
|
||||
self.buildEnv.run_command(command, self.source_path, context, allow_wrapper=False)
|
||||
|
||||
def _compile(self, context):
|
||||
command = "{} -v".format(self.buildEnv.ninja_command)
|
||||
self.buildEnv.run_command(command, self.build_path, context, allow_wrapper=False)
|
||||
|
||||
def _install(self, context):
|
||||
command = "{} -v install".format(self.buildEnv.ninja_command)
|
||||
self.buildEnv.run_command(command, self.build_path, context, allow_wrapper=False)
|
||||
|
||||
|
||||
# *************************************
|
||||
# Missing dependencies
|
||||
# Is this ok to assume that those libs
|
||||
# exist in your "distri" (linux/mac) ?
|
||||
# If not, we need to compile them here
|
||||
# *************************************
|
||||
# Zlib
|
||||
# LZMA
|
||||
# aria2
|
||||
# Argtable
|
||||
# MSVirtual
|
||||
# Android
|
||||
# libiconv
|
||||
# gettext
|
||||
# *************************************
|
||||
|
||||
class zlib(Dependency):
|
||||
name = 'zlib'
|
||||
version = '1.2.8'
|
||||
|
||||
class Source(ReleaseDownload):
|
||||
archive = Remotefile('zlib-1.2.8.tar.gz',
|
||||
'36658cb768a54c1d4dec43c3116c27ed893e88b02ecfcb44f2166f9c0b7f2a0d')
|
||||
patches = ['zlib_std_libname.patch']
|
||||
|
||||
class Builder(CMakeBuilder):
|
||||
@property
|
||||
def configure_option(self):
|
||||
return "-DINSTALL_PKGCONFIG_DIR={}".format(pj(self.buildEnv.install_dir, self.buildEnv.libprefix, 'pkgconfig'))
|
||||
|
||||
class UUID(Dependency):
|
||||
name = 'uuid'
|
||||
version = "1.43.4"
|
||||
|
||||
class Source(ReleaseDownload):
|
||||
archive = Remotefile('e2fsprogs-1.43.4.tar.gz',
|
||||
'1644db4fc58300c363ba1ab688cf9ca1e46157323aee1029f8255889be4bc856',
|
||||
'https://www.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v1.43.4/e2fsprogs-1.43.4.tar.gz')
|
||||
extract_dir = 'e2fsprogs-1.43.4'
|
||||
|
||||
class Builder(MakeBuilder):
|
||||
configure_option = "--enable-libuuid"
|
||||
configure_env = {'_format_CFLAGS' : "{env.CFLAGS} -fPIC"}
|
||||
make_target = 'libs'
|
||||
make_install_target = 'install-libs'
|
||||
|
||||
|
||||
class Xapian(Dependency):
|
||||
name = "xapian-core"
|
||||
version = "1.4.2"
|
||||
|
||||
class Source(ReleaseDownload):
|
||||
archive = Remotefile('xapian-core-1.4.2.tar.xz',
|
||||
'aec2c4352998127a2f2316218bf70f48cef0a466a87af3939f5f547c5246e1ce')
|
||||
patches = ["xapian_pkgconfig.patch"]
|
||||
|
||||
class Builder(MakeBuilder):
|
||||
configure_option = "--disable-sse --disable-backend-inmemory --disable-documentation"
|
||||
dynamic_configure_option = "--enable-shared --disable-static"
|
||||
static_configure_option = "--enable-static --disable-shared"
|
||||
configure_env = {'_format_LDFLAGS' : "-L{buildEnv.install_dir}/{buildEnv.libprefix}",
|
||||
'_format_CXXFLAGS' : "-I{buildEnv.install_dir}/include"}
|
||||
|
||||
@property
|
||||
def dependencies(self):
|
||||
deps = ['zlib']
|
||||
if self.buildEnv.build_target == 'win32':
|
||||
return deps
|
||||
return deps + ['UUID']
|
||||
|
||||
|
||||
class CTPP2(Dependency):
|
||||
name = "ctpp2"
|
||||
version = "2.8.3"
|
||||
|
||||
class Source(ReleaseDownload):
|
||||
archive = Remotefile('ctpp2-2.8.3.tar.gz',
|
||||
'a83ffd07817adb575295ef40fbf759892512e5a63059c520f9062d9ab8fb42fc')
|
||||
patches = ["ctpp2_include.patch",
|
||||
"ctpp2_no_src_modification.patch",
|
||||
"ctpp2_fix-static-libname.patch",
|
||||
"ctpp2_mingw32.patch",
|
||||
"ctpp2_dll_export_VMExecutable.patch",
|
||||
"ctpp2_win_install_lib_in_lib_dir.patch",
|
||||
"ctpp2_iconv_support.patch"]
|
||||
|
||||
class Builder(CMakeBuilder):
|
||||
configure_option = "-DMD5_SUPPORT=OFF"
|
||||
|
||||
|
||||
class Pugixml(Dependency):
|
||||
name = "pugixml"
|
||||
version = "1.2"
|
||||
|
||||
class Source(ReleaseDownload):
|
||||
archive = Remotefile('pugixml-1.2.tar.gz',
|
||||
'0f422dad86da0a2e56a37fb2a88376aae6e931f22cc8b956978460c9db06136b')
|
||||
patches = ["pugixml_meson.patch"]
|
||||
|
||||
Builder = MesonBuilder
|
||||
|
||||
|
||||
class MicroHttpd(Dependency):
|
||||
name = "libmicrohttpd"
|
||||
version = "0.9.46"
|
||||
|
||||
class Source(ReleaseDownload):
|
||||
archive = Remotefile('libmicrohttpd-0.9.46.tar.gz',
|
||||
'06dbd2654f390fa1e8196fe063fc1449a6c2ed65a38199a49bf29ad8a93b8979',
|
||||
'http://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.46.tar.gz')
|
||||
|
||||
class Builder(MakeBuilder):
|
||||
configure_option = "--disable-https --without-libgcrypt --without-libcurl"
|
||||
dynamic_configure_option = "--enable-shared --disable-static"
|
||||
static_configure_option = "--enable-static --disable-shared"
|
||||
|
||||
|
||||
class Icu(Dependency):
|
||||
name = "icu4c"
|
||||
version = "56_1"
|
||||
|
||||
class Source(ReleaseDownload):
|
||||
archive = Remotefile('icu4c-56_1-src.tgz',
|
||||
'3a64e9105c734dcf631c0b3ed60404531bce6c0f5a64bfe1a6402a4cc2314816'
|
||||
)
|
||||
patches = ["icu4c_fix_static_lib_name_mingw.patch"]
|
||||
data = Remotefile('icudt56l.dat',
|
||||
'e23d85eee008f335fc49e8ef37b1bc2b222db105476111e3d16f0007d371cbca')
|
||||
|
||||
def _download_data(self, context):
|
||||
self.buildEnv.download(self.data)
|
||||
|
||||
def _copy_data(self, context):
|
||||
context.try_skip(self.extract_path)
|
||||
shutil.copyfile(pj(self.buildEnv.archive_dir, self.data.name), pj(self.extract_path, 'source', 'data', 'in', self.data.name))
|
||||
|
||||
def prepare(self):
|
||||
super().prepare()
|
||||
self.command("download_data", self._download_data)
|
||||
self.command("copy_data", self._copy_data)
|
||||
|
||||
class Builder(MakeBuilder):
|
||||
subsource_dir = "source"
|
||||
configure_option = "--disable-samples --disable-tests --disable-extras --disable-dyload"
|
||||
dynamic_configure_option = "--enable-shared --disable-static"
|
||||
static_configure_option = "--enable-static --disable-shared"
|
||||
|
||||
|
||||
class Icu_native(Icu):
|
||||
force_native_build = True
|
||||
|
||||
class Builder(Icu.Builder):
|
||||
@property
|
||||
def build_path(self):
|
||||
return super().build_path+"_native"
|
||||
|
||||
def _install(self, context):
|
||||
raise SkipCommand()
|
||||
|
||||
|
||||
class Icu_cross_compile(Icu):
|
||||
dependencies = ['Icu_native']
|
||||
|
||||
class Builder(Icu.Builder):
|
||||
@property
|
||||
def configure_option(self):
|
||||
Icu_native = self.buildEnv.targetsDict['Icu_native']
|
||||
return super().configure_option + " --with-cross-build=" + Icu_native.builder.build_path
|
||||
|
||||
|
||||
class Zimlib(Dependency):
|
||||
name = "zimlib"
|
||||
|
||||
class Source(GitClone):
|
||||
#git_remote = "https://gerrit.wikimedia.org/r/p/openzim.git"
|
||||
git_remote = "https://github.com/mgautierfr/openzim"
|
||||
git_dir = "openzim"
|
||||
git_ref = "meson"
|
||||
|
||||
class Builder(MesonBuilder):
|
||||
subsource_dir = "zimlib"
|
||||
|
||||
|
||||
class Kiwixlib(Dependency):
|
||||
name = "kiwix-lib"
|
||||
dependencies = ['zlib']
|
||||
|
||||
@property
|
||||
def dependencies(self):
|
||||
if self.buildEnv.build_target == 'win32':
|
||||
return ["Xapian", "CTPP2", "Pugixml", "Icu_cross_compile", "Zimlib"]
|
||||
return ["Xapian", "CTPP2", "Pugixml", "Icu", "Zimlib"]
|
||||
|
||||
class Source(GitClone):
|
||||
git_remote = "https://github.com/kiwix/kiwix-lib.git"
|
||||
git_dir = "kiwix-lib"
|
||||
|
||||
class Builder(MesonBuilder):
|
||||
configure_option = "-Dctpp2-install-prefix={buildEnv.install_dir}"
|
||||
|
||||
|
||||
class KiwixTools(Dependency):
|
||||
name = "kiwix-tools"
|
||||
dependencies = ["Kiwixlib", "MicroHttpd", "zlib"]
|
||||
|
||||
class Source(GitClone):
|
||||
git_remote = "https://github.com/kiwix/kiwix-tools.git"
|
||||
git_dir = "kiwix-tools"
|
||||
|
||||
class Builder(MesonBuilder):
|
||||
@property
|
||||
def configure_option(self):
|
||||
base_options = "-Dctpp2-install-prefix={buildEnv.install_dir}"
|
||||
if self.buildEnv.build_static:
|
||||
base_options += " -Dstatic-linkage=true"
|
||||
return base_options
|
||||
with open(autoskip_file, 'w'):
|
||||
pass
|
||||
|
||||
|
||||
class Builder:
|
||||
|
|
@ -989,7 +414,7 @@ class Builder:
|
|||
dependencies = list(remove_duplicates(dependencies))
|
||||
|
||||
for dep in dependencies:
|
||||
self.targets[dep] = _targets[dep]
|
||||
self.targets[dep] = _targets[dep]
|
||||
|
||||
def add_targets(self, targetName, targets):
|
||||
if targetName in targets:
|
||||
|
|
@ -1011,7 +436,7 @@ class Builder:
|
|||
|
||||
def prepare_sources(self):
|
||||
sources = (dep.source for dep in self.targets.values() if not dep.skip)
|
||||
sources = remove_duplicates(sources, lambda s:s.__class__)
|
||||
sources = remove_duplicates(sources, lambda s: s.__class__)
|
||||
for source in sources:
|
||||
print("prepare sources {} :".format(source.name))
|
||||
source.prepare()
|
||||
|
|
@ -1036,6 +461,7 @@ class Builder:
|
|||
except StopBuild:
|
||||
sys.exit("Stopping build due to errors")
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('working_dir', default=".", nargs='?')
|
||||
|
|
@ -1049,6 +475,7 @@ def parse_args():
|
|||
help="Skip SSL certificate verification during download")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
options = parse_args()
|
||||
options.working_dir = os.path.abspath(options.working_dir)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,90 @@
|
|||
import os.path
|
||||
import hashlib
|
||||
import tarfile
|
||||
import tempfile
|
||||
from collections import namedtuple, defaultdict
|
||||
|
||||
pj = os.path.join
|
||||
|
||||
|
||||
class Defaultdict(defaultdict):
|
||||
def __getattr__(self, name):
|
||||
return self[name]
|
||||
|
||||
|
||||
def remove_duplicates(iterable, key_function=None):
|
||||
seen = set()
|
||||
if key_function is None:
|
||||
key_function = lambda e: e
|
||||
for elem in iterable:
|
||||
key = key_function(elem)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
yield elem
|
||||
|
||||
|
||||
def get_sha256(path):
|
||||
sha256 = hashlib.sha256()
|
||||
with open(path, 'br') as f:
|
||||
sha256.update(f.read())
|
||||
return sha256.hexdigest()
|
||||
|
||||
|
||||
class SkipCommand(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class StopBuild(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class Remotefile(namedtuple('Remotefile', ('name', 'sha256', 'url'))):
|
||||
def __new__(cls, name, sha256, url=None):
|
||||
return super().__new__(cls, name, sha256, url)
|
||||
|
||||
|
||||
class Context:
|
||||
def __init__(self, command_name, log_file):
|
||||
self.command_name = command_name
|
||||
self.log_file = log_file
|
||||
self.autoskip_file = None
|
||||
|
||||
def try_skip(self, path):
|
||||
self.autoskip_file = pj(path, ".{}_ok".format(self.command_name))
|
||||
if os.path.exists(self.autoskip_file):
|
||||
raise SkipCommand()
|
||||
|
||||
def _finalise(self):
|
||||
if self.autoskip_file is not None:
|
||||
with open(self.autoskip_file, 'w'):
|
||||
pass
|
||||
|
||||
|
||||
def extract_archive(archive_path, dest_dir, topdir=None, name=None):
|
||||
with tarfile.open(archive_path) as archive:
|
||||
members = archive.getmembers()
|
||||
if not topdir:
|
||||
for d in (m for m in members if m.isdir()):
|
||||
if not os.path.dirname(d.name):
|
||||
if topdir:
|
||||
# There is already a top dir.
|
||||
# Two topdirs in the same archive.
|
||||
# Extract all
|
||||
topdir = None
|
||||
break
|
||||
topdir = d
|
||||
else:
|
||||
topdir = archive.getmember(topdir)
|
||||
if topdir:
|
||||
members_to_extract = [m for m in members if m.name.startswith(topdir.name+'/')]
|
||||
os.makedirs(dest_dir, exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(prefix=os.path.basename(archive_path), dir=dest_dir) as tmpdir:
|
||||
archive.extractall(path=tmpdir, members=members_to_extract)
|
||||
name = name or topdir.name
|
||||
os.rename(pj(tmpdir, topdir.name), pj(dest_dir, name))
|
||||
else:
|
||||
if name:
|
||||
dest_dir = pj(dest_dir, name)
|
||||
os.makedirs(dest_dir)
|
||||
archive.extractall(path=dest_dir)
|
||||
Loading…
Reference in New Issue