smallsolar
/
kiwix-build
mirror of https://github.com/kiwix/kiwix-build.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge toolchain and dependencies. 

Toolchain now became "simple" dependencies.
They are referenced by Platform.

Platform are now created at demande and a neutralPlatform now exists.
This commit is contained in:
Matthieu Gautier 2018-05-29 11:40:42 +02:00
parent 9496ffb86c
commit 4d6b6994bc
17 changed files with 87 additions and 174 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
kiwixbuild/_global.py
View File

@ -3,7 +3,6 @@ from collections import OrderedDict as _OrderedDict
_neutralEnv = None
_options = None
_target_steps = _OrderedDict()
_plt_steps = _OrderedDict()
def set_neutralEnv(env):
global _neutralEnv
@ -31,16 +30,3 @@ def get_target_step(key, default_context=None):
def target_steps():
return _target_steps
def add_plt_step(key, what):
_plt_steps[key] = what
def get_plt_step(key, default_context=None):
try:
context, target = key
except ValueError:
context, target = default_context, key
return _plt_steps[(context, target)]
def plt_steps():
return _plt_steps

3
kiwixbuild/buildenv.py
View File

@ -3,7 +3,6 @@ import os, sys, shutil
import subprocess
import platform
from .toolchains import Toolchain
from .utils import pj, download_remote, Defaultdict
from ._global import neutralEnv, option
@ -77,9 +76,11 @@ class BuildEnv:
self.platformInfo = platformInfo
self.build_dir = pj(option('working_dir'), build_dir)
self.install_dir = pj(self.build_dir, "INSTALL")
self.toolchain_dir = pj(self.build_dir, "TOOLCHAINS")
self.log_dir = pj(self.build_dir, 'LOGS')
for d in (self.build_dir,
self.install_dir,
self.toolchain_dir,
self.log_dir):
os.makedirs(d, exist_ok=True)

63
kiwixbuild/builder.py
View File

@ -8,8 +8,7 @@ from .utils import remove_duplicates, StopBuild
from .dependencies import Dependency
from ._global import (
neutralEnv, option,
add_target_step, get_target_step, target_steps,
add_plt_step, get_plt_step, plt_steps)
add_target_step, get_target_step, target_steps)
from . import _global
class Builder:
@ -20,6 +19,7 @@ class Builder:
dependencies = self.order_dependencies(_targets, targetDef)
dependencies = list(remove_duplicates(dependencies))
PlatformInfo.get_platform('neutral', _targets)
if option('build_nodeps'):
add_target_step(targetDef, _targets[targetDef])
else:
@ -27,12 +27,13 @@ class Builder:
if option('build_deps_only') and dep == targetDef:
continue
add_target_step(dep, _targets[dep])
self.instanciate_steps()
def add_targets(self, targetDef, targets):
if targetDef in targets:
return
targetPlatformName, targetName = targetDef
targetPlatform = PlatformInfo.get_platform(targetPlatformName)
targetPlatform = PlatformInfo.get_platform(targetPlatformName, targets)
targetClass = Dependency.all_deps[targetName]
targets[('source', targetName)] = targetClass.Source
targets[targetDef] = targetClass.Builder
@ -44,6 +45,12 @@ class Builder:
self.add_targets((depPlatform, depName), targets)
def order_dependencies(self, _targets, targetDef):
for pltName in PlatformInfo.all_running_platforms:
plt = PlatformInfo.all_platforms[pltName]
for tlcName in plt.toolchain_names:
tlc = Dependency.all_deps[tlcName]
yield('source', tlcName)
yield('neutral' if tlc.neutral else pltName, tlcName)
targetPlatformName, targetName = targetDef
if targetPlatformName == 'source':
# Do not try to order sources, they will be added as dep by the
@ -60,14 +67,18 @@ class Builder:
yield from self.order_dependencies(_targets, (depPlatform, depName))
yield targetDef
def prepare_toolchain_sources(self):
tlsourceDefs = (tlDef for tlDef in plt_steps() if tlDef[0]=='source')
for tlsourceDef in tlsourceDefs:
print("prepare sources for toolchain {} :".format(tlsourceDef[1]))
toolchainClass = Toolchain.all_toolchains[tlsourceDef[1]]
source = get_plt_step(tlsourceDef)(toolchainClass)
add_plt_step(tlsourceDef, source)
source.prepare()
def instanciate_steps(self):
for stepDef in list(target_steps()):
stepPlatform, stepName = stepDef
stepClass = Dependency.all_deps[stepName]
if stepPlatform == 'source':
source = get_target_step(stepDef)(stepClass)
add_target_step(stepDef, source)
else:
source = get_target_step(stepName, 'source')
env = PlatformInfo.get_platform(stepPlatform).buildEnv
builder = get_target_step(stepDef)(stepClass, source, env)
add_target_step(stepDef, builder)
def prepare_sources(self):
if option('skip_source_prepare'):
@ -76,36 +87,16 @@ class Builder:
sourceDefs = remove_duplicates(tDef for tDef in target_steps() if tDef[0]=='source')
for sourceDef in sourceDefs:
print("prepare sources {} :".format(sourceDef[1]))
depClass = Dependency.all_deps[sourceDef[1]]
source = get_target_step(sourceDef)(depClass)
add_target_step(sourceDef, source)
source = get_target_step(sourceDef)
source.prepare()
def build_toolchains(self):
tlbuilderDefs = (tlDef for tlDef in plt_steps() if tlDef[0] != 'source')
for tlbuilderDef in tlbuilderDefs:
print("build toolchain {} :".format(tlbuilderDef[1]))
toolchainClass = Toolchain.all_toolchains[tlbuilderDef[1]]
source = get_plt_step(tlbuilderDef[1], 'source')
if tlbuilderDef[0] == 'neutral':
env = _global._neutralEnv
else:
env = PlatformInfo.get_platform(tlbuilderDef[0]).buildEnv
builder = get_plt_step(tlbuilderDef)(toolchainClass, source, env)
add_plt_step(tlbuilderDef, builder)
builder.build()
def build(self):
builderDefs = (tDef for tDef in target_steps() if tDef[0] != 'source')
for builderDef in builderDefs:
depClass = Dependency.all_deps[builderDef[1]]
source = get_target_step(builderDef[1], 'source')
env = PlatformInfo.get_platform(builderDef[0]).buildEnv
builder = get_target_step(builderDef)(depClass, source, env)
if option('make_dist') and builderDef[1] == option('targets'):
print("make dist {}:".format(builder.name))
builder = get_target_step(builderDef)
if option('make_dist') and builderName == option('targets'):
print("make dist {} ({}):".format(builder.name, builderDef[0]))
builder.make_dist()
continue
print("build {} ({}):".format(builder.name, builderDef[0]))
@ -116,8 +107,6 @@ class Builder:
def run(self):
try:
print("[SETUP PLATFORMS]")
self.prepare_toolchain_sources()
self.build_toolchains()
for platform in PlatformInfo.all_running_platforms.values():
platform.finalize_setup()
print("[INSTALL PACKAGES]")

3
kiwixbuild/dependencies/__init__.py
View File

@ -2,6 +2,9 @@
from .base import *
from . import (
all_dependencies,
android_ndk,
android_sdk,
armhf,
ctpp2,
gradle,
gumbo,

16
kiwixbuild/toolchains/android_ndk.py → kiwixbuild/dependencies/android_ndk.py
View File

@ -1,23 +1,15 @@
import os
from .base_toolchain import Toolchain
from kiwixbuild.dependencies import ReleaseDownload, Builder
from .base import Dependency, ReleaseDownload, Builder
from kiwixbuild.utils import Remotefile, add_execution_right, run_command
pj = os.path.join
class android_ndk(Toolchain):
class android_ndk(Dependency):
neutral = False
name = 'android-ndk'
version = 'r13b'
gccver = '4.9.x'
@classmethod
def full_name(cls):
return "{name}-{version}".format(
name = cls.name,
version = cls.version)
class Source(ReleaseDownload):
archive = Remotefile('android-ndk-r13b-linux-x86_64.zip',
'3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c',
@ -25,9 +17,7 @@ class android_ndk(Toolchain):
@property
def source_dir(self):
return "{}-{}".format(
self.target.name,
self.target.version)
return self.target.full_name()
class Builder(Builder):

7
kiwixbuild/toolchains/android_sdk.py → kiwixbuild/dependencies/android_sdk.py
View File

@ -1,15 +1,14 @@
import os
import shutil
from .base_toolchain import Toolchain
from kiwixbuild.dependencies import ReleaseDownload, Builder
from .base import Dependency, ReleaseDownload, Builder
from kiwixbuild.utils import Remotefile, run_command
pj = os.path.join
class android_sdk(Toolchain):
class android_sdk(Dependency):
neutral = True
name = 'android-sdk'
version = 'r25.2.3'
class Source(ReleaseDownload):
archive = Remotefile('tools_r25.2.3-linux.zip',

11
kiwixbuild/dependencies/armhf.py Normal file
View File

@ -0,0 +1,11 @@
from .base import Dependency, GitClone, NoopBuilder
class armhf_toolchain(Dependency):
neutral = True
name = 'armhf'
class Source(GitClone):
git_remote = "https://github.com/raspberrypi/tools"
git_dir = "raspberrypi-tools"
Builder = NoopBuilder

2
kiwixbuild/dependencies/base.py
View File

@ -350,7 +350,7 @@ class CMakeBuilder(MakeBuilder):
" {source_path}"
" {cross_option}")
command = command.format(
configure_option="{} {}".format(self.buildEnv.cmake_option, self.configure_option),
configure_option=self.configure_option,
install_dir=self.buildEnv.install_dir,
libdir=self.buildEnv.libprefix,
source_path=self.source_path,

1
kiwixbuild/platforms/__init__.py
View File

@ -7,5 +7,6 @@ from . import (
i586,
ios,
native,
neutral,
win32
)

6
kiwixbuild/platforms/android.py
View File

@ -1,6 +1,6 @@
from .base import PlatformInfo
from kiwixbuild.utils import pj
from kiwixbuild._global import get_plt_step
from kiwixbuild._global import get_target_step
class AndroidPlatformInfo(PlatformInfo):
@ -27,11 +27,11 @@ class AndroidPlatformInfo(PlatformInfo):
@property
def ndk_builder(self):
return get_plt_step('android_ndk', self.name)
return get_target_step('android-ndk', self.name)
@property
def sdk_builder(self):
return get_plt_step('android_sdk', 'neutral')
return get_target_step('android-sdk', 'neutral')
def get_cross_config(self):
install_path = self.ndk_builder.install_path

4
kiwixbuild/platforms/armhf.py
View File

@ -1,7 +1,7 @@
from .base import PlatformInfo
from kiwixbuild.utils import pj
from kiwixbuild._global import get_plt_step
from kiwixbuild._global import get_target_step
class ArmhfPlatformInfo(PlatformInfo):
@ -29,7 +29,7 @@ class ArmhfPlatformInfo(PlatformInfo):
@property
def tlc_source(self):
return get_plt_step('armhf', 'source')
return get_target_step('armhf', 'source')
@property
def root_path(self):

35
kiwixbuild/platforms/base.py
View File

@ -1,13 +1,12 @@
import os
import os, sys
import subprocess
from kiwixbuild.dependencies import Dependency
from kiwixbuild.toolchains import Toolchain
from kiwixbuild.packages import PACKAGE_NAME_MAPPERS
from kiwixbuild.utils import pj, remove_duplicates
from kiwixbuild.buildenv import BuildEnv
from kiwixbuild._global import neutralEnv, option, add_plt_step, target_steps
from kiwixbuild._global import neutralEnv, option, target_steps
_SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
TEMPLATES_DIR = pj(os.path.dirname(_SCRIPT_DIR), 'templates')
@ -28,12 +27,17 @@ class PlatformInfo(metaclass=_MetaPlatform):
configure_option = ""
@classmethod
def get_platform(cls, name):
def get_platform(cls, name, targets=None):
if name not in cls.all_running_platforms:
cls.all_running_platforms[name] = cls.all_platforms[name]()
if targets is None:
print("Should not got there.")
print(cls.all_running_platforms)
raise KeyError(name)
sys.exit(-1)
cls.all_running_platforms[name] = cls.all_platforms[name](targets)
return cls.all_running_platforms[name]
def __init__(self):
def __init__(self, targets):
if neutralEnv('distname') not in self.compatible_hosts:
print(('ERROR: The target platform {} cannot be build on host {}.\n'
'Select another target platform, or change your host system.'
@ -41,19 +45,17 @@ class PlatformInfo(metaclass=_MetaPlatform):
sys.exit(-1)
self.all_running_platforms[self.name] = self
self.buildEnv = BuildEnv(self)
self.setup_toolchains()
self.setup_toolchains(targets)
def __str__(self):
return "{}_{}".format(self.build, 'static' if self.static else 'dyn')
def setup_toolchains(self):
def setup_toolchains(self, targets):
for tlc_name in self.toolchain_names:
ToolchainClass = Toolchain.all_toolchains[tlc_name]
if ToolchainClass.Source is not None:
add_plt_step(('source', tlc_name), ToolchainClass.Source)
if ToolchainClass.Builder is not None:
plt_name = 'neutral' if ToolchainClass.neutral else self.name
add_plt_step((plt_name, tlc_name), ToolchainClass.Builder)
ToolchainClass = Dependency.all_deps[tlc_name]
targets[('source', tlc_name)] = ToolchainClass.Source
plt_name = 'neutral' if ToolchainClass.neutral else self.name
targets[(plt_name, tlc_name)] = ToolchainClass.Builder
def get_cross_config(self):
return {}
@ -93,9 +95,10 @@ class PlatformInfo(metaclass=_MetaPlatform):
except KeyError:
print("SKIP : We don't know which packages we must install to compile"
" a {target} {build_type} version on a {host} host.".format(
target=self.platform_info,
target=self,
build_type=self.static,
host=neutralEnv('distname')))
return
return []
packages_list = package_name_mapper.get('COMMON', [])
to_drop = []

9
kiwixbuild/platforms/neutral.py Normal file
View File

@ -0,0 +1,9 @@
from .base import PlatformInfo
class NeutralPlatformInfo(PlatformInfo):
name = 'neutral'
static = ''
compatible_hosts = ['fedora', 'debian', 'Darwin']
def __str__(self):
return "neutral"

4
kiwixbuild/toolchains/__init__.py
View File

@ -1,4 +0,0 @@
from . import android_ndk, android_sdk, armhf
from .base_toolchain import Toolchain

14
kiwixbuild/toolchains/armhf.py
View File

@ -1,14 +0,0 @@
import os
import subprocess
from .base_toolchain import Toolchain
from kiwixbuild.dependencies import GitClone
from kiwixbuild.utils import which
pj = os.path.join
class armhf_toolchain(Toolchain):
name = 'armhf'
class Source(GitClone):
git_remote = "https://github.com/raspberrypi/tools"
git_dir = "raspberrypi-tools"

63
kiwixbuild/toolchains/base_toolchain.py
View File

@ -1,63 +0,0 @@
import os
import subprocess
pj = os.path.join
from kiwixbuild.utils import Context, SkipCommand, StopBuild
from kiwixbuild._global import neutralEnv
class _MetaToolchain(type):
def __new__(cls, name, bases, dct):
_class = type.__new__(cls, name, bases, dct)
if name != 'Toolchain':
Toolchain.all_toolchains[dct['name']] = _class
return _class
class Toolchain(metaclass=_MetaToolchain):
force_native_build = False
neutral = True
all_toolchains = {}
configure_option = ""
cmake_option = ""
exec_wrapper_def = ""
Builder = None
Source = None
@classmethod
def full_name(cls):
return "{name}-{version}".format(
name = cls.name,
version = cls.version)
@property
def source_path(self):
return pj(neutralEnv('source_dir'), self.source.source_dir)
@property
def _log_dir(self):
return neutralEnv('log_dir')
def command(self, name, function, *args):
print(" {} {} : ".format(name, self.name), end="", flush=True)
log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
context = Context(name, log, True)
try:
ret = function(*args, context=context)
context._finalise()
print("OK")
return ret
except SkipCommand:
print("SKIP")
except subprocess.CalledProcessError:
print("ERROR")
try:
with open(log, 'r') as f:
print(f.read())
except:
pass
raise StopBuild()
except:
print("ERROR")
raise

6
kiwixbuild/versions.py
View File

@ -10,7 +10,7 @@ main_project_versions = {
# This is the "version" of the whole base_deps_versions dict.
# Change this when you change base_deps_versions.
base_deps_meta_version = '3'
base_deps_meta_version = '4'
base_deps_versions = {
@ -25,5 +25,7 @@ base_deps_versions = {
'icu4c' : '58.2',
'Gradle' : '4.6',
'libaria2' : '1.33.1',
'libmagic' : '5.33'
'libmagic' : '5.33',
'android-sdk' : 'r25.2.3',
'android-ndk' : 'r13b'
}