smallsolar
/
kiwix-build
mirror of https://github.com/kiwix/kiwix-build.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Correctly install packages.

This commit is contained in:
Matthieu Gautier 2018-05-29 12:32:55 +02:00
parent 4d6b6994bc
commit 7e8ed1a623
3 changed files with 82 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
kiwixbuild/__init__.py
View File

@ -35,8 +35,8 @@ def parse_args():
help="Skip SSL certificate verification during download")
subgroup.add_argument('--clean-at-end', action='store_true',
help="Clean all intermediate files after the (successfull) build")
subgroup.add_argument('--force-install-packages', action='store_true',
help="Allways check for needed packages before compiling")
subgroup.add_argument('--dont-install-packages', action='store_true',
help="Do not try to install packages before compiling")
subgroup = parser.add_argument_group('custom app',
description="Android custom app specific options")
subgroup.add_argument('--android-custom-app',

95
kiwixbuild/builder.py
View File

@ -6,6 +6,7 @@ from .buildenv import *
from .platforms import PlatformInfo
from .utils import remove_duplicates, StopBuild
from .dependencies import Dependency
from .packages import PACKAGE_NAME_MAPPERS
from ._global import (
neutralEnv, option,
add_target_step, get_target_step, target_steps)
@ -13,20 +14,23 @@ from . import _global
class Builder:
def __init__(self):
_targets = {}
targetDef = (option('target_platform'), option('targets'))
self.add_targets(targetDef, _targets)
dependencies = self.order_dependencies(_targets, targetDef)
self._targets = {}
PlatformInfo.get_platform('neutral', self._targets)
self.targetDef = (option('target_platform'), option('targets'))
self.add_targets(self.targetDef, self._targets)
def finalize_target_steps(self):
dependencies = self.order_dependencies(self.targetDef)
dependencies = list(remove_duplicates(dependencies))
PlatformInfo.get_platform('neutral', _targets)
if option('build_nodeps'):
add_target_step(targetDef, _targets[targetDef])
add_target_step(self.targetDef, self._targets[self.targetDef])
else:
for dep in dependencies:
if option('build_deps_only') and dep == targetDef:
if option('build_deps_only') and dep == self.targetDef:
continue
add_target_step(dep, _targets[dep])
add_target_step(dep, self._targets[dep])
self.instanciate_steps()
def add_targets(self, targetDef, targets):
@ -44,7 +48,7 @@ class Builder:
depPlatform, depName = targetPlatformName, dep
self.add_targets((depPlatform, depName), targets)
def order_dependencies(self, _targets, targetDef):
def order_dependencies(self, targetDef):
for pltName in PlatformInfo.all_running_platforms:
plt = PlatformInfo.all_platforms[pltName]
for tlcName in plt.toolchain_names:
@ -56,15 +60,16 @@ class Builder:
# Do not try to order sources, they will be added as dep by the
# build step two lines later.
return
target = _targets[targetDef]
target = self._targets[targetDef]
targetPlatform = PlatformInfo.get_platform(targetPlatformName)
yield ('source', targetName)
for dep in target.get_dependencies(targetPlatform):
try:
depPlatform, depName = dep
except ValueError:
depPlatform, depName = targetPlatformName, dep
yield from self.order_dependencies(_targets, (depPlatform, depName))
if (depPlatform, depName) in self._targets:
yield from self.order_dependencies((depPlatform, depName))
yield ('source', targetName)
yield targetDef
def instanciate_steps(self):
@ -103,15 +108,75 @@ class Builder:
add_target_step(builderDef, builder)
builder.build()
def _get_packages(self):
packages_list = []
for platform in PlatformInfo.all_running_platforms.values():
mapper_name = "{host}_{target}".format(
host=neutralEnv('distname'),
target=platform)
package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {})
packages_list += package_name_mapper.get('COMMON', [])
to_drop = []
for builderDef in self._targets:
platformName, builderName = builderDef
mapper_name = "{host}_{target}".format(
host=neutralEnv('distname'),
target=platformName)
package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {})
packages = package_name_mapper.get(builderName)
if packages:
packages_list += packages
to_drop.append(builderDef)
for dep in to_drop:
del self._targets[dep]
return packages_list
def install_packages(self):
packages_to_have = self._get_packages()
packages_to_have = remove_duplicates(packages_to_have)
distname = neutralEnv('distname')
if distname in ('fedora', 'redhat', 'centos'):
package_installer = 'sudo dnf install {}'
package_checker = 'rpm -q --quiet {}'
elif distname in ('debian', 'Ubuntu'):
package_installer = 'sudo apt-get install {}'
package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1'
elif distname == 'Darwin':
package_installer = 'brew install {}'
package_checker = 'brew list -1 | grep -q {}'
packages_to_install = []
for package in packages_to_have:
print(" - {} : ".format(package), end="")
command = package_checker.format(package)
try:
subprocess.check_call(command, shell=True)
except subprocess.CalledProcessError:
print("NEEDED")
packages_to_install.append(package)
else:
print("SKIP")
if packages_to_install:
command = package_installer.format(" ".join(packages_to_install))
print(command)
subprocess.check_call(command, shell=True)
else:
print("SKIP, No package to install.")
def run(self):
try:
print("[INSTALL PACKAGES]")
if option('dont_install_packages'):
print("SKIP")
else:
self.install_packages()
self.finalize_target_steps()
print("[SETUP PLATFORMS]")
for platform in PlatformInfo.all_running_platforms.values():
platform.finalize_setup()
print("[INSTALL PACKAGES]")
for platform in PlatformInfo.all_running_platforms.values():
platform.install_packages()
print("[PREPARE]")
self.prepare_sources()
print("[BUILD]")

80
kiwixbuild/platforms/base.py
View File

@ -3,7 +3,6 @@ import os, sys
import subprocess
from kiwixbuild.dependencies import Dependency
from kiwixbuild.packages import PACKAGE_NAME_MAPPERS
from kiwixbuild.utils import pj, remove_duplicates
from kiwixbuild.buildenv import BuildEnv
from kiwixbuild._global import neutralEnv, option, target_steps
@ -86,85 +85,6 @@ class PlatformInfo(metaclass=_MetaPlatform):
self.buildEnv.meson_crossfile = None
self.buildEnv.cmake_crossfile = None
def get_packages(self):
mapper_name = "{host}_{target}".format(
host=neutralEnv('distname'),
target=self)
try:
package_name_mapper = PACKAGE_NAME_MAPPERS[mapper_name]
except KeyError:
print("SKIP : We don't know which packages we must install to compile"
" a {target} {build_type} version on a {host} host.".format(
target=self,
build_type=self.static,
host=neutralEnv('distname')))
return []
packages_list = package_name_mapper.get('COMMON', [])
to_drop = []
build_steps = (step for step in target_steps() if step[0] == self.name)
for dep in build_steps:
packages = package_name_mapper.get(dep[1])
if packages:
packages_list += packages
to_drop.append(dep)
for dep in to_drop:
del target_steps()[dep]
source_dep = [dep[1] for dep in target_steps() if dep[0] == 'source']
for dep in source_dep:
it_build = (True for dep in target_steps() if dep[0] != 'source' and dep[1] == dep)
if next(it_build, False):
print("removing source ", dep)
del target_steps()[('source', dep)]
build_steps = (step for step in target_steps() if step[0] == self.name)
for dep in build_steps:
packages = getattr(dep, 'extra_packages', [])
for package in packages:
packages_list += package_name_mapper.get(package, [])
return packages_list
def install_packages(self):
autoskip_file = pj(self.buildEnv.build_dir, ".install_packages_ok")
packages_to_have = self.get_packages()
packages_to_have = remove_duplicates(packages_to_have)
if not option('force_install_packages') and os.path.exists(autoskip_file):
print("SKIP")
return
distname = neutralEnv('distname')
if distname in ('fedora', 'redhat', 'centos'):
package_installer = 'sudo dnf install {}'
package_checker = 'rpm -q --quiet {}'
elif distname in ('debian', 'Ubuntu'):
package_installer = 'sudo apt-get install {}'
package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1'
elif distname == 'Darwin':
package_installer = 'brew install {}'
package_checker = 'brew list -1 | grep -q {}'
packages_to_install = []
for package in packages_to_have:
print(" - {} : ".format(package), end="")
command = package_checker.format(package)
try:
subprocess.check_call(command, shell=True)
except subprocess.CalledProcessError:
print("NEEDED")
packages_to_install.append(package)
else:
print("SKIP")
if packages_to_install:
command = package_installer.format(" ".join(packages_to_install))
print(command)
subprocess.check_call(command, shell=True)
else:
print("SKIP, No package to install.")
with open(autoskip_file, 'w'):
pass
def clean_intermediate_directories(self):
self.buildEnv.clean_intermediate_directories()