Correctly install packages.

kiwixbuild/__init__.py

@@ -35,8 +35,8 @@ def parse_args():
                           help="Skip SSL certificate verification during download")
     subgroup.add_argument('--clean-at-end', action='store_true',
                           help="Clean all intermediate files after the (successfull) build")
-    subgroup.add_argument('--force-install-packages', action='store_true',
+    subgroup.add_argument('--dont-install-packages', action='store_true',
-                          help="Allways check for needed packages before compiling")
+                          help="Do not try to install packages before compiling")
     subgroup = parser.add_argument_group('custom app',
                                          description="Android custom app specific options")
     subgroup.add_argument('--android-custom-app',

kiwixbuild/builder.py

@@ -6,6 +6,7 @@ from .buildenv import *
 from .platforms import PlatformInfo
 from .utils import remove_duplicates, StopBuild
 from .dependencies import Dependency
+from .packages import PACKAGE_NAME_MAPPERS
 from ._global import (
     neutralEnv, option,
     add_target_step, get_target_step, target_steps)
@@ -13,20 +14,23 @@ from . import _global
 
 class Builder:
     def __init__(self):
-        _targets = {}
+        self._targets = {}
-        targetDef = (option('target_platform'), option('targets'))
+        PlatformInfo.get_platform('neutral', self._targets)
-        self.add_targets(targetDef, _targets)
+
-        dependencies = self.order_dependencies(_targets, targetDef)
+        self.targetDef = (option('target_platform'), option('targets'))
+        self.add_targets(self.targetDef, self._targets)
+
+    def finalize_target_steps(self):
+        dependencies = self.order_dependencies(self.targetDef)
         dependencies = list(remove_duplicates(dependencies))
 
-        PlatformInfo.get_platform('neutral', _targets)
         if option('build_nodeps'):
-            add_target_step(targetDef, _targets[targetDef])
+            add_target_step(self.targetDef, self._targets[self.targetDef])
         else:
             for dep in dependencies:
-                if option('build_deps_only') and dep == targetDef:
+                if option('build_deps_only') and dep == self.targetDef:
                     continue
-                add_target_step(dep, _targets[dep])
+                add_target_step(dep, self._targets[dep])
         self.instanciate_steps()
 
     def add_targets(self, targetDef, targets):
@@ -44,7 +48,7 @@ class Builder:
                 depPlatform, depName = targetPlatformName, dep
             self.add_targets((depPlatform, depName), targets)
 
-    def order_dependencies(self, _targets, targetDef):
+    def order_dependencies(self, targetDef):
         for pltName in PlatformInfo.all_running_platforms:
             plt = PlatformInfo.all_platforms[pltName]
             for tlcName in plt.toolchain_names:
@@ -56,15 +60,16 @@ class Builder:
             # Do not try to order sources, they will be added as dep by the
             # build step two lines later.
             return
-        target = _targets[targetDef]
+        target = self._targets[targetDef]
         targetPlatform = PlatformInfo.get_platform(targetPlatformName)
-        yield ('source', targetName)
         for dep in target.get_dependencies(targetPlatform):
             try:
                 depPlatform, depName = dep
             except ValueError:
                 depPlatform, depName = targetPlatformName, dep
-            yield from self.order_dependencies(_targets, (depPlatform, depName))
+            if (depPlatform, depName) in self._targets:
+                yield from self.order_dependencies((depPlatform, depName))
+        yield ('source', targetName)
         yield targetDef
 
     def instanciate_steps(self):
@@ -103,15 +108,75 @@ class Builder:
             add_target_step(builderDef, builder)
             builder.build()
 
+    def _get_packages(self):
+        packages_list = []
+        for platform in PlatformInfo.all_running_platforms.values():
+            mapper_name = "{host}_{target}".format(
+                host=neutralEnv('distname'),
+                target=platform)
+            package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {})
+            packages_list += package_name_mapper.get('COMMON', [])
+
+        to_drop = []
+        for builderDef in self._targets:
+            platformName, builderName = builderDef
+            mapper_name = "{host}_{target}".format(
+                host=neutralEnv('distname'),
+                target=platformName)
+            package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {})
+            packages = package_name_mapper.get(builderName)
+            if packages:
+                packages_list += packages
+                to_drop.append(builderDef)
+        for dep in to_drop:
+            del self._targets[dep]
+        return packages_list
+
+    def install_packages(self):
+        packages_to_have = self._get_packages()
+        packages_to_have = remove_duplicates(packages_to_have)
+
+        distname = neutralEnv('distname')
+        if distname in ('fedora', 'redhat', 'centos'):
+            package_installer = 'sudo dnf install {}'
+            package_checker = 'rpm -q --quiet {}'
+        elif distname in ('debian', 'Ubuntu'):
+            package_installer = 'sudo apt-get install {}'
+            package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1'
+        elif distname == 'Darwin':
+            package_installer = 'brew install {}'
+            package_checker = 'brew list -1 | grep -q {}'
+
+        packages_to_install = []
+        for package in packages_to_have:
+            print(" - {} : ".format(package), end="")
+            command = package_checker.format(package)
+            try:
+                subprocess.check_call(command, shell=True)
+            except subprocess.CalledProcessError:
+                print("NEEDED")
+                packages_to_install.append(package)
+            else:
+                print("SKIP")
+
+        if packages_to_install:
+            command = package_installer.format(" ".join(packages_to_install))
+            print(command)
+            subprocess.check_call(command, shell=True)
+        else:
+            print("SKIP, No package to install.")
 
     def run(self):
         try:
+            print("[INSTALL PACKAGES]")
+            if option('dont_install_packages'):
+                print("SKIP")
+            else:
+                self.install_packages()
+            self.finalize_target_steps()
             print("[SETUP PLATFORMS]")
             for platform in PlatformInfo.all_running_platforms.values():
                 platform.finalize_setup()
-            print("[INSTALL PACKAGES]")
-            for platform in PlatformInfo.all_running_platforms.values():
-                platform.install_packages()
             print("[PREPARE]")
             self.prepare_sources()
             print("[BUILD]")

kiwixbuild/platforms/base.py

@@ -3,7 +3,6 @@ import os, sys
 import subprocess
 
 from kiwixbuild.dependencies import Dependency
-from kiwixbuild.packages import PACKAGE_NAME_MAPPERS
 from kiwixbuild.utils import pj, remove_duplicates
 from kiwixbuild.buildenv import BuildEnv
 from kiwixbuild._global import neutralEnv, option, target_steps
@@ -86,85 +85,6 @@ class PlatformInfo(metaclass=_MetaPlatform):
         self.buildEnv.meson_crossfile = None
         self.buildEnv.cmake_crossfile = None
 
-    def get_packages(self):
-        mapper_name = "{host}_{target}".format(
-            host=neutralEnv('distname'),
-            target=self)
-        try:
-            package_name_mapper = PACKAGE_NAME_MAPPERS[mapper_name]
-        except KeyError:
-            print("SKIP : We don't know which packages we must install to compile"
-                  " a {target} {build_type} version on a {host} host.".format(
-                      target=self,
-                      build_type=self.static,
-                      host=neutralEnv('distname')))
-            return []
-
-        packages_list = package_name_mapper.get('COMMON', [])
-        to_drop = []
-        build_steps = (step for step in target_steps() if step[0] == self.name)
-        for dep in build_steps:
-            packages = package_name_mapper.get(dep[1])
-            if packages:
-                packages_list += packages
-                to_drop.append(dep)
-        for dep in to_drop:
-            del target_steps()[dep]
-        source_dep = [dep[1] for dep in target_steps() if dep[0] == 'source']
-        for dep in source_dep:
-            it_build = (True for dep in target_steps() if dep[0] != 'source' and dep[1] == dep)
-            if next(it_build, False):
-                print("removing source ", dep)
-                del target_steps()[('source', dep)]
-        build_steps = (step for step in target_steps() if step[0] == self.name)
-        for dep in build_steps:
-            packages = getattr(dep, 'extra_packages', [])
-            for package in packages:
-                packages_list += package_name_mapper.get(package, [])
-        return packages_list
-
-    def install_packages(self):
-        autoskip_file = pj(self.buildEnv.build_dir, ".install_packages_ok")
-        packages_to_have = self.get_packages()
-        packages_to_have = remove_duplicates(packages_to_have)
-
-        if not option('force_install_packages') and os.path.exists(autoskip_file):
-            print("SKIP")
-            return
-
-        distname = neutralEnv('distname')
-        if distname in ('fedora', 'redhat', 'centos'):
-            package_installer = 'sudo dnf install {}'
-            package_checker = 'rpm -q --quiet {}'
-        elif distname in ('debian', 'Ubuntu'):
-            package_installer = 'sudo apt-get install {}'
-            package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1'
-        elif distname == 'Darwin':
-            package_installer = 'brew install {}'
-            package_checker = 'brew list -1 | grep -q {}'
-
-        packages_to_install = []
-        for package in packages_to_have:
-            print(" - {} : ".format(package), end="")
-            command = package_checker.format(package)
-            try:
-                subprocess.check_call(command, shell=True)
-            except subprocess.CalledProcessError:
-                print("NEEDED")
-                packages_to_install.append(package)
-            else:
-                print("SKIP")
-
-        if packages_to_install:
-            command = package_installer.format(" ".join(packages_to_install))
-            print(command)
-            subprocess.check_call(command, shell=True)
-        else:
-            print("SKIP, No package to install.")
-
-        with open(autoskip_file, 'w'):
-            pass
-
     def clean_intermediate_directories(self):
         self.buildEnv.clean_intermediate_directories()