diff --git a/.github/scripts/common.py b/.github/scripts/common.py index de52aa9..b47fe5a 100644 --- a/.github/scripts/common.py +++ b/.github/scripts/common.py @@ -68,6 +68,7 @@ def run_kiwix_build( command = ["kiwix-build"] command.append(target) command.append("--hide-progress") + command.append("--fast-clone") if platform == "flatpak" or platform.startswith("win32_"): command.append("--assume-packages-installed") if target == "kiwix-lib-app" and platform.startswith("android_"): diff --git a/kiwixbuild/__init__.py b/kiwixbuild/__init__.py index 144fdff..b319eaa 100644 --- a/kiwixbuild/__init__.py +++ b/kiwixbuild/__init__.py @@ -48,6 +48,10 @@ def parse_args(): help=("Specify the architecture to build for ios application/libraries.\n" "Can be specified several times to build for several architectures.\n" "If not specified, all architectures will be build.")) + subgroup.add_argument('--fast-clone', action='store_true', + help=("Do not clone the whole repository.\n" + "This is useful for one shot build but it is not recommended if you want " + "to develop with the cloned sources.")) options = parser.parse_args() if not options.android_arch: diff --git a/kiwixbuild/builder.py b/kiwixbuild/builder.py index b70c139..2452a1b 100644 --- a/kiwixbuild/builder.py +++ b/kiwixbuild/builder.py @@ -4,7 +4,7 @@ from collections import OrderedDict from .buildenv import * from .platforms import PlatformInfo -from .utils import remove_duplicates, StopBuild +from .utils import remove_duplicates, StopBuild, colorize from .dependencies import Dependency from .packages import PACKAGE_NAME_MAPPERS from ._global import ( @@ -21,7 +21,7 @@ class Builder: target_platform = option('target_platform') platform = PlatformInfo.get_platform(target_platform, self._targets) if neutralEnv('distname') not in platform.compatible_hosts: - print(('ERROR: The target platform {} cannot be build on host {}.\n' + print((colorize('ERROR')+': The target platform {} cannot be build on host {}.\n' 'Select another target platform or change your host system.' ).format(platform.name, neutralEnv('distname'))) self.targetDefs = platform.add_targets(option('target'), self._targets) @@ -97,7 +97,7 @@ class Builder: def prepare_sources(self): if option('skip_source_prepare'): - print("SKIP") + print(colorize("SKIP")) return sourceDefs = remove_duplicates(tDef for tDef in target_steps() if tDef[0]=='source') @@ -149,7 +149,7 @@ class Builder: packages_to_have = remove_duplicates(packages_to_have) if option('assume_packages_installed'): - print("SKIP, Assume package installed") + print(colorize("SKIP") + ", Assume package installed") return distname = neutralEnv('distname') @@ -170,23 +170,23 @@ class Builder: try: subprocess.check_call(command, shell=True) except subprocess.CalledProcessError: - print("NEEDED") + print(colorize("NEEDED")) packages_to_install.append(package) else: - print("SKIP") + print(colorize("SKIP")) if packages_to_install: command = package_installer.format(" ".join(packages_to_install)) print(command) subprocess.check_call(command, shell=True) else: - print("SKIP, No package to install.") + print(colorize("SKIP")+ ", No package to install.") def run(self): try: print("[INSTALL PACKAGES]") if option('dont_install_packages'): - print("SKIP") + print(colorize("SKIP")) else: self.install_packages() self.finalize_target_steps() @@ -203,7 +203,7 @@ class Builder: for platform in PlatformInfo.all_running_platforms.values(): platform.clean_intermediate_directories() else: - print("SKIP") + print(colorize("SKIP")) except StopBuild as e: print(e) sys.exit("Stopping build due to errors") diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index 09a5db8..c8ee4ed 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -3,7 +3,7 @@ import os import shutil import time -from kiwixbuild.utils import pj, Context, SkipCommand, extract_archive, Defaultdict, StopBuild, run_command +from kiwixbuild.utils import pj, Context, SkipCommand, WarningMessage, extract_archive, Defaultdict, StopBuild, run_command, colorize from kiwixbuild.versions import main_project_versions, base_deps_versions from kiwixbuild._global import neutralEnv, option @@ -76,12 +76,14 @@ class Source: ret = function(*args, context=context) context._finalise() duration = time.time() - start_time - print("OK ({:.1f}s)".format(duration)) + print(colorize("OK"), "({:.1f}s)".format(duration)) return ret - except SkipCommand: - print("SKIP") + except WarningMessage as e: + print(e) + except SkipCommand as e: + print(e) except subprocess.CalledProcessError: - print("ERROR") + print(colorize("ERROR")) try: with open(log, 'r') as f: print(f.read()) @@ -89,7 +91,7 @@ class Source: pass raise StopBuild() except: - print("ERROR") + print(colorize("ERROR")) raise @@ -152,22 +154,31 @@ class GitClone(Source): else: return self.base_git_ref - def _git_clone(self, context): - if os.path.exists(self.git_path): - raise SkipCommand() - command = "git clone --depth=1 --branch {} {} {}".format( - self.git_ref, self.git_remote, self.source_dir) - run_command(command, neutralEnv('source_dir'), context) + def _git_init(self, context): + if option('fast_clone'): + command = "git clone --depth=1 --branch {} {} {}".format( + self.git_ref, self.git_remote, self.source_dir) + run_command(command, neutralEnv('source_dir'), context) + else: + command = "git clone {} {}".format(self.git_remote, self.source_dir) + run_command(command, neutralEnv('source_dir'), context) + command = "git checkout {}".format(self.git_ref) + run_command(command, self.git_path, context) def _git_update(self, context): - command = "git fetch origin {}".format( - self.git_ref) + command = "git fetch origin {}".format(self.git_ref) run_command(command, self.git_path, context) - run_command("git checkout "+self.git_ref, self.git_path, context) + try: + command = "git merge --ff-only origin/{}".format(self.git_ref) + run_command(command, self.git_path, context) + except subprocess.CalledProcessError: + raise WarningMessage("Cannot update, please check log for information") def prepare(self): - self.command('gitclone', self._git_clone) - self.command('gitupdate', self._git_update) + if not os.path.exists(self.git_path): + self.command('gitinit', self._git_init) + else: + self.command('gitupdate', self._git_update) if hasattr(self, '_post_prepare_script'): self.command('post_prepare_script', self._post_prepare_script) @@ -234,12 +245,14 @@ class Builder: ret = function(*args, context=context) context._finalise() duration = time.time() - start_time - print("OK ({:.1f}s)".format(duration)) + print(colorize("OK"), "({:.1f}s)".format(duration)) return ret - except SkipCommand: - print("SKIP") + except SkipCommand as e: + print(e) + except WarningMessage as e: + print(e) except subprocess.CalledProcessError: - print("ERROR") + print(colorize("ERROR")) try: with open(log, 'r') as f: print(f.read()) @@ -247,7 +260,7 @@ class Builder: pass raise StopBuild() except: - print("ERROR") + print(colorize("ERROR")) raise def build(self): diff --git a/kiwixbuild/utils.py b/kiwixbuild/utils.py index 2405a21..5070d1e 100644 --- a/kiwixbuild/utils.py +++ b/kiwixbuild/utils.py @@ -15,6 +15,16 @@ from kiwixbuild._global import neutralEnv, option pj = os.path.join +COLORS = { + 'OK': '\033[92m', + 'WARNING': '\033[93m', + 'NEEDED': '\033[93m', + 'SKIP': '\033[34m', + 'ERROR': '\033[91m', + '': '\033[0m', +} + + REMOTE_PREFIX = 'http://download.kiwix.org/dev/' @@ -66,6 +76,12 @@ def get_sha256(path): return sha256.hexdigest() +def colorize(text, color=None): + if color is None: + color = text + return "{}{}{}".format(COLORS[color], text, COLORS['']) + + def print_progress(progress): if option('show_progress'): text = "{}\033[{}D".format(progress, len(progress)) @@ -134,11 +150,7 @@ def download_remote(what, where): raise StopBuild("Sha 256 doesn't correspond") -class SkipCommand(Exception): - pass - - -class StopBuild(Exception): +class BaseCommandResult(Exception): def __init__(self, msg=""): self.msg = msg @@ -146,6 +158,22 @@ class StopBuild(Exception): return self.msg +class SkipCommand(BaseCommandResult): + def __str__(self): + if self.msg: + return colorize("SKIP") + " : {}".format(self.msg) + return colorize("SKIP") + + +class WarningMessage(BaseCommandResult): + def __str__(self): + return colorize("WARNING") + " : {}".format(self.msg) + + +class StopBuild(BaseCommandResult): + pass + + class Remotefile(namedtuple('Remotefile', ('name', 'sha256', 'url'))): def __new__(cls, name, sha256, url=None): if url is None: diff --git a/travis/compile_all.py b/travis/compile_all.py index 31dda69..6f3cfdf 100755 --- a/travis/compile_all.py +++ b/travis/compile_all.py @@ -89,6 +89,7 @@ def run_kiwix_build(target, platform, command = ['kiwix-build'] command.append(target) command.append('--hide-progress') + command.append('--fast-clone') if platform == 'flatpak' or platform.startswith('win32_'): command.append('--assume-packages-installed') if target == 'kiwix-lib-app' and platform.startswith('android_'):