Merge pull request #413 from kiwix/no_git_update

No git update

.github/scripts/common.py

@@ -68,6 +68,7 @@ def run_kiwix_build(
     command = ["kiwix-build"]
     command.append(target)
     command.append("--hide-progress")
+    command.append("--fast-clone")
     if platform == "flatpak" or platform.startswith("win32_"):
         command.append("--assume-packages-installed")
     if target == "kiwix-lib-app" and platform.startswith("android_"):

kiwixbuild/__init__.py

@@ -48,6 +48,10 @@ def parse_args():
                           help=("Specify the architecture to build for ios application/libraries.\n"
                                 "Can be specified several times to build for several architectures.\n"
                                 "If not specified, all architectures will be build."))
+    subgroup.add_argument('--fast-clone', action='store_true',
+                          help=("Do not clone the whole repository.\n"
+                                "This is useful for one shot build but it is not recommended if you want "
+                                "to develop with the cloned sources."))
     options = parser.parse_args()
 
     if not options.android_arch:

kiwixbuild/builder.py

@@ -4,7 +4,7 @@ from collections import OrderedDict
 from .buildenv import *
 
 from .platforms import PlatformInfo
-from .utils import remove_duplicates, StopBuild
+from .utils import remove_duplicates, StopBuild, colorize
 from .dependencies import Dependency
 from .packages import PACKAGE_NAME_MAPPERS
 from ._global import (
@@ -21,7 +21,7 @@ class Builder:
         target_platform = option('target_platform')
         platform = PlatformInfo.get_platform(target_platform, self._targets)
         if neutralEnv('distname') not in platform.compatible_hosts:
-            print(('ERROR: The target platform {} cannot be build on host {}.\n'
+            print((colorize('ERROR')+': The target platform {} cannot be build on host {}.\n'
                    'Select another target platform or change your host system.'
                   ).format(platform.name, neutralEnv('distname')))
         self.targetDefs = platform.add_targets(option('target'), self._targets)
@@ -97,7 +97,7 @@ class Builder:
 
     def prepare_sources(self):
         if option('skip_source_prepare'):
-            print("SKIP")
+            print(colorize("SKIP"))
             return
 
         sourceDefs = remove_duplicates(tDef for tDef in target_steps() if tDef[0]=='source')
@@ -149,7 +149,7 @@ class Builder:
         packages_to_have = remove_duplicates(packages_to_have)
 
         if option('assume_packages_installed'):
-            print("SKIP, Assume package installed")
+            print(colorize("SKIP") + ", Assume package installed")
             return
 
         distname = neutralEnv('distname')
@@ -170,23 +170,23 @@ class Builder:
             try:
                 subprocess.check_call(command, shell=True)
             except subprocess.CalledProcessError:
-                print("NEEDED")
+                print(colorize("NEEDED"))
                 packages_to_install.append(package)
             else:
-                print("SKIP")
+                print(colorize("SKIP"))
 
         if packages_to_install:
             command = package_installer.format(" ".join(packages_to_install))
             print(command)
             subprocess.check_call(command, shell=True)
         else:
-            print("SKIP, No package to install.")
+            print(colorize("SKIP")+ ", No package to install.")
 
     def run(self):
         try:
             print("[INSTALL PACKAGES]")
             if option('dont_install_packages'):
-                print("SKIP")
+                print(colorize("SKIP"))
             else:
                 self.install_packages()
             self.finalize_target_steps()
@@ -203,7 +203,7 @@ class Builder:
                 for platform in PlatformInfo.all_running_platforms.values():
                     platform.clean_intermediate_directories()
             else:
-                print("SKIP")
+                print(colorize("SKIP"))
         except StopBuild as e:
             print(e)
             sys.exit("Stopping build due to errors")

kiwixbuild/dependencies/base.py

@@ -3,7 +3,7 @@ import os
 import shutil
 import time
 
-from kiwixbuild.utils import pj, Context, SkipCommand, extract_archive, Defaultdict, StopBuild, run_command
+from kiwixbuild.utils import pj, Context, SkipCommand, WarningMessage, extract_archive, Defaultdict, StopBuild, run_command, colorize
 from kiwixbuild.versions import main_project_versions, base_deps_versions
 from kiwixbuild._global import neutralEnv, option
 
@@ -76,12 +76,14 @@ class Source:
             ret = function(*args, context=context)
             context._finalise()
             duration = time.time() - start_time
-            print("OK ({:.1f}s)".format(duration))
+            print(colorize("OK"), "({:.1f}s)".format(duration))
             return ret
-        except SkipCommand:
+        except WarningMessage as e:
-            print("SKIP")
+            print(e)
+        except SkipCommand as e:
+            print(e)
         except subprocess.CalledProcessError:
-            print("ERROR")
+            print(colorize("ERROR"))
             try:
                 with open(log, 'r') as f:
                     print(f.read())
@@ -89,7 +91,7 @@ class Source:
                 pass
             raise StopBuild()
         except:
-            print("ERROR")
+            print(colorize("ERROR"))
             raise
 
 
@@ -152,21 +154,30 @@ class GitClone(Source):
         else:
             return self.base_git_ref
 
-    def _git_clone(self, context):
+    def _git_init(self, context):
-        if os.path.exists(self.git_path):
+        if option('fast_clone'):
-            raise SkipCommand()
             command = "git clone --depth=1 --branch {} {} {}".format(
                 self.git_ref, self.git_remote, self.source_dir)
             run_command(command, neutralEnv('source_dir'), context)
+        else:
+            command = "git clone {} {}".format(self.git_remote, self.source_dir)
+            run_command(command, neutralEnv('source_dir'), context)
+            command = "git checkout {}".format(self.git_ref)
+            run_command(command, self.git_path, context)
 
     def _git_update(self, context):
-        command = "git fetch origin {}".format(
+        command = "git fetch origin {}".format(self.git_ref)
-            self.git_ref)
         run_command(command, self.git_path, context)
-        run_command("git checkout "+self.git_ref, self.git_path, context)
+        try:
+            command = "git merge --ff-only origin/{}".format(self.git_ref)
+            run_command(command, self.git_path, context)
+        except subprocess.CalledProcessError:
+            raise WarningMessage("Cannot update, please check log for information")
 
     def prepare(self):
-        self.command('gitclone', self._git_clone)
+        if not os.path.exists(self.git_path):
+            self.command('gitinit', self._git_init)
+        else:
             self.command('gitupdate', self._git_update)
         if hasattr(self, '_post_prepare_script'):
             self.command('post_prepare_script', self._post_prepare_script)
@@ -234,12 +245,14 @@ class Builder:
             ret = function(*args, context=context)
             context._finalise()
             duration = time.time() - start_time
-            print("OK ({:.1f}s)".format(duration))
+            print(colorize("OK"), "({:.1f}s)".format(duration))
             return ret
-        except SkipCommand:
+        except SkipCommand as e:
-            print("SKIP")
+            print(e)
+        except WarningMessage as e:
+            print(e)
         except subprocess.CalledProcessError:
-            print("ERROR")
+            print(colorize("ERROR"))
             try:
                 with open(log, 'r') as f:
                     print(f.read())
@@ -247,7 +260,7 @@ class Builder:
                 pass
             raise StopBuild()
         except:
-            print("ERROR")
+            print(colorize("ERROR"))
             raise
 
     def build(self):

kiwixbuild/utils.py

@@ -15,6 +15,16 @@ from kiwixbuild._global import neutralEnv, option
 pj = os.path.join
 
 
+COLORS = {
+    'OK': '\033[92m',
+    'WARNING': '\033[93m',
+    'NEEDED': '\033[93m',
+    'SKIP': '\033[34m',
+    'ERROR': '\033[91m',
+    '': '\033[0m',
+}
+
+
 REMOTE_PREFIX = 'http://download.kiwix.org/dev/'
 
 
@@ -66,6 +76,12 @@ def get_sha256(path):
     return sha256.hexdigest()
 
 
+def colorize(text, color=None):
+    if color is None:
+        color = text
+    return "{}{}{}".format(COLORS[color], text, COLORS[''])
+
+
 def print_progress(progress):
     if option('show_progress'):
         text = "{}\033[{}D".format(progress, len(progress))
@@ -134,11 +150,7 @@ def download_remote(what, where):
         raise StopBuild("Sha 256 doesn't correspond")
 
 
-class SkipCommand(Exception):
+class BaseCommandResult(Exception):
-    pass
-
-
-class StopBuild(Exception):
     def __init__(self, msg=""):
         self.msg = msg
 
@@ -146,6 +158,22 @@ class StopBuild(Exception):
         return self.msg
 
 
+class SkipCommand(BaseCommandResult):
+    def __str__(self):
+        if self.msg:
+            return colorize("SKIP") + " : {}".format(self.msg)
+        return colorize("SKIP")
+
+
+class WarningMessage(BaseCommandResult):
+    def __str__(self):
+        return colorize("WARNING") + " : {}".format(self.msg)
+
+
+class StopBuild(BaseCommandResult):
+    pass
+
+
 class Remotefile(namedtuple('Remotefile', ('name', 'sha256', 'url'))):
     def __new__(cls, name, sha256, url=None):
         if url is None:

travis/compile_all.py

@@ -89,6 +89,7 @@ def run_kiwix_build(target, platform,
     command = ['kiwix-build']
     command.append(target)
     command.append('--hide-progress')
+    command.append('--fast-clone')
     if platform == 'flatpak' or platform.startswith('win32_'):
         command.append('--assume-packages-installed')
     if target == 'kiwix-lib-app' and platform.startswith('android_'):