smallsolar
/
kiwix-build
mirror of https://github.com/kiwix/kiwix-build.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #413 from kiwix/no_git_update 

No git update
This commit is contained in:
Matthieu Gautier 2020-02-25 16:34:53 +01:00 committed by GitHub
parent 3f50b1b069 08ed650c78
commit d30ec7219e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 83 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.github/scripts/common.py vendored
View File

@ -68,6 +68,7 @@ def run_kiwix_build(
command = ["kiwix-build"]
command.append(target)
command.append("--hide-progress")
command.append("--fast-clone")
if platform == "flatpak" or platform.startswith("win32_"):
command.append("--assume-packages-installed")
if target == "kiwix-lib-app" and platform.startswith("android_"):

4
kiwixbuild/__init__.py
View File

@ -48,6 +48,10 @@ def parse_args():
help=("Specify the architecture to build for ios application/libraries.\n"
"Can be specified several times to build for several architectures.\n"
"If not specified, all architectures will be build."))
subgroup.add_argument('--fast-clone', action='store_true',
help=("Do not clone the whole repository.\n"
"This is useful for one shot build but it is not recommended if you want "
"to develop with the cloned sources."))
options = parser.parse_args()
if not options.android_arch:

18
kiwixbuild/builder.py
View File

@ -4,7 +4,7 @@ from collections import OrderedDict
from .buildenv import *
from .platforms import PlatformInfo
from .utils import remove_duplicates, StopBuild
from .utils import remove_duplicates, StopBuild, colorize
from .dependencies import Dependency
from .packages import PACKAGE_NAME_MAPPERS
from ._global import (
@ -21,7 +21,7 @@ class Builder:
target_platform = option('target_platform')
platform = PlatformInfo.get_platform(target_platform, self._targets)
if neutralEnv('distname') not in platform.compatible_hosts:
print(('ERROR: The target platform {} cannot be build on host {}.\n'
print((colorize('ERROR')+': The target platform {} cannot be build on host {}.\n'
'Select another target platform or change your host system.'
).format(platform.name, neutralEnv('distname')))
self.targetDefs = platform.add_targets(option('target'), self._targets)
@ -97,7 +97,7 @@ class Builder:
def prepare_sources(self):
if option('skip_source_prepare'):
print("SKIP")
print(colorize("SKIP"))
return
sourceDefs = remove_duplicates(tDef for tDef in target_steps() if tDef[0]=='source')
@ -149,7 +149,7 @@ class Builder:
packages_to_have = remove_duplicates(packages_to_have)
if option('assume_packages_installed'):
print("SKIP, Assume package installed")
print(colorize("SKIP") + ", Assume package installed")
return
distname = neutralEnv('distname')
@ -170,23 +170,23 @@ class Builder:
try:
subprocess.check_call(command, shell=True)
except subprocess.CalledProcessError:
print("NEEDED")
print(colorize("NEEDED"))
packages_to_install.append(package)
else:
print("SKIP")
print(colorize("SKIP"))
if packages_to_install:
command = package_installer.format(" ".join(packages_to_install))
print(command)
subprocess.check_call(command, shell=True)
else:
print("SKIP, No package to install.")
print(colorize("SKIP")+ ", No package to install.")
def run(self):
try:
print("[INSTALL PACKAGES]")
if option('dont_install_packages'):
print("SKIP")
print(colorize("SKIP"))
else:
self.install_packages()
self.finalize_target_steps()
@ -203,7 +203,7 @@ class Builder:
for platform in PlatformInfo.all_running_platforms.values():
platform.clean_intermediate_directories()
else:
print("SKIP")
print(colorize("SKIP"))
except StopBuild as e:
print(e)
sys.exit("Stopping build due to errors")

57
kiwixbuild/dependencies/base.py
View File

@ -3,7 +3,7 @@ import os
import shutil
import time
from kiwixbuild.utils import pj, Context, SkipCommand, extract_archive, Defaultdict, StopBuild, run_command
from kiwixbuild.utils import pj, Context, SkipCommand, WarningMessage, extract_archive, Defaultdict, StopBuild, run_command, colorize
from kiwixbuild.versions import main_project_versions, base_deps_versions
from kiwixbuild._global import neutralEnv, option
@ -76,12 +76,14 @@ class Source:
ret = function(*args, context=context)
context._finalise()
duration = time.time() - start_time
print("OK ({:.1f}s)".format(duration))
print(colorize("OK"), "({:.1f}s)".format(duration))
return ret
except SkipCommand:
print("SKIP")
except WarningMessage as e:
print(e)
except SkipCommand as e:
print(e)
except subprocess.CalledProcessError:
print("ERROR")
print(colorize("ERROR"))
try:
with open(log, 'r') as f:
print(f.read())
@ -89,7 +91,7 @@ class Source:
pass
raise StopBuild()
except:
print("ERROR")
print(colorize("ERROR"))
raise
@ -152,22 +154,31 @@ class GitClone(Source):
else:
return self.base_git_ref
def _git_clone(self, context):
if os.path.exists(self.git_path):
raise SkipCommand()
command = "git clone --depth=1 --branch {} {} {}".format(
self.git_ref, self.git_remote, self.source_dir)
run_command(command, neutralEnv('source_dir'), context)
def _git_init(self, context):
if option('fast_clone'):
command = "git clone --depth=1 --branch {} {} {}".format(
self.git_ref, self.git_remote, self.source_dir)
run_command(command, neutralEnv('source_dir'), context)
else:
command = "git clone {} {}".format(self.git_remote, self.source_dir)
run_command(command, neutralEnv('source_dir'), context)
command = "git checkout {}".format(self.git_ref)
run_command(command, self.git_path, context)
def _git_update(self, context):
command = "git fetch origin {}".format(
self.git_ref)
command = "git fetch origin {}".format(self.git_ref)
run_command(command, self.git_path, context)
run_command("git checkout "+self.git_ref, self.git_path, context)
try:
command = "git merge --ff-only origin/{}".format(self.git_ref)
run_command(command, self.git_path, context)
except subprocess.CalledProcessError:
raise WarningMessage("Cannot update, please check log for information")
def prepare(self):
self.command('gitclone', self._git_clone)
self.command('gitupdate', self._git_update)
if not os.path.exists(self.git_path):
self.command('gitinit', self._git_init)
else:
self.command('gitupdate', self._git_update)
if hasattr(self, '_post_prepare_script'):
self.command('post_prepare_script', self._post_prepare_script)
@ -234,12 +245,14 @@ class Builder:
ret = function(*args, context=context)
context._finalise()
duration = time.time() - start_time
print("OK ({:.1f}s)".format(duration))
print(colorize("OK"), "({:.1f}s)".format(duration))
return ret
except SkipCommand:
print("SKIP")
except SkipCommand as e:
print(e)
except WarningMessage as e:
print(e)
except subprocess.CalledProcessError:
print("ERROR")
print(colorize("ERROR"))
try:
with open(log, 'r') as f:
print(f.read())
@ -247,7 +260,7 @@ class Builder:
pass
raise StopBuild()
except:
print("ERROR")
print(colorize("ERROR"))
raise
def build(self):

38
kiwixbuild/utils.py
View File

@ -15,6 +15,16 @@ from kiwixbuild._global import neutralEnv, option
pj = os.path.join
COLORS = {
'OK': '\033[92m',
'WARNING': '\033[93m',
'NEEDED': '\033[93m',
'SKIP': '\033[34m',
'ERROR': '\033[91m',
'': '\033[0m',
}
REMOTE_PREFIX = 'http://download.kiwix.org/dev/'
@ -66,6 +76,12 @@ def get_sha256(path):
return sha256.hexdigest()
def colorize(text, color=None):
if color is None:
color = text
return "{}{}{}".format(COLORS[color], text, COLORS[''])
def print_progress(progress):
if option('show_progress'):
text = "{}\033[{}D".format(progress, len(progress))
@ -134,11 +150,7 @@ def download_remote(what, where):
raise StopBuild("Sha 256 doesn't correspond")
class SkipCommand(Exception):
pass
class StopBuild(Exception):
class BaseCommandResult(Exception):
def __init__(self, msg=""):
self.msg = msg
@ -146,6 +158,22 @@ class StopBuild(Exception):
return self.msg
class SkipCommand(BaseCommandResult):
def __str__(self):
if self.msg:
return colorize("SKIP") + " : {}".format(self.msg)
return colorize("SKIP")
class WarningMessage(BaseCommandResult):
def __str__(self):
return colorize("WARNING") + " : {}".format(self.msg)
class StopBuild(BaseCommandResult):
pass
class Remotefile(namedtuple('Remotefile', ('name', 'sha256', 'url'))):
def __new__(cls, name, sha256, url=None):
if url is None:

1
travis/compile_all.py
View File

@ -89,6 +89,7 @@ def run_kiwix_build(target, platform,
command = ['kiwix-build']
command.append(target)
command.append('--hide-progress')
command.append('--fast-clone')
if platform == 'flatpak' or platform.startswith('win32_'):
command.append('--assume-packages-installed')
if target == 'kiwix-lib-app' and platform.startswith('android_'):