From 22fffcee511e4f2933e983a4ca20a6fa732b5f88 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 24 May 2017 14:31:05 +0200 Subject: [PATCH] Add script to build customapp --- build_custom_app.py | 390 ++++++++++++++++++ dependencies.py | 25 +- dependency_utils.py | 7 +- kiwix-build.py | 53 +-- requirements_build_custom_app.txt | 3 + travis/compile_custom_app.sh | 13 + travis/deploy_apk.sh | 27 ++ ...ay_android_developer-5a411156212c.json.enc | Bin 0 -> 2368 bytes travis/install_extra_deps.sh | 3 +- travis/make_release.sh | 28 ++ travis/test_ks.ks.enc | Bin 0 -> 2256 bytes utils.py | 48 ++- 12 files changed, 552 insertions(+), 45 deletions(-) create mode 100755 build_custom_app.py create mode 100644 requirements_build_custom_app.txt create mode 100755 travis/compile_custom_app.sh create mode 100755 travis/deploy_apk.sh create mode 100644 travis/googleplay_android_developer-5a411156212c.json.enc create mode 100755 travis/make_release.sh create mode 100644 travis/test_ks.ks.enc diff --git a/build_custom_app.py b/build_custom_app.py new file mode 100755 index 0000000..5a32462 --- /dev/null +++ b/build_custom_app.py @@ -0,0 +1,390 @@ +#!/usr/bin/env python3 + +import \ + argparse, \ + datetime, \ + glob, \ + json, \ + os, \ + ssl, \ + sys, \ + tempfile, \ + time, \ + urllib +from uuid import uuid4 +from contextlib import contextmanager +from urllib.parse import urlparse + +from utils import ( + Remotefile, + download_remote +) + +import requests +import httplib2 +from apiclient.discovery import build +from apiclient.errors import HttpError +from oauth2client import client +from oauth2client.service_account import ServiceAccountCredentials + +tmpl_request_url = "https://api.travis-ci.org/repo/{organisation}%2F{repository}/{endpoint}" +tmpl_message = """Build of custom app {app} with zim file {zim}. + +UUID:#{uuid}#""" + +description = """Launch a custom application build. +This command will launch a custom application build on Travis-CI. +Travis-CI jobs will compile the application and upload the build apks on +google play, using tha 'alpha' track of the application. + +You will need to have a valid TRAVIS_TOKEN associated to your personnal account +and the kiwix-build repository on travis. +Use the 'travis' command line tool (https://github.com/travis-ci/travis.rb) +to generate a token. +""" + +def parse_args(): + parser = argparse.ArgumentParser(description=description, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('--custom-app') + parser.add_argument('--travis-token') + parser.add_argument('--zim-url') + + advance = parser.add_argument_group('advance', "Some advanced options.") + advance.add_argument('--extra-code', type=int, default=0) + advance.add_argument('--check-certificate', default=True) + + # Hidden options + parser.add_argument('--step', default='launch', choices=['launch', 'publish'], help=argparse.SUPPRESS) + parser.add_argument('--apks-dir', help=argparse.SUPPRESS) + parser.add_argument('--version', default="0", help=argparse.SUPPRESS) + parser.add_argument('--content-version-code', type=int) + parser.add_argument('--package-name', default=None, help=argparse.SUPPRESS) + parser.add_argument('--google-api-key', help=argparse.SUPPRESS) + + options = parser.parse_args() + + if not options.package_name: + print("Try to get package name from info.json file") + request_url = ('https://raw.githubusercontent.com/kiwix/kiwix-android-custom/master/{}/info.json' + .format(options.custom_app)) + json_request = requests.get(request_url) + if json_request.status_code != 200: + print("Error while getting json file.") + print("Reason is '{}'".format(json_request.reason)) + sys.exit(-1) + json_data = json.loads(json_request.text) + print("Found package_name '{}'".format(json_data['package'])) + options.package_name = json_data['package'] + + options.base_version = "{}{}".format( + datetime.date.today().strftime('%y%j'), + options.extra_code) + + return options + + +def download_zim_file(zim_url, dest_dir=None): + if dest_dir is None: + dest_dir = os.getcwd() + out_filename = urlparse(zim_url).path + out_filename = os.path.basename(out_filename) + zim_file = Remotefile(out_filename, '', zim_url) + download_remote(zim_file, dest_dir) + return os.path.join(dest_dir, out_filename) + + +def get_zim_size(zim_url, check_certificate=True): + print("Try to get zim size") + if not check_certificate: + context = ssl.create_default_context() + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + else: + context = None + extra_args = {'context':context} if sys.version_info >= (3, 4, 3) else {} + with urllib.request.urlopen(zim_url, **extra_args) as resource: + size = resource.getheader('Content-Length', None) + if size is not None: + size = int(size) + print("Zim size is {}".format(size)) + return size + else: + print("No 'Content-Length' header in http answer from the server.\n" + "We need to download the zim file to get its size.") + zim_path = download_zim_file(zim_url, tempfile.gettempdir()) + size = os.path.getsize(zim_path) + print("Zim size is {}".format(size)) + return size + + +def do_launch(options): + zim_size = get_zim_size(options.zim_url, options.check_certificate) + travis_launch_build('kiwix', 'kiwix-build', options, zim_size) + print("Travis build has been launch.") + + +def do_publish(options): + zim_path = download_zim_file(options.zim_url) + googleService = Google(options) + with googleService.new_request(): + versionCodes = [] + for apk in glob.iglob(options.apks_dir+"/*.apk"): + result = googleService.upload_apk(apk) + versionCodes.append(result['versionCode']) + googleService.upload_expansionfile( + zim_path, options.content_version_code, versionCodes) + googleService.publish_release(options, versionCodes) + + +def travis_launch_build(organisation, repository, options, zim_size): + request_url = tmpl_request_url.format( + organisation=organisation, + repository=repository, + endpoint='requests') + headers = { + "Travis-API-Version": "3", + "Content-Type": "application/json", + "Authorization": "token {}".format(options.travis_token), + "User-Agent": "kiwix-build" + } + uuid = uuid4() + envs = [] + for platform_index, platform in enumerate(['arm', 'arm64', 'x86', 'x86_64', 'mips', 'mips64']): + d = { + 'PLATFORM': "android_{}".format(platform), + 'VERSION_CODE': gen_version_code(platform_index, options.base_version) + } + envs.append(d) + + global_env = [ + { 'CUSTOM_APP': options.custom_app}, + { 'ZIM_FILE_SIZE': zim_size}, + { 'PACKAGE_NAME': options.package_name}, + { 'ZIM_URL': options.zim_url}, + { 'EXTRA_CODE': options.extra_code}, + { 'BASE_VERSION': options.base_version}, + { 'CONTENT_VERSION_CODE': gen_version_code(0, options.base_version)}, + { 'VERSION': options.version}, + { 'secure': ('u6MCvCQhAlvU0jNojLVatNXHr6AYj4rjU9CY9JcUUG9CKMsls1t3ERz' + 'aab0vY1H7EIYl8uMgJcE71lF+61/sVOz7DPJVbKBorivnOlvMvyG2xU' + 'su/q3ddMcAkuyixtW2Yo7oq8Tug5PbL5qybrZ+TD0BplS/XcpgljMTH' + 'gxWDYion5Y1/Udz5zg1VIxHGAcv3lKYqtbPzA5gR7scLhFVMVulpFDC' + 'Ps9MS/jEpwJtHrSZPrRmkOdLCJJNgOnQmRe0ib6NsmhO9KgpeEZRgUx' + 'bSaOOiXKBwN7MdWHCZKczZbwVBEA9Izw29hVh0fQZpZNuC3GAICydyK' + 'XmbGLNy+GqIPOJPXbAgfT8zRK2IN+PuUHL0xizK4QALMX1jTiiJ65i1' + 'vWy1NZ3fS7kbgulPUXvkIs2PoSULVEAqhhanp+1ICcEVKs5ffc3e0WP' + '/p7NSTIL8QF58QYPOIqWXX17IP3kfrWURIWCdPHCHKEGkg0S+HyqKa6' + '/pdR5CeBCeECLraupzJ+ZfKIfsFFyrOgCWRYe1pIAD5C1f/0jM6T+9h' + 'Vsc2ktgjM8GpujTztzMKNrVJ80mNBtdXbFGYn01B/zhVSsxPIrbROVp' + '5UL/lFxJVPIrSe2nIs4AFhanhSDK5ro9qCI6GotPqhyd+RhuJMY/p36' + 'fs/srp3KL+aNd8JFxWj45M0=') }, + { 'secure': ('xotOZyUbkcgoZhS4ZAaNtqaxhxRV03rgWRSxaccI+INczLNK18xiVUj' + 'I/1zdAil+ck8rXDMYVuXGo860UIaxt7Sxa7j24+nu6KNpDmlB2a1xyd' + 'Ls8RNL9tUpx4t+DooTHmUuUJSipUcchK6ZRAKFYOHB9Lumkbz2H2Ifs' + 'U1AYnLZfcGC7U63DojGFYJmgXwCgHJ25Zq00SCYdHq1b5vpvQ3ZVviI' + '0+OTXCK804sSiLu3g6EamsQn7+P5F4RnMrKtm6kP2iTZ3sbED6L6adA' + 'mtOWKDfgrMIbYO3kgYemfNE12KG+Lfey10RiheC/CbUZHV3jSl+LE1z' + 'hY8Nqcf6rYQZ8QceE9gW53MrILpUxi7oSltR/pwenNNS5uEox1SrMtY' + '2/O0K0dixHHgF96RV5YuqJQurzTEXTt4T9Uu1ZItp3rauSwcj6wbTYu' + 'fWnGeUzNVItZkSruysBY5mrcvkH6lQpCMk5x6kAzvHCQM4DtIWcep3P' + 'LG0lj034OuZc7MxIh9GzReeYw/gawF2xEgmiitlBaPVvYUff0BaJ8K7' + 'R0bbbYOuIibxrXsiAIZwa0d9vS2PBoerORD/hiE3QD+c0o3uEn8btw+' + 'pebokk6SizfS1q44tcWWRpaovm8uD3rhJ0alm6Ht0V7ErGzXafAzsiR' + 'B/o4trHiAeJUJuSIKDyvmWk=') } + ] + + env = { + 'matrix': envs, + 'global': global_env + } + + data = { + 'request': { + 'message' : tmpl_message.format(app=options.custom_app, zim=options.zim_url, uuid=uuid), + 'branch' : "custom_app", + 'config' : { + 'before_install' : [ + ( 'pip3 install pyOpenSSl google-api-python-client' + ' httplib2 apiclient requests'), + ( 'openssl aes-256-cbc -k $google_key' + ' -in travis/googleplay_android_developer-5a411156212c.json.enc' + ' -out travis/googleplay_android_developer-5a411156212c.json' + ' -d'), + ( 'openssl aes-256-cbc -k $google_key' + ' -in travis/test_ks.ks.enc' + ' -out travis/test_ks.ks -d'), + ( 'openssl aes-256-cbc -K $encrypted_eba2f7543984_key' + ' -iv $encrypted_eba2f7543984_iv' + ' -in travis/travisci_builder_id_key.enc' + ' -out travis/travisci_builder_id_key -d'), + 'chmod 600 travis/travisci_builder_id_key' + ], + 'env' : env, + 'script' : 'travis_wait 30 travis/compile_custom_app.sh', + 'deploy' : { + 'provider': 'script', + 'skip_cleanup': True, + 'script': 'travis/deploy_apk.sh', + 'on': { + 'branch': 'custom_app' + } + }, + 'jobs': { + 'include': [ + { + 'stage' : 'make_release', + 'install': 'pip3 install -r requirements_build_custom_app.txt', + 'script': True, + 'env': global_env, + 'deploy' : { + 'provider': 'script', + 'skip_cleanup': True, + 'script': 'travis/make_release.sh', + 'on': { + 'branch': 'custom_app' + } + } + } + ] + } + } + } + } + + + r = requests.post(request_url, headers=headers, json=data) + if r.status_code != 202: + print("Error while requesting build:") + print(r.reason) + print("Have you forget to give the travis token ?") + sys.exit(-1) + else: + request_id = r.json()['request']['id'] + print("Request {} has been schedule.".format(request_id)) + request_left = 10 + found = False + request_url = tmpl_request_url.format( + organisation=organisation, + repository=repository, + endpoint='builds') + while request_left: + time.sleep(1) + print("Try to get associated build.") + r = requests.get(request_url, headers=headers) + json_data = json.loads(r.text) + for build in json_data['builds']: + if build['event_type'] != 'api': + continue + message = build['commit']['message'] + if str(uuid) in message: + found = True + break + if found: + break + print("Cannot find build. Wait 1 second and try again") + print("{} tries left".format(request_left)) + request_left -= 1 + if found: + print("Associated build found: {}.".format(build['number'])) + print("https://travis-ci.org/kiwix/kiwix-build/builds/{}".format(build['id'])) + else: + print("Request has been accepted by travis-ci but I cannot found " + "the associated build. Have a look here " + "https://travis-ci.org/kiwix/kiwix-build/builds" + "if you found it.") + + +ERROR_MSG_EDIT_CHANGE = "A change was made to the application outside of this Edit, please create a new edit." + +class Google: + def __init__(self, options): + scope = 'https://www.googleapis.com/auth/androidpublisher' + key = options.google_api_key + credentials = ServiceAccountCredentials.from_json_keyfile_name( + key, + scopes=[scope]) + + http = httplib2.Http() + http = credentials.authorize(http) + + self.service = build('androidpublisher', 'v2', http=http) + self.packageName = options.package_name + self.edit_id = None + + @contextmanager + def new_request(self): + edit_request = self.service.edits().insert( + packageName=self.packageName, + body={}) + result = edit_request.execute() + print("create", result) + self.edit_id = result['id'] + + yield + + commit_request = self.service.edits().commit( + editId=self.edit_id, + packageName=self.packageName) + result = commit_request.execute() + print("commit", result) + + self.edit_id = None + + def make_request(self, section, method, **kwargs): + request_content = self._build_request_content(kwargs) + _section = getattr(self._edits, section)() + _method = getattr(_section, method) + print(">", request_content) + request = _method(**request_content) + result = request.execute() + print("<", result) + return result + + def _build_request_content(self, kwargs): + d = kwargs.copy() + d['editId'] = self.edit_id + d['packageName'] = self.packageName + return d + + @property + def _edits(self): + return self.service.edits() + + def upload_expansionfile(self, comp_file, contentVersionCode, versionCodes): + versionCodes = [int(v) for v in versionCodes] + self.make_request('expansionfiles', 'upload', + expansionFileType='main', + apkVersionCode=contentVersionCode, + media_body=comp_file, + media_mime_type='application/octet-stream') + for versionCode in versionCodes: + if versionCode == contentVersionCode: + continue + self.make_request('expansionfiles', 'update', + expansionFileType='main', + apkVersionCode=versionCode, + body={'referencesVersion': contentVersionCode} + ) + + def upload_apk(self, apk_file): + return self.make_request('apks', 'upload', + media_body=apk_file) + + def publish_release(self, options, versionCodes): + return self.make_request('tracks', 'update', + track="alpha", + body={'versionCodes': versionCodes}) + + +def gen_version_code(platform_index, base_version): + str_version = "{platform}{base_version}".format( + platform=platform_index, + base_version=base_version + ) + return int(str_version) + +if __name__ == "__main__": + options = parse_args() + func = globals()['do_{}'.format(options.step)] + func(options) diff --git a/dependencies.py b/dependencies.py index 8bf08dc..b22ce2e 100644 --- a/dependencies.py +++ b/dependencies.py @@ -345,6 +345,12 @@ class KiwixAndroid(Dependency): git_dir = "kiwix-android" class Builder(GradleBuilder): + def build(self): + if self.buildEnv.options.targets == 'kiwix-android-custom': + print("SKIP") + else: + super().build() + def _configure(self, context): if not os.path.exists(self.build_path): shutil.copytree(self.source_path, self.build_path) @@ -375,7 +381,15 @@ class KiwixCustomApp(Dependency): @property def gradle_option(self): - return "-i -P customDir={}".format(pj(self.build_path, 'custom')) + template = ("-i -P customDir={customDir}" + " -P zim_file_size={zim_size}" + " -P version_code={version_code}" + " -P content_version_code={content_version_code}") + return template.format( + customDir=pj(self.build_path, 'custom'), + zim_size=self._get_zim_size(), + version_code=os.environ['VERSION_CODE'], + content_version_code=os.environ['CONTENT_VERSION_CODE']) @property def build_path(self): @@ -385,6 +399,15 @@ class KiwixCustomApp(Dependency): def custom_build_path(self): return pj(self.build_path, 'custom', self.target.custom_name) + def _get_zim_size(self): + try: + zim_size = self.buildEnv.options.zim_file_size + except AttributeError: + with open(pj(self.source_path, self.target.custom_name, 'info.json')) as f: + app_info = json.load(f) + zim_size = os.path.getsize(pj(self.custom_build_path, app_info['zim_file'])) + return zim_size + def build(self): self.command('configure', self._configure) self.command('download_zim', self._download_zim) diff --git a/dependency_utils.py b/dependency_utils.py index de6d82c..c538d57 100644 --- a/dependency_utils.py +++ b/dependency_utils.py @@ -135,12 +135,15 @@ class GitClone(Source): context.force_native_build = True if os.path.exists(self.git_path): raise SkipCommand() - command = "git clone --depth=1 {} {}".format(self.git_remote, self.git_dir) + command = "git clone --depth=1 --branch {} {} {}".format( + self.git_ref, self.git_remote, self.git_dir) self.buildEnv.run_command(command, self.buildEnv.source_dir, context) def _git_update(self, context): context.force_native_build = True - self.buildEnv.run_command("git fetch", self.git_path, context) + command = "git fetch origin {}".format( + self.git_ref) + self.buildEnv.run_command(command, self.git_path, context) self.buildEnv.run_command("git checkout "+self.git_ref, self.git_path, context) def prepare(self): diff --git a/kiwix-build.py b/kiwix-build.py index 2a57056..2692abd 100755 --- a/kiwix-build.py +++ b/kiwix-build.py @@ -17,6 +17,7 @@ from utils import ( get_sha256, print_progress, setup_print_progress, + download_remote, StopBuild, SkipCommand, Defaultdict, @@ -432,44 +433,7 @@ class BuildEnv: def download(self, what, where=None): where = where or self.archive_dir - file_path = pj(where, what.name) - file_url = what.url or (REMOTE_PREFIX + what.name) - if os.path.exists(file_path): - if what.sha256 == get_sha256(file_path): - raise SkipCommand() - os.remove(file_path) - - if options.no_cert_check == True: - context = ssl.create_default_context() - context.check_hostname = False - context.verify_mode = ssl.CERT_NONE - else: - context = None - batch_size = 1024 * 8 - extra_args = {'context':context} if sys.version_info >= (3, 4, 3) else {} - progress_chars = "/-\|" - with urllib.request.urlopen(file_url, **extra_args) as resource, open(file_path, 'wb') as file: - tsize = resource.getheader('Content-Length', None) - if tsize is not None: - tsize = int(tsize) - current = 0 - while True: - batch = resource.read(batch_size) - if not batch: - break - if tsize: - current += batch_size - print_progress("{:.2%}".format(current/tsize)) - else: - print_progress(progress_chars[current]) - current = (current+1)%4 - file.write(batch) - - if not what.sha256: - print('Sha256 for {} not set, do no verify download'.format(what.name)) - elif what.sha256 != get_sha256(file_path): - os.remove(file_path) - raise StopBuild() + download_remote(what, where, not self.options.no_cert_check) def install_packages(self): autoskip_file = pj(self.build_dir, ".install_packages_ok") @@ -945,12 +909,21 @@ def parse_args(): help="The custom android app to build") subgroup.add_argument('--zim-file-url', help="The url of the zim file to download") + subgroup.add_argument('--zim-file-size', + help="The size of the zim file.") options = parser.parse_args() if options.targets == 'kiwix-android-custom': - if not options.android_custom_app or not options.zim_file_url: - print("You need to specify ANDROID_CUSTOM_APP and ZIM_FILE_URL if " + err = False + if not options.android_custom_app: + print("You need to specify ANDROID_CUSTOM_APP if you " "want to build a kiwix-android-custom target") + err = True + if not options.zim_file_url and not options.zim_file_size: + print("You need to specify ZIM_FILE_SIZE or ZIM_FILE_URL if you " + "want to build a kiwix-android-custom target") + err = True + if err: sys.exit(1) return options diff --git a/requirements_build_custom_app.txt b/requirements_build_custom_app.txt new file mode 100644 index 0000000..8892948 --- /dev/null +++ b/requirements_build_custom_app.txt @@ -0,0 +1,3 @@ +requests==2.10.0 +apiclient==1.0.3 + diff --git a/travis/compile_custom_app.sh b/travis/compile_custom_app.sh new file mode 100755 index 0000000..6bde800 --- /dev/null +++ b/travis/compile_custom_app.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -e + +cd ${HOME} + +${TRAVIS_BUILD_DIR}/kiwix-build.py \ + --target-platform $PLATFORM \ + --hide-progress \ + --android-custom-app $CUSTOM_APP \ + --zim-file-size $ZIM_FILE_SIZE \ + kiwix-android-custom + diff --git a/travis/deploy_apk.sh b/travis/deploy_apk.sh new file mode 100755 index 0000000..896d498 --- /dev/null +++ b/travis/deploy_apk.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -e + +KEYSTORE_FILE=${TRAVIS_BUILD_DIR}/travis/test_ks.ks +GOOGLE_API_KEY=${TRAVIS_BUILD_DIR}/travis/googleplay_android_developer-5a411156212c.json +SSH_KEY=${TRAVIS_BUILD_DIR}/travis/travisci_builder_id_key + +cd ${HOME} + +# Sign apk file + +BASE_DIR="BUILD_${PLATFORM}" +INPUT_APK_FILE=${BASE_DIR}/kiwix-android-custom_${CUSTOM_APP}/app/build/outputs/apk/app-${CUSTOM_APP}-release-unsigned.apk +SIGNED_APK=${BASE_DIR}/app-${CUSTOM_APP}_${VERSION_CODE}-release-signed.apk + +TOOLCHAINS/android-sdk-r25.2.3/build-tools/25.0.2/apksigner sign \ + --ks ${KEYSTORE_FILE} \ + --ks-pass env:KEYSTORE_PASS \ + --out ${SIGNED_APK} \ + ${INPUT_APK_FILE} + +ssh -i ${SSH_KEY} nightlybot@download.kiwix.org "mkdir -p ~/apks/${CUSTOM_APP}_${BASE_VERSION}" + +scp -i ${SSH_KEY} \ + ${SIGNED_APK} \ + nightlybot@download.kiwix.org:~/apks/${CUSTOM_APP}_${BASE_VERSION} diff --git a/travis/googleplay_android_developer-5a411156212c.json.enc b/travis/googleplay_android_developer-5a411156212c.json.enc new file mode 100644 index 0000000000000000000000000000000000000000..960996e9e063da0c72ddc283de0397002f57ff62 GIT binary patch literal 2368 zcmV-G3BUGJVQh3|WM5x>Y!D|MOsKK5eD5tFko_=4R0Ge*6#F(_v2;Z9?TYg*jHxDz zo(8&nSUcN#WOBkzNKoCoQc1SO`uYp0{yu;J_QT#)pTkpnsFCX%x!$fcN5TQQwPWo^ zCY%g&jWaRrfQ=nu)yrwl(PJ}pZw2bNHw3@&V+$g3Esrjms*%vDA{h3WPnB5GqZt5Q z`Zeb(-5-)3M&t4n9*YcEhR4D8j!G@l2*&b?%#w(_`?U(FneIdp8!`Q#+#fPJ`Q<$% zyz0OiWYOyi5D^NS)xaRV7h?>c0|6Lu*Yy2>?i@%yG$QN5#&6V3whs~6okde>%(x?d ztP+*!SZSqJ2tKUe*}1BPTM@OnVhL}F(|=TZ}p zy_Oh=WID2u%aKqI$0X{xm{$AnGXqL3Z@~y**|Zs~kRO;}o*tl!ruw{@qHJ{y*RaR^;f`hD5W0-;BpY__1!BvToJ=-*ldqTVZ84sm3%fG~`v<^C6QFN9 z`WnFJtac9d@~c6xFI!-xUm?H|m0S)G-pJPVIH6i`YN8CMP1m9txq&Dsa5rIi%-YHqGwa1vXZI z5K2P|kJ@#9YcZzEQJB#Z6V9s}Ed&`2N-$bDw+>$T^UjJl;Aqihowr+fpW03%K!3mY z314MmXDp5>0p1uJ0wbBt0!bA2i}oM_9{A~BNkw}ITGgT zq~@wM*-6t6`ASEFZ1#g7#i=R7l;2txPS|<8LYo4Dj;fZHxidk9xUS`lJKR)^q9zar zD*bkKOX7sQc*usQ* z}qFIDyvp<*S#0y_A^1VA2S9(-e@YxpEN0?aeM2e}% zdcAcy%Y2i0+`PgaK=z{cz*u zlgTDR8Q<|qasLmD1$|nde~uw4GT|~=!k-?YC?90Ixi(2Jf63s%gtrYMkSUPSrC!8# zC#JK6YXvT=lOG_(Lnw1roUXw_CE!lG3l&a7$@3BKXFM2OoION3?lWqwSU$aS&BSVK zagB*VU1r4wWq?(agcm*;sp>gC8nw8Q#|g@t+@b)GkEEAD0^#-G0ctLvN_qt9u6;PS~7i&Q}gEPeE2{d(q&NQgN9Q_Nabhi#XWk_1t{{I_trmH;ZGh~I2&kgt5P2XakgZ!9T@<&l8}Ms z{nBL%Mc1^5$p-&AMWt<$HZinw2xlwPgZ-xmc&9fb@*a0gm}Sbqt*ZI9O~YF)3)-e; z0-%Btl4XUPd*{YICGbS54*|~2>f)07@EuW$5%A-L&y3g%nXc+ru4P#m`JUovjElEU zkojbr;klyI4M8X2rCX4)cHDx8CFO)HbJ{XAZl(ytKyewVS^ zO6IuDGBmg^U>+g=Cyf#Ganw988n?2S7p-4BwquY}q02$xdLZSnE#Q~fDr0{a6LPs5 zY*JxkPut`xlH9XN>*|K|lU3p8?O*UT_a zT)I|qB$=CE8#}uM7Wa@*X~)ZV{3dluHqE5k-%OFs@Eu{*m>uNDDMK+e(PTfC-6w~b z8c$jEoPXH&MNboyGhs~b&Upyugy(9gZe<`ZggoBNvp52dcI5kBzQej|9h2rjzqmy*pOJQ3LCPb)>!UjdiaI*TI8zuJ(xiN literal 0 HcmV?d00001 diff --git a/travis/install_extra_deps.sh b/travis/install_extra_deps.sh index b888ae6..a42c30b 100755 --- a/travis/install_extra_deps.sh +++ b/travis/install_extra_deps.sh @@ -5,8 +5,9 @@ set -e orig_dir=$(pwd) sudo apt-get update -qq -sudo apt-get install -qq python3-pip +sudo apt-get install -qq python3-pip zlib1g-dev libjpeg-dev pip3 install meson +pip3 install pillow # ninja git clone git://github.com/ninja-build/ninja.git diff --git a/travis/make_release.sh b/travis/make_release.sh new file mode 100755 index 0000000..33b50ab --- /dev/null +++ b/travis/make_release.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -e + +KEYSTORE_FILE=${TRAVIS_BUILD_DIR}/travis/test_ks.ks +GOOGLE_API_KEY=${TRAVIS_BUILD_DIR}/travis/googleplay_android_developer-5a411156212c.json +SSH_KEY=${TRAVIS_BUILD_DIR}/travis/travisci_builder_id_key + +cd ${HOME} + + +BASE_DIR="BUILD_${PLATFORM}" + +mkdir -p ${HOME}/APKS + +scp -i ${SSH_KEY} nightlybot@download.kiwix.org:~/apks/${CUSTOM_APP}_${BASE_VERSION}/* ${HOME}/APKS + +ssh -i ${SSH_KEY} nightlybot@download.kiwix.org "rm -rf ~/apks/${CUSTOM_APP}_${BASE_VERSION}" + +${TRAVIS_BUILD_DIR}/build_custom_app.py \ + --step publish \ + --custom-app ${CUSTOM_APP} \ + --package-name ${PACKAGE_NAME} \ + --google-api-key ${GOOGLE_API_KEY} \ + --version ${VERSION} \ + --zim-url ${ZIM_URL} \ + --apks-dir ${HOME}/APKS \ + --content-version-code ${CONTENT_VERSION_CODE} diff --git a/travis/test_ks.ks.enc b/travis/test_ks.ks.enc new file mode 100644 index 0000000000000000000000000000000000000000..e86609b1d47f79617fade07e8d2cbe993d1039b5 GIT binary patch literal 2256 zcmV;>2ru_jVQh3|WM5wk4eu{f*c%`2)&RFHf4)x3**_4R@(XvxhbOavgD9e&XX0I| z;ppQ^C%u8v_fqnZ$r%BU2>mt*`j&#M+jwpU@st zd0mngWI>{D-<4g!wU3PhaA6kH&$v}_ORi5VGoG`tD9-VoOMjGFst+JSykjL+xgP0j z*=2vZ3ZBqpMwdVPeIT(vuxIQ%GQ#>ZE0oly!AvGs)y~M}v^zkHd4U>lVlXRh>mj=_ zYbN#(%~o3v1cGm>+DZDf|1Jq0h4Z{o2qMoYMX2e?50RXq9)fnbFW)0>*vv!Kw>|~@ zquF4o=l;;=kP4pnP$C(Hsr5o6J{fx%y1t;~N}p*H^fuhdqucIFFS4E)U>! zL<_IB-lFAey>fK+H0HnS4){n1rmwH8Aj@CWm!ZH5OHTS;+96!H!n|%a`v8aFX&5}( zO5j=Jx517;(iAlyLX*-KYpjw#a&m{ah}^CWlvZi=CYk9`iNp8K^^HW8;DzZ}BNKV${QPQf7CN=~BekK6e=uldT{=Nnx34hUPowm5jR|*r)Jv zy<-as0`F)^Dj&-B%k?424Cl+h21%jfEoxMzRD~C% zXjGUgV)oQbP5;;RWIJ~oJM!5-bcT>fqLX`=f@F0>BU5{gkB$Co6SY-1U^j{RghW9N z(a@7xB))up6CpDyg2fE6gpn<2?PJ>UZ34(}EBf@b;s4XxF2`}BaJY$Pmde3;@VfZU%}jJ+n=R6x&DnPL8F2mpND%`&e&qb-mzm2_P;pny$rjA^EzM#JP!V zKI2|r?igJeAY4hU5bpWHao`0*?hYU@Zcfu5+WdMVL(%>?b1o8YZX{@h)(~s=7*y`# zJ-c3T*ux=vq&G2LeYimc`^atv?&6rA+<%P%J5SMZnA3R&&TzgOmz0c!9WE_X&7Z^Y z>jMJUo+XDK$QuDrp%%Su`!lhTw=WhA6Fn`_VUs!UJumTXwuEHT5bXgkQWH%nTXa?s z;+FP$iR9xGgr%6-@q3L_zkHz=!yR?^I_tI|o&~|o?D!7>TFeMZ_@+)7UUpI>L)`G6 zCq%BTU6VeL-r)~(;hLOsj2#{~HsSc)u@jNCecZP$t0i4!6uPAr@5$vwuKS}H3+gnm{ zi3u__Cb6W_T@&wKM6D_nC`(!QRc>+@)_t1|KV^x2Sx@kkD$wN}hC63hF#K<=PgWVB zl3YDp!1S{P@xCvLssfgGe1R_AYB#Wx`6LFKlk>~m{{XxIKSZz$%=E#SwTF|9=NkzL znVCVsgXYOyq#6y6qjVGMZD#wAfo^MkN@miNoPf|k(Q3sj<@h7%XylAhPf$ounKm_U z6?ABa6+~KkE>e8pt*(vo7^=)m)`&DV3fw&6S56t_GXw1>XN&eC%nvy+o!_$_*ZU`( zSn#%U1sM-^6v9Nf=mxz`#juM@K2iQEth$JED|O8XW|QkcUO@|1`GqUO%!Q)arMV)4NZ0~yNxYVB!ecRb2I zN{2mewL)sMG^KZSHZ?y`By_D)3Um+_6mBbb=Ev@$Kb(>s*teOJ^(-<3?=RrcCTkW| zEwD>EphQByL0g(sJj1(cNAFw84hc-zJ#7(AgP@T|19j4zotzDS17A?gONG!OX7O9) zF#U?nP`d7gsmgii<$Fz?GLZVmS%l5JE|`nmE^apFDzM_c)d}4F3P$$VyTCn*Lfcck z6+jSF7ZERz%2jE2{p6Lz9`hS6C@VgqgTYg3a?ceLXpBg;_WOV5uM@oLdcq(OI@X{fVrSN5YXibc69``f+_StZq zd)k%mF!+bzu;64JPL664X)lo<@vn2aaf<(8T*MhY z_OOZ_6~34T^p37X(%~trrGe|~?BCh}!;Hv2QW`)Xc1l)?I2Z%GJsk`G=*z z0BhAgp0C6Pq&4g!a`5$=o%!;TzH9AeOmW(?*5W&jaHt~oJv{Qaz e!rTc3wvoOoT4aKW4RamuO(lTuZt-VbZzanTT}}l6 literal 0 HcmV?d00001 diff --git a/utils.py b/utils.py index a371dd3..f4ace50 100644 --- a/utils.py +++ b/utils.py @@ -3,13 +3,17 @@ import hashlib import tarfile, zipfile import tempfile import shutil -import os, stat +import os, stat, sys +import urllib +import ssl from collections import namedtuple, defaultdict pj = os.path.join g_print_progress = True +REMOTE_PREFIX = 'http://download.kiwix.org/dev/' + def setup_print_progress(print_progress): global g_print_progress @@ -71,6 +75,48 @@ def copy_tree(src, dst, post_copy_function=None): if post_copy_function is not None: post_copy_function(dstfile) + +def download_remote(what, where, check_certificate=True): + file_path = pj(where, what.name) + file_url = what.url or (REMOTE_PREFIX + what.name) + if os.path.exists(file_path): + if what.sha256 == get_sha256(file_path): + raise SkipCommand() + os.remove(file_path) + + if not check_certificate: + context = ssl.create_default_context() + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + else: + context = None + batch_size = 1024 * 8 + extra_args = {'context':context} if sys.version_info >= (3, 4, 3) else {} + progress_chars = "/-\|" + with urllib.request.urlopen(file_url, **extra_args) as resource, open(file_path, 'wb') as file: + tsize = resource.getheader('Content-Length', None) + if tsize is not None: + tsize = int(tsize) + current = 0 + while True: + batch = resource.read(batch_size) + if not batch: + break + if tsize: + current += batch_size + print_progress("{:.2%}".format(current/tsize)) + else: + print_progress(progress_chars[current]) + current = (current+1)%4 + file.write(batch) + + if not what.sha256: + print('Sha256 for {} not set, do no verify download'.format(what.name)) + elif what.sha256 != get_sha256(file_path): + os.remove(file_path) + raise StopBuild() + + class SkipCommand(Exception): pass