Merge pull request #47 from kiwix/custom_app

Add script to build customapp

build_custom_app.py

@@ -0,0 +1,415 @@
+#!/usr/bin/env python3
+
+import \
+    argparse, \
+    datetime, \
+    glob, \
+    json, \
+    os, \
+    ssl, \
+    sys, \
+    tempfile, \
+    time, \
+    urllib
+from uuid import uuid4
+from contextlib import contextmanager
+from urllib.parse import urlparse
+
+from utils import (
+    Remotefile,
+    download_remote
+)
+
+import requests
+import httplib2
+from apiclient.discovery import build
+from apiclient.errors import HttpError
+from oauth2client import client
+from oauth2client.service_account import ServiceAccountCredentials
+
+tmpl_request_url = "https://api.travis-ci.org/repo/{organisation}%2F{repository}/{endpoint}"
+tmpl_message = """Build of custom app {app} with zim file {zim}.
+
+UUID:#{uuid}#"""
+
+description = """Launch a custom application build.
+This command will launch a custom application build on Travis-CI.
+Travis-CI jobs will compile the application and upload the build apks on
+google play, using tha 'alpha' track of the application.
+
+You will need to have a valid TRAVIS_TOKEN associated to your personnal account
+and the kiwix-build repository on travis.
+Use the 'travis' command line tool (https://github.com/travis-ci/travis.rb)
+to generate a token.
+"""
+
+def parse_args():
+    parser = argparse.ArgumentParser(description=description,
+        formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument('--custom-app')
+    parser.add_argument('--travis-token')
+
+    advance = parser.add_argument_group('advance', "Some advanced options.")
+    advance.add_argument('--extra-code', type=int, default=0)
+    advance.add_argument('--check-certificate', default=True)
+    advance.add_argument('--zim-url', default=None)
+    advance.add_argument('--no-android-upload', action='store_false', dest='android_upload')
+
+    # Hidden options
+    parser.add_argument('--step', default='launch', choices=['launch', 'publish'], help=argparse.SUPPRESS)
+    parser.add_argument('--apks-dir', help=argparse.SUPPRESS)
+    parser.add_argument('--version', default="0", help=argparse.SUPPRESS)
+    parser.add_argument('--content-version-code', type=int)
+    parser.add_argument('--package-name', default=None, help=argparse.SUPPRESS)
+    parser.add_argument('--google-api-key', help=argparse.SUPPRESS)
+
+    options = parser.parse_args()
+
+    if not options.package_name or not options.zim_url:
+        if not options.package_name:
+            print("Try to get package name from info.json file")
+        if not options.zim_url:
+            print("Try to get zim url from info.json file")
+        request_url = ('https://raw.githubusercontent.com/kiwix/kiwix-android-custom/master/{}/info.json'
+                      .format(options.custom_app))
+        json_request = requests.get(request_url)
+        if json_request.status_code != 200:
+            print("Error while getting json file.")
+            print("Reason is '{}'".format(json_request.reason))
+            sys.exit(-1)
+        json_data = json.loads(json_request.text)
+        if not options.package_name:
+            print("Found package_name '{}'".format(json_data['package']))
+            options.package_name = json_data['package']
+        if not options.zim_url:
+            print("Found zim_url '{}'".format(json_data['zim_url']))
+            options.zim_url = json_data['zim_url']
+
+    options.base_version = "{}{}".format(
+        datetime.date.today().strftime('%y%j'),
+        options.extra_code)
+
+    return options
+
+
+def download_zim_file(zim_url, dest_dir=None):
+    if dest_dir is None:
+        dest_dir = os.getcwd()
+    out_filename = urlparse(zim_url).path
+    out_filename = os.path.basename(out_filename)
+    zim_file = Remotefile(out_filename, '', zim_url)
+    download_remote(zim_file, dest_dir)
+    return os.path.join(dest_dir, out_filename)
+
+
+def get_zim_size(zim_url, check_certificate=True):
+    print("Try to get zim size")
+    if not check_certificate:
+        context = ssl.create_default_context()
+        context.check_hostname = False
+        context.verify_mode = ssl.CERT_NONE
+    else:
+        context = None
+    extra_args = {'context':context} if sys.version_info >= (3, 4, 3) else {}
+    with urllib.request.urlopen(zim_url, **extra_args) as resource:
+        size = resource.getheader('Content-Length', None)
+    if size is not None:
+        size = int(size)
+        print("Zim size is {}".format(size))
+        return size
+    else:
+        print("No 'Content-Length' header in http answer from the server.\n"
+              "We need to download the zim file to get its size.")
+        zim_path = download_zim_file(zim_url, tempfile.gettempdir())
+        size = os.path.getsize(zim_path)
+        print("Zim size is {}".format(size))
+        return size
+
+
+def do_launch(options):
+    zim_size = get_zim_size(options.zim_url, options.check_certificate)
+    travis_launch_build('kiwix', 'kiwix-build', options, zim_size)
+    print("Travis build has been launch.")
+
+
+def do_publish(options):
+    zim_path = download_zim_file(options.zim_url)
+    googleService = Google(options)
+    with googleService.new_request():
+        versionCodes = []
+        for apk in glob.iglob(options.apks_dir+"/*.apk"):
+            result = googleService.upload_apk(apk)
+            versionCodes.append(result['versionCode'])
+        googleService.upload_expansionfile(
+            zim_path, options.content_version_code, versionCodes)
+        googleService.publish_release(options, versionCodes)
+
+
+def travis_launch_build(organisation, repository, options, zim_size):
+    request_url = tmpl_request_url.format(
+        organisation=organisation,
+        repository=repository,
+        endpoint='requests')
+    headers = {
+        "Travis-API-Version": "3",
+        "Content-Type": "application/json",
+        "Authorization": "token {}".format(options.travis_token),
+        "User-Agent": "kiwix-build"
+    }
+    uuid = uuid4()
+    envs = []
+    for platform_index, platform in enumerate(['arm', 'arm64', 'x86', 'x86_64', 'mips', 'mips64']):
+        d = {
+            'PLATFORM': "android_{}".format(platform),
+            'VERSION_CODE': gen_version_code(platform_index, options.base_version)
+        }
+        envs.append(d)
+
+    global_env = [
+        { 'CUSTOM_APP': options.custom_app},
+        { 'ZIM_FILE_SIZE': zim_size},
+        { 'PACKAGE_NAME': options.package_name},
+        { 'ZIM_URL': options.zim_url},
+        { 'EXTRA_CODE': options.extra_code},
+        { 'CONTENT_VERSION_CODE': gen_version_code(0, options.base_version)},
+        { 'VERSION': options.version},
+        # google_key
+        { 'secure': ('VAgKBMx0KEIyJlSnpM4YrHKLALIbaibkhlsgiv19ITa6dODoEIqeYHz'
+                     'wFTiL3mRHU6HwtXtdNb/JeMle9NfHJVFSV56ZgFzX7ev9zr0YG0qZQv'
+                     'tl8vHQlFPBErARder/L2tblOTM194/TiJk/q89a0XWDanKswXExwjcW'
+                     'Z0tnDYQXTHSAKEt+xW8hjbnhqqB/v16lX6dUjZI+sVlsw+qAM4VT/qf'
+                     'FCyDO5eJCzWIEL2LDUWI7jKSETNih5hl5fMMvCCNRPnkgGnytw5kF/t'
+                     'Lw8YAbLRxkGsO4FCx5mB7HF5pNHyWOCCalMTKheyg/qUV/VcXW9Unlr'
+                     'puMu0+d3hpLZESplS/NkvDxSrx16ank7EORS8OxLOufu56TW2hDuBzz'
+                     'w1CBAj1p6s+Z6Kc4RMYYdxRgR1TjXg/ZVUn3T69d9igdS/5lAPFx2Ww'
+                     '8x82FWCLSaiXymxXRNsNcKx5ifuvtv307r4yh31QjlKFYwadOCaCHHZ'
+                     'zGE1mXcOu3j6W9WIaZfYRTpxmOrcfDIHxZSdLf11hOSZEUUFpj9hQlV'
+                     'Op0RHkDEJUMNs6vkXUhZq9yPuqgrcb6GaN+UhOT8iHlijKmlG8NJEPk'
+                     'Hp8RnL1hsr44N57ZzLqmSUZtvC83u/5e+YUb7beUDGsMyJV/fcMiGMM'
+                     'LVtRnuPCFyNVNQUf2CphtG0=') },
+        # pass
+        { 'secure': ('AtbgKUukES2uJPpEWNEDHLg0WcghLlCGL171Ah3+4CckBI8y1Fn+VpH'
+                     'U2vEXzsV8tKoxX1IyB2tFivzuyo6CQXHSuWGJYQexwkBeGCgOfzKJLj'
+                     'MAy75ATYA6JnFrikV+UcqdEz/9Dow3J1K7Slp3jpsQhERHbNeqkr4I+'
+                     'XCL1LLnpewfOZo9OIEu93p6b6YlqvIPXJHyQe5xnMd8jFWg3/uIYqFn'
+                     'XPvigeZqC2lhNp48mj4JdwwF2tmiArgyXOmgxiuHJNVVI7okbhc7kmI'
+                     'Y3MmCSFgG0XPUEBU3Kdr4o/2hy8DDP6Gff+rUZW8nPI/2UWXRLWtOxv'
+                     'XGGRyjHHTxGWzI4JyZbli9dls5M32MMjsXVKtciSFVwsMM8qn7wFnRi'
+                     'q248a1Sg5fDNX/WYowmsHlWjffHZ7+UqUqJxAKtZ0vpQL+4SPIALPnK'
+                     'V6j6CoorQp+VhMF01EFlZ0c8bkNmk4YW7R7RyNLIcaHKfd1ud8QF9PD'
+                     'AnQ7Jr1GRBxzkjHvHfFrE14WPUu+FjVvDO7UPVMNQX7RS1IVACpKSRu'
+                     '7N8KnIK3vSnLpn5GXKsbx0JB2vtyoTaFZvC9c3qyAw1nlpn7Lp3sPs3'
+                     'bgIBU4tWOzg5g46eHbc4ad5nyB9Soz715lbMdECvKs2HHJUG3tubLKj'
+                     'L0S/LRGRQ+IDgC7xrjQj8aA=') },
+    ]
+
+    env = {
+        'matrix': envs,
+        'global': global_env
+    }
+
+    data = {
+        'request': {
+            'message' : tmpl_message.format(app=options.custom_app, zim=options.zim_url, uuid=uuid),
+            'branch'  : "custom_app",
+            'config'  : {
+                'before_install' : [
+                    ( 'pip3 install pyOpenSSl google-api-python-client'
+                      ' httplib2 apiclient requests'),
+                    ( 'openssl aes-256-cbc -k $google_key'
+                      ' -in travis/googleplay_android_developer-5a411156212c.json.enc'
+                      ' -out travis/googleplay_android_developer-5a411156212c.json'
+                      ' -d'),
+                    ( 'openssl aes-256-cbc -k $google_key'
+                      ' -in travis/kiwix-android.keystore.enc'
+                      ' -out travis/kiwix-android.keystore -d'),
+                    ( 'openssl aes-256-cbc -K $encrypted_eba2f7543984_key'
+                      ' -iv $encrypted_eba2f7543984_iv'
+                      ' -in travis/travisci_builder_id_key.enc'
+                      ' -out travis/travisci_builder_id_key -d'),
+                    'chmod 600 travis/travisci_builder_id_key'
+                ],
+                'env' : env,
+                'script' : 'travis_wait 30 travis/compile_custom_app.sh',
+                'deploy' : {
+                     'provider': 'script',
+                     'skip_cleanup': True,
+                     'script': 'travis/deploy_apk.sh',
+                     'on': {
+                         'branch': 'custom_app'
+                     }
+                }
+            }
+        }
+    }
+
+    if options.android_upload:
+        data['request']['config']['jobs'] = {
+            'include': [
+                {
+                    'stage' : 'make_release',
+                    'install': 'pip3 install -r requirements_build_custom_app.txt',
+                    'script': True,
+                    'env': global_env,
+                    'deploy' : {
+                        'provider': 'script',
+                        'skip_cleanup': True,
+                        'script': 'travis/make_release.sh',
+                        'on': {
+                            'branch': 'custom_app'
+                        }
+                    }
+                }
+            ]
+        }
+        global_env.append({
+            'DEPLOY_DIR' : '/home/nightlybot/apks/{}_{}'.format(
+                options.custom_app, options.base_version)
+        })
+    else:
+        global_env.append({
+            'DEPLOY_DIR' : '/var/www/tmp.kiwix.org/custom_apps/{}_{}'.format(
+                options.custom_app, options.base_version)
+        })
+
+
+    r = requests.post(request_url, headers=headers, json=data)
+    if r.status_code != 202:
+        print("Error while requesting build:")
+        print(r.reason)
+        print("Have you forget to give the travis token ?")
+        sys.exit(-1)
+    else:
+        request_id = r.json()['request']['id']
+        print("Request {} has been schedule.".format(request_id))
+        request_left = 10
+        found = False
+        request_url = tmpl_request_url.format(
+            organisation=organisation,
+            repository=repository,
+            endpoint='builds')
+        while request_left:
+            time.sleep(1)
+            print("Try to get associated build.")
+            r = requests.get(request_url, headers=headers)
+            json_data = json.loads(r.text)
+            for build in json_data['builds']:
+                if build['event_type'] != 'api':
+                    continue
+                message = build['commit']['message']
+                if str(uuid) in message:
+                    found = True
+                    break
+            if found:
+                break
+            print("Cannot find build. Wait 1 second and try again")
+            print("{} tries left".format(request_left))
+            request_left -= 1
+        if found:
+            print("Associated build found:  {}.".format(build['number']))
+            print("https://travis-ci.org/kiwix/kiwix-build/builds/{}".format(build['id']))
+        else:
+            print("Request has been accepted by travis-ci but I cannot found "
+                  "the associated build. Have a look here "
+                  "https://travis-ci.org/kiwix/kiwix-build/builds"
+                  "if you found it.")
+        if not options.android_upload:
+            print(("Automatic upload to android play store has been deactivated.\n"
+                   "You will find the apks at this address once they have been compiled :"
+                   " http://tmp.kiwix.org/custom_apps/{}_{}").format(
+                       options.custom_app, options.base_version))
+
+
+ERROR_MSG_EDIT_CHANGE = "A change was made to the application outside of this Edit, please create a new edit."
+
+class Google:
+    def __init__(self, options):
+        scope = 'https://www.googleapis.com/auth/androidpublisher'
+        key = options.google_api_key
+        credentials = ServiceAccountCredentials.from_json_keyfile_name(
+            key,
+            scopes=[scope])
+
+        http = httplib2.Http()
+        http = credentials.authorize(http)
+
+        self.service = build('androidpublisher', 'v2', http=http)
+        self.packageName = options.package_name
+        self.edit_id = None
+
+    @contextmanager
+    def new_request(self):
+        edit_request = self.service.edits().insert(
+            packageName=self.packageName,
+            body={})
+        result = edit_request.execute()
+        print("create", result)
+        self.edit_id = result['id']
+
+        yield
+
+        commit_request = self.service.edits().commit(
+            editId=self.edit_id,
+            packageName=self.packageName)
+        result = commit_request.execute()
+        print("commit", result)
+
+        self.edit_id = None
+
+    def make_request(self, section, method, **kwargs):
+        request_content = self._build_request_content(kwargs)
+        _section = getattr(self._edits, section)()
+        _method = getattr(_section, method)
+        print(">", request_content)
+        request = _method(**request_content)
+        result = request.execute()
+        print("<", result)
+        return result
+
+    def _build_request_content(self, kwargs):
+        d = kwargs.copy()
+        d['editId'] = self.edit_id
+        d['packageName'] = self.packageName
+        return d
+
+    @property
+    def _edits(self):
+        return self.service.edits()
+
+    def upload_expansionfile(self, comp_file, contentVersionCode, versionCodes):
+        versionCodes = [int(v) for v in versionCodes]
+        self.make_request('expansionfiles', 'upload',
+            expansionFileType='main',
+            apkVersionCode=contentVersionCode,
+            media_body=comp_file,
+            media_mime_type='application/octet-stream')
+        for versionCode in versionCodes:
+            if versionCode == contentVersionCode:
+                continue
+            self.make_request('expansionfiles', 'update',
+                expansionFileType='main',
+                apkVersionCode=versionCode,
+                body={'referencesVersion': contentVersionCode}
+            )
+
+    def upload_apk(self, apk_file):
+        return self.make_request('apks', 'upload',
+            media_body=apk_file)
+
+    def publish_release(self, options, versionCodes):
+        return self.make_request('tracks', 'update',
+            track="alpha",
+            body={'versionCodes': versionCodes})
+
+
+def gen_version_code(platform_index, base_version):
+    str_version = "{platform}{base_version}".format(
+        platform=platform_index,
+        base_version=base_version
+    )
+    return int(str_version)
+
+if __name__ == "__main__":
+    options = parse_args()
+    func = globals()['do_{}'.format(options.step)]
+    func(options)

dependencies.py

@@ -345,6 +345,12 @@ class KiwixAndroid(Dependency):
         git_dir = "kiwix-android"
 
     class Builder(GradleBuilder):
+        def build(self):
+            if self.buildEnv.options.targets == 'kiwix-android-custom':
+                print("SKIP")
+            else:
+                super().build()
+
         def _configure(self, context):
             if not os.path.exists(self.build_path):
                 shutil.copytree(self.source_path, self.build_path)
@@ -375,7 +381,15 @@ class KiwixCustomApp(Dependency):
 
         @property
         def gradle_option(self):
-            return "-i -P customDir={}".format(pj(self.build_path, 'custom'))
+            template = ("-i -P customDir={customDir}"
+                        " -P zim_file_size={zim_size}"
+                        " -P version_code={version_code}"
+                        " -P content_version_code={content_version_code}")
+            return template.format(
+                customDir=pj(self.build_path, 'custom'),
+                zim_size=self._get_zim_size(),
+                version_code=os.environ['VERSION_CODE'],
+                content_version_code=os.environ['CONTENT_VERSION_CODE'])
 
         @property
         def build_path(self):
@@ -385,6 +399,15 @@ class KiwixCustomApp(Dependency):
         def custom_build_path(self):
             return pj(self.build_path, 'custom', self.target.custom_name)
 
+        def _get_zim_size(self):
+            try:
+                zim_size = self.buildEnv.options.zim_file_size
+            except AttributeError:
+                with open(pj(self.source_path, self.target.custom_name, 'info.json')) as f:
+                    app_info = json.load(f)
+                zim_size = os.path.getsize(pj(self.custom_build_path, app_info['zim_file']))
+            return zim_size
+
         def build(self):
             self.command('configure', self._configure)
             self.command('download_zim', self._download_zim)

dependency_utils.py

@@ -135,12 +135,15 @@ class GitClone(Source):
         context.force_native_build = True
         if os.path.exists(self.git_path):
             raise SkipCommand()
-        command = "git clone --depth=1 {} {}".format(self.git_remote, self.git_dir)
+        command = "git clone --depth=1 --branch {} {} {}".format(
+            self.git_ref, self.git_remote, self.git_dir)
         self.buildEnv.run_command(command, self.buildEnv.source_dir, context)
 
     def _git_update(self, context):
         context.force_native_build = True
-        self.buildEnv.run_command("git fetch", self.git_path, context)
+        command = "git fetch origin {}".format(
+            self.git_ref)
+        self.buildEnv.run_command(command, self.git_path, context)
         self.buildEnv.run_command("git checkout "+self.git_ref, self.git_path, context)
 
     def prepare(self):

kiwix-build.py

@@ -17,6 +17,7 @@ from utils import (
     get_sha256,
     print_progress,
     setup_print_progress,
+    download_remote,
     StopBuild,
     SkipCommand,
     Defaultdict,
@@ -432,44 +433,7 @@ class BuildEnv:
 
     def download(self, what, where=None):
         where = where or self.archive_dir
-        file_path = pj(where, what.name)
+        download_remote(what, where, not self.options.no_cert_check)
-        file_url = what.url or (REMOTE_PREFIX + what.name)
-        if os.path.exists(file_path):
-            if what.sha256 == get_sha256(file_path):
-                raise SkipCommand()
-            os.remove(file_path)
-
-        if options.no_cert_check == True:
-            context = ssl.create_default_context()
-            context.check_hostname = False
-            context.verify_mode = ssl.CERT_NONE
-        else:
-            context = None
-        batch_size = 1024 * 8
-        extra_args = {'context':context} if sys.version_info >= (3, 4, 3) else {}
-        progress_chars = "/-\|"
-        with urllib.request.urlopen(file_url, **extra_args) as resource, open(file_path, 'wb') as file:
-            tsize = resource.getheader('Content-Length', None)
-            if tsize is not None:
-                tsize = int(tsize)
-            current = 0
-            while True:
-                batch = resource.read(batch_size)
-                if not batch:
-                    break
-                if tsize:
-                    current += batch_size
-                    print_progress("{:.2%}".format(current/tsize))
-                else:
-                    print_progress(progress_chars[current])
-                    current = (current+1)%4
-                file.write(batch)
-
-        if not what.sha256:
-            print('Sha256 for {} not set, do no verify download'.format(what.name))
-        elif what.sha256 != get_sha256(file_path):
-            os.remove(file_path)
-            raise StopBuild()
 
     def install_packages(self):
         autoskip_file = pj(self.build_dir, ".install_packages_ok")
@@ -945,12 +909,21 @@ def parse_args():
                           help="The custom android app to build")
     subgroup.add_argument('--zim-file-url',
                           help="The url of the zim file to download")
+    subgroup.add_argument('--zim-file-size',
+                          help="The size of the zim file.")
     options = parser.parse_args()
 
     if options.targets == 'kiwix-android-custom':
-        if not options.android_custom_app or not options.zim_file_url:
+        err = False
-            print("You need to specify ANDROID_CUSTOM_APP and ZIM_FILE_URL if "
+        if not options.android_custom_app:
+            print("You need to specify ANDROID_CUSTOM_APP if you "
                   "want to build a kiwix-android-custom target")
+            err = True
+        if not options.zim_file_url and not options.zim_file_size:
+            print("You need to specify ZIM_FILE_SIZE or ZIM_FILE_URL if you "
+                  "want to build a kiwix-android-custom target")
+            err = True
+        if err:
             sys.exit(1)
     return options
 

requirements_build_custom_app.txt

@@ -0,0 +1,3 @@
+requests==2.10.0
+apiclient==1.0.3
+

travis/compile_custom_app.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -e
+
+cd ${HOME}
+
+${TRAVIS_BUILD_DIR}/kiwix-build.py \
+  --target-platform $PLATFORM \
+  --hide-progress \
+  --android-custom-app $CUSTOM_APP \
+  --zim-file-size $ZIM_FILE_SIZE \
+  kiwix-android-custom
+

travis/deploy_apk.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+set -e
+
+KEYSTORE_FILE=${TRAVIS_BUILD_DIR}/travis/kiwix-android.keystore
+GOOGLE_API_KEY=${TRAVIS_BUILD_DIR}/travis/googleplay_android_developer-5a411156212c.json
+SSH_KEY=${TRAVIS_BUILD_DIR}/travis/travisci_builder_id_key
+
+cd ${HOME}
+
+# Sign apk file
+
+BASE_DIR="BUILD_${PLATFORM}"
+INPUT_APK_FILE=${BASE_DIR}/kiwix-android-custom_${CUSTOM_APP}/app/build/outputs/apk/app-${CUSTOM_APP}-release-unsigned.apk
+SIGNED_APK=${BASE_DIR}/app-${CUSTOM_APP}_${VERSION_CODE}-release-signed.apk
+
+TOOLCHAINS/android-sdk-r25.2.3/build-tools/25.0.2/apksigner sign \
+  --ks ${KEYSTORE_FILE} \
+  --ks-pass env:KEYSTORE_PASS \
+  --out ${SIGNED_APK} \
+  ${INPUT_APK_FILE}
+
+ssh -i ${SSH_KEY} nightlybot@download.kiwix.org "mkdir -p ${DEPLOY_DIR}"
+
+scp -i ${SSH_KEY} \
+  ${SIGNED_APK} \
+  nightlybot@download.kiwix.org:${DEPLOY_DIR}

travis/install_extra_deps.sh

@@ -5,8 +5,9 @@ set -e
 orig_dir=$(pwd)
 
 sudo apt-get update -qq
-sudo apt-get install -qq python3-pip
+sudo apt-get install -qq python3-pip zlib1g-dev libjpeg-dev
 pip3 install meson
+pip3 install pillow
 
 # ninja
 git clone git://github.com/ninja-build/ninja.git

travis/make_release.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+set -e
+
+KEYSTORE_FILE=${TRAVIS_BUILD_DIR}/travis/test_ks.ks
+GOOGLE_API_KEY=${TRAVIS_BUILD_DIR}/travis/googleplay_android_developer-5a411156212c.json
+SSH_KEY=${TRAVIS_BUILD_DIR}/travis/travisci_builder_id_key
+
+cd ${HOME}
+
+
+BASE_DIR="BUILD_${PLATFORM}"
+
+mkdir -p ${HOME}/APKS
+
+scp -i ${SSH_KEY} nightlybot@download.kiwix.org:${DEPLOY_DIR}/* ${HOME}/APKS
+
+ssh -i ${SSH_KEY} nightlybot@download.kiwix.org "rm -rf ${DEPLOY_DIR}"
+
+${TRAVIS_BUILD_DIR}/build_custom_app.py \
+  --step publish \
+  --custom-app ${CUSTOM_APP} \
+  --package-name ${PACKAGE_NAME} \
+  --google-api-key ${GOOGLE_API_KEY} \
+  --version ${VERSION} \
+  --zim-url ${ZIM_URL} \
+  --apks-dir ${HOME}/APKS \
+  --content-version-code ${CONTENT_VERSION_CODE}

utils.py

@@ -3,13 +3,17 @@ import hashlib
 import tarfile, zipfile
 import tempfile
 import shutil
-import os, stat
+import os, stat, sys
+import urllib
+import ssl
 from collections import namedtuple, defaultdict
 
 pj = os.path.join
 
 g_print_progress = True
 
+REMOTE_PREFIX = 'http://download.kiwix.org/dev/'
+
 
 def setup_print_progress(print_progress):
     global g_print_progress
@@ -71,6 +75,48 @@ def copy_tree(src, dst, post_copy_function=None):
             if post_copy_function is not None:
                 post_copy_function(dstfile)
 
+
+def download_remote(what, where, check_certificate=True):
+    file_path = pj(where, what.name)
+    file_url = what.url or (REMOTE_PREFIX + what.name)
+    if os.path.exists(file_path):
+        if what.sha256 == get_sha256(file_path):
+            raise SkipCommand()
+        os.remove(file_path)
+
+    if not check_certificate:
+        context = ssl.create_default_context()
+        context.check_hostname = False
+        context.verify_mode = ssl.CERT_NONE
+    else:
+        context = None
+    batch_size = 1024 * 8
+    extra_args = {'context':context} if sys.version_info >= (3, 4, 3) else {}
+    progress_chars = "/-\|"
+    with urllib.request.urlopen(file_url, **extra_args) as resource, open(file_path, 'wb') as file:
+        tsize = resource.getheader('Content-Length', None)
+        if tsize is not None:
+            tsize = int(tsize)
+        current = 0
+        while True:
+            batch = resource.read(batch_size)
+            if not batch:
+                break
+            if tsize:
+                current += batch_size
+                print_progress("{:.2%}".format(current/tsize))
+            else:
+                print_progress(progress_chars[current])
+                current = (current+1)%4
+            file.write(batch)
+
+    if not what.sha256:
+        print('Sha256 for {} not set, do no verify download'.format(what.name))
+    elif what.sha256 != get_sha256(file_path):
+        os.remove(file_path)
+        raise StopBuild()
+
+
 class SkipCommand(Exception):
     pass