diff --git a/build_custom_app.pl b/build_custom_app.pl deleted file mode 100755 index 18acdec..0000000 --- a/build_custom_app.pl +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/perl - -use utf8; -use strict; -use warnings; -use Getopt::Long; - -# get the params -my $zim_url; -my $custom_app; -my $keystore; -my $api_key; -my $version = "0"; -my $cmd; - -# Get console line arguments -GetOptions('zim_url=s' => \$zim_url, - 'custom_app=s' => \$custom_app, - 'keystore=s' => \$keystore, - 'api_key=s' => \$api_key, - 'version=s' => \$version - ); - -# Print usage() if necessary -if (!$zim_url || !$custom_app || !$keystore || !$api_key) { - print "usage: ./build_custom_app.pl --keystore=kiwix-android.keystore --api_key=google.json --zim_url=\"https://download.kiwix.org/zim/wikipedia_en_medicine_novid.zim\" --custom_app=wikimed [--version=1]\n"; - exit; -} - -# Clean signed ap -$cmd = "rm ./signed_apks/*apk"; `$cmd`; - -# Download ZIM file -$cmd = "wget \"$zim_url\" -O content.zim"; `$cmd`; - -# Get ZIM file size -$cmd = "stat -c %s content.zim"; -my $zim_size = `$cmd` =~ s/\n//gr ; -$ENV{ZIM_SIZE} = $zim_size; - -# Compute version code base -$cmd = "date +%y%j"; -my $version_code_base = `$cmd` =~ s/\n//gr . $version; - -# Compute content version code -my $content_version_code = $version_code_base; -$ENV{CONTENT_VERSION_CODE} = $content_version_code; - -# Compute custom app date -$cmd = "date +%Y-%m"; -my $date = `$cmd` =~ s/\n//gr; -$ENV{VERSION_NAME} = $date; - -# Compile apps -$ENV{VERSION_CODE} = $version_code_base; -$cmd = "./kiwix-build.py --target-platform android_arm --android-custom-app $custom_app --zim-file-size $zim_size kiwix-android-custom"; system $cmd; - -$ENV{VERSION_CODE} = "1" . $version_code_base; -$cmd = "./kiwix-build.py --target-platform android_arm64 --android-custom-app $custom_app --zim-file-size $zim_size kiwix-android-custom"; system $cmd; - -$ENV{VERSION_CODE} = "2" . $version_code_base; -$cmd = "./kiwix-build.py --target-platform android_x86 --android-custom-app $custom_app --zim-file-size $zim_size kiwix-android-custom"; system $cmd; - -$ENV{VERSION_CODE} = "3" . $version_code_base; -$cmd = "./kiwix-build.py --target-platform android_x86_64 --android-custom-app $custom_app --zim-file-size $zim_size kiwix-android-custom"; system $cmd; - -# Sign apps -$cmd = "./TOOLCHAINS/android-sdk-r25.2.3/build-tools/25.0.2/apksigner sign -ks \"${keystore}\" --out signed_apks/app-${version_code_base}-release-signed.apk BUILD_android_arm/kiwix-android-custom_${custom_app}/app/build/outputs/apk/${custom_app}/release/app-${custom_app}-release-unsigned.apk"; -system $cmd; - -$cmd = "./TOOLCHAINS/android-sdk-r25.2.3/build-tools/25.0.2/apksigner sign -ks \"${keystore}\" --out signed_apks/app-1${version_code_base}-release-signed.apk BUILD_android_arm64/kiwix-android-custom_${custom_app}/app/build/outputs/apk/${custom_app}/release/app-${custom_app}-release-unsigned.apk"; -system $cmd; - -$cmd = "./TOOLCHAINS/android-sdk-r25.2.3/build-tools/25.0.2/apksigner sign -ks \"${keystore}\" --out signed_apks/app-2${version_code_base}-release-signed.apk BUILD_android_x86/kiwix-android-custom_${custom_app}/app/build/outputs/apk/${custom_app}/release/app-${custom_app}-release-unsigned.apk"; -system $cmd; - -$cmd = "./TOOLCHAINS/android-sdk-r25.2.3/build-tools/25.0.2/apksigner sign -ks \"${keystore}\" --out signed_apks/app-3${version_code_base}-release-signed.apk BUILD_android_x86_64/kiwix-android-custom_${custom_app}/app/build/outputs/apk/${custom_app}/release/app-${custom_app}-release-unsigned.apk"; -system $cmd; - -# Upload - -$cmd = "./build_custom_app.py --step publish --custom-app ${custom_app} --google-api-key ${api_key} --zim-path content.zim --apks-dir signed_apks --content-version-code ${content_version_code}"; -system $cmd; - -exit; diff --git a/build_custom_app.py b/build_custom_app.py deleted file mode 100755 index 35590fd..0000000 --- a/build_custom_app.py +++ /dev/null @@ -1,430 +0,0 @@ -#!/usr/bin/env python3 - -import \ - argparse, \ - datetime, \ - glob, \ - json, \ - os, \ - ssl, \ - sys, \ - tempfile, \ - time, \ - urllib -from uuid import uuid4 -from contextlib import contextmanager -from urllib.parse import urlparse - -from utils import ( - Remotefile, - download_remote -) - -import requests -import httplib2 -from apiclient.discovery import build -from apiclient.errors import HttpError -from oauth2client import client -from oauth2client.service_account import ServiceAccountCredentials - -tmpl_request_url = "https://api.travis-ci.org/repo/{organisation}%2F{repository}/{endpoint}" -tmpl_message = """Build of custom app {app} with zim file {zim}. - -UUID:#{uuid}#""" - -description = """Launch a custom application build. -This command will launch a custom application build on Travis-CI. -Travis-CI jobs will compile the application and upload the build apks on -google play, using tha 'alpha' track of the application. - -You will need to have a valid TRAVIS_TOKEN associated to your personnal account -and the kiwix-build repository on travis. -Use the 'travis' command line tool (https://github.com/travis-ci/travis.rb) -to generate a token. -""" - -def parse_args(): - parser = argparse.ArgumentParser(description=description, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument('--custom-app') - parser.add_argument('--travis-token') - - advance = parser.add_argument_group('advance', "Some advanced options.") - advance.add_argument('--extra-code', type=int, default=0) - advance.add_argument('--version-name', default=None, - help="The version of the application (seen by user). Get from json info file by default.") - advance.add_argument('--check-certificate', default=True) - advance.add_argument('--zim-url', default=None, help="Get from json info file by default.") - advance.add_argument('--no-android-upload', action='store_false', dest='android_upload') - - # Hidden options - parser.add_argument('--step', default='launch', choices=['launch', 'publish'], help=argparse.SUPPRESS) - parser.add_argument('--apks-dir', help=argparse.SUPPRESS) - parser.add_argument('--zim-path', default=None, help=argparse.SUPPRESS) - parser.add_argument('--content-version-code', type=int, help=argparse.SUPPRESS) - parser.add_argument('--package-name', default=None, help=argparse.SUPPRESS) - parser.add_argument('--google-api-key', help=argparse.SUPPRESS) - - options = parser.parse_args() - - if (not options.package_name - or not (options.zim_url or options.zim_path) - or not options.version_name): - if not options.package_name: - print("Try to get package name from info.json file") - if not options.zim_url: - print("Try to get zim url from info.json file") - if not options.version_name: - print("Try to get version_name form info.json file") - request_url = ('https://raw.githubusercontent.com/kiwix/kiwix-android-custom/master/{}/info.json' - .format(options.custom_app)) - json_request = requests.get(request_url) - if json_request.status_code != 200: - print("Error while getting json file.") - print("Reason is '{}'".format(json_request.reason)) - sys.exit(-1) - json_data = json.loads(json_request.text) - if not options.package_name: - print("Found package_name '{}'".format(json_data['package'])) - options.package_name = json_data['package'] - if not options.zim_url: - print("Found zim_url '{}'".format(json_data['zim_url'])) - options.zim_url = json_data['zim_url'] - if not options.version_name: - print("Found version_name '{}'".format(json_data['version_name'])) - options.version_name = json_data['version_name'] - - options.base_version = "{}{}".format( - datetime.date.today().strftime('%y%j'), - options.extra_code) - - return options - - -def download_zim_file(zim_url, dest_dir=None): - if dest_dir is None: - dest_dir = os.getcwd() - out_filename = urlparse(zim_url).path - out_filename = os.path.basename(out_filename) - zim_file = Remotefile(out_filename, '', zim_url) - download_remote(zim_file, dest_dir) - return os.path.join(dest_dir, out_filename) - - -def get_zim_size(*, zim_url=None, zim_path=None, check_certificate=True): - print("Try to get zim size") - if not zim_path: - if not check_certificate: - context = ssl.create_default_context() - context.check_hostname = False - context.verify_mode = ssl.CERT_NONE - else: - context = None - extra_args = {'context':context} if sys.version_info >= (3, 4, 3) else {} - with urllib.request.urlopen(zim_url, **extra_args) as resource: - size = resource.getheader('Content-Length', None) - if size is not None: - size = int(size) - print("Zim size is {}".format(size)) - return size - else: - print("No 'Content-Length' header in http answer from the server.\n" - "We need to download the zim file to get its size.") - zim_path = download_zim_file(zim_url, tempfile.gettempdir()) - - size = os.path.getsize(zim_path) - print("Zim size is {}".format(size)) - return size - - -def do_launch(options): - if options.zim_path: - zim_size = get_zim_size(zim_path=options.zim_path) - else: - zim_size = get_zim_size(zim_url=options.zim_url, - check_certificate=options.check_certificate) - travis_launch_build('kiwix', 'kiwix-build', options, zim_size) - print("Travis build has been launch.") - - -def do_publish(options): - zim_path = options.zim_path or download_zim_file(options.zim_url) - googleService = Google(options) - with googleService.new_request(): - versionCodes = [] - for apk in glob.iglob(options.apks_dir+"/*.apk"): - result = googleService.upload_apk(apk) - versionCodes.append(result['versionCode']) - googleService.upload_expansionfile( - zim_path, options.content_version_code, versionCodes) - googleService.publish_release(options, versionCodes) - - -def travis_launch_build(organisation, repository, options, zim_size): - request_url = tmpl_request_url.format( - organisation=organisation, - repository=repository, - endpoint='requests') - headers = { - "Travis-API-Version": "3", - "Content-Type": "application/json", - "Authorization": "token {}".format(options.travis_token), - "User-Agent": "kiwix-build" - } - uuid = uuid4() - envs = [] - for platform_index, platform in enumerate(['arm', 'arm64', 'x86', 'x86_64']): - d = { - 'PLATFORM': "android_{}".format(platform), - 'VERSION_CODE': gen_version_code(platform_index, options.base_version) - } - envs.append(d) - - global_env = [ - { 'CUSTOM_APP': options.custom_app}, - { 'ZIM_FILE_SIZE': zim_size}, - { 'PACKAGE_NAME': options.package_name}, - { 'ZIM_URL': options.zim_url}, - { 'EXTRA_CODE': options.extra_code}, - { 'CONTENT_VERSION_CODE': gen_version_code(0, options.base_version)}, - { 'VERSION_NAME': options.version_name}, - # google_key - { 'secure': ('VAgKBMx0KEIyJlSnpM4YrHKLALIbaibkhlsgiv19ITa6dODoEIqeYHz' - 'wFTiL3mRHU6HwtXtdNb/JeMle9NfHJVFSV56ZgFzX7ev9zr0YG0qZQv' - 'tl8vHQlFPBErARder/L2tblOTM194/TiJk/q89a0XWDanKswXExwjcW' - 'Z0tnDYQXTHSAKEt+xW8hjbnhqqB/v16lX6dUjZI+sVlsw+qAM4VT/qf' - 'FCyDO5eJCzWIEL2LDUWI7jKSETNih5hl5fMMvCCNRPnkgGnytw5kF/t' - 'Lw8YAbLRxkGsO4FCx5mB7HF5pNHyWOCCalMTKheyg/qUV/VcXW9Unlr' - 'puMu0+d3hpLZESplS/NkvDxSrx16ank7EORS8OxLOufu56TW2hDuBzz' - 'w1CBAj1p6s+Z6Kc4RMYYdxRgR1TjXg/ZVUn3T69d9igdS/5lAPFx2Ww' - '8x82FWCLSaiXymxXRNsNcKx5ifuvtv307r4yh31QjlKFYwadOCaCHHZ' - 'zGE1mXcOu3j6W9WIaZfYRTpxmOrcfDIHxZSdLf11hOSZEUUFpj9hQlV' - 'Op0RHkDEJUMNs6vkXUhZq9yPuqgrcb6GaN+UhOT8iHlijKmlG8NJEPk' - 'Hp8RnL1hsr44N57ZzLqmSUZtvC83u/5e+YUb7beUDGsMyJV/fcMiGMM' - 'LVtRnuPCFyNVNQUf2CphtG0=') }, - # pass - { 'secure': ('AtbgKUukES2uJPpEWNEDHLg0WcghLlCGL171Ah3+4CckBI8y1Fn+VpH' - 'U2vEXzsV8tKoxX1IyB2tFivzuyo6CQXHSuWGJYQexwkBeGCgOfzKJLj' - 'MAy75ATYA6JnFrikV+UcqdEz/9Dow3J1K7Slp3jpsQhERHbNeqkr4I+' - 'XCL1LLnpewfOZo9OIEu93p6b6YlqvIPXJHyQe5xnMd8jFWg3/uIYqFn' - 'XPvigeZqC2lhNp48mj4JdwwF2tmiArgyXOmgxiuHJNVVI7okbhc7kmI' - 'Y3MmCSFgG0XPUEBU3Kdr4o/2hy8DDP6Gff+rUZW8nPI/2UWXRLWtOxv' - 'XGGRyjHHTxGWzI4JyZbli9dls5M32MMjsXVKtciSFVwsMM8qn7wFnRi' - 'q248a1Sg5fDNX/WYowmsHlWjffHZ7+UqUqJxAKtZ0vpQL+4SPIALPnK' - 'V6j6CoorQp+VhMF01EFlZ0c8bkNmk4YW7R7RyNLIcaHKfd1ud8QF9PD' - 'AnQ7Jr1GRBxzkjHvHfFrE14WPUu+FjVvDO7UPVMNQX7RS1IVACpKSRu' - '7N8KnIK3vSnLpn5GXKsbx0JB2vtyoTaFZvC9c3qyAw1nlpn7Lp3sPs3' - 'bgIBU4tWOzg5g46eHbc4ad5nyB9Soz715lbMdECvKs2HHJUG3tubLKj' - 'L0S/LRGRQ+IDgC7xrjQj8aA=') }, - ] - - env = { - 'matrix': envs, - 'global': global_env - } - - data = { - 'request': { - 'message' : tmpl_message.format(app=options.custom_app, zim=options.zim_url, uuid=uuid), - 'branch' : "custom_app", - 'config' : { - 'before_install' : [ - ( 'pip3 install pyOpenSSl google-api-python-client' - ' httplib2 apiclient requests'), - ( 'openssl aes-256-cbc -k $google_key' - ' -in travis/googleplay_android_developer-5a411156212c.json.enc' - ' -out travis/googleplay_android_developer-5a411156212c.json' - ' -d'), - ( 'openssl aes-256-cbc -k $google_key' - ' -in travis/kiwix-android.keystore.enc' - ' -out travis/kiwix-android.keystore -d'), - ( 'openssl aes-256-cbc -K $encrypted_eba2f7543984_key' - ' -iv $encrypted_eba2f7543984_iv' - ' -in travis/travisci_builder_id_key.enc' - ' -out travis/travisci_builder_id_key -d'), - 'chmod 600 travis/travisci_builder_id_key' - ], - 'env' : env, - 'script' : 'travis_wait 30 travis/compile_custom_app.sh', - 'deploy' : { - 'provider': 'script', - 'skip_cleanup': True, - 'script': 'travis/deploy_apk.sh', - 'on': { - 'branch': 'custom_app' - } - } - } - } - } - - if options.android_upload: - data['request']['config']['jobs'] = { - 'include': [ - { - 'stage' : 'make_release', - 'install': 'pip3 install -r requirements_build_custom_app.txt', - 'script': True, - 'env': global_env, - 'deploy' : { - 'provider': 'script', - 'skip_cleanup': True, - 'script': 'travis/make_release.sh', - 'on': { - 'branch': 'custom_app' - } - } - } - ] - } - global_env.append({ - 'DEPLOY_DIR' : '/home/nightlybot/apks/{}_{}'.format( - options.custom_app, options.base_version) - }) - else: - global_env.append({ - 'DEPLOY_DIR' : '/data/tmp/custom_apps/{}_{}'.format( - options.custom_app, options.base_version) - }) - - - r = requests.post(request_url, headers=headers, json=data) - if r.status_code != 202: - print("Error while requesting build:") - print(r.reason) - print("Have you forget to give the travis token ?") - sys.exit(-1) - else: - request_id = r.json()['request']['id'] - print("Request {} has been schedule.".format(request_id)) - request_left = 10 - found = False - request_url = tmpl_request_url.format( - organisation=organisation, - repository=repository, - endpoint='builds') - while request_left: - time.sleep(1) - print("Try to get associated build.") - r = requests.get(request_url, headers=headers) - json_data = json.loads(r.text) - for build in json_data['builds']: - if build['event_type'] != 'api': - continue - message = build['commit']['message'] - if str(uuid) in message: - found = True - break - if found: - break - print("Cannot find build. Wait 1 second and try again") - print("{} tries left".format(request_left)) - request_left -= 1 - if found: - print("Associated build found: {}.".format(build['number'])) - print("https://travis-ci.org/kiwix/kiwix-build/builds/{}".format(build['id'])) - else: - print("Request has been accepted by travis-ci but I cannot found " - "the associated build. Have a look here " - "https://travis-ci.org/kiwix/kiwix-build/builds" - "if you found it.") - if not options.android_upload: - print(("Automatic upload to android play store has been deactivated.\n" - "You will find the apks at this address once they have been compiled :" - " http://tmp.kiwix.org/custom_apps/{}_{}").format( - options.custom_app, options.base_version)) - - -ERROR_MSG_EDIT_CHANGE = "A change was made to the application outside of this Edit, please create a new edit." - -class Google: - def __init__(self, options): - scope = 'https://www.googleapis.com/auth/androidpublisher' - key = options.google_api_key - credentials = ServiceAccountCredentials.from_json_keyfile_name( - key, - scopes=[scope]) - - http = httplib2.Http() - http = credentials.authorize(http) - - self.service = build('androidpublisher', 'v2', http=http) - self.packageName = options.package_name - self.edit_id = None - - @contextmanager - def new_request(self): - edit_request = self.service.edits().insert( - packageName=self.packageName, - body={}) - result = edit_request.execute() - print("create", result) - self.edit_id = result['id'] - - yield - - commit_request = self.service.edits().commit( - editId=self.edit_id, - packageName=self.packageName) - result = commit_request.execute() - print("commit", result) - - self.edit_id = None - - def make_request(self, section, method, **kwargs): - request_content = self._build_request_content(kwargs) - _section = getattr(self._edits, section)() - _method = getattr(_section, method) - print(">", request_content) - request = _method(**request_content) - result = request.execute() - print("<", result) - return result - - def _build_request_content(self, kwargs): - d = kwargs.copy() - d['editId'] = self.edit_id - d['packageName'] = self.packageName - return d - - @property - def _edits(self): - return self.service.edits() - - def upload_expansionfile(self, comp_file, contentVersionCode, versionCodes): - versionCodes = [int(v) for v in versionCodes] - self.make_request('expansionfiles', 'upload', - expansionFileType='main', - apkVersionCode=contentVersionCode, - media_body=comp_file, - media_mime_type='application/octet-stream') - for versionCode in versionCodes: - if versionCode == contentVersionCode: - continue - self.make_request('expansionfiles', 'update', - expansionFileType='main', - apkVersionCode=versionCode, - body={'referencesVersion': contentVersionCode} - ) - - def upload_apk(self, apk_file): - return self.make_request('apks', 'upload', - media_body=apk_file) - - def publish_release(self, options, versionCodes): - return self.make_request('tracks', 'update', - track="alpha", - body={'versionCodes': versionCodes}) - - -def gen_version_code(platform_index, base_version): - str_version = "{platform}{base_version}".format( - platform=platform_index, - base_version=base_version - ) - return int(str_version) - -if __name__ == "__main__": - options = parse_args() - func = globals()['do_{}'.format(options.step)] - func(options) diff --git a/requirements_build_custom_app.txt b/requirements_build_custom_app.txt deleted file mode 100644 index 769b2a7..0000000 --- a/requirements_build_custom_app.txt +++ /dev/null @@ -1,3 +0,0 @@ -requests==2.20.0 -apiclient==1.0.3 - diff --git a/travis/compile_custom_app.sh b/travis/compile_custom_app.sh deleted file mode 100755 index 6bde800..0000000 --- a/travis/compile_custom_app.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -set -e - -cd ${HOME} - -${TRAVIS_BUILD_DIR}/kiwix-build.py \ - --target-platform $PLATFORM \ - --hide-progress \ - --android-custom-app $CUSTOM_APP \ - --zim-file-size $ZIM_FILE_SIZE \ - kiwix-android-custom - diff --git a/travis/deploy_apk.sh b/travis/deploy_apk.sh deleted file mode 100755 index c17d748..0000000 --- a/travis/deploy_apk.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -set -e - -KEYSTORE_FILE=${TRAVIS_BUILD_DIR}/travis/kiwix-android.keystore -GOOGLE_API_KEY=${TRAVIS_BUILD_DIR}/travis/googleplay_android_developer-5a411156212c.json - -cd ${HOME} - -# Sign apk file - -BASE_DIR="BUILD_${PLATFORM}" -INPUT_APK_FILE=${BASE_DIR}/kiwix-android-custom_${CUSTOM_APP}/app/build/outputs/apk/${CUSTOM_APP}/release/app-${CUSTOM_APP}-release-unsigned.apk -SIGNED_APK=${BASE_DIR}/app-${CUSTOM_APP}_${VERSION_CODE}-release-signed.apk - -TOOLCHAINS/android-sdk-r25.2.3/build-tools/25.0.2/apksigner sign \ - --ks ${KEYSTORE_FILE} \ - --ks-pass env:KEYSTORE_PASS \ - --out ${SIGNED_APK} \ - ${INPUT_APK_FILE} - -ssh -i ${SSH_KEY} ci@download.kiwix.org "mkdir -p ${DEPLOY_DIR}" - -scp -i ${SSH_KEY} \ - ${SIGNED_APK} \ - ci@download.kiwix.org:${DEPLOY_DIR} diff --git a/travis/googleplay_android_developer-5a411156212c.json.enc b/travis/googleplay_android_developer-5a411156212c.json.enc deleted file mode 100644 index d64c3de..0000000 Binary files a/travis/googleplay_android_developer-5a411156212c.json.enc and /dev/null differ diff --git a/travis/kiwix-android.keystore.enc b/travis/kiwix-android.keystore.enc deleted file mode 100644 index d3ca4d7..0000000 Binary files a/travis/kiwix-android.keystore.enc and /dev/null differ diff --git a/travis/make_release.sh b/travis/make_release.sh deleted file mode 100755 index 54be0b8..0000000 --- a/travis/make_release.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -set -e - -KEYSTORE_FILE=${TRAVIS_BUILD_DIR}/travis/test_ks.ks -GOOGLE_API_KEY=${TRAVIS_BUILD_DIR}/travis/googleplay_android_developer-5a411156212c.json - -cd ${HOME} - - -BASE_DIR="BUILD_${PLATFORM}" - -mkdir -p ${HOME}/APKS - -scp -i ${SSH_KEY} ci@download.kiwix.org:${DEPLOY_DIR}/* ${HOME}/APKS - -ssh -i ${SSH_KEY} ci@download.kiwix.org "rm -rf ${DEPLOY_DIR}" - -${TRAVIS_BUILD_DIR}/build_custom_app.py \ - --step publish \ - --custom-app ${CUSTOM_APP} \ - --package-name ${PACKAGE_NAME} \ - --google-api-key ${GOOGLE_API_KEY} \ - --zim-url ${ZIM_URL} \ - --apks-dir ${HOME}/APKS \ - --content-version-code ${CONTENT_VERSION_CODE}