407 lines
16 KiB
Python
Executable File
407 lines
16 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
|
||
import \
|
||
argparse, \
|
||
datetime, \
|
||
glob, \
|
||
json, \
|
||
os, \
|
||
ssl, \
|
||
sys, \
|
||
tempfile, \
|
||
time, \
|
||
urllib
|
||
from uuid import uuid4
|
||
from contextlib import contextmanager
|
||
from urllib.parse import urlparse
|
||
|
||
from utils import (
|
||
Remotefile,
|
||
download_remote
|
||
)
|
||
|
||
import requests
|
||
import httplib2
|
||
from apiclient.discovery import build
|
||
from apiclient.errors import HttpError
|
||
from oauth2client import client
|
||
from oauth2client.service_account import ServiceAccountCredentials
|
||
|
||
tmpl_request_url = "https://api.travis-ci.org/repo/{organisation}%2F{repository}/{endpoint}"
|
||
tmpl_message = """Build of custom app {app} with zim file {zim}.
|
||
|
||
UUID:#{uuid}#"""
|
||
|
||
description = """Launch a custom application build.
|
||
This command will launch a custom application build on Travis-CI.
|
||
Travis-CI jobs will compile the application and upload the build apks on
|
||
google play, using tha 'alpha' track of the application.
|
||
|
||
You will need to have a valid TRAVIS_TOKEN associated to your personnal account
|
||
and the kiwix-build repository on travis.
|
||
Use the 'travis' command line tool (https://github.com/travis-ci/travis.rb)
|
||
to generate a token.
|
||
"""
|
||
|
||
def parse_args():
|
||
parser = argparse.ArgumentParser(description=description,
|
||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||
parser.add_argument('--custom-app')
|
||
parser.add_argument('--travis-token')
|
||
parser.add_argument('--zim-url')
|
||
|
||
advance = parser.add_argument_group('advance', "Some advanced options.")
|
||
advance.add_argument('--extra-code', type=int, default=0)
|
||
advance.add_argument('--check-certificate', default=True)
|
||
advance.add_argument('--no-android-upload', action='store_false', dest='android_upload')
|
||
|
||
# Hidden options
|
||
parser.add_argument('--step', default='launch', choices=['launch', 'publish'], help=argparse.SUPPRESS)
|
||
parser.add_argument('--apks-dir', help=argparse.SUPPRESS)
|
||
parser.add_argument('--version', default="0", help=argparse.SUPPRESS)
|
||
parser.add_argument('--content-version-code', type=int)
|
||
parser.add_argument('--package-name', default=None, help=argparse.SUPPRESS)
|
||
parser.add_argument('--google-api-key', help=argparse.SUPPRESS)
|
||
|
||
options = parser.parse_args()
|
||
|
||
if not options.package_name:
|
||
print("Try to get package name from info.json file")
|
||
request_url = ('https://raw.githubusercontent.com/kiwix/kiwix-android-custom/master/{}/info.json'
|
||
.format(options.custom_app))
|
||
json_request = requests.get(request_url)
|
||
if json_request.status_code != 200:
|
||
print("Error while getting json file.")
|
||
print("Reason is '{}'".format(json_request.reason))
|
||
sys.exit(-1)
|
||
json_data = json.loads(json_request.text)
|
||
print("Found package_name '{}'".format(json_data['package']))
|
||
options.package_name = json_data['package']
|
||
|
||
options.base_version = "{}{}".format(
|
||
datetime.date.today().strftime('%y%j'),
|
||
options.extra_code)
|
||
|
||
return options
|
||
|
||
|
||
def download_zim_file(zim_url, dest_dir=None):
|
||
if dest_dir is None:
|
||
dest_dir = os.getcwd()
|
||
out_filename = urlparse(zim_url).path
|
||
out_filename = os.path.basename(out_filename)
|
||
zim_file = Remotefile(out_filename, '', zim_url)
|
||
download_remote(zim_file, dest_dir)
|
||
return os.path.join(dest_dir, out_filename)
|
||
|
||
|
||
def get_zim_size(zim_url, check_certificate=True):
|
||
print("Try to get zim size")
|
||
if not check_certificate:
|
||
context = ssl.create_default_context()
|
||
context.check_hostname = False
|
||
context.verify_mode = ssl.CERT_NONE
|
||
else:
|
||
context = None
|
||
extra_args = {'context':context} if sys.version_info >= (3, 4, 3) else {}
|
||
with urllib.request.urlopen(zim_url, **extra_args) as resource:
|
||
size = resource.getheader('Content-Length', None)
|
||
if size is not None:
|
||
size = int(size)
|
||
print("Zim size is {}".format(size))
|
||
return size
|
||
else:
|
||
print("No 'Content-Length' header in http answer from the server.\n"
|
||
"We need to download the zim file to get its size.")
|
||
zim_path = download_zim_file(zim_url, tempfile.gettempdir())
|
||
size = os.path.getsize(zim_path)
|
||
print("Zim size is {}".format(size))
|
||
return size
|
||
|
||
|
||
def do_launch(options):
|
||
zim_size = get_zim_size(options.zim_url, options.check_certificate)
|
||
travis_launch_build('kiwix', 'kiwix-build', options, zim_size)
|
||
print("Travis build has been launch.")
|
||
|
||
|
||
def do_publish(options):
|
||
zim_path = download_zim_file(options.zim_url)
|
||
googleService = Google(options)
|
||
with googleService.new_request():
|
||
versionCodes = []
|
||
for apk in glob.iglob(options.apks_dir+"/*.apk"):
|
||
result = googleService.upload_apk(apk)
|
||
versionCodes.append(result['versionCode'])
|
||
googleService.upload_expansionfile(
|
||
zim_path, options.content_version_code, versionCodes)
|
||
googleService.publish_release(options, versionCodes)
|
||
|
||
|
||
def travis_launch_build(organisation, repository, options, zim_size):
|
||
request_url = tmpl_request_url.format(
|
||
organisation=organisation,
|
||
repository=repository,
|
||
endpoint='requests')
|
||
headers = {
|
||
"Travis-API-Version": "3",
|
||
"Content-Type": "application/json",
|
||
"Authorization": "token {}".format(options.travis_token),
|
||
"User-Agent": "kiwix-build"
|
||
}
|
||
uuid = uuid4()
|
||
envs = []
|
||
for platform_index, platform in enumerate(['arm', 'arm64', 'x86', 'x86_64', 'mips', 'mips64']):
|
||
d = {
|
||
'PLATFORM': "android_{}".format(platform),
|
||
'VERSION_CODE': gen_version_code(platform_index, options.base_version)
|
||
}
|
||
envs.append(d)
|
||
|
||
global_env = [
|
||
{ 'CUSTOM_APP': options.custom_app},
|
||
{ 'ZIM_FILE_SIZE': zim_size},
|
||
{ 'PACKAGE_NAME': options.package_name},
|
||
{ 'ZIM_URL': options.zim_url},
|
||
{ 'EXTRA_CODE': options.extra_code},
|
||
{ 'CONTENT_VERSION_CODE': gen_version_code(0, options.base_version)},
|
||
{ 'VERSION': options.version},
|
||
{ 'secure': ('u6MCvCQhAlvU0jNojLVatNXHr6AYj4rjU9CY9JcUUG9CKMsls1t3ERz'
|
||
'aab0vY1H7EIYl8uMgJcE71lF+61/sVOz7DPJVbKBorivnOlvMvyG2xU'
|
||
'su/q3ddMcAkuyixtW2Yo7oq8Tug5PbL5qybrZ+TD0BplS/XcpgljMTH'
|
||
'gxWDYion5Y1/Udz5zg1VIxHGAcv3lKYqtbPzA5gR7scLhFVMVulpFDC'
|
||
'Ps9MS/jEpwJtHrSZPrRmkOdLCJJNgOnQmRe0ib6NsmhO9KgpeEZRgUx'
|
||
'bSaOOiXKBwN7MdWHCZKczZbwVBEA9Izw29hVh0fQZpZNuC3GAICydyK'
|
||
'XmbGLNy+GqIPOJPXbAgfT8zRK2IN+PuUHL0xizK4QALMX1jTiiJ65i1'
|
||
'vWy1NZ3fS7kbgulPUXvkIs2PoSULVEAqhhanp+1ICcEVKs5ffc3e0WP'
|
||
'/p7NSTIL8QF58QYPOIqWXX17IP3kfrWURIWCdPHCHKEGkg0S+HyqKa6'
|
||
'/pdR5CeBCeECLraupzJ+ZfKIfsFFyrOgCWRYe1pIAD5C1f/0jM6T+9h'
|
||
'Vsc2ktgjM8GpujTztzMKNrVJ80mNBtdXbFGYn01B/zhVSsxPIrbROVp'
|
||
'5UL/lFxJVPIrSe2nIs4AFhanhSDK5ro9qCI6GotPqhyd+RhuJMY/p36'
|
||
'fs/srp3KL+aNd8JFxWj45M0=') },
|
||
{ 'secure': ('xotOZyUbkcgoZhS4ZAaNtqaxhxRV03rgWRSxaccI+INczLNK18xiVUj'
|
||
'I/1zdAil+ck8rXDMYVuXGo860UIaxt7Sxa7j24+nu6KNpDmlB2a1xyd'
|
||
'Ls8RNL9tUpx4t+DooTHmUuUJSipUcchK6ZRAKFYOHB9Lumkbz2H2Ifs'
|
||
'U1AYnLZfcGC7U63DojGFYJmgXwCgHJ25Zq00SCYdHq1b5vpvQ3ZVviI'
|
||
'0+OTXCK804sSiLu3g6EamsQn7+P5F4RnMrKtm6kP2iTZ3sbED6L6adA'
|
||
'mtOWKDfgrMIbYO3kgYemfNE12KG+Lfey10RiheC/CbUZHV3jSl+LE1z'
|
||
'hY8Nqcf6rYQZ8QceE9gW53MrILpUxi7oSltR/pwenNNS5uEox1SrMtY'
|
||
'2/O0K0dixHHgF96RV5YuqJQurzTEXTt4T9Uu1ZItp3rauSwcj6wbTYu'
|
||
'fWnGeUzNVItZkSruysBY5mrcvkH6lQpCMk5x6kAzvHCQM4DtIWcep3P'
|
||
'LG0lj034OuZc7MxIh9GzReeYw/gawF2xEgmiitlBaPVvYUff0BaJ8K7'
|
||
'R0bbbYOuIibxrXsiAIZwa0d9vS2PBoerORD/hiE3QD+c0o3uEn8btw+'
|
||
'pebokk6SizfS1q44tcWWRpaovm8uD3rhJ0alm6Ht0V7ErGzXafAzsiR'
|
||
'B/o4trHiAeJUJuSIKDyvmWk=') }
|
||
]
|
||
|
||
env = {
|
||
'matrix': envs,
|
||
'global': global_env
|
||
}
|
||
|
||
data = {
|
||
'request': {
|
||
'message' : tmpl_message.format(app=options.custom_app, zim=options.zim_url, uuid=uuid),
|
||
'branch' : "custom_app",
|
||
'config' : {
|
||
'before_install' : [
|
||
( 'pip3 install pyOpenSSl google-api-python-client'
|
||
' httplib2 apiclient requests'),
|
||
( 'openssl aes-256-cbc -k $google_key'
|
||
' -in travis/googleplay_android_developer-5a411156212c.json.enc'
|
||
' -out travis/googleplay_android_developer-5a411156212c.json'
|
||
' -d'),
|
||
( 'openssl aes-256-cbc -k $google_key'
|
||
' -in travis/test_ks.ks.enc'
|
||
' -out travis/test_ks.ks -d'),
|
||
( 'openssl aes-256-cbc -K $encrypted_eba2f7543984_key'
|
||
' -iv $encrypted_eba2f7543984_iv'
|
||
' -in travis/travisci_builder_id_key.enc'
|
||
' -out travis/travisci_builder_id_key -d'),
|
||
'chmod 600 travis/travisci_builder_id_key'
|
||
],
|
||
'env' : env,
|
||
'script' : 'travis_wait 30 travis/compile_custom_app.sh',
|
||
'deploy' : {
|
||
'provider': 'script',
|
||
'skip_cleanup': True,
|
||
'script': 'travis/deploy_apk.sh',
|
||
'on': {
|
||
'branch': 'custom_app'
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
if options.android_upload:
|
||
data['request']['config']['jobs'] = {
|
||
'include': [
|
||
{
|
||
'stage' : 'make_release',
|
||
'install': 'pip3 install -r requirements_build_custom_app.txt',
|
||
'script': True,
|
||
'env': global_env,
|
||
'deploy' : {
|
||
'provider': 'script',
|
||
'skip_cleanup': True,
|
||
'script': 'travis/make_release.sh',
|
||
'on': {
|
||
'branch': 'custom_app'
|
||
}
|
||
}
|
||
}
|
||
]
|
||
}
|
||
global_env.append({
|
||
'DEPLOY_DIR' : '/home/nightlybot/apks/{}_{}'.format(
|
||
options.custom_app, options.base_version)
|
||
})
|
||
else:
|
||
global_env.append({
|
||
'DEPLOY_DIR' : '/var/www/tmp.kiwix.org/custom_apps/{}_{}'.format(
|
||
options.custom_app, options.base_version)
|
||
})
|
||
|
||
|
||
r = requests.post(request_url, headers=headers, json=data)
|
||
if r.status_code != 202:
|
||
print("Error while requesting build:")
|
||
print(r.reason)
|
||
print("Have you forget to give the travis token ?")
|
||
sys.exit(-1)
|
||
else:
|
||
request_id = r.json()['request']['id']
|
||
print("Request {} has been schedule.".format(request_id))
|
||
request_left = 10
|
||
found = False
|
||
request_url = tmpl_request_url.format(
|
||
organisation=organisation,
|
||
repository=repository,
|
||
endpoint='builds')
|
||
while request_left:
|
||
time.sleep(1)
|
||
print("Try to get associated build.")
|
||
r = requests.get(request_url, headers=headers)
|
||
json_data = json.loads(r.text)
|
||
for build in json_data['builds']:
|
||
if build['event_type'] != 'api':
|
||
continue
|
||
message = build['commit']['message']
|
||
if str(uuid) in message:
|
||
found = True
|
||
break
|
||
if found:
|
||
break
|
||
print("Cannot find build. Wait 1 second and try again")
|
||
print("{} tries left".format(request_left))
|
||
request_left -= 1
|
||
if found:
|
||
print("Associated build found: {}.".format(build['number']))
|
||
print("https://travis-ci.org/kiwix/kiwix-build/builds/{}".format(build['id']))
|
||
else:
|
||
print("Request has been accepted by travis-ci but I cannot found "
|
||
"the associated build. Have a look here "
|
||
"https://travis-ci.org/kiwix/kiwix-build/builds"
|
||
"if you found it.")
|
||
if not options.android_upload:
|
||
print(("Automatic upload to android play store has been deactivated.\n"
|
||
"You will find the apks at this address once they have been compiled :"
|
||
" http://tmp.kiwix.org/custom_apps/{}_{}").format(
|
||
options.custom_app, options.base_version))
|
||
|
||
|
||
ERROR_MSG_EDIT_CHANGE = "A change was made to the application outside of this Edit, please create a new edit."
|
||
|
||
class Google:
|
||
def __init__(self, options):
|
||
scope = 'https://www.googleapis.com/auth/androidpublisher'
|
||
key = options.google_api_key
|
||
credentials = ServiceAccountCredentials.from_json_keyfile_name(
|
||
key,
|
||
scopes=[scope])
|
||
|
||
http = httplib2.Http()
|
||
http = credentials.authorize(http)
|
||
|
||
self.service = build('androidpublisher', 'v2', http=http)
|
||
self.packageName = options.package_name
|
||
self.edit_id = None
|
||
|
||
@contextmanager
|
||
def new_request(self):
|
||
edit_request = self.service.edits().insert(
|
||
packageName=self.packageName,
|
||
body={})
|
||
result = edit_request.execute()
|
||
print("create", result)
|
||
self.edit_id = result['id']
|
||
|
||
yield
|
||
|
||
commit_request = self.service.edits().commit(
|
||
editId=self.edit_id,
|
||
packageName=self.packageName)
|
||
result = commit_request.execute()
|
||
print("commit", result)
|
||
|
||
self.edit_id = None
|
||
|
||
def make_request(self, section, method, **kwargs):
|
||
request_content = self._build_request_content(kwargs)
|
||
_section = getattr(self._edits, section)()
|
||
_method = getattr(_section, method)
|
||
print(">", request_content)
|
||
request = _method(**request_content)
|
||
result = request.execute()
|
||
print("<", result)
|
||
return result
|
||
|
||
def _build_request_content(self, kwargs):
|
||
d = kwargs.copy()
|
||
d['editId'] = self.edit_id
|
||
d['packageName'] = self.packageName
|
||
return d
|
||
|
||
@property
|
||
def _edits(self):
|
||
return self.service.edits()
|
||
|
||
def upload_expansionfile(self, comp_file, contentVersionCode, versionCodes):
|
||
versionCodes = [int(v) for v in versionCodes]
|
||
self.make_request('expansionfiles', 'upload',
|
||
expansionFileType='main',
|
||
apkVersionCode=contentVersionCode,
|
||
media_body=comp_file,
|
||
media_mime_type='application/octet-stream')
|
||
for versionCode in versionCodes:
|
||
if versionCode == contentVersionCode:
|
||
continue
|
||
self.make_request('expansionfiles', 'update',
|
||
expansionFileType='main',
|
||
apkVersionCode=versionCode,
|
||
body={'referencesVersion': contentVersionCode}
|
||
)
|
||
|
||
def upload_apk(self, apk_file):
|
||
return self.make_request('apks', 'upload',
|
||
media_body=apk_file)
|
||
|
||
def publish_release(self, options, versionCodes):
|
||
return self.make_request('tracks', 'update',
|
||
track="alpha",
|
||
body={'versionCodes': versionCodes})
|
||
|
||
|
||
def gen_version_code(platform_index, base_version):
|
||
str_version = "{platform}{base_version}".format(
|
||
platform=platform_index,
|
||
base_version=base_version
|
||
)
|
||
return int(str_version)
|
||
|
||
if __name__ == "__main__":
|
||
options = parse_args()
|
||
func = globals()['do_{}'.format(options.step)]
|
||
func(options)
|