Merge pull request #343 from kiwix/kiwix-lib_app

Kiwix lib app

kiwixbuild/dependencies/base.py

@@ -1,6 +1,7 @@
 import subprocess
 import os
 import shutil
+import time
 
 from kiwixbuild.utils import pj, Context, SkipCommand, extract_archive, Defaultdict, StopBuild, run_command
 from kiwixbuild.versions import main_project_versions, base_deps_versions
@@ -72,9 +73,11 @@ class Source:
         log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
         context = Context(name, log, True)
         try:
+            start_time = time.time()
             ret = function(*args, context=context)
             context._finalise()
-            print("OK")
+            duration = time.time() - start_time
+            print("OK ({:.1f}s)".format(duration))
             return ret
         except SkipCommand:
             print("SKIP")
@@ -228,9 +231,11 @@ class Builder:
         log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
         context = Context(name, log, self.target.force_native_build)
         try:
+            start_time = time.time()
             ret = function(*args, context=context)
             context._finalise()
-            print("OK")
+            duration = time.time() - start_time
+            print("OK ({:.1f}s)".format(duration))
             return ret
         except SkipCommand:
             print("SKIP")

kiwixbuild/dependencies/kiwix_lib.py

@@ -1,8 +1,12 @@
+import shutil, os
+
 from .base import (
     Dependency,
     GitClone,
-    MesonBuilder)
+    MesonBuilder,
-from kiwixbuild._global import neutralEnv
+    GradleBuilder)
+from kiwixbuild.utils import pj, copy_tree
+from kiwixbuild._global import option, get_target_step, neutralEnv
 
 class Kiwixlib(Dependency):
     name = "kiwix-lib"
@@ -28,3 +32,56 @@ class Kiwixlib(Dependency):
             if self.buildEnv.platformInfo.build == 'android':
                 return 'shared'
             return super().library_type
+
+
+class KiwixlibApp(Dependency):
+    name = "kiwix-lib-app"
+
+    Source = Kiwixlib.Source
+
+    class Builder(GradleBuilder):
+        dependencies = ["kiwix-lib"]
+        gradle_target = "assembleRelease"
+
+        @classmethod
+        def get_dependencies(cls, platformInfo, allDeps):
+            if not allDeps:
+                return super().get_dependencies(platformInfo, allDeps)
+            else:
+                deps = [('android_{}'.format(arch), 'kiwix-lib')
+                    for arch in option('android_arch')]
+                return deps
+
+        def _configure(self, context):
+            try:
+                shutil.rmtree(self.build_path)
+            except FileNotFoundError:
+                pass
+            if not os.path.exists(self.build_path):
+                shutil.copytree(pj(self.source_path, 'android-kiwix-lib-publisher'), self.build_path, symlinks=True)
+            for arch in option('android_arch'):
+                try:
+                    kiwix_builder = get_target_step('kiwix-lib', 'android_{}'.format(arch))
+                except KeyError:
+                    pass
+                else:
+                    copy_tree(pj(kiwix_builder.buildEnv.install_dir, 'kiwix-lib'),
+                              pj(self.build_path, 'kiwixLibAndroid', 'src', 'main'))
+
+#            The ICU dat file should be embedded with the kiwix-lib application
+#            but for now it is package with kiwix-android app and use there.
+#            We must fix this at a time (before we update the icu version).
+#            os.makedirs(
+#                pj(self.build_path, 'app', 'src', 'main', 'assets', 'icu'),
+#                exist_ok=True)
+#            for arch in option('android_arch'):
+#                try:
+#                    kiwix_builder = get_target_step('kiwix-lib', 'android_{}'.format(arch))
+#                except KeyError:
+#                    pass
+#                else:
+#                    shutil.copy2(pj(kiwix_builder.buildEnv.install_dir, 'share', 'icu', '58.2',
+#                                    'icudt58l.dat'),
+#                                 pj(self.build_path, 'app', 'src', 'main', 'assets',
+#                                    'icu', 'icudt58l.dat'))
+#                    break

kiwixbuild/platforms/android.py

@@ -131,7 +131,7 @@ class Android(MetaPlatformInfo):
         return ['android_{}'.format(arch) for arch in option('android_arch')]
 
     def add_targets(self, targetName, targets):
-        if targetName not in ('kiwix-android', 'kiwix-android-custom'):
+        if targetName not in ('kiwix-android', 'kiwix-android-custom', 'kiwix-lib-app'):
             return super().add_targets(targetName, targets)
         else:
             return AndroidPlatformInfo.add_targets(self, targetName, targets)

scripts/upload_kiwix_lib_android_to_bintray.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+
+import requests
+from requests.auth import HTTPBasicAuth
+import os, sys
+import json
+
+bintray_auth = (os.environ['BINTRAY_USER'], os.environ['BINTRAY_PASS'])
+
+
+def generate_pom_file(version):
+    template = """<?xml version="1.0" encoding="UTF-8"?>
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>org.kiwix.kiwixlib</groupId>
+  <artifactId>kiwixlib</artifactId>
+  <version>{version}</version>
+  <packaging>aar</packaging>
+  <name>kiwixlib</name>
+  <description>kiwixlib</description>
+  <url>https://github.com/kiwix/kiwix-lib</url>
+  <licenses>
+    <license>
+      <name>GPLv3</name>
+      <url>https://www.gnu.org/licenses/gpl-3.0.en.html</url>
+    </license>
+  </licenses>
+  <developers>
+    <developer>
+      <id>kiwix</id>
+      <name>kiwix</name>
+      <email>contact@kiwix.org</email>
+    </developer>
+  </developers>
+  <scm>
+    <connection>https://github.com/kiwix/kiwix-lib.git</connection>
+    <developerConnection>https://github.com/kiwix/kiwix-lib.git</developerConnection>
+    <url>https://github.com/kiwix/kiwix-lib</url>
+  </scm>
+</project>"""
+    return template.format(version=version)
+
+
+def create_version(version):
+    url = "https://api.bintray.com/packages/kiwix/kiwix/kiwixlib/versions"
+    payload = {
+        'name': version,
+        'desc': 'Release of kiwix-lib'
+    }
+    headers = {
+        'Content-Type': 'application/json'
+    }
+
+    r = requests.post(url, data=json.dumps(payload), headers=headers, auth=bintray_auth)
+    rcode = r.status_code
+
+    if rcode == 409:
+        print("Bintray version %s already exists, skipping." % version)
+        return True
+
+    rcode_family = rcode // 100
+    if rcode_family in (2, 3):
+        print("Bintray Version created!")
+        return True
+
+    print("ERROR : Bintray API response {}".format(code))
+    return False
+
+
+def upload_archive(version, filename, artefact):
+    url_template = "https://api.bintray.com/content/kiwix/kiwix/kiwixlib/{version}/org/kiwix/kiwixlib/kiwixlib/{version}/{artefact}"
+    parameters = {
+        'publish': 1,
+        'override': 1
+    }
+
+    # Upload the main artefact
+    url = url_template.format(version=version, artefact=artefact)
+    with open(filename, 'rb') as f:
+        r = requests.put(url, data=f, auth=bintray_auth, params=parameters)
+
+    rcode = r.status_code
+    rcode_family = rcode // 100
+    if rcode_family not in (2, 3):
+        print("ERROR: Fail to upload artefact")
+        return False
+
+    # Upload the pom file
+    pom_artefact = os.path.splitext(artefact) + ".pom"
+    url = url_template.format(version=version, artefact=pom_artefact)
+    data = generate_pom_file(version)
+    r = requests.put(url, data=data, auth=bintray_auth, params=parameters)
+
+    rcode = r.status_code
+    rcode_family = rcode // 100
+    if rcode_family not in (2, 3):
+       print("ERROR: Fail to upload pom artefact")
+       return False
+
+    return True
+
+
+
+if __name__ == "__main__":
+    try:
+       info_file = sys.argv[1]
+    except IndexError:
+       print("Usage {} infofile".format(sys.argv[0]))
+       sys.exit(-1)
+
+    print("Use info file {}".format(info_file))
+    with open(info_file) as f:
+        options = json.load(f)
+
+    if not create_version(options['version']):
+         sys.exit("Cannot create version")
+
+    filepath = os.path.join(os.path.dirname(sys.argv[1]), options['filename'])
+    if not upload_archive(options['version'], filepath, options['artefact']):
+         sys.exit("Cannot upload artefact")

travis/compile_all.py

@@ -8,6 +8,7 @@ from datetime import date
 import tarfile, zipfile
 import subprocess
 import re
+import json
 from urllib.request import urlopen
 from urllib.error import URLError
 
@@ -37,6 +38,7 @@ NIGHTLY_ZIM_ARCHIVES_DIR = EXPORT_DIR/'NIGHTLY_ZIM_ARCHIVES'/NIGHTLY_DATE
 RELEASE_ZIM_ARCHIVES_DIR = EXPORT_DIR/'RELEASE_ZIM_ARCHIVES'
 DIST_KIWIX_ARCHIVES_DIR = EXPORT_DIR/'DIST_KIWIX_ARCHIVES'
 DIST_ZIM_ARCHIVES_DIR = EXPORT_DIR/'DIST_ZIM_ARCHIVES'
+BINTRAY_ARCHIVES_DIR = EXPORT_DIR/'BINTRAY_ARCHIVES'
 
 BIN_EXT = '.exe' if PLATFORM.startswith('win32_') else ''
 
@@ -286,6 +288,7 @@ for p in (NIGHTLY_KIWIX_ARCHIVES_DIR,
           RELEASE_ZIM_ARCHIVES_DIR,
           DIST_KIWIX_ARCHIVES_DIR,
           DIST_ZIM_ARCHIVES_DIR,
+          BINTRAY_ARCHIVES_DIR,
           BASE_EXPORT_DIR,
           GIT_EXPORT_DIR):
     try:
@@ -397,7 +400,9 @@ if environ['TRAVIS_EVENT_TYPE'] != 'cron' and not make_release:
 
 
 TARGETS = tuple()
-if PLATFORM.startswith('android'):
+if PLATFORM == 'android':
+    TARGETS = ('kiwix-lib-app',)
+elif PLATFORM.startswith('android_'):
     TARGETS = ('libzim', 'kiwix-lib')
 elif PLATFORM.startswith('iOS'):
     TARGETS = ('libzim', 'kiwix-lib')
@@ -472,12 +477,25 @@ elif PLATFORM == 'i586_static':
     make_archive('kiwix-tools', 'linux-i586')
 elif make_release and PLATFORM == 'flatpak':
     update_flathub_git()
-elif PLATFORM.startswith('android') and 'kiwix-android' in TARGETS:
+elif PLATFORM == 'android' and 'kiwix-lib-app' in TARGETS:
-    APK_NAME = "kiwix-{}".format(PLATFORM)
+    if make_release and release_versions.get(project) is not None:
-    source_debug_dir = HOME/'BUILD_android'/'kiwix-android'/'app'/'build'/'outputs'/'apk'/'kiwix'/'debug'
+        postfix = main_project_versions[project]
-    source_release_dir = HOME/'BUILD_android'/'kiwix-android'/'app'/'build'/'outputs'/'apk'/'kiwix'/'release'
+        extra_postfix = release_versions.get(project)
-    shutil.copy(str(source_debug_dir/'app-kiwix-debug.apk'),
+        if extra_postfix:
-                str(NIGHTLY_KIWIX_ARCHIVES_DIR/"{}-debug.apk".format(APK_NAME)))
+            postfix = "{}-{}".format(postfix, extra_postfix)
-    shutil.copy(str(source_release_dir/'app-kiwix-release-unsigned.apk'),
+
-                str(NIGHTLY_KIWIX_ARCHIVES_DIR/"{}-release_signed".format(APK_NAME)))
+        arr_name = "kiwixlib-{}.aar".format(postfix)
+
+        source_release_dir = HOME/'BUILD_android'/'kiwix-lib-app'/'kiwixLibAndroid'/'build'/'outputs'/'aar'
+        shutil.copy(str(source_release_dir/'kiwixLibAndroid-release.aar'),
+                    str(BINTRAY_ARCHIVES_DIR/arr_name))
+
+        json_filename = '{}_bintray_info.json'.format(arr_name)
+        data = {
+            version: main_project_versions[project],
+            filename : arr_name,
+            artefact: 'kiwixlib-5.1.0.aar'
+        }
+        with open(str(BINTRAY_ARCHIVES_DIR/json_filename), 'w') as f:
+            json.dump(data, f)
 

travis/deploy.sh

@@ -69,7 +69,7 @@ then
     done
   fi
 
-  cd ${EXPORT_DIR}/GIT
+  (cd ${EXPORT_DIR}/GIT
   GIT_REPOS=$(ls -l | awk '/^d/ { print $9 }')
   if [[ "x$GIT_REPOS" != "x" ]]
   then
@@ -80,5 +80,16 @@ then
       )
     done
   fi
+  )
+
+  BINTRAY_ARCHIVES=$(find $BINTRAY_ARCHIVES_DIR/*_bintray_info.json -type f)
+  if [[ "x$BINTRAY_ARCHIVES" != "x" ]]
+  then
+    for archive_info in $BINTRAY_ARCHIVES
+    do
+      ${HOME}/kiwix-build/scripts/upload_kiwix_lib_android_to_bintray.py $archive_info
+    done
+ fi
+
 fi