Merge pull request #20 from kiwix/android_apk

Android apk

.gitignore

@@ -2,3 +2,4 @@ ARCHIVE
 BUILD_*
 LOGS
 SOURCE
+TOOLCHAINS

dependencies.py

@@ -1,4 +1,4 @@
-import shutil
+import shutil,os
 
 from dependency_utils import (
     Dependency,
@@ -6,9 +6,11 @@ from dependency_utils import (
     GitClone,
     MakeBuilder,
     CMakeBuilder,
-    MesonBuilder)
+    MesonBuilder,
+    GradleBuilder,
+    Builder as BaseBuilder)
 
-from utils import Remotefile, pj, SkipCommand
+from utils import Remotefile, pj, SkipCommand, copy_tree, add_execution_right
 
 # *************************************
 # Missing dependencies
@@ -318,3 +320,42 @@ class KiwixTools(Dependency):
             if self.buildEnv.platform_info.static:
                 return "-Dstatic-linkage=true"
             return ""
+
+
+class Gradle(Dependency):
+    name = "Gradle"
+    version = "3.4"
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('gradle-3.4-bin.zip',
+                             '72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205',
+                             'https://services.gradle.org/distributions/gradle-3.4-bin.zip')
+
+    class Builder(BaseBuilder):
+        def build(self):
+            self.command('install', self._install)
+
+        def _install(self, context):
+            copy_tree(
+                pj(self.source_path, "bin"),
+                pj(self.buildEnv.install_dir, "bin"),
+                post_copy_function = add_execution_right)
+            copy_tree(
+                pj(self.source_path, "lib"),
+                pj(self.buildEnv.install_dir, "lib"))
+
+
+class KiwixAndroid(Dependency):
+    name = "kiwix-android"
+    dependencies = ["Gradle", "kiwix-lib"]
+
+    class Source(GitClone):
+        git_remote = "https://github.com/kiwix/kiwix-android"
+        git_dir = "kiwix-android"
+
+    class Builder(GradleBuilder):
+        def _configure(self, context):
+            if not os.path.exists(self.build_path):
+                shutil.copytree(self.source_path, self.build_path)
+            shutil.rmtree(pj(self.build_path, 'kiwixlib', 'src', 'main'))
+            shutil.copytree(pj(self.buildEnv.install_dir, 'kiwix-lib'), pj(self.build_path, 'kiwixlib', 'src', 'main'))

dependency_utils.py

@@ -110,7 +110,7 @@ class ReleaseDownload(Source):
         context.force_native_build = True
         for p in self.patches:
             with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input:
-                self.buildEnv.run_command("patch -p1", self.extract_path, context, input=patch_input)
+                self.buildEnv.run_command("patch -p1", self.extract_path, context, input=patch_input.read())
 
     def prepare(self):
         self.command('download', self._download)
@@ -305,3 +305,28 @@ class MesonBuilder(Builder):
     def _install(self, context):
         command = "{} -v install".format(self.buildEnv.ninja_command)
         self.buildEnv.run_command(command, self.build_path, context)
+
+
+class GradleBuilder(Builder):
+    def build(self):
+        self.command('configure', self._configure)
+        if hasattr(self, '_pre_compile_script'):
+            self.command('pre_compile_script', self._pre_compile_script)
+        self.command('compile', self._compile)
+        self.command('install', self._install)
+
+    def _configure(self, context):
+        # We don't have a lot to configure by itself
+        context.try_skip(self.build_path)
+        if os.path.exists(self.build_path):
+            shutil.rmtree(self.build_path)
+        shutil.copytree(self.source_path, self.build_path)
+
+    def _compile(self, context):
+        command = "gradle clean assemble --info"
+        self.buildEnv.run_command(command, self.build_path, context)
+        command = "gradle build  --info"
+        self.buildEnv.run_command(command, self.build_path, context)
+
+    def _install(self, context):
+        pass

kiwix-build.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-import os, sys, stat
+import os, sys, shutil
 import argparse
 import ssl
 import urllib.request
@@ -13,6 +13,7 @@ from dependency_utils import ReleaseDownload, Builder
 from utils import (
     pj,
     remove_duplicates,
+    add_execution_right,
     get_sha256,
     StopBuild,
     SkipCommand,
@@ -38,7 +39,7 @@ CROSS_ENV = {
         }
     },
     'fedora_android': {
-        'toolchain_names': ['android_ndk'],
+        'toolchain_names': ['android_ndk', 'android_sdk'],
         'extra_libs': [],
         'extra_cflags': [],
         'host_machine': {
@@ -61,7 +62,7 @@ CROSS_ENV = {
         }
     },
     'debian_android': {
-        'toolchain_names': ['android_ndk'],
+        'toolchain_names': ['android_ndk', 'android_sdk'],
         'extra_libs': [],
         'extra_cflags': [],
         'host_machine': {
@@ -189,11 +190,13 @@ class BuildEnv:
         build_dir = "BUILD_{}".format(options.target_platform)
         self.build_dir = pj(options.working_dir, build_dir)
         self.archive_dir = pj(options.working_dir, "ARCHIVE")
+        self.toolchain_dir = pj(options.working_dir, "TOOLCHAINS")
         self.log_dir = pj(self.build_dir, 'LOGS')
         self.install_dir = pj(self.build_dir, "INSTALL")
         for d in (self.source_dir,
                   self.build_dir,
                   self.archive_dir,
+                  self.toolchain_dir,
                   self.log_dir,
                   self.install_dir):
             os.makedirs(d, exist_ok=True)
@@ -210,6 +213,16 @@ class BuildEnv:
         self.libprefix = options.libprefix or self._detect_libdir()
         self.targetsDict = targetsDict
 
+    def clean_intermediate_directories(self):
+        for subdir in os.listdir(self.build_dir):
+            subpath = pj(self.build_dir, subdir)
+            if subpath == self.install_dir:
+                continue
+            if os.path.isdir(subpath):
+                shutil.rmtree(subpath)
+            else:
+                os.remove(subpath)
+
     def detect_platform(self):
         _platform = platform.system()
         self.distname = _platform
@@ -397,8 +410,13 @@ class BuildEnv:
 
             kwargs = dict()
             if input:
-                kwargs['stdin'] = input
+                kwargs['stdin'] = subprocess.PIPE
-            return subprocess.check_call(command, shell=True, cwd=cwd, env=env, stdout=log or sys.stdout, stderr=subprocess.STDOUT, **kwargs)
+            process = subprocess.Popen(command, shell=True, cwd=cwd, env=env, stdout=log or sys.stdout, stderr=subprocess.STDOUT, **kwargs)
+            if input:
+                process.communicate(input.encode())
+            retcode = process.wait()
+            if retcode:
+                raise subprocess.CalledProcessError(retcode, command)
         finally:
             if log:
                 log.close()
@@ -662,8 +680,7 @@ class android_ndk(Toolchain):
         def _build_platform(self, context):
             context.try_skip(self.build_path)
             script = pj(self.source_path, 'build/tools/make_standalone_toolchain.py')
-            current_permissions = stat.S_IMODE(os.lstat(script).st_mode)
+            add_execution_right(script)
-            os.chmod(script, current_permissions | stat.S_IXUSR)
             command = '{script} --arch={arch} --api={api} --install-dir={install_dir} --force'
             command = command.format(
                 script=script,
@@ -687,8 +704,7 @@ class android_ndk(Toolchain):
                     file_path = pj(root, file_)
                     if os.path.islink(file_path):
                         continue
-                    current_permissions = stat.S_IMODE(os.lstat(file_path).st_mode)
+                    add_execution_right(file_path)
-                    os.chmod(file_path, current_permissions | stat.S_IXUSR)
 
         def build(self):
             self.command('build_platform', self._build_platform)
@@ -713,6 +729,56 @@ class android_ndk(Toolchain):
         env['NDK_DEBUG'] = '0'
 
 
+class android_sdk(Toolchain):
+    name = 'android-sdk'
+    version = 'r25.2.3'
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('tools_r25.2.3-linux.zip',
+                             '1b35bcb94e9a686dff6460c8bca903aa0281c6696001067f34ec00093145b560',
+                             'https://dl.google.com/android/repository/tools_r25.2.3-linux.zip')
+
+    class Builder(Builder):
+
+        @property
+        def install_path(self):
+            return pj(self.buildEnv.toolchain_dir, self.target.full_name)
+
+        def _build_platform(self, context):
+            context.try_skip(self.install_path)
+            tools_dir = pj(self.install_path, 'tools')
+            shutil.copytree(self.source_path, tools_dir)
+            script = pj(tools_dir, 'android')
+            command = '{script} --verbose update sdk -a --no-ui --filter {packages}'
+            command = command.format(
+                script=script,
+                packages = ','.join(str(i) for i in [1,2,8,34,162])
+            )
+            # packages correspond to :
+            # - 1 : Android SDK Tools, revision 25.2.5
+            # - 2 : Android SDK Platform-tools, revision 25.0.3
+            # - 8 : Android SDK Build-tools, revision 24.0.1
+            # - 34 : SDK Platform Android 7.0, API 24, revision 2
+            # - 162 : Android Support Repository, revision 44
+            self.buildEnv.run_command(command, self.install_path, context, input="y\n")
+
+        def _fix_licenses(self, context):
+            context.try_skip(self.install_path)
+            os.makedirs(pj(self.install_path, 'licenses'), exist_ok=True)
+            with open(pj(self.install_path, 'licenses', 'android-sdk-license'), 'w') as f:
+                f.write("\n8933bad161af4178b1185d1a37fbf41ea5269c55")
+
+        def build(self):
+            self.command('build_platform', self._build_platform)
+            self.command('fix_licenses', self._fix_licenses)
+
+    def get_bin_dir(self):
+        return []
+
+    def set_env(self, env):
+        env['ANDROID_HOME'] = self.builder.install_path
+
+
 class Builder:
     def __init__(self, options):
         self.options = options
@@ -781,6 +847,12 @@ class Builder:
             self.prepare_sources()
             print("[BUILD]")
             self.build()
+            # No error, clean intermediate file at end of build if needed.
+            print("[CLEAN]")
+            if self.buildEnv.options.clean_at_end:
+                self.buildEnv.clean_intermediate_directories()
+            else:
+                print("SKIP")
         except StopBuild:
             sys.exit("Stopping build due to errors")
 
@@ -801,6 +873,8 @@ def parse_args():
                         help="Skip the source download part")
     parser.add_argument('--build-deps-only', action='store_true',
                         help=("Build only the dependencies of the specified targets."))
+    parser.add_argument('--clean-at-end', action='store_true',
+                        help="Clean all intermediate files after the (successfull) build")
 
     return parser.parse_args()
 

travis/compile_all.sh

@@ -14,7 +14,7 @@ if [[ "$TRAVIS_EVENT_TYPE" = "cron" ]]
 then
   if [[ ${PLATFORM} = android* ]]
   then
-    TARGETS="libzim kiwix-lib"
+    TARGETS="libzim kiwix-lib kiwix-android"
   else
     TARGETS="libzim kiwix-lib kiwix-tools"
   fi
@@ -71,13 +71,18 @@ EOF
         tar -czf "${NIGHTLY_ARCHIVES_DIR}/$ARCHIVE_NAME" $FILES_LIST
       )
       ;;
+    android_*)
+      APK_NAME="kiwix-${PLATFORM}"
+      cp ${BASE_DIR}/kiwix-android/app/build/outputs/apk/app-kiwix-debug.apk ${NIGHTLY_ARCHIVES_DIR}/${APK_NAME}-debug.apk
+      cp ${BASE_DIR}/kiwix-android/app/build/outputs/apk/app-kiwix-release-unsigned.apk ${NIGHTLY_ARCHIVES_DIR}/${APK_NAME}-release-unsigned.apk
+      ;;
   esac
 
 else
   # No a cron job, we just have to build to be sure nothing is broken.
   if [[ ${PLATFORM} = android* ]]
   then
-    TARGET=kiwix-lib
+    TARGET=kiwix-android
   else
     TARGET=kiwix-tools
   fi

utils.py

@@ -2,7 +2,8 @@ import os.path
 import hashlib
 import tarfile, zipfile
 import tempfile
-import os
+import shutil
+import os, stat
 from collections import namedtuple, defaultdict
 
 pj = os.path.join
@@ -32,6 +33,22 @@ def get_sha256(path):
     return sha256.hexdigest()
 
 
+def add_execution_right(file_path):
+    current_permissions = stat.S_IMODE(os.lstat(file_path).st_mode)
+    os.chmod(file_path, current_permissions | stat.S_IXUSR)
+
+def copy_tree(src, dst, post_copy_function=None):
+    os.makedirs(dst, exist_ok=True)
+    for root, dirs, files in os.walk(src):
+        r = os.path.relpath(root, src)
+        dstdir = pj(dst, r)
+        os.makedirs(dstdir, exist_ok=True)
+        for f in files:
+            dstfile = pj(dstdir, f)
+            shutil.copy2(pj(root, f), dstfile)
+            if post_copy_function is not None:
+                post_copy_function(dstfile)
+
 class SkipCommand(Exception):
     pass