diff --git a/.travis.yml b/.travis.yml index 9938aec..39a5ffe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,8 +32,13 @@ script: - | if [[ $TRAVIS_OS_NAME = "linux" && $DESKTOP_ONLY == 1 ]] then - docker build -t kiwix/build -f travis/Dockerfile . - docker run -e PLATFORM -e NIGHTLY_DATE -e TRAVIS_EVENT_TYPE -e TRAVIS_BUILD_DIR -e DESKTOP_ONLY -e TRAVIS_TAG --device /dev/fuse --cap-add SYS_ADMIN kiwix/build + if [[ $PLATFORM = "flatpak" ]] + then + docker build -t kiwix/build -f travis/Dockerfile_flatpak . + else + docker build -t kiwix/build -f travis/Dockerfile . + fi + docker run -e PLATFORM -e NIGHTLY_DATE -e TRAVIS_EVENT_TYPE -e TRAVIS_BUILD_DIR -e DESKTOP_ONLY -e TRAVIS_TAG --device /dev/fuse --cap-add ALL kiwix/build else travis/compile_all.py fi @@ -83,6 +88,10 @@ matrix: addons: apt: packages: [] + - env: PLATFORM="flatpak" DESKTOP_ONLY=1 + addons: + apt: + packages: [] - env: PLATFORM="native_static" addons: apt: diff --git a/kiwixbuild/__init__.py b/kiwixbuild/__init__.py index fd95d98..bb7d6c0 100644 --- a/kiwixbuild/__init__.py +++ b/kiwixbuild/__init__.py @@ -6,6 +6,7 @@ import argparse from .dependencies import Dependency from .platforms import PlatformInfo from .builder import Builder +from .flatpak_builder import FlatpakBuilder from . import _global def parse_args(): @@ -37,6 +38,8 @@ def parse_args(): help="Clean all intermediate files after the (successfull) build") subgroup.add_argument('--dont-install-packages', action='store_true', help="Do not try to install packages before compiling") + subgroup.add_argument('--assume-packages-installed', action='store_true', + help="Assume the package to install to be aleady installed") subgroup.add_argument('--android-arch', action='append', help=("Specify the architecture to build for android application/libraries.\n" "Can be specified several times to build for several architectures.\n" @@ -86,6 +89,9 @@ def main(): _global.set_options(options) neutralEnv = buildenv.PlatformNeutralEnv() _global.set_neutralEnv(neutralEnv) - builder = Builder() + if options.target_platform == 'flatpak': + builder = FlatpakBuilder() + else: + builder = Builder() builder.run() diff --git a/kiwixbuild/_global.py b/kiwixbuild/_global.py index 8efafee..c476c09 100644 --- a/kiwixbuild/_global.py +++ b/kiwixbuild/_global.py @@ -1,4 +1,5 @@ from collections import OrderedDict as _OrderedDict +import platform _neutralEnv = None _options = None @@ -30,3 +31,22 @@ def get_target_step(key, default_context=None): def target_steps(): return _target_steps + + +def backend(): + global _backend + if _backend is not None: + return _backend + + _platform = platform.system() + if _platform == 'Windows': + print('ERROR: kiwix-build is not intented to run on Windows platform.\n' + 'There is no backend for Windows, so we can\'t launch any commands.') + sys.exit(0) + if _platform == 'Linux': + _platform, _, _ = platform.linux_distribution() + _platform = _platform.lower() + _backend = backends.Linux() + + return _backend + diff --git a/kiwixbuild/builder.py b/kiwixbuild/builder.py index e8442f8..9df8762 100644 --- a/kiwixbuild/builder.py +++ b/kiwixbuild/builder.py @@ -9,7 +9,8 @@ from .dependencies import Dependency from .packages import PACKAGE_NAME_MAPPERS from ._global import ( neutralEnv, option, - add_target_step, get_target_step, target_steps) + add_target_step, get_target_step, target_steps, + backend) from . import _global class Builder: @@ -148,6 +149,10 @@ class Builder: packages_to_have = self._get_packages() packages_to_have = remove_duplicates(packages_to_have) + if option('assume_packages_installed'): + print("SKIP, Assume package installed") + return + distname = neutralEnv('distname') if distname in ('fedora', 'redhat', 'centos'): package_installer = 'sudo dnf install {}' diff --git a/kiwixbuild/dependencies/__init__.py b/kiwixbuild/dependencies/__init__.py index d1efa7d..c7d13e4 100644 --- a/kiwixbuild/dependencies/__init__.py +++ b/kiwixbuild/dependencies/__init__.py @@ -4,8 +4,10 @@ from . import ( all_dependencies, android_ndk, android_sdk, + aria2, armhf, ctpp2, + flatpak, gradle, gumbo, icu4c, diff --git a/kiwixbuild/dependencies/aria2.py b/kiwixbuild/dependencies/aria2.py new file mode 100644 index 0000000..8432058 --- /dev/null +++ b/kiwixbuild/dependencies/aria2.py @@ -0,0 +1,26 @@ +from .base import ( + Dependency, + ReleaseDownload, + MakeBuilder +) + +from kiwixbuild.utils import Remotefile, run_command + +class Aria2(Dependency): + name = "aria2" + + class Source(ReleaseDownload): + archive = Remotefile('aria2-1.34.0.tar.xz', + '3a44a802631606e138a9e172a3e9f5bcbaac43ce2895c1d8e2b46f30487e77a3', + 'https://github.com/aria2/aria2/releases/download/release-1.34.0/aria2-1.34.0.tar.xz') + + patches = ["libaria2_android.patch"] + + def _post_prepare_script(self, context): + context.try_skip(self.extract_path) + command = "autoreconf -i" + run_command(command, self.extract_path, context) + + class Builder(MakeBuilder): + dependencies = ['zlib'] + configure_option = "--disable-libaria2 --without-sqlite3" diff --git a/kiwixbuild/dependencies/flatpak.py b/kiwixbuild/dependencies/flatpak.py new file mode 100644 index 0000000..c17e71a --- /dev/null +++ b/kiwixbuild/dependencies/flatpak.py @@ -0,0 +1,37 @@ +import os + +from .base import Dependency, NoopSource, Builder +from kiwixbuild.utils import Remotefile, add_execution_right, run_command + +pj = os.path.join + +class org_kde(Dependency): + neutral = False + name = 'org.kde' + version = '5.11' + + Source = NoopSource + + class Builder(Builder): + def _setup_remote(self, context): + command = "flatpak --user remote-add --if-not-exists {remote_name} {remote_url}" + command = command.format( + remote_name = 'flathub', + remote_url = 'https://flathub.org/repo/flathub.flatpakrepo' + ) + run_command(command, self.buildEnv.build_dir, context, buildEnv=self.buildEnv) + + def _install_sdk(self, context): + command = "flatpak --user install -y {remote_name} {name}.Sdk//{version} {name}.Platform//{version}" + command = command.format( + remote_name = 'flathub', + name = self.target.name, + version = self.target.version + ) + run_command(command, self.buildEnv.build_dir, context, buildEnv=self.buildEnv) + + def build(self): + self.command('setup_remote', self._setup_remote) + self.command('install_sdk', self._install_sdk) + + diff --git a/kiwixbuild/dependencies/kiwix_desktop.py b/kiwixbuild/dependencies/kiwix_desktop.py index 66e810a..3bafc5b 100644 --- a/kiwixbuild/dependencies/kiwix_desktop.py +++ b/kiwixbuild/dependencies/kiwix_desktop.py @@ -11,9 +11,11 @@ class KiwixDesktop(Dependency): git_dir = "kiwix-desktop" class Builder(QMakeBuilder): - dependencies = ["qt", "qtwebengine", "kiwix-lib"] + dependencies = ["qt", "qtwebengine", "kiwix-lib", "aria2"] @property def configure_option(self): + if self.buildEnv.platformInfo.name == 'flatpak': + return [] options = ["PREFIX={}".format(self.buildEnv.install_dir)] if self.buildEnv.platformInfo.static: options.append('"CONFIG+=static"') diff --git a/kiwixbuild/dependencies/pugixml.py b/kiwixbuild/dependencies/pugixml.py index 94ac080..57904ae 100644 --- a/kiwixbuild/dependencies/pugixml.py +++ b/kiwixbuild/dependencies/pugixml.py @@ -12,5 +12,6 @@ class Pugixml(Dependency): archive = Remotefile('pugixml-1.2.tar.gz', '0f422dad86da0a2e56a37fb2a88376aae6e931f22cc8b956978460c9db06136b') patches = ["pugixml_meson.patch"] + flatpak_dest = "src" Builder = MesonBuilder diff --git a/kiwixbuild/flatpak_builder.py b/kiwixbuild/flatpak_builder.py new file mode 100644 index 0000000..4f44a61 --- /dev/null +++ b/kiwixbuild/flatpak_builder.py @@ -0,0 +1,283 @@ + +import sys +from collections import OrderedDict +from .buildenv import * + +from .platforms import PlatformInfo +from .utils import remove_duplicates, run_command, StopBuild, Context +from .dependencies import Dependency +from .packages import PACKAGE_NAME_MAPPERS +from ._global import ( + neutralEnv, option, + add_target_step, get_target_step, target_steps, + backend) +from . import _global +from .dependencies.base import ( + Source, + Builder, + ReleaseDownload, + GitClone, + MesonBuilder, + CMakeBuilder, + QMakeBuilder, + MakeBuilder, + SCRIPT_DIR) +import json +from shutil import copyfile + +MANIFEST = { + 'app-id': 'org.kiwix.Client', + 'runtime': 'org.kde.Platform', + 'runtime-version': '5.11', + 'sdk': 'org.kde.Sdk', + 'command': 'kiwix-desktop', + 'rename-desktop-file' : 'kiwix-desktop.desktop', + 'rename-icon': 'kiwix-desktop', + 'finish-args': [ + '--socket=wayland', + '--socket=x11', + '--share=network', + '--share=ipc', + '--device=dri', + '--socket=pulseaudio', + '--filesystem=xdg-data', + ], + 'cleanup': [ + '/include', + '/lib/pkgconfig', + '/lib/cmake', + '/lib/*.la', + '/bin/curl', + '/bin/copydatabase', + '/bin/kiwix-compile-resources', + '/bin/quest', + '/bin/simple*', + '/bin/xapian-*', + '/share/aclocal', + '/share/doc', + '/share/man' + ] +} + + + +class FlatpakBuilder: + def __init__(self): + self._targets = {} + PlatformInfo.get_platform('neutral', self._targets) + self.platform = PlatformInfo.get_platform('flatpak', self._targets) + if neutralEnv('distname') not in self.platform.compatible_hosts: + print(('ERROR: The target platform {} cannot be build on host {}.\n' + 'Select another target platform or change your host system.' + ).format(self.platform.name, neutralEnv('distname'))) + self.targetDefs = self.platform.add_targets(option('target'), self._targets) + + def finalize_target_steps(self): + steps = [] + for targetDef in self.targetDefs: + steps += self.order_steps(targetDef) + steps = list(remove_duplicates(steps)) + + for pltName in PlatformInfo.all_running_platforms: + plt = PlatformInfo.all_platforms[pltName] + for tlcName in plt.toolchain_names: + tlc = Dependency.all_deps[tlcName] + src_plt_step = ('source', tlcName) + add_target_step(src_plt_step, self._targets[src_plt_step]) + blt_plt_step = ('neutral' if tlc.neutral else pltName, tlcName) + add_target_step(blt_plt_step, self._targets[blt_plt_step]) + + for dep in steps: + add_target_step(dep, self._targets[dep]) + self.instanciate_steps() + + def order_steps(self, targetDef): + _targets = dict(self._targets) + yield from self.order_dependencies(targetDef, _targets) + + def order_dependencies(self, targetDef, targets): + targetPlatformName, targetName = targetDef + if targetPlatformName == 'source': + # Do not try to order sources, they will be added as dep by the + # build step two lines later. + return + try: + target = targets.pop(targetDef) + except KeyError: + return + + targetPlatform = PlatformInfo.get_platform(targetPlatformName) + for dep in target.get_dependencies(targetPlatform, True): + if isinstance(dep, tuple): + depPlatform, depName = dep + else: + depPlatform, depName = targetPlatformName, dep + if (depPlatform, depName) in targets: + yield from self.order_dependencies((depPlatform, depName), targets) + yield ('source', targetName) + yield targetDef + + def instanciate_steps(self): + for stepDef in list(target_steps()): + stepPlatform, stepName = stepDef + stepClass = Dependency.all_deps[stepName] + if stepPlatform == 'source': + source = get_target_step(stepDef)(stepClass) + add_target_step(stepDef, source) + else: + source = get_target_step(stepName, 'source') + env = PlatformInfo.get_platform(stepPlatform).buildEnv + builder = get_target_step(stepDef)(stepClass, source, env) + add_target_step(stepDef, builder) + + def configure(self): + steps = remove_duplicates(target_steps()) + modules = {} + for stepDef in steps: + module = modules.setdefault(stepDef[1], {}) + module['name'] = stepDef[1] + if stepDef[0] == 'source': + source = get_target_step(stepDef) + module['no-autogen'] = getattr(source, 'flatpack_no_autogen', False) + module_sources = module.setdefault('sources', []) + if isinstance(source, ReleaseDownload): + src = { + 'type': 'archive', + 'sha256': source.archive.sha256, + 'url': source.archive.url + } + if hasattr(source, 'flatpak_dest'): + src['dest'] = source.flatpak_dest + module_sources.append(src) + elif isinstance(source, GitClone): + src = { + 'type': 'git', + 'url': source.git_remote, + 'tag': source.git_ref + } + module_sources.append(src) + for p in getattr(source, 'patches', []): + patch = { + 'type': 'patch', + 'path': 'patches/' + p + } + module_sources.append(patch) + + if hasattr(source, 'flatpak_command'): + command = { + 'type': 'shell', + 'commands': [source.flatpak_command] + } + module_sources.append(command) + + else: + builder = get_target_step(stepDef) + if isinstance(builder, MesonBuilder): + module['buildsystem'] = 'meson' + module['builddir'] = True + elif isinstance(builder, CMakeBuilder): + module['buildsystem'] = 'cmake' + module['builddir'] = True + elif isinstance(builder, QMakeBuilder): + module['buildsystem'] = 'qmake' + # config-opts + print(builder) + if getattr(builder, 'configure_option', ''): + module['config-opts'] = builder.configure_option.split(' ') + + manifest = MANIFEST.copy() + manifest['modules'] = list(modules.values()) + with open(pj(self.platform.buildEnv.build_dir, 'manifest.json'), 'w') as f: + f.write(json.dumps(manifest, indent=4)) + + def copy_patches(self): + sourceDefs = (tDef for tDef in target_steps() if tDef[0] == 'source') + for sourceDef in sourceDefs: + source = get_target_step(sourceDef) + if not hasattr(source, 'patches'): + continue + for p in source.patches: + path = pj(SCRIPT_DIR, 'patches', p) + os.makedirs(pj(self.platform.buildEnv.build_dir, 'patches'), exist_ok=True) + dest = pj(self.platform.buildEnv.build_dir, 'patches', p) + copyfile(path, dest) + + + def build(self): + log = pj(self.platform.buildEnv.log_dir, 'cmd_build_flatpak.log') + context = Context('build', log, False) + command = "flatpak-builder --user --ccache --force-clean --repo=repo builddir manifest.json" + try: + run_command(command, self.platform.buildEnv.build_dir, context, self.platform.buildEnv) + context._finalise() + except subprocess.CalledProcessError: + try: + with open(log, 'r') as f: + print(f.read()) + except: + pass + + def bundle(self): + log = pj(self.platform.buildEnv.log_dir, 'cmd_bundle_flatpak.log') + context = Context('bundle', log, False) + command = "flatpak build-bundle repo {id}.flatpak {id}" + command = command.format(id = MANIFEST['app-id']) + try: + run_command(command, self.platform.buildEnv.build_dir, context, self.platform.buildEnv) + context._finalise() + except subprocess.CalledProcessError: + try: + with open(log, 'r') as f: + print(f.read()) + except: + pass + + + def _get_packages(self): + package_name_mapper = PACKAGE_NAME_MAPPERS.get('flatpak', {}) + + to_drop = [] + for builderDef in self._targets: + platformName, builderName = builderDef + packages = package_name_mapper.get(builderName) + if packages: + to_drop.append(builderDef) + for dep in to_drop: + del self._targets[dep] + + def run(self): + try: + # This is a small hack, we don't need the list of packages to + # install in a flatpak sdk, but _get_packages() will drop the + # dependencies we already have in the sdk. + self._get_packages() + self.finalize_target_steps() + print("[SETUP PLATFORMS]") + for platform in PlatformInfo.all_running_platforms.values(): + platform.finalize_setup() + for pltName in PlatformInfo.all_running_platforms: + plt = PlatformInfo.all_platforms[pltName] + for tlcName in plt.toolchain_names: + tlc = Dependency.all_deps[tlcName] + builderDef = (pltName, tlcName) + builder = get_target_step(builderDef) + print("build {} ({}):".format(builder.name, pltName[0])) + add_target_step(builderDef, builder) + builder.build() + print("[GENERATE FLATPAK MANIFEST]") + self.configure() + self.copy_patches() + print("[BUILD FLATBACK]") + self.build() + print("[BUNDLE]") + self.bundle() + # No error, clean intermediate file at end of build if needed. + print("[CLEAN]") + if option('clean_at_end'): + for platform in PlatformInfo.all_running_platforms.values(): + platform.clean_intermediate_directories() + else: + print("SKIP") + except StopBuild: + sys.exit("Stopping build due to errors") + diff --git a/kiwixbuild/packages.py b/kiwixbuild/packages.py index c5d2f72..54fb6bd 100644 --- a/kiwixbuild/packages.py +++ b/kiwixbuild/packages.py @@ -3,6 +3,18 @@ _fedora_common = ['automake', 'libtool', 'cmake', 'git', 'subversion', 'ccache', 'pkgconfig', 'gcc-c++', 'gettext-devel'] _debian_common = ['automake', 'libtool', 'cmake', 'git', 'subversion', 'ccache', 'pkg-config', 'gcc', 'autopoint'] PACKAGE_NAME_MAPPERS = { + 'flatpak': { + 'zlib': True, + 'lzma': True, + 'icu4c': True, + 'qt': True, + 'qtwebengine': True, + 'ctpp2': True, + 'ctpp2c': True, + 'uuid': True, + 'libxml2': True, + 'libssl': True, + }, 'fedora_native_dyn': { 'COMMON': _fedora_common, 'uuid': ['libuuid-devel'], @@ -16,6 +28,7 @@ PACKAGE_NAME_MAPPERS = { 'zimlib': None, 'file' : ['file-devel'], 'gumbo' : ['gumbo-parser-devel'], + 'aria2': ['aria2'], }, 'fedora_native_static': { 'COMMON': _fedora_common + ['glibc-static', 'libstdc++-static'], @@ -60,6 +73,7 @@ PACKAGE_NAME_MAPPERS = { 'libmicrohttpd': ['libmicrohttpd-dev', 'ccache'], 'qt' : ['libqt5gui5', 'qtbase5-dev', 'qt5-default'], 'qtwebengine' : ['qtwebengine5-dev'], + 'aria2': ['aria2'], }, 'debian_native_static': { 'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'], diff --git a/kiwixbuild/platforms/__init__.py b/kiwixbuild/platforms/__init__.py index 4b63719..c76338b 100644 --- a/kiwixbuild/platforms/__init__.py +++ b/kiwixbuild/platforms/__init__.py @@ -4,6 +4,7 @@ from .base import * from . import ( android, armhf, + flatpak, i586, ios, native, diff --git a/kiwixbuild/platforms/flatpak.py b/kiwixbuild/platforms/flatpak.py new file mode 100644 index 0000000..8f3ff89 --- /dev/null +++ b/kiwixbuild/platforms/flatpak.py @@ -0,0 +1,17 @@ +from .base import PlatformInfo +from kiwixbuild._global import option, neutralEnv +from kiwixbuild.utils import run_command + +class FlatpakPlatformInfo(PlatformInfo): + name = 'flatpak' + build = 'flatpak' + static = '' + toolchain_names = ['org.kde'] + compatible_hosts = ['debian', 'fedora'] + + def __str__(self): + return "flatpak" + + def set_env(self, env): + env['FLATPAK_USER_DIR'] = self.buildEnv.build_dir + diff --git a/kiwixbuild/utils.py b/kiwixbuild/utils.py index a02d33a..223e69c 100644 --- a/kiwixbuild/utils.py +++ b/kiwixbuild/utils.py @@ -88,7 +88,6 @@ def copy_tree(src, dst, post_copy_function=None): def download_remote(what, where): file_path = pj(where, what.name) - file_url = what.url or (REMOTE_PREFIX + what.name) if os.path.exists(file_path): if what.sha256 == get_sha256(file_path): raise SkipCommand() @@ -104,7 +103,7 @@ def download_remote(what, where): extra_args = {'context':context} if sys.version_info >= (3, 4, 3) else {} progress_chars = "/-\|" try: - with urllib.request.urlopen(file_url, **extra_args) as resource, open(file_path, 'wb') as file: + with urllib.request.urlopen(what.url, **extra_args) as resource, open(file_path, 'wb') as file: tsize = resource.info().get('Content-Length', None) if tsize is not None: tsize = int(tsize) @@ -121,7 +120,7 @@ def download_remote(what, where): current = (current+1)%4 file.write(batch) except urllib.error.URLError as e: - print("Cannot download url {}:\n{}".format(file_url, e.reason)) + print("Cannot download url {}:\n{}".format(what.url, e.reason)) raise StopBuild() if not what.sha256: @@ -141,6 +140,8 @@ class StopBuild(Exception): class Remotefile(namedtuple('Remotefile', ('name', 'sha256', 'url'))): def __new__(cls, name, sha256, url=None): + if url is None: + url = REMOTE_PREFIX + name return super().__new__(cls, name, sha256, url) diff --git a/scripts/create_kiwix-desktop_appImage.sh b/scripts/create_kiwix-desktop_appImage.sh index a2f8c19..23e7873 100755 --- a/scripts/create_kiwix-desktop_appImage.sh +++ b/scripts/create_kiwix-desktop_appImage.sh @@ -8,7 +8,7 @@ APPDIR=${3:-$PWD/AppDir} #TODO We should have our icon ICONFILE=$SOURCEDIR/resources/icons/kiwix/app_icon.svg -DESKTOPFILE=$SOURCEDIR/kiwix.desktop +DESKTOPFILE=$SOURCEDIR/resources/kiwix-desktop.desktop # Create structure mkdir -p $APPDIR/usr/{bin,lib,share} $APPDIR/usr/share/applications $APPDIR/usr/share/icons/hicolor/48x48/apps @@ -24,7 +24,7 @@ cp -a /usr/lib/x86_64-linux-gnu/libc.so* $APPDIR/usr/lib cp -a /usr/lib/x86_64-linux-gnu/libz.so* $APPDIR/usr/lib cp $ICONFILE $APPDIR/usr/share/icons/hicolor/48x48/apps/kiwix-desktop.svg mkdir -p $APPDIR/usr/share/applications -cp $DESKTOPFILE $APPDIR/usr/share/applications/kiwix.desktop +cp $DESKTOPFILE $APPDIR/usr/share/applications/kiwix-desktop.desktop # get the aria2 wget https://github.com/q3aql/aria2-static-builds/releases/download/v1.34.0/aria2-1.34.0-linux-gnu-64bit-build1.tar.bz2 @@ -40,4 +40,4 @@ chmod a+x linuxdeployqt-continuous-x86_64.AppImage # Fix the RPATH of QtWebEngineProcess [TODO] Fill a issue ? patchelf --set-rpath '$ORIGIN/../lib' $APPDIR/usr/libexec/QtWebEngineProcess # Build the image. -./linuxdeployqt-continuous-x86_64.AppImage $APPDIR/usr/share/applications/kiwix.desktop -verbose=3 -bundle-non-qt-libs -extra-plugins=imageformats,iconengines -appimage +./linuxdeployqt-continuous-x86_64.AppImage $APPDIR/usr/share/applications/kiwix-desktop.desktop -verbose=3 -bundle-non-qt-libs -extra-plugins=imageformats,iconengines -appimage diff --git a/travis/Dockerfile b/travis/Dockerfile index c2c4bda..8535e3b 100644 --- a/travis/Dockerfile +++ b/travis/Dockerfile @@ -33,6 +33,7 @@ RUN \ ctpp2-utils \ libctpp2-dev \ libmicrohttpd-dev \ + aria2 \ # Qt packages libqt5gui5 \ qtbase5-dev \ diff --git a/travis/Dockerfile_flatpak b/travis/Dockerfile_flatpak new file mode 100644 index 0000000..8518e8c --- /dev/null +++ b/travis/Dockerfile_flatpak @@ -0,0 +1,63 @@ +FROM ubuntu:bionic + +ENV LANG C.UTF-8 + +RUN \ + apt update -q; \ + apt full-upgrade --purge -q -y; \ + apt install -q -y --no-install-recommends \ +# Base build tools + build-essential \ + automake \ + libtool \ + cmake \ + ccache \ + pkg-config \ + autopoint \ + patch \ + python \ + python3 \ + python3-pip \ + python3-setuptools \ + git \ + subversion \ + wget \ + unzip \ + sudo \ +# Flatpak tools + elfutils \ + flatpak \ + flatpak-builder \ +# Some helper tools + vim \ + less \ + grep \ + openssh-client \ + ; \ + apt-get clean -y; \ + rm -rf \ + /usr/share/doc/* \ + /var/cache/debconf/* + +RUN useradd --create-home travis -G sudo +RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers +USER travis + +WORKDIR /home/travis + +RUN \ + mkdir -p /home/travis/.local/bin ;\ + wget https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip ;\ + unzip ninja-linux.zip ninja ;\ + mv ninja /home/travis/.local/bin ;\ + rm ninja-linux.zip +ENV PATH="/home/travis/.local/bin:${PATH}" + +COPY . kiwix-build/ +RUN sudo chown -R travis:travis /home/travis/kiwix-build +RUN pip3 install --user -e kiwix-build + +ENV TRAVISCI_SSH_KEY /home/travis/kiwix-build/travis/travisci_builder_id_key +ENV TRAVIS_OS_NAME linux_artful + +CMD kiwix-build/travis/compile_all.py && kiwix-build/travis/deploy.sh diff --git a/travis/compile_all.py b/travis/compile_all.py index 6606edb..3ba00b9 100755 --- a/travis/compile_all.py +++ b/travis/compile_all.py @@ -75,6 +75,8 @@ def run_kiwix_build(target, platform, command = ['kiwix-build'] command.append(target) command.append('--hide-progress') + if platform == 'flatpak': + command.append('--assume-packages-installed') if target == 'kiwix-android' and platform.startswith('android_'): command.extend(['--target-platform', 'android', '--android-arch', platform[8:]]) elif platform == 'android': @@ -96,7 +98,7 @@ def run_kiwix_build(target, platform, subprocess.check_call(command, cwd=str(HOME)) -def create_app_image(): +def create_desktop_image(): if make_release: postfix = main_project_versions['kiwix-desktop'] extra_postfix = release_versions.get('kiwix-desktop') @@ -112,19 +114,24 @@ def create_app_image(): archive_dir = NIGHTLY_KIWIX_ARCHIVES_DIR src_dir = SOURCE_DIR/'kiwix-desktop' - command = ['kiwix-build/scripts/create_kiwix-desktop_appImage.sh', - str(BASE_DIR/'INSTALL'), str(src_dir), str(HOME/'AppDir')] - print_message("Build AppImage of kiwix-desktop") - subprocess.check_call(command, cwd=str(HOME)) + if PLATFORM == 'flatpak': + build_path = BASE_DIR/'BUILD_flatpak'/'org.kiwix.Client.flatpak' + app_name = 'org.kiwix.Client.{}.flatpak'.format(postfix) + else: + build_path = HOME/'Kiwix-x86_64.AppImage' + app_name = "kiwix-desktop_x86_64_{}.appimage".format(postfix) + command = ['kiwix-build/scripts/create_kiwix-desktop_appImage.sh', + str(BASE_DIR/'INSTALL'), str(src_dir), str(HOME/'AppDir')] + print_message("Build AppImage of kiwix-desktop") + subprocess.check_call(command, cwd=str(HOME)) try: archive_dir.mkdir(parents=True) except FileExistsError: pass - app_name = "kiwix-desktop_x86_64_{}.appimage".format(postfix) - print_message("Copy AppImage to {}".format(archive_dir/app_name)) - shutil.copy(str(HOME/'Kiwix-x86_64.AppImage'), str(archive_dir/app_name)) + print_message("Copy Build to {}".format(archive_dir/app_name)) + shutil.copy(str(build_path), str(archive_dir/app_name)) def make_archive(project, platform): @@ -242,40 +249,41 @@ for p in (NIGHTLY_KIWIX_ARCHIVES_DIR, make_release = re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", environ.get('TRAVIS_TAG', '')) is not None -# The first thing we need to do is to (potentially) download already compiled base dependencies. -base_dep_archive_name = "base_deps_{os}_{platform}_{version}.tar.gz".format( - os=TRAVIS_OS_NAME, - platform=PLATFORM, - version=base_deps_meta_version) +if PLATFORM != 'flatpak': + # The first thing we need to do is to (potentially) download already compiled base dependencies. + base_dep_archive_name = "base_deps_{os}_{platform}_{version}.tar.gz".format( + os=TRAVIS_OS_NAME, + platform=PLATFORM, + version=base_deps_meta_version) -print_message("Getting archive {}", base_dep_archive_name) -try: - local_filename, _ = urlretrieve( - 'http://tmp.kiwix.org/ci/{}'.format(base_dep_archive_name)) - with tarfile.open(local_filename) as f: - f.extractall(str(HOME)) -except URLError: - print_message("Cannot get archive. Build dependencies") - if PLATFORM == 'android': - for arch in ('arm', 'arm64', 'x86', 'x86_64'): - archive_name = "base_deps_{os}_android_{arch}_{version}.tar.gz".format( - os=TRAVIS_OS_NAME, - arch=arch, - version=base_deps_meta_version) - print_message("Getting archive {}", archive_name) - try: - local_filename, _ = urlretrieve( - 'http://tmp.kiwix.org/ci/{}'.format(archive_name)) - with tarfile.open(local_filename) as f: - f.extractall(str(HOME)) - except URLError: - pass - run_kiwix_build('alldependencies', platform=PLATFORM) - if SSH_KEY.exists(): - archive = make_deps_archive('alldependencies', full=True) - destination = 'ci@tmp.kiwix.org:/data/tmp/ci/{}' - destination = destination.format(base_dep_archive_name) - scp(archive, destination) + print_message("Getting archive {}", base_dep_archive_name) + try: + local_filename, _ = urlretrieve( + 'http://tmp.kiwix.org/ci/{}'.format(base_dep_archive_name)) + with tarfile.open(local_filename) as f: + f.extractall(str(HOME)) + except URLError: + print_message("Cannot get archive. Build dependencies") + if PLATFORM == 'android': + for arch in ('arm', 'arm64', 'x86', 'x86_64'): + archive_name = "base_deps_{os}_android_{arch}_{version}.tar.gz".format( + os=TRAVIS_OS_NAME, + arch=arch, + version=base_deps_meta_version) + print_message("Getting archive {}", archive_name) + try: + local_filename, _ = urlretrieve( + 'http://tmp.kiwix.org/ci/{}'.format(archive_name)) + with tarfile.open(local_filename) as f: + f.extractall(str(HOME)) + except URLError: + pass + run_kiwix_build('alldependencies', platform=PLATFORM) + if SSH_KEY.exists(): + archive = make_deps_archive('alldependencies', full=True) + destination = 'ci@tmp.kiwix.org:/data/tmp/ci/{}' + destination = destination.format(base_dep_archive_name) + scp(archive, destination) # A basic compilation to be sure everything is working (for a PR) @@ -291,6 +299,8 @@ if environ['TRAVIS_EVENT_TYPE'] != 'cron' and not make_release: TARGETS = ('kiwix-desktop', ) else: TARGETS = ('kiwix-tools', 'zim-tools', 'zimwriterfs') + elif PLATFORM == 'flatpak': + TARGETS = ('kiwix-desktop', ) else: TARGETS = ('kiwix-tools', ) @@ -319,6 +329,8 @@ elif PLATFORM.startswith('native_'): TARGETS = ('kiwix-desktop', ) else: TARGETS = ('libzim', 'zimwriterfs', 'zim-tools', 'kiwix-lib', 'kiwix-tools') +elif PLATFORM == 'flatpak': + TARGETS = ('kiwix-desktop', ) else: TARGETS = ('libzim', 'zim-tools', 'kiwix-lib', 'kiwix-tools') @@ -334,7 +346,7 @@ for target in TARGETS: platform=PLATFORM, make_release=make_release) if target == 'kiwix-desktop': - create_app_image() + create_desktop_image() if make_release and PLATFORM == 'native_dyn' and release_versions.get(target) == 0: run_kiwix_build(target, platform=PLATFORM,