From 8d58d8d7cbd78e03725d62cd311fe63278735d14 Mon Sep 17 00:00:00 2001 From: birros Date: Thu, 15 Nov 2018 17:29:32 +0100 Subject: [PATCH] First version of a flatpak builder. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit add a first version code to build a flatpak of kiwix-desktop. It is mainly based on the PR #254 of @birros (hence he is the author of this commit) However there is some differences : - I (@mgautier) create a new builder to run the flatpak build instead of using a new dependency. - Use the flatpak platform to install org.kde.Platform and org.kde.Sdk This code version doesn't correctly work but I wanted to commit the birros' code without too many modification (even if there is a lot). --- kiwixbuild/__init__.py | 6 +- kiwixbuild/_global.py | 20 ++ kiwixbuild/builder.py | 3 +- kiwixbuild/dependencies/__init__.py | 1 + kiwixbuild/dependencies/flatpak.py | 37 ++++ kiwixbuild/flatpak_builder.py | 291 ++++++++++++++++++++++++++++ kiwixbuild/platforms/__init__.py | 1 + kiwixbuild/platforms/flatpak.py | 17 ++ kiwixbuild/utils.py | 6 +- 9 files changed, 376 insertions(+), 6 deletions(-) create mode 100644 kiwixbuild/dependencies/flatpak.py create mode 100644 kiwixbuild/flatpak_builder.py create mode 100644 kiwixbuild/platforms/flatpak.py diff --git a/kiwixbuild/__init__.py b/kiwixbuild/__init__.py index fd95d98..5872732 100644 --- a/kiwixbuild/__init__.py +++ b/kiwixbuild/__init__.py @@ -6,6 +6,7 @@ import argparse from .dependencies import Dependency from .platforms import PlatformInfo from .builder import Builder +from .flatpak_builder import FlatpakBuilder from . import _global def parse_args(): @@ -86,6 +87,9 @@ def main(): _global.set_options(options) neutralEnv = buildenv.PlatformNeutralEnv() _global.set_neutralEnv(neutralEnv) - builder = Builder() + if options.target_platform == 'flatpak': + builder = FlatpakBuilder() + else: + builder = Builder() builder.run() diff --git a/kiwixbuild/_global.py b/kiwixbuild/_global.py index 8efafee..c476c09 100644 --- a/kiwixbuild/_global.py +++ b/kiwixbuild/_global.py @@ -1,4 +1,5 @@ from collections import OrderedDict as _OrderedDict +import platform _neutralEnv = None _options = None @@ -30,3 +31,22 @@ def get_target_step(key, default_context=None): def target_steps(): return _target_steps + + +def backend(): + global _backend + if _backend is not None: + return _backend + + _platform = platform.system() + if _platform == 'Windows': + print('ERROR: kiwix-build is not intented to run on Windows platform.\n' + 'There is no backend for Windows, so we can\'t launch any commands.') + sys.exit(0) + if _platform == 'Linux': + _platform, _, _ = platform.linux_distribution() + _platform = _platform.lower() + _backend = backends.Linux() + + return _backend + diff --git a/kiwixbuild/builder.py b/kiwixbuild/builder.py index e8442f8..1b60056 100644 --- a/kiwixbuild/builder.py +++ b/kiwixbuild/builder.py @@ -9,7 +9,8 @@ from .dependencies import Dependency from .packages import PACKAGE_NAME_MAPPERS from ._global import ( neutralEnv, option, - add_target_step, get_target_step, target_steps) + add_target_step, get_target_step, target_steps, + backend) from . import _global class Builder: diff --git a/kiwixbuild/dependencies/__init__.py b/kiwixbuild/dependencies/__init__.py index d1efa7d..d2d8681 100644 --- a/kiwixbuild/dependencies/__init__.py +++ b/kiwixbuild/dependencies/__init__.py @@ -6,6 +6,7 @@ from . import ( android_sdk, armhf, ctpp2, + flatpak, gradle, gumbo, icu4c, diff --git a/kiwixbuild/dependencies/flatpak.py b/kiwixbuild/dependencies/flatpak.py new file mode 100644 index 0000000..c17e71a --- /dev/null +++ b/kiwixbuild/dependencies/flatpak.py @@ -0,0 +1,37 @@ +import os + +from .base import Dependency, NoopSource, Builder +from kiwixbuild.utils import Remotefile, add_execution_right, run_command + +pj = os.path.join + +class org_kde(Dependency): + neutral = False + name = 'org.kde' + version = '5.11' + + Source = NoopSource + + class Builder(Builder): + def _setup_remote(self, context): + command = "flatpak --user remote-add --if-not-exists {remote_name} {remote_url}" + command = command.format( + remote_name = 'flathub', + remote_url = 'https://flathub.org/repo/flathub.flatpakrepo' + ) + run_command(command, self.buildEnv.build_dir, context, buildEnv=self.buildEnv) + + def _install_sdk(self, context): + command = "flatpak --user install -y {remote_name} {name}.Sdk//{version} {name}.Platform//{version}" + command = command.format( + remote_name = 'flathub', + name = self.target.name, + version = self.target.version + ) + run_command(command, self.buildEnv.build_dir, context, buildEnv=self.buildEnv) + + def build(self): + self.command('setup_remote', self._setup_remote) + self.command('install_sdk', self._install_sdk) + + diff --git a/kiwixbuild/flatpak_builder.py b/kiwixbuild/flatpak_builder.py new file mode 100644 index 0000000..1366061 --- /dev/null +++ b/kiwixbuild/flatpak_builder.py @@ -0,0 +1,291 @@ + +import sys +from collections import OrderedDict +from .buildenv import * + +from .platforms import PlatformInfo +from .utils import remove_duplicates, run_command, StopBuild, Context +from .dependencies import Dependency +from .packages import PACKAGE_NAME_MAPPERS +from ._global import ( + neutralEnv, option, + add_target_step, get_target_step, target_steps, + backend) +from . import _global +from .dependencies.base import ( + Source, + Builder, + ReleaseDownload, + GitClone, + MesonBuilder, + CMakeBuilder, + QMakeBuilder, + MakeBuilder, + SCRIPT_DIR) +import json +from shutil import copyfile + +MANIFEST = { + 'id': 'org.kiwix.Client', + 'runtime': 'org.kde.Platform', + 'runtime-version': '5.11', + 'sdk': 'org.kde.Sdk', + 'command': 'kiwix-desktop', + 'finish-args': [ + '--socket=wayland', + '--socket=x11', + '--share=ipc', + '--device=dri', + '--socket=pulseaudio' + ], + 'cleanup': [ + '/include', + '/lib/pkgconfig', + '/lib/cmake', + '/lib/*.la', + '/bin/aria2c', + '/bin/copydatabase', + '/bin/kiwix-compile-resources', + '/bin/quest', + '/bin/simple*', + '/bin/xapian-*', + '/share/aclocal', + '/share/doc', + '/share/man' + ] +} + + + +class FlatpakBuilder: + def __init__(self): + self._targets = {} + PlatformInfo.get_platform('neutral', self._targets) + self.platform = PlatformInfo.get_platform('flatpak', self._targets) + if neutralEnv('distname') not in self.platform.compatible_hosts: + print(('ERROR: The target platform {} cannot be build on host {}.\n' + 'Select another target platform or change your host system.' + ).format(self.platform.name, neutralEnv('distname'))) + self.targetDefs = self.platform.add_targets(option('target'), self._targets) + + def finalize_target_steps(self): + steps = [] + for targetDef in self.targetDefs: + steps += self.order_steps(targetDef) + steps = list(remove_duplicates(steps)) + + for pltName in PlatformInfo.all_running_platforms: + plt = PlatformInfo.all_platforms[pltName] + for tlcName in plt.toolchain_names: + tlc = Dependency.all_deps[tlcName] + src_plt_step = ('source', tlcName) + add_target_step(src_plt_step, self._targets[src_plt_step]) + blt_plt_step = ('neutral' if tlc.neutral else pltName, tlcName) + add_target_step(blt_plt_step, self._targets[blt_plt_step]) + + for dep in steps: + add_target_step(dep, self._targets[dep]) + self.instanciate_steps() + + def order_steps(self, targetDef): + _targets = dict(self._targets) + yield from self.order_dependencies(targetDef, _targets) + + def order_dependencies(self, targetDef, targets): + targetPlatformName, targetName = targetDef + if targetPlatformName == 'source': + # Do not try to order sources, they will be added as dep by the + # build step two lines later. + return + try: + target = targets.pop(targetDef) + except KeyError: + return + + targetPlatform = PlatformInfo.get_platform(targetPlatformName) + for dep in target.get_dependencies(targetPlatform, True): + if isinstance(dep, tuple): + depPlatform, depName = dep + else: + depPlatform, depName = targetPlatformName, dep + if (depPlatform, depName) in targets: + yield from self.order_dependencies((depPlatform, depName), targets) + yield ('source', targetName) + yield targetDef + + def instanciate_steps(self): + for stepDef in list(target_steps()): + stepPlatform, stepName = stepDef + stepClass = Dependency.all_deps[stepName] + if stepPlatform == 'source': + source = get_target_step(stepDef)(stepClass) + add_target_step(stepDef, source) + else: + source = get_target_step(stepName, 'source') + env = PlatformInfo.get_platform(stepPlatform).buildEnv + builder = get_target_step(stepDef)(stepClass, source, env) + add_target_step(stepDef, builder) + + def configure(self): + steps = remove_duplicates(target_steps()) + modules = {} + for stepDef in steps: + module = modules.setdefault(stepDef[1], {}) + module['name'] = stepDef[1] + if stepDef[0] == 'source': + source = get_target_step(stepDef) + module['no-autogen'] = getattr(source, 'flatpack_no_autogen', False) + module_sources = module.setdefault('sources', []) + if isinstance(source, ReleaseDownload): + src = { + 'type': 'archive', + 'sha256': source.archive.sha256, + 'url': source.archive.url + } + if hasattr(source, 'flatpak_dest'): + src['dest'] = source.flatpak_dest + module_sources.append(src) + elif isinstance(source, GitClone): + src = { + 'type': 'git', + 'url': source.git_remote, + 'tag': source.git_ref + } + module_sources.append(src) + for p in getattr(source, 'patches', []): + patch = { + 'type': 'patch', + 'path': 'patches/' + p + } + module_sources.append(patch) + + if hasattr(source, 'flatpak_command'): + command = { + 'type': 'shell', + 'commands': [source.flatpak_command] + } + module_sources.append(command) + + else: + builder = get_target_step(stepDef) + if isinstance(builder, MesonBuilder): + module['buildsystem'] = 'meson' + module['builddir'] = True + elif isinstance(builder, CMakeBuilder): + module['buildsystem'] = 'cmake' + module['builddir'] = True + elif isinstance(builder, QMakeBuilder): + module['buildsystem'] = 'qmake' + # config-opts + print(builder) + if getattr(builder, 'configure_option', ''): + module['config-opts'] = builder.configure_option.split(' ') + + manifest = MANIFEST.copy() + manifest['modules'] = list(modules.values()) + with open(pj(self.platform.buildEnv.build_dir, 'manifest.json'), 'w') as f: + f.write(json.dumps(manifest, indent=4)) + + def copy_patches(self): + sourceDefs = (tDef for tDef in target_steps() if tDef[0] == 'source') + for sourceDef in sourceDefs: + source = get_target_step(sourceDef) + if not hasattr(source, 'patches'): + continue + for p in source.patches: + path = pj(SCRIPT_DIR, 'patches', p) + os.makedirs(pj(self.platform.buildEnv.build_dir, 'patches'), exist_ok=True) + dest = pj(self.platform.buildEnv.build_dir, 'patches', p) + copyfile(path, dest) + + + def build(self): + log = pj(self.platform.buildEnv.log_dir, 'cmd_build_flatpak.log') + context = Context('build', log, False) + command = "flatpak-builder --ccache --force-clean --repo=repo builddir manifest.json" + try: + run_command(command, self.platform.buildEnv.build_dir, context, self.platform.buildEnv) + context._finalise() + except subprocess.CalledProcessError: + try: + with open(log, 'r') as f: + print(f.read()) + except: + pass + + def bundle(self): + log = pj(self.platform.buildEnv.log_dir, 'cmd_bundle_flatpak.log') + context = Context('bundle', log, False) + command = "flatpak build-bundle repo {id}.flatpak {id}" + command = command.format(id = self.target.flatpak['manifest']['id']) + try: + run_command(command, self.platform.buildEnv.build_dir, context, self.platform.buildEnv) + context._finalise() + except subprocess.CalledProcessError: + try: + with open(log, 'r') as f: + print(f.read()) + except: + pass + + + def _get_packages(self): + packages_list = [] + for platform in PlatformInfo.all_running_platforms.values(): + mapper_name = "{host}_{target}".format( + host=neutralEnv('distname'), + target=platform) + package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {}) + packages_list += package_name_mapper.get('COMMON', []) + + to_drop = [] + for builderDef in self._targets: + platformName, builderName = builderDef + mapper_name = "{host}_{target}".format( + host=neutralEnv('distname'), + target=platformName) + package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {}) + packages = package_name_mapper.get(builderName) + if packages: + packages_list += packages + to_drop.append(builderDef) + for dep in to_drop: + del self._targets[dep] + return packages_list + + def run(self): + try: + # This is a small hack, we don't need the list of packages to + # install in a flatpak sdk, but _get_packages() will drop the + # dependencies we already have in the sdk. + self._get_packages() + self.finalize_target_steps() + print("[SETUP PLATFORMS]") + for platform in PlatformInfo.all_running_platforms.values(): + platform.finalize_setup() + for pltName in PlatformInfo.all_running_platforms: + plt = PlatformInfo.all_platforms[pltName] + for tlcName in plt.toolchain_names: + tlc = Dependency.all_deps[tlcName] + builderDef = (pltName, tlcName) + builder = get_target_step(builderDef) + print("build {} ({}):".format(builder.name, pltName[0])) + add_target_step(builderDef, builder) + builder.build() + print("[GENERATE FLATPAK MANIFEST]") + self.configure() + self.copy_patches() + print("[BUILD FLATBACK]") + self.build() + print("[BUNDLE]") + self.bundle() + # No error, clean intermediate file at end of build if needed. + print("[CLEAN]") + if option('clean_at_end'): + for platform in PlatformInfo.all_running_platforms.values(): + platform.clean_intermediate_directories() + else: + print("SKIP") + except StopBuild: + sys.exit("Stopping build due to errors") + diff --git a/kiwixbuild/platforms/__init__.py b/kiwixbuild/platforms/__init__.py index 4b63719..c76338b 100644 --- a/kiwixbuild/platforms/__init__.py +++ b/kiwixbuild/platforms/__init__.py @@ -4,6 +4,7 @@ from .base import * from . import ( android, armhf, + flatpak, i586, ios, native, diff --git a/kiwixbuild/platforms/flatpak.py b/kiwixbuild/platforms/flatpak.py new file mode 100644 index 0000000..8f3ff89 --- /dev/null +++ b/kiwixbuild/platforms/flatpak.py @@ -0,0 +1,17 @@ +from .base import PlatformInfo +from kiwixbuild._global import option, neutralEnv +from kiwixbuild.utils import run_command + +class FlatpakPlatformInfo(PlatformInfo): + name = 'flatpak' + build = 'flatpak' + static = '' + toolchain_names = ['org.kde'] + compatible_hosts = ['debian', 'fedora'] + + def __str__(self): + return "flatpak" + + def set_env(self, env): + env['FLATPAK_USER_DIR'] = self.buildEnv.build_dir + diff --git a/kiwixbuild/utils.py b/kiwixbuild/utils.py index 0b37627..223e69c 100644 --- a/kiwixbuild/utils.py +++ b/kiwixbuild/utils.py @@ -140,12 +140,10 @@ class StopBuild(Exception): class Remotefile(namedtuple('Remotefile', ('name', 'sha256', 'url'))): def __new__(cls, name, sha256, url=None): + if url is None: + url = REMOTE_PREFIX + name return super().__new__(cls, name, sha256, url) - @property - def url(self): - return self.url or (REMOTE_PREFIX + self.name) - class Context: def __init__(self, command_name, log_file, force_native_build):