From 63003d5bcedc637facc67f2e6811f630eb36787e Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 11 Jun 2018 18:02:29 +0200 Subject: [PATCH 1/6] Correctly extract platform from the dependency. If the `dep` is a two char string (as "qt"), the `platform, name = dep` will split the string and search for 't' dependency in 'q' platform. So, we have to be sure that the dep is a tuple before splitting it. --- kiwixbuild/_global.py | 4 ++-- kiwixbuild/builder.py | 4 ++-- kiwixbuild/platforms/base.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/kiwixbuild/_global.py b/kiwixbuild/_global.py index f513eda..8efafee 100644 --- a/kiwixbuild/_global.py +++ b/kiwixbuild/_global.py @@ -22,9 +22,9 @@ def add_target_step(key, what): _target_steps[key] = what def get_target_step(key, default_context=None): - try: + if isinstance(key, tuple): context, target = key - except ValueError: + else: context, target = default_context, key return _target_steps[(context, target)] diff --git a/kiwixbuild/builder.py b/kiwixbuild/builder.py index 4ee7dd5..9113078 100644 --- a/kiwixbuild/builder.py +++ b/kiwixbuild/builder.py @@ -68,9 +68,9 @@ class Builder: targetPlatform = PlatformInfo.get_platform(targetPlatformName) for dep in target.get_dependencies(targetPlatform, True): - try: + if isinstance(dep, tuple): depPlatform, depName = dep - except ValueError: + else: depPlatform, depName = targetPlatformName, dep if (depPlatform, depName) in targets: yield from self.order_dependencies((depPlatform, depName), targets) diff --git a/kiwixbuild/platforms/base.py b/kiwixbuild/platforms/base.py index 71edc13..4252195 100644 --- a/kiwixbuild/platforms/base.py +++ b/kiwixbuild/platforms/base.py @@ -57,9 +57,9 @@ class PlatformInfo(metaclass=_MetaPlatform): targets[('source', targetName)] = targetClass.Source targets[(self.name, targetName)] = targetClass.Builder for dep in targetClass.Builder.get_dependencies(self, False): - try: + if isinstance(dep, tuple): depPlatformName, depName = dep - except ValueError: + else: depPlatformName, depName = self.name, dep depPlatform = self.get_platform(depPlatformName, targets) depPlatform.add_targets(depName, targets) From cf8f303362e80669b20a98fcd65837e18cc2daae Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 17 May 2018 17:59:44 +0200 Subject: [PATCH 2/6] Add qt(webengine) dependency. Add qt dependency and QMakeBuilder. --- kiwixbuild/buildenv.py | 4 ++ kiwixbuild/dependencies/__init__.py | 1 + kiwixbuild/dependencies/base.py | 24 ++++++++++ kiwixbuild/dependencies/qt.py | 70 +++++++++++++++++++++++++++++ kiwixbuild/packages.py | 4 +- kiwixbuild/versions.py | 4 +- 6 files changed, 105 insertions(+), 2 deletions(-) create mode 100644 kiwixbuild/dependencies/qt.py diff --git a/kiwixbuild/buildenv.py b/kiwixbuild/buildenv.py index 51fc163..1568090 100644 --- a/kiwixbuild/buildenv.py +++ b/kiwixbuild/buildenv.py @@ -138,7 +138,11 @@ class BuildEnv: pj(self.install_dir, self.libprefix) ]) + env['QMAKE_CXXFLAGS'] = " ".join(['-I'+pj(self.install_dir, 'include'), env['QMAKE_CXXFLAGS']]) env['CPPFLAGS'] = " ".join(['-I'+pj(self.install_dir, 'include'), env['CPPFLAGS']]) + env['QMAKE_LFLAGS'] = " ".join(['-L'+pj(self.install_dir, 'lib'), + '-L'+pj(self.install_dir, self.libprefix), + env['QMAKE_LFLAGS']]) env['LDFLAGS'] = " ".join(['-L'+pj(self.install_dir, 'lib'), '-L'+pj(self.install_dir, self.libprefix), env['LDFLAGS']]) diff --git a/kiwixbuild/dependencies/__init__.py b/kiwixbuild/dependencies/__init__.py index 5850450..e2a59ee 100644 --- a/kiwixbuild/dependencies/__init__.py +++ b/kiwixbuild/dependencies/__init__.py @@ -19,6 +19,7 @@ from . import ( libmicrohttpd, libzim, lzma, + qt, pugixml, uuid, xapian, diff --git a/kiwixbuild/dependencies/base.py b/kiwixbuild/dependencies/base.py index 13d6de5..cd3453e 100644 --- a/kiwixbuild/dependencies/base.py +++ b/kiwixbuild/dependencies/base.py @@ -369,6 +369,30 @@ class CMakeBuilder(MakeBuilder): run_command(command, self.build_path, context, env=env, buildEnv=self.buildEnv, cross_env_only=True) +class QMakeBuilder(MakeBuilder): + qmake_target = "" + def _configure(self, context): + context.try_skip(self.build_path) + cross_option = "" + command = ("qmake {configure_option}" + " {source_path}" + " {cross_option}") + command = command.format( + configure_option=self.configure_option, + source_path=self.source_path, + cross_option=cross_option + ) + run_command(command, self.build_path, context, buildEnv=self.buildEnv, cross_env_only=True) + + def _install(self, context): + context.try_skip(self.build_path) + command = "make {make_install_target} {make_option}".format( + make_install_target=self.make_install_target, + make_option=self.make_option + ) + run_command(command, self.build_path, context, buildEnv=self.buildEnv) + + class MesonBuilder(Builder): configure_option = "" test_option = "" diff --git a/kiwixbuild/dependencies/qt.py b/kiwixbuild/dependencies/qt.py new file mode 100644 index 0000000..d6f0883 --- /dev/null +++ b/kiwixbuild/dependencies/qt.py @@ -0,0 +1,70 @@ +import shutil + +from .base import ( + Dependency, + ReleaseDownload, + MakeBuilder, + QMakeBuilder) + +from kiwixbuild.utils import Remotefile, pj, SkipCommand + + +class Qt(Dependency): + name = 'qt' + + class Source(ReleaseDownload): + name = "qt" + source_dir = "qt-5.10.1" + archive = Remotefile('qt-everywhere-src-5.10.1.tar.xz', + '', + 'http://ftp1.nluug.nl/languages/qt/archive/qt/5.10/5.10.1/single/qt-everywhere-src-5.10.1.tar.xz') + + class Builder(MakeBuilder): + dependencies = ['icu4c', 'zlib'] + configure_option_template = "{dep_options} {static_option} {env_option} -prefix {install_dir} -libdir {libdir}" + dynamic_configure_option = "-shared" + static_configure_option = "-static" + + @property + def configure_option(self): + skip_modules = [ + 'qt3d', + 'qtcanvas3d', + 'qtcharts', + 'qtconnectivity', + 'qtdatavis3d', + # 'qtdeclarative', + 'qtdoc', + 'qtgamepad', + 'qtgraphicaleffects', + 'qtlocation', + 'qtmultimedia', + 'qtnetworkauth', + 'qtpurchasing', + # 'qtquickcontrols', + 'qtquickcontrols2', + 'qtremoteobjects', + 'qtscript', + 'qtscxml', + 'qtsensors', + 'qtserialbus', + 'qtserialport', + 'qtspeech', + 'qtvirtualkeyboard', + 'qtwayland', + 'qtwebglplugin', + 'qtwebsockets', +# 'qtwebview', +] + skip_modules = " ".join("-skip {}".format(m) for m in skip_modules) + options = "-recheck -opensource -confirm-license -ccache -make libs {}".format(skip_modules) + return options + +class QtWebEngine(Dependency): + name = "qtwebengine" + + Source = Qt.Source + + class Builder(QMakeBuilder): + dependencies = ['qt'] + subsource_dir = "qtwebengine" diff --git a/kiwixbuild/packages.py b/kiwixbuild/packages.py index 6d33089..d12ec57 100644 --- a/kiwixbuild/packages.py +++ b/kiwixbuild/packages.py @@ -57,7 +57,9 @@ PACKAGE_NAME_MAPPERS = { 'uuid': ['uuid-dev'], 'ctpp2': ['libctpp2-dev'], 'ctpp2c': ['ctpp2-utils'], - 'libmicrohttpd': ['libmicrohttpd-dev', 'ccache'] + 'libmicrohttpd': ['libmicrohttpd-dev', 'ccache'], + 'qt' : ['libqt5gui5', 'qtbase5-dev', 'qt5-default'], + 'qtwebengine' : ['qtwebengine5-dev'] }, 'debian_native_static': { 'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'], diff --git a/kiwixbuild/versions.py b/kiwixbuild/versions.py index b29ed25..bc3e7e4 100644 --- a/kiwixbuild/versions.py +++ b/kiwixbuild/versions.py @@ -27,5 +27,7 @@ base_deps_versions = { 'libaria2' : '1.33.1', 'libmagic' : '5.33', 'android-sdk' : 'r25.2.3', - 'android-ndk' : 'r13b' + 'android-ndk' : 'r13b', + 'qt' : '5.10.1', + 'qtwebengine' : '5.10.1', } From 398f77f9f3f3911086aed7092b29b04346b7b79d Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 17 May 2018 18:00:07 +0200 Subject: [PATCH 3/6] Add kiwix-desktop dependency. --- kiwixbuild/dependencies/__init__.py | 1 + kiwixbuild/dependencies/kiwix_desktop.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 kiwixbuild/dependencies/kiwix_desktop.py diff --git a/kiwixbuild/dependencies/__init__.py b/kiwixbuild/dependencies/__init__.py index e2a59ee..915967e 100644 --- a/kiwixbuild/dependencies/__init__.py +++ b/kiwixbuild/dependencies/__init__.py @@ -12,6 +12,7 @@ from . import ( ios_fat_lib, kiwix_android, kiwix_custom_app, + kiwix_desktop, kiwix_lib, kiwix_tools, libaria2, diff --git a/kiwixbuild/dependencies/kiwix_desktop.py b/kiwixbuild/dependencies/kiwix_desktop.py new file mode 100644 index 0000000..d309310 --- /dev/null +++ b/kiwixbuild/dependencies/kiwix_desktop.py @@ -0,0 +1,20 @@ +from .base import ( + Dependency, + GitClone, + QMakeBuilder) + +class KiwixDesktop(Dependency): + name = "kiwix-desktop" + dependencies = ["qt", "qtwebengine", "kiwix-lib"] + + class Source(GitClone): + git_remote = "https://github.com/kiwix/kiwix-desktop.git" + git_dir = "kiwix-desktop" + + class Builder(QMakeBuilder): + @property + def configure_option(self): + options = ["PREFIX={}".format(self.buildEnv.install_dir)] + if self.buildEnv.platformInfo.static: + options.append('"CONFIG+=static"') + return " ".join(options) From 8c3a8be65d1eb363c6aab48fbbbb598eb29f6cfd Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 4 Jun 2018 19:16:55 +0200 Subject: [PATCH 4/6] Build kiwix-desktop in travis. --- Dockerfile | 5 +++++ travis/compile_all.py | 7 ++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4ed82f4..70f8218 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,6 +40,11 @@ RUN \ ctpp2-utils \ libctpp2-dev \ libmicrohttpd-dev \ +# Qt packages + libqt5gui5 \ + qtbase5-dev \ + qtwebengine5-dev \ + qt5-default \ # Some helper tools vim \ less \ diff --git a/travis/compile_all.py b/travis/compile_all.py index f73af5d..a84c6eb 100755 --- a/travis/compile_all.py +++ b/travis/compile_all.py @@ -211,6 +211,8 @@ if environ['TRAVIS_EVENT_TYPE'] != 'cron' and not make_release: elif PLATFORM.startswith('native_'): if TRAVIS_OS_NAME == "osx": TARGETS = ('kiwix-lib', 'zim-tools', 'zimwriterfs') + elif PLATFORM == 'native_dyn': + TARGETS = ('kiwix-tools', 'kiwix-desktop', 'zim-tools', 'zimwriterfs') else: TARGETS = ('kiwix-tools', 'zim-tools', 'zimwriterfs') else: @@ -235,7 +237,10 @@ elif PLATFORM.startswith('native_'): if TRAVIS_OS_NAME == "osx": TARGETS = ('libzim', 'zimwriterfs', 'zim-tools', 'kiwix-lib') else: - TARGETS = ('libzim', 'zimwriterfs', 'zim-tools', 'kiwix-lib', 'kiwix-tools') + if make_release: + TARGETS = ('libzim', 'zimwriterfs', 'zim-tools', 'kiwix-lib', 'kiwix-tools') + else: + TARGETS = ('libzim', 'kiwix-lib', 'kiwix-desktop', 'zimwriterfs', 'zim-tools', 'kiwix-tools') else: TARGETS = ('libzim', 'zim-tools', 'kiwix-lib', 'kiwix-tools') From 894ab4ba06daa9e78072cefdf04c5e132165df34 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 12 Jun 2018 16:09:56 +0200 Subject: [PATCH 5/6] Add a script to package kiwix-desktop in a AppImage. AppImage is a tools to create application running in all linux. See https://appimage.org/ --- scripts/create_kiwix-desktop_appImage.sh | 42 ++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100755 scripts/create_kiwix-desktop_appImage.sh diff --git a/scripts/create_kiwix-desktop_appImage.sh b/scripts/create_kiwix-desktop_appImage.sh new file mode 100755 index 0000000..f7064df --- /dev/null +++ b/scripts/create_kiwix-desktop_appImage.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -e + +INSTALLDIR=${1:-$PWD/BUILD_native_dyn/INSTALL} +APPDIR=${2:-$PWD/AppDir} + +#TODO We should have our icon +ICONFILE=/usr/share/icons/hicolor/48x48/apps/gvim.png + +# Create structure +mkdir -p $APPDIR/usr/{bin,lib,share} $APPDIR/usr/share/applications $APPDIR/usr/share/icons/hicolor/48x48/apps +# Copy our files +cp $INSTALLDIR/bin/kiwix-desktop $APPDIR/usr/bin/ +cp $INSTALLDIR/lib/x86_64-linux-gnu/*.so* $APPDIR/usr/lib +# Remove it as it break with linuxdeployqt (should we compile without it) ? +rm $APPDIR/usr/lib/libmagic.so* +# Copy nss lib (to not conflict with host's ones) +cp -a /usr/lib/x86_64-linux-gnu/nss $APPDIR/usr/lib +cp $ICONFILE $APPDIR/usr/share/icons/hicolor/48x48/apps/kiwix-desktop.png +# TODO we should have our .desktop +tee $APPDIR/usr/share/applications/kiwix-desktop.desktop < /dev/null +[Desktop Entry] +Name=Kiwix +Exec=kiwix-desktop %F +Icon=kiwix-desktop +Type=Application +Terminal=false +StartupNotify=true +NoDisplay=false +EOF + +# Get linuxdeployqt +wget https://github.com/probonopd/linuxdeployqt/releases/download/continuous/linuxdeployqt-continuous-x86_64.AppImage +chmod a+x linuxdeployqt-continuous-x86_64.AppImage + +# Fill with all deps libs and so +./linuxdeployqt-continuous-x86_64.AppImage $APPDIR/usr/bin/kiwix-desktop -verbose=3 -bundle-non-qt-libs +# Fix the RPATH of QtWebEngineProcess [TODO] Fill a issue ? +patchelf --set-rpath '$ORIGIN/../lib' $APPDIR/usr/libexec/QtWebEngineProcess +# Build the image. +./linuxdeployqt-continuous-x86_64.AppImage $APPDIR/usr/share/applications/kiwix-desktop.desktop -verbose=3 -bundle-non-qt-libs -appimage # From 269e2b39a65143cf7424a3c80e302c95966d85c6 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 12 Jun 2018 16:11:22 +0200 Subject: [PATCH 6/6] Build the kiwix-desktop app image in Travis. --- .travis.yml | 2 +- Dockerfile | 4 ++++ travis/compile_all.py | 24 ++++++++++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f000177..575b15d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,7 @@ cache: install: travis/install_extra_deps.sh script: - if [ $TRAVIS_OS_NAME == "osx" ] || [ $PLATFORM != "native_dyn" ]; then travis/compile_all.py; fi -- if [ $TRAVIS_OS_NAME == "linux" ] && [ $PLATFORM == "native_dyn" ]; then docker build -t kiwix/build . && docker run -e PLATFORM -e NIGHTLY_DATE -e TRAVIS_EVENT_TYPE -e TRAVIS_BUILD_DIR kiwix/build; fi +- if [ $TRAVIS_OS_NAME == "linux" ] && [ $PLATFORM == "native_dyn" ]; then docker build -t kiwix/build . && docker run -e PLATFORM -e NIGHTLY_DATE -e TRAVIS_EVENT_TYPE -e TRAVIS_BUILD_DIR --device /dev/fuse --cap-add SYS_ADMIN kiwix/build; fi after_failure: travis/upload_all_log.sh deploy: - provider: script diff --git a/Dockerfile b/Dockerfile index 70f8218..c213dda 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,6 +45,10 @@ RUN \ qtbase5-dev \ qtwebengine5-dev \ qt5-default \ +# To create the appimage of kiwix-desktop + libfuse2 \ + fuse \ + patchelf \ # Some helper tools vim \ less \ diff --git a/travis/compile_all.py b/travis/compile_all.py index a84c6eb..f12db82 100755 --- a/travis/compile_all.py +++ b/travis/compile_all.py @@ -85,6 +85,28 @@ def run_kiwix_build(target, platform, build_deps_only=False, make_release=False, subprocess.check_call(command, cwd=str(HOME)) +def create_app_image(): + command = ['kiwix-build/scripts/create_kiwix-desktop_appImage.sh', + str(BASE_DIR/'INSTALL'), str(HOME/'AppDir')] + print_message("Build AppImage of kiwix-desktop") + subprocess.check_call(command, cwd=str(HOME)) + if make_release: + postfix = main_project_versions['kiwix-desktop'] + archive_dir = RELEASE_KIWIX_ARCHIVES_DIR/'kiwix-desktop' + else: + postfix = _date + archive_dir = NIGHTLY_KIWIX_ARCHIVES_DIR + + try: + archive_dir.mkdir(parents=True) + except FileExistsError: + pass + + app_name = "kiwix-desktop_x86_64_{}".format(postfix) + print_message("Copy AppImage to {}".format(archive_dir/app_name)) + shutil.copy(str(HOME/'Kiwix-x86_64.AppImage'), str(archive_dir/app_name)) + + def make_archive(project, platform): file_to_archives = BINARIES[project] base_bin_dir = BASE_DIR/'INSTALL'/'bin' @@ -255,6 +277,8 @@ for target in TARGETS: run_kiwix_build(target, platform=PLATFORM, make_release=make_release) + if target == 'kiwix-desktop': + create_app_image() if make_release and PLATFORM == 'native_dyn': run_kiwix_build(target, platform=PLATFORM,