smallsolar
/
kiwix-build
mirror of https://github.com/kiwix/kiwix-build.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #318 from kiwix/podman 

Docker
This commit is contained in:
Matthieu Gautier 2019-05-03 17:05:06 +02:00 committed by GitHub
parent 88bb9231ea 0142dc94d4
commit 6eb4c5a8b6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 178 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

169
.travis.yml
View File

@ -1,6 +1,6 @@
language: cpp
dist: trusty
dist: xenial
sudo: required
services:
- docker
@ -10,13 +10,51 @@ branches:
- /\d+\.\d+\.\d+$/
if: type != push OR tag IS present
stages:
- build_docker_images
- build
- android_build
jobs:
include:
- stage: build_docker_images
script: &build_images
- |
if [ ! $(curl -sflL https://hub.docker.com/v2/repositories/kiwix/kiwix-build_ci/tags/${VARIANT}-${TRAVIS_COMMIT}) ]
then
echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
docker build -t kiwix-build_ci -f travis/${VARIANT}_builder.dockerfile .
docker tag kiwix-build_ci kiwix/kiwix-build_ci:${VARIANT}-${TRAVIS_COMMIT}
docker push kiwix/kiwix-build_ci:${VARIANT}-${TRAVIS_COMMIT}
fi
env: VARIANT=xenial
- stage: build_docker_images
script: *build_images
env: VARIANT=flatpak
- stage: build_docker_images
script: *build_images
env: VARIANT=appimage
- stage: android_build
env: PLATFORM="android"
if: type != pull_request
- env: PLATFORM="native_dyn"
os: osx
- env: PLATFORM="iOS_arm64"
os: osx
- env: PLATFORM="iOS_i386"
os: osx
before_install:
- PATH=$PATH:${HOME}/bin:${HOME}/.local/bin
- mkdir -p /tmp/private $HOME/EXPORT/BASE
- chmod -R a+w $HOME/EXPORT
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then PATH=$PATH:$(brew --prefix)/opt/gettext/bin; fi
- '[ -n "$encrypted_eba2f7543984_iv" ] && openssl aes-256-cbc
-K $encrypted_eba2f7543984_key -iv $encrypted_eba2f7543984_iv
-in travis/travisci_builder_id_key.enc -out travis/travisci_builder_id_key -d'
- chmod 600 travis/travisci_builder_id_key
-in travis/travisci_builder_id_key.enc -out $SSH_KEY -d'
- chmod 600 $SSH_KEY
before_cache:
- rm -f $HOME/.gradle/caches/modules-2/modules-2.lock
- rm -fr $HOME/.gradle/caches/*/plugin-resolution/
@ -27,27 +65,33 @@ cache:
- $HOME/.gradle/caches/
- $HOME/.gradle/wrapper/
- $HOME/.android/build-cache
install: if [[ $TRAVIS_OS_NAME != "linux" || $DESKTOP_ONLY != 1 ]]; then travis/install_extra_deps.sh; fi
install: if [[ $TRAVIS_OS_NAME != "linux" ]]; then travis/install_extra_deps.sh; fi
stage: build
script:
- |
if [[ $TRAVIS_OS_NAME = "linux" && $DESKTOP_ONLY == 1 ]]
if [[ $TRAVIS_OS_NAME = "linux" ]]
then
if [[ $PLATFORM = "flatpak" ]]
then
docker build -t kiwix/build -f travis/Dockerfile_flatpak .
else
docker build -t kiwix/build -f travis/Dockerfile .
fi
docker run -e PLATFORM -e NIGHTLY_DATE -e TRAVIS_EVENT_TYPE -e DESKTOP_ONLY -e TRAVIS_TAG --device /dev/fuse --cap-add ALL --privileged kiwix/build
else
if [[ $TRAVIS_OS_NAME = "linux" ]]
then
python3.5 travis/compile_all.py
else
python3 travis/compile_all.py
fi
VARIANT=xenial
if [[ $PLATFORM = "flatpak" && $DESKTOP_ONLY = 1 ]]; then VARIANT=flatpak; fi
if [[ $PLATFORM = "native_dyn" && $DESKTOP_ONLY = 1 ]]; then VARIANT=appimage; fi
fi
after_failure: travis/upload_all_log.sh
- |
if [[ $TRAVIS_OS_NAME = "linux" ]]
then
docker run --rm \
-e PLATFORM -e NIGHTLY_DATE -e TRAVIS_EVENT_TYPE -e DESKTOP_ONLY -e TRAVIS_TAG \
--device /dev/fuse --cap-add ALL --privileged \
--mount=type=bind,src=$HOME/EXPORT,dst=/home/builder/EXPORT \
--mount=type=bind,src=$HOME/.cache,dst=/home/builder/.cache \
--mount=type=bind,src=$HOME/.gradle/caches,dst=/home/builder/.gradle/caches \
kiwix/kiwix-build_ci:${VARIANT}-${TRAVIS_COMMIT}
fi
- if [[ $TRAVIS_OS_NAME != "linux" ]]; then python3 travis/compile_all.py; fi
after_success:
- travis/upload_base_deps.sh
after_failure:
- travis/upload_base_deps.sh
- travis/upload_all_log.sh
deploy:
- provider: script
skip_cleanup: true
@ -64,87 +108,28 @@ deploy:
env:
global:
- NIGHTLY_DATE=$(date +%Y-%m-%d)
- SSH_KEY=/tmp/private/travisci_builder_id_key
matrix:
- PLATFORM="flatpak" DESKTOP_ONLY=1
- PLATFORM="native_dyn"
- PLATFORM="native_static"
- PLATFORM="native_mixed"
- PLATFORM="armhf_dyn"
- PLATFORM="armhf_static"
- PLATFORM="native_mixed"
- PLATFORM="win32_dyn"
- PLATFORM="win32_static"
- PLATFORM="i586_dyn"
- PLATFORM="i586_static"
- PLATFORM="native_dyn" DESKTOP_ONLY=1
- PLATFORM="android_arm"
- PLATFORM="android_arm64"
- PLATFORM="android_x86"
- PLATFORM="android_x86_64"
addons:
ssh_known_hosts:
- download.kiwix.org
- tmp.kiwix.org
- download.openzim.org
apt:
packages: &common
- cmake
- autopoint
- python3.5
- python3-pip
- ctpp2-utils
- libmagic-dev
matrix:
include:
- env: PLATFORM="native_dyn" DESKTOP_ONLY=1
addons:
apt:
packages: []
- env: PLATFORM="flatpak" DESKTOP_ONLY=1
addons:
apt:
packages: []
- env: PLATFORM="native_static"
addons:
apt:
packages:
- *common
- libbz2-dev
- zlib1g-dev
- uuid-dev
- libctpp2-dev
- env: PLATFORM="win32_dyn"
addons: &win32_addons
apt:
packages:
- *common
- g++-mingw-w64-i686
- gcc-mingw-w64-i686
- gcc-mingw-w64-base
- mingw-w64-tools
- env: PLATFORM="win32_static"
addons: *win32_addons
- env: PLATFORM="i586_dyn"
addons: &i586_addons
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- *common
- gcc-multilib
- g++-multilib
- libc6-dev:i386
- libstdc++-6-dev:i386
- env: PLATFORM="i586_static"
addons: *i586_addons
- env: PLATFORM="android"
addons: &android_addons
apt:
packages:
- *common
- default-jdk
- env: PLATFORM="android_arm"
addons: *android_addons
- env: PLATFORM="android_arm64"
addons: *android_addons
- env: PLATFORM="android_x86"
addons: *android_addons
- env: PLATFORM="android_x86_64"
addons: *android_addons
- env: PLATFORM="native_dyn"
os: osx
- env: PLATFORM="iOS_arm64"
os: osx
- env: PLATFORM="iOS_i386"
os: osx
notifications:
irc:

9
kiwixbuild/dependencies/armhf.py
View File

@ -1,11 +1,12 @@
from .base import Dependency, GitClone, NoopBuilder
from .base import Dependency, ReleaseDownload, NoopBuilder
from kiwixbuild.utils import Remotefile
class armhf_toolchain(Dependency):
neutral = True
name = 'armhf'
class Source(GitClone):
git_remote = "https://github.com/raspberrypi/tools"
git_dir = "raspberrypi-tools"
class Source(ReleaseDownload):
archive = Remotefile('raspberrypi-tools.tar.gz',
'e72b35436f2f23f2f7df322d6c318b9be57b21596b5ff0b8936af4ad94e04f2e')
Builder = NoopBuilder

3
kiwixbuild/dependencies/libmagic.py
View File

@ -17,8 +17,7 @@ class LibMagic(Dependency):
source_dir = "libmagic"
archive_top_dir = 'file-5.35'
archive = Remotefile('file-5.35.tar.gz',
'30c45e817440779be7aac523a905b123cba2a6ed0bf4f5439e1e99ba940b5546',
'https://fossies.org/linux/misc/file-5.35.tar.gz')
'30c45e817440779be7aac523a905b123cba2a6ed0bf4f5439e1e99ba940b5546')
class Builder(MakeBuilder):

4
kiwixbuild/packages.py
View File

@ -76,10 +76,10 @@ PACKAGE_NAME_MAPPERS = {
'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'],
},
'debian_i586_dyn': {
'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib'],
'COMMON': _debian_common + ['libc6-dev-i386', 'lib32stdc++6', 'gcc-multilib', 'g++-multilib'],
},
'debian_i586_static': {
'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib'],
'COMMON': _debian_common + ['libc6-dev-i386', 'lib32stdc++6', 'gcc-multilib', 'g++-multilib'],
},
'debian_win32_dyn': {
'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'],

1
kiwixbuild/platforms/armhf.py
View File

@ -34,6 +34,7 @@ class ArmhfPlatformInfo(PlatformInfo):
@property
def root_path(self):
return pj(self.tlc_source.source_path,
'raspberrypi-tools',
'arm-bcm2708',
'gcc-linaro-{}-raspbian-x64'.format(self.arch_full))

2
kiwixbuild/versions.py
View File

@ -35,7 +35,7 @@ release_versions = {
# This is the "version" of the whole base_deps_versions dict.
# Change this when you change base_deps_versions.
base_deps_meta_version = '18'
base_deps_meta_version = '51'
base_deps_versions = {

0
travis/Dockerfile → travis/appimage_builder.dockerfile
View File

58
travis/compile_all.py
View File

@ -25,6 +25,8 @@ KBUILD_SOURCE_DIR = Path(environ['TRAVIS_BUILD_DIR'])
KIWIX_DESKTOP_ONLY = environ.get('DESKTOP_ONLY') == '1'
BASE_DIR = HOME/"BUILD_{}".format(PLATFORM)
BASE_EXPORT_DIR = HOME/"EXPORT"/"BASE"
GIT_EXPORT_DIR = HOME/"EXPORT"/"GIT"
SOURCE_DIR = HOME/"SOURCE"
ARCHIVE_DIR = HOME/"ARCHIVE"
INSTALL_DIR = BASE_DIR/"INSTALL"
@ -34,7 +36,6 @@ NIGHTLY_ZIM_ARCHIVES_DIR = HOME/'NIGHTLY_ZIM_ARCHIVES'/NIGHTLY_DATE
RELEASE_ZIM_ARCHIVES_DIR = HOME/'RELEASE_ZIM_ARCHIVES'
DIST_KIWIX_ARCHIVES_DIR = HOME/'DIST_KIWIX_ARCHIVES'
DIST_ZIM_ARCHIVES_DIR = HOME/'DIST_ZIM_ARCHIVES'
SSH_KEY = KBUILD_SOURCE_DIR/'travis'/'travisci_builder_id_key'
BIN_EXT = '.exe' if PLATFORM.startswith('win32_') else ''
@ -193,8 +194,8 @@ def make_archive(project, platform):
archive_add(arch, f)
def make_deps_archive(target, full=False):
archive_name = "deps_{}_{}_{}.tar.xz".format(
def make_deps_archive(target=None, name=None, full=False):
archive_name = name or "deps_{}_{}_{}.tar.xz".format(
TRAVIS_OS_NAME, PLATFORM, target)
print_message("Create archive {}.", archive_name)
files_to_archive = [INSTALL_DIR]
@ -230,57 +231,50 @@ def make_deps_archive(target, full=False):
base_deps_versions['pugixml'])]
files_to_archive += HOME.glob('BUILD_*/pugixml-{}'.format(
base_deps_versions['pugixml']))
toolchains_subdirs = HOME.glob('**/TOOLCHAINS/*/*')
if PLATFORM.startswith('armhf'):
files_to_archive += [SOURCE_DIR/'raspberrypi-tools']
files_to_archive += (SOURCE_DIR/'armhf').glob('*')
toolchains_subdirs = HOME.glob('**/TOOLCHAINS/*/*')
for subdir in toolchains_subdirs:
if not subdir.match('tools'):
files_to_archive.append(subdir)
with tarfile.open(str(relative_path/archive_name), 'w:xz') as tar:
with tarfile.open(str(BASE_EXPORT_DIR/archive_name), 'w:xz') as tar:
for name in set(files_to_archive):
print('.', end='', flush=True)
print('.{}'.format(name), flush=True)
tar.add(str(name), arcname=str(name.relative_to(relative_path)))
return relative_path/archive_name
def update_flathub_git():
GIT_REPO_DIR = GIT_EXPORT_DIR/"org.kiwix.desktop"
env = dict(os.environ)
env['GIT_SSH_COMMAND'] = 'ssh -o StrictHostKeyChecking=no -i {}'.format(SSH_KEY)
env['GIT_AUTHOR_NAME'] = env['GIT_COMMITTER_NAME'] = "KiwixBot"
env['GIT_AUTHOR_EMAIL'] = env['GIT_COMMITTER_EMAIL'] = "kiwixbot@kymeria.fr"
def call(command, cwd=None):
cwd = cwd or GIT_REPO_DIR
subprocess.check_call(command, env=env, cwd=str(cwd))
command = ['git', 'clone', FLATPAK_GIT_REMOTE]
subprocess.check_call(command, env=env, cwd=str(HOME))
shutil.copy(str(BASE_DIR/'org.kiwix.desktop.json'),
str(HOME/'org.kiwix.desktop'))
call(command, cwd=GIT_EXPORT_DIR)
shutil.copy(str(BASE_DIR/'org.kiwix.desktop.json'), str(GIT_REPO_DIR))
patch_dir = KBUILD_SOURCE_DIR/'kiwixbuild'/'patches'
for dep in ('libaria2', 'mustache', 'pugixml', 'xapian'):
for f in patch_dir.glob('{}_*.patch'.format(dep)):
shutil.copy(str(f), str(HOME/'org.kiwix.desktop'/'patches'))
shutil.copy(str(f), str(GIT_REPO_DIR/'patches'))
command = ['git', 'add', '-A', '.']
subprocess.check_call(command, env=env, cwd=str(HOME/'org.kiwix.desktop'))
call(command)
command = ['git', 'commit', '-m',
'Update to version {}'.format(main_project_versions['kiwix-desktop'])]
subprocess.check_call(command, env=env, cwd=str(HOME/'org.kiwix.desktop'))
command = ['git', 'push']
subprocess.check_call(command, env=env, cwd=str(HOME/'org.kiwix.desktop'))
call(command)
def scp(what, where):
print_message("Copy {} to {}", what, where)
command = ['scp', '-o', 'StrictHostKeyChecking=no',
'-i', str(SSH_KEY),
str(what), str(where)]
subprocess.check_call(command)
for p in (NIGHTLY_KIWIX_ARCHIVES_DIR,
NIGHTLY_ZIM_ARCHIVES_DIR,
RELEASE_KIWIX_ARCHIVES_DIR,
RELEASE_ZIM_ARCHIVES_DIR,
DIST_KIWIX_ARCHIVES_DIR,
DIST_ZIM_ARCHIVES_DIR):
DIST_ZIM_ARCHIVES_DIR,
BASE_EXPORT_DIR,
GIT_EXPORT_DIR):
try:
p.mkdir(parents=True)
except FileExistsError:
@ -330,12 +324,9 @@ if PLATFORM != 'flatpak':
f.extractall(str(HOME))
except URLError:
pass
run_kiwix_build('alldependencies', platform=PLATFORM)
if SSH_KEY.exists():
archive = make_deps_archive('alldependencies', full=True)
destination = 'ci@tmp.kiwix.org:/data/tmp/ci/{}'
destination = destination.format(base_dep_archive_name)
scp(archive, destination)
else:
run_kiwix_build('alldependencies', platform=PLATFORM)
make_deps_archive(name=base_dep_archive_name, full=True)
# A basic compilation to be sure everything is working (for a PR)
@ -407,8 +398,7 @@ for target in TARGETS:
run_kiwix_build(target,
platform=PLATFORM,
build_deps_only=True)
archive = make_deps_archive(target)
scp(archive, 'ci@tmp.kiwix.org:/data/tmp/ci/')
make_deps_archive(target=target)
run_kiwix_build(target,
platform=PLATFORM,

12
travis/deploy.sh
View File

@ -8,7 +8,6 @@ RELEASE_KIWIX_ARCHIVES_DIR=${HOME}/RELEASE_KIWIX_ARCHIVES
RELEASE_ZIM_ARCHIVES_DIR=${HOME}/RELEASE_ZIM_ARCHIVES
DIST_KIWIX_ARCHIVES_DIR=${HOME}/DIST_KIWIX_ARCHIVES
DIST_ZIM_ARCHIVES_DIR=${HOME}/DIST_ZIM_ARCHIVES
SSH_KEY=${TRAVIS_BUILD_DIR}/travis/travisci_builder_id_key
if [[ "$TRAVIS_EVENT_TYPE" = "cron" ]]
then
@ -68,5 +67,16 @@ then
ci@download.openzim.org:/data/openzim/release/${subdir}
done
fi
GIT_REPOS=$(ls -d */)
if [[ "x$GIT_REPOS" != "x" ]]
then
for repo in $GIT_REPOS
do
(cd repo;
GIT_SSH_COMMAND"ssh -o StrictHostKeyChecking=no -i $SSH_KEY" git push
)
done
fi
fi

1
travis/deploy_apk.sh
View File

@ -4,7 +4,6 @@ set -e
KEYSTORE_FILE=${TRAVIS_BUILD_DIR}/travis/kiwix-android.keystore
GOOGLE_API_KEY=${TRAVIS_BUILD_DIR}/travis/googleplay_android_developer-5a411156212c.json
SSH_KEY=${TRAVIS_BUILD_DIR}/travis/travisci_builder_id_key
cd ${HOME}

0
travis/Dockerfile_flatpak → travis/flatpak_builder.dockerfile
View File

1
travis/make_release.sh
View File

@ -4,7 +4,6 @@ set -e
KEYSTORE_FILE=${TRAVIS_BUILD_DIR}/travis/test_ks.ks
GOOGLE_API_KEY=${TRAVIS_BUILD_DIR}/travis/googleplay_android_developer-5a411156212c.json
SSH_KEY=${TRAVIS_BUILD_DIR}/travis/travisci_builder_id_key
cd ${HOME}

2
travis/upload_all_log.sh
View File

@ -2,8 +2,6 @@
set -e
SSH_KEY=$(pwd)/travis/travisci_builder_id_key
cd $HOME
tar -czf fail_log_${PLATFORM}.tar.gz BUILD_${PLATFORM}

15
travis/upload_base_deps.sh Executable file
View File

@ -0,0 +1,15 @@
#!/usr/bin/env bash
set -e
BASE_EXPORT_DIR=$HOME/EXPORT/BASE
ls $BASE_EXPORT_DIR
BASE_ARCHIVES=$(find $BASE_EXPORT_DIR -type f)
echo $BASE_ARCHIVES
if [[ "x$BASE_ARCHIVES" != "x" ]]
then
scp -vrp -i ${SSH_KEY} -o StrictHostKeyChecking=no \
$BASE_ARCHIVES \
ci@tmp.kiwix.org:/data/tmp/ci
fi

41
travis/xenial_builder.dockerfile Normal file
View File

@ -0,0 +1,41 @@
FROM ubuntu:xenial
ENV LANG C.UTF-8
RUN apt update -q
RUN \
dpkg --add-architecture i386 && \
apt install -q -y --no-install-recommends \
# Base build tools
build-essential automake libtool cmake ccache pkg-config autopoint patch \
python3-pip python3-setuptools python3-wheel git subversion wget unzip \
ninja-build \
# Python (2) is needed to install android-ndk
python \
# Packaged dependencies
libbz2-dev libmagic-dev uuid-dev zlib1g-dev default-jdk \
libmicrohttpd-dev \
# Cross win32 compiler
g++-mingw-w64-i686 gcc-mingw-w64-i686 gcc-mingw-w64-base mingw-w64-tools \
# Cross compile i586
libc6-dev-i386 lib32stdc++6 gcc-multilib g++-multilib \
# Other tools (to remove)
vim less grep && \
apt-get clean -y && \
rm -rf /usr/share/doc/* /var/cache/debconf/*
# Create user
RUN useradd --create-home builder
USER builder
WORKDIR /home/builder
ENV PATH="/home/builder/.local/bin:${PATH}"
# Install kiwix-build
COPY --chown=builder:builder . kiwix-build
RUN pip3 install --user -e ./kiwix-build
ENV TRAVIS_BUILD_DIR /home/builder/kiwix-build
ENV GRADLE_USER_HOME /home/builder
ENV TRAVIS_OS_NAME linux_xenial
CMD kiwix-build/travis/compile_all.py