Build basedeps archives on GithubAction.
Use a specific script to download/build the base deps. Mainly based on script in travis/compile_all.py
This commit is contained in:
Matthieu Gautier
parent
ce45966d40
commit
d37acc2cbe
|
|
@ -0,0 +1,162 @@
|
||||||
|
import os
|
||||||
|
from os import environ as _environ
|
||||||
|
from pathlib import Path
|
||||||
|
from datetime import date
|
||||||
|
import tarfile
|
||||||
|
import subprocess
|
||||||
|
import re
|
||||||
|
|
||||||
|
from kiwixbuild.versions import base_deps_versions
|
||||||
|
|
||||||
|
|
||||||
|
PLATFORM_TARGET = _environ["PLATFORM_TARGET"]
|
||||||
|
OS_NAME = _environ["OS_NAME"]
|
||||||
|
HOME = Path(os.path.expanduser("~"))
|
||||||
|
|
||||||
|
BASE_DIR = HOME / "BUILD_{}".format(PLATFORM_TARGET)
|
||||||
|
SOURCE_DIR = HOME / "SOURCE"
|
||||||
|
ARCHIVE_DIR = HOME / "ARCHIVE"
|
||||||
|
INSTALL_DIR = BASE_DIR / "INSTALL"
|
||||||
|
TMP_DIR = Path("/tmp")
|
||||||
|
|
||||||
|
# [TODO]
|
||||||
|
KIWIX_DESKTOP_ONLY = False
|
||||||
|
|
||||||
|
_ref = _environ.get("GITHUB_REF", "").split("/")[-1]
|
||||||
|
MAKE_RELEASE = re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", _ref) is not None
|
||||||
|
|
||||||
|
|
||||||
|
def print_message(message, *args, **kwargs):
|
||||||
|
message = message.format(*args, **kwargs)
|
||||||
|
message = "{0} {1} {0}".format("-" * 3, message)
|
||||||
|
print(message, flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
MANIFEST_TEMPLATE = """{archive_name}
|
||||||
|
***************************
|
||||||
|
|
||||||
|
Dependencies archive for {target} on platform {platform}
|
||||||
|
Generated at {date}
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def write_manifest(manifest_file, archive_name, target, platform):
|
||||||
|
with manifest_file.open(mode="w") as f:
|
||||||
|
f.write(
|
||||||
|
MANIFEST_TEMPLATE.format(
|
||||||
|
archive_name=archive_name,
|
||||||
|
target=target,
|
||||||
|
platform=platform,
|
||||||
|
date=date.today().isoformat(),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def run_kiwix_build(
|
||||||
|
target,
|
||||||
|
platform,
|
||||||
|
build_deps_only=False,
|
||||||
|
target_only=False,
|
||||||
|
make_release=False,
|
||||||
|
make_dist=False,
|
||||||
|
):
|
||||||
|
command = ["kiwix-build"]
|
||||||
|
command.append(target)
|
||||||
|
command.append("--hide-progress")
|
||||||
|
if platform == "flatpak" or platform.startswith("win32_"):
|
||||||
|
command.append("--assume-packages-installed")
|
||||||
|
if target == "kiwix-lib-app" and platform.startswith("android_"):
|
||||||
|
command.extend(["--target-platform", "android", "--android-arch", platform[8:]])
|
||||||
|
elif platform == "android":
|
||||||
|
command.extend(["--target-platform", "android"])
|
||||||
|
for arch in ("arm", "arm64", "x86", "x86_64"):
|
||||||
|
command.extend(["--android-arch", arch])
|
||||||
|
else:
|
||||||
|
command.extend(["--target-platform", platform])
|
||||||
|
if build_deps_only:
|
||||||
|
command.append("--build-deps-only")
|
||||||
|
if target_only:
|
||||||
|
command.append("--build-nodeps")
|
||||||
|
if make_release:
|
||||||
|
command.append("--make-release")
|
||||||
|
if make_dist:
|
||||||
|
command.append("--make-dist")
|
||||||
|
print_message(
|
||||||
|
"Build {} (deps={}, release={}, dist={})",
|
||||||
|
target,
|
||||||
|
build_deps_only,
|
||||||
|
make_release,
|
||||||
|
make_dist,
|
||||||
|
)
|
||||||
|
subprocess.check_call(command, cwd=str(HOME))
|
||||||
|
print_message("Build ended")
|
||||||
|
|
||||||
|
|
||||||
|
def upload(file_to_upload, host, dest_path):
|
||||||
|
command = [
|
||||||
|
"scp",
|
||||||
|
"-i",
|
||||||
|
_environ.get("SSH_KEY"),
|
||||||
|
"-o",
|
||||||
|
"StrictHostKeyChecking=no",
|
||||||
|
str(file_to_upload),
|
||||||
|
"{}:{}".format(host, dest_path),
|
||||||
|
]
|
||||||
|
print_message("Sending archive with command {}", command)
|
||||||
|
subprocess.check_call(command)
|
||||||
|
|
||||||
|
|
||||||
|
def make_deps_archive(target=None, name=None, full=False):
|
||||||
|
archive_name = name or "deps2_{}_{}_{}.tar.xz".format(
|
||||||
|
OS_NAME, PLATFORM_TARGET, target
|
||||||
|
)
|
||||||
|
print_message("Create archive {}.", archive_name)
|
||||||
|
files_to_archive = [INSTALL_DIR]
|
||||||
|
files_to_archive += HOME.glob("BUILD_*/LOGS")
|
||||||
|
if PLATFORM_TARGET == "native_mixed":
|
||||||
|
files_to_archive += [HOME / "BUILD_native_static" / "INSTALL"]
|
||||||
|
if PLATFORM_TARGET.startswith("android"):
|
||||||
|
files_to_archive.append(HOME / "BUILD_neutral" / "INSTALL")
|
||||||
|
if PLATFORM_TARGET == "android":
|
||||||
|
for arch in ("arm", "arm64", "x86", "x86_64"):
|
||||||
|
base_dir = HOME / "BUILD_android_{}".format(arch)
|
||||||
|
files_to_archive.append(base_dir / "INSTALL")
|
||||||
|
if (base_dir / "meson_cross_file.txt").exists():
|
||||||
|
files_to_archive.append(base_dir / "meson_cross_file.txt")
|
||||||
|
files_to_archive += HOME.glob("BUILD_*/android-ndk*")
|
||||||
|
files_to_archive += HOME.glob("BUILD_*/android-sdk*")
|
||||||
|
if (BASE_DIR / "meson_cross_file.txt").exists():
|
||||||
|
files_to_archive.append(BASE_DIR / "meson_cross_file.txt")
|
||||||
|
|
||||||
|
manifest_file = BASE_DIR / "manifest.txt"
|
||||||
|
write_manifest(manifest_file, archive_name, target, PLATFORM_TARGET)
|
||||||
|
files_to_archive.append(manifest_file)
|
||||||
|
|
||||||
|
relative_path = HOME
|
||||||
|
if full:
|
||||||
|
files_to_archive += ARCHIVE_DIR.glob(".*_ok")
|
||||||
|
files_to_archive += BASE_DIR.glob("*/.*_ok")
|
||||||
|
files_to_archive += (HOME / "BUILD_native_dyn").glob("*/.*_ok")
|
||||||
|
files_to_archive += (HOME / "BUILD_native_static").glob("*/.*_ok")
|
||||||
|
files_to_archive += HOME.glob("BUILD_android*/**/.*_ok")
|
||||||
|
files_to_archive += SOURCE_DIR.glob("*/.*_ok")
|
||||||
|
files_to_archive += [
|
||||||
|
SOURCE_DIR / "pugixml-{}".format(base_deps_versions["pugixml"])
|
||||||
|
]
|
||||||
|
files_to_archive += HOME.glob(
|
||||||
|
"BUILD_*/pugixml-{}".format(base_deps_versions["pugixml"])
|
||||||
|
)
|
||||||
|
if PLATFORM_TARGET.startswith("armhf"):
|
||||||
|
files_to_archive += (SOURCE_DIR / "armhf").glob("*")
|
||||||
|
toolchains_subdirs = HOME.glob("BUILD_*/TOOLCHAINS/*/*")
|
||||||
|
for subdir in toolchains_subdirs:
|
||||||
|
if not subdir.match("tools"):
|
||||||
|
files_to_archive.append(subdir)
|
||||||
|
|
||||||
|
archive_file = TMP_DIR / archive_name
|
||||||
|
with tarfile.open(str(archive_file), "w:xz") as tar:
|
||||||
|
for name in set(files_to_archive):
|
||||||
|
print(".{}".format(name), flush=True)
|
||||||
|
tar.add(str(name), arcname=str(name.relative_to(relative_path)))
|
||||||
|
|
||||||
|
return archive_file
|
||||||
|
|
@ -0,0 +1,72 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import os
|
||||||
|
import tarfile
|
||||||
|
from urllib.request import urlopen
|
||||||
|
from urllib.error import URLError
|
||||||
|
|
||||||
|
from kiwixbuild.versions import base_deps_meta_version
|
||||||
|
|
||||||
|
from common import (
|
||||||
|
print_message,
|
||||||
|
run_kiwix_build,
|
||||||
|
upload,
|
||||||
|
make_deps_archive,
|
||||||
|
HOME,
|
||||||
|
PLATFORM_TARGET,
|
||||||
|
OS_NAME,
|
||||||
|
MAKE_RELEASE,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def download_base_archive(base_name):
|
||||||
|
url = "http://tmp.kiwix.org/ci/{}".format(base_name)
|
||||||
|
file_path = str(HOME / base_name)
|
||||||
|
batch_size = 1024 * 1024 * 8
|
||||||
|
with urlopen(url) as resource, open(file_path, "wb") as file:
|
||||||
|
while True:
|
||||||
|
batch = resource.read(batch_size)
|
||||||
|
if not batch:
|
||||||
|
break
|
||||||
|
print(".", end="", flush=True)
|
||||||
|
file.write(batch)
|
||||||
|
return file_path
|
||||||
|
|
||||||
|
|
||||||
|
ARCHIVE_NAME_TEMPLATE = "base_deps2_{os}_{platform}_{version}_{release}.tar.xz"
|
||||||
|
base_dep_archive_name = ARCHIVE_NAME_TEMPLATE.format(
|
||||||
|
os=OS_NAME,
|
||||||
|
platform=PLATFORM_TARGET,
|
||||||
|
version=base_deps_meta_version,
|
||||||
|
release="release" if MAKE_RELEASE else "debug",
|
||||||
|
)
|
||||||
|
|
||||||
|
print_message("Getting archive {}", base_dep_archive_name)
|
||||||
|
try:
|
||||||
|
local_filename = download_base_archive(base_dep_archive_name)
|
||||||
|
with tarfile.open(local_filename) as f:
|
||||||
|
f.extractall(str(HOME))
|
||||||
|
os.remove(str(local_filename))
|
||||||
|
except URLError:
|
||||||
|
print_message("Cannot get archive. Build dependencies")
|
||||||
|
if PLATFORM_TARGET == "android":
|
||||||
|
for arch in ("arm", "arm64", "x86", "x86_64"):
|
||||||
|
archive_name = ARCHIVE_NAME_TEMPLATE.format(
|
||||||
|
os=OS_NAME,
|
||||||
|
platform="android_{}".format(arch),
|
||||||
|
version=base_deps_meta_version,
|
||||||
|
release="release" if MAKE_RELEASE else "debug",
|
||||||
|
)
|
||||||
|
print_message("Getting archive {}", archive_name)
|
||||||
|
try:
|
||||||
|
local_filename = download_base_archive(archive_name)
|
||||||
|
with tarfile.open(local_filename) as f:
|
||||||
|
f.extractall(str(HOME))
|
||||||
|
os.remove(str(local_filename))
|
||||||
|
except URLError:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
run_kiwix_build("alldependencies", platform=PLATFORM_TARGET)
|
||||||
|
archive_file = make_deps_archive(name=base_dep_archive_name, full=True)
|
||||||
|
upload(archive_file, "ci@tmp.kiwix.org", "/data/tmp/ci")
|
||||||
|
os.remove(str(archive_file))
|
||||||
|
|
@ -26,3 +26,61 @@ jobs:
|
||||||
docker push ${FULLTAGNAME}
|
docker push ${FULLTAGNAME}
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
Linux:
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
target:
|
||||||
|
- native_static
|
||||||
|
- native_dyn
|
||||||
|
- android_arm
|
||||||
|
- android_arm64
|
||||||
|
- win32_static
|
||||||
|
- win32_dyn
|
||||||
|
include:
|
||||||
|
- target: native_static
|
||||||
|
image_variant: xenial
|
||||||
|
lib_postfix: '/x86_64-linux-gnu'
|
||||||
|
- target: native_dyn
|
||||||
|
image_variant: xenial
|
||||||
|
lib_postfix: '/x86_64-linux-gnu'
|
||||||
|
- target: android_arm
|
||||||
|
image_variant: xenial
|
||||||
|
lib_postfix: '/x86_64-linux-gnu'
|
||||||
|
- target: android_arm64
|
||||||
|
image_variant: xenial
|
||||||
|
lib_postfix: '/x86_64-linux-gnu'
|
||||||
|
- target: win32_static
|
||||||
|
image_variant: f30
|
||||||
|
lib_postfix: '64'
|
||||||
|
- target: win32_dyn
|
||||||
|
image_variant: f30
|
||||||
|
lib_postfix: '64'
|
||||||
|
env:
|
||||||
|
HOME: /home/runner
|
||||||
|
SSH_KEY: /tmp/id_rsa
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
needs: Docker
|
||||||
|
container:
|
||||||
|
image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-25"
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
cd $HOME
|
||||||
|
git clone https://github.com/${REP} --depth=1 --branch ${GITHUB_REF##*/}
|
||||||
|
pip3 install --user --no-deps ./${REP##*/}
|
||||||
|
env:
|
||||||
|
REP: ${{github.repository}}
|
||||||
|
- name: secret
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
echo "${{secrets.ssh_key}}" > $SSH_KEY
|
||||||
|
chmod 600 $SSH_KEY
|
||||||
|
- name: Ensure base deps
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
cd $HOME
|
||||||
|
kiwix-build/.github/scripts/ensure_base_deps.py
|
||||||
|
env:
|
||||||
|
PLATFORM_TARGET: ${{matrix.target}}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue