Build basedeps archives on GithubAction.
Use a specific script to download/build the base deps. Mainly based on script in travis/compile_all.py
This commit is contained in:
Matthieu Gautier
parent
ce45966d40
commit
d37acc2cbe
|
|
@ -0,0 +1,162 @@
|
|||
import os
|
||||
from os import environ as _environ
|
||||
from pathlib import Path
|
||||
from datetime import date
|
||||
import tarfile
|
||||
import subprocess
|
||||
import re
|
||||
|
||||
from kiwixbuild.versions import base_deps_versions
|
||||
|
||||
|
||||
PLATFORM_TARGET = _environ["PLATFORM_TARGET"]
|
||||
OS_NAME = _environ["OS_NAME"]
|
||||
HOME = Path(os.path.expanduser("~"))
|
||||
|
||||
BASE_DIR = HOME / "BUILD_{}".format(PLATFORM_TARGET)
|
||||
SOURCE_DIR = HOME / "SOURCE"
|
||||
ARCHIVE_DIR = HOME / "ARCHIVE"
|
||||
INSTALL_DIR = BASE_DIR / "INSTALL"
|
||||
TMP_DIR = Path("/tmp")
|
||||
|
||||
# [TODO]
|
||||
KIWIX_DESKTOP_ONLY = False
|
||||
|
||||
_ref = _environ.get("GITHUB_REF", "").split("/")[-1]
|
||||
MAKE_RELEASE = re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", _ref) is not None
|
||||
|
||||
|
||||
def print_message(message, *args, **kwargs):
|
||||
message = message.format(*args, **kwargs)
|
||||
message = "{0} {1} {0}".format("-" * 3, message)
|
||||
print(message, flush=True)
|
||||
|
||||
|
||||
MANIFEST_TEMPLATE = """{archive_name}
|
||||
***************************
|
||||
|
||||
Dependencies archive for {target} on platform {platform}
|
||||
Generated at {date}
|
||||
"""
|
||||
|
||||
|
||||
def write_manifest(manifest_file, archive_name, target, platform):
|
||||
with manifest_file.open(mode="w") as f:
|
||||
f.write(
|
||||
MANIFEST_TEMPLATE.format(
|
||||
archive_name=archive_name,
|
||||
target=target,
|
||||
platform=platform,
|
||||
date=date.today().isoformat(),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def run_kiwix_build(
|
||||
target,
|
||||
platform,
|
||||
build_deps_only=False,
|
||||
target_only=False,
|
||||
make_release=False,
|
||||
make_dist=False,
|
||||
):
|
||||
command = ["kiwix-build"]
|
||||
command.append(target)
|
||||
command.append("--hide-progress")
|
||||
if platform == "flatpak" or platform.startswith("win32_"):
|
||||
command.append("--assume-packages-installed")
|
||||
if target == "kiwix-lib-app" and platform.startswith("android_"):
|
||||
command.extend(["--target-platform", "android", "--android-arch", platform[8:]])
|
||||
elif platform == "android":
|
||||
command.extend(["--target-platform", "android"])
|
||||
for arch in ("arm", "arm64", "x86", "x86_64"):
|
||||
command.extend(["--android-arch", arch])
|
||||
else:
|
||||
command.extend(["--target-platform", platform])
|
||||
if build_deps_only:
|
||||
command.append("--build-deps-only")
|
||||
if target_only:
|
||||
command.append("--build-nodeps")
|
||||
if make_release:
|
||||
command.append("--make-release")
|
||||
if make_dist:
|
||||
command.append("--make-dist")
|
||||
print_message(
|
||||
"Build {} (deps={}, release={}, dist={})",
|
||||
target,
|
||||
build_deps_only,
|
||||
make_release,
|
||||
make_dist,
|
||||
)
|
||||
subprocess.check_call(command, cwd=str(HOME))
|
||||
print_message("Build ended")
|
||||
|
||||
|
||||
def upload(file_to_upload, host, dest_path):
|
||||
command = [
|
||||
"scp",
|
||||
"-i",
|
||||
_environ.get("SSH_KEY"),
|
||||
"-o",
|
||||
"StrictHostKeyChecking=no",
|
||||
str(file_to_upload),
|
||||
"{}:{}".format(host, dest_path),
|
||||
]
|
||||
print_message("Sending archive with command {}", command)
|
||||
subprocess.check_call(command)
|
||||
|
||||
|
||||
def make_deps_archive(target=None, name=None, full=False):
|
||||
archive_name = name or "deps2_{}_{}_{}.tar.xz".format(
|
||||
OS_NAME, PLATFORM_TARGET, target
|
||||
)
|
||||
print_message("Create archive {}.", archive_name)
|
||||
files_to_archive = [INSTALL_DIR]
|
||||
files_to_archive += HOME.glob("BUILD_*/LOGS")
|
||||
if PLATFORM_TARGET == "native_mixed":
|
||||
files_to_archive += [HOME / "BUILD_native_static" / "INSTALL"]
|
||||
if PLATFORM_TARGET.startswith("android"):
|
||||
files_to_archive.append(HOME / "BUILD_neutral" / "INSTALL")
|
||||
if PLATFORM_TARGET == "android":
|
||||
for arch in ("arm", "arm64", "x86", "x86_64"):
|
||||
base_dir = HOME / "BUILD_android_{}".format(arch)
|
||||
files_to_archive.append(base_dir / "INSTALL")
|
||||
if (base_dir / "meson_cross_file.txt").exists():
|
||||
files_to_archive.append(base_dir / "meson_cross_file.txt")
|
||||
files_to_archive += HOME.glob("BUILD_*/android-ndk*")
|
||||
files_to_archive += HOME.glob("BUILD_*/android-sdk*")
|
||||
if (BASE_DIR / "meson_cross_file.txt").exists():
|
||||
files_to_archive.append(BASE_DIR / "meson_cross_file.txt")
|
||||
|
||||
manifest_file = BASE_DIR / "manifest.txt"
|
||||
write_manifest(manifest_file, archive_name, target, PLATFORM_TARGET)
|
||||
files_to_archive.append(manifest_file)
|
||||
|
||||
relative_path = HOME
|
||||
if full:
|
||||
files_to_archive += ARCHIVE_DIR.glob(".*_ok")
|
||||
files_to_archive += BASE_DIR.glob("*/.*_ok")
|
||||
files_to_archive += (HOME / "BUILD_native_dyn").glob("*/.*_ok")
|
||||
files_to_archive += (HOME / "BUILD_native_static").glob("*/.*_ok")
|
||||
files_to_archive += HOME.glob("BUILD_android*/**/.*_ok")
|
||||
files_to_archive += SOURCE_DIR.glob("*/.*_ok")
|
||||
files_to_archive += [
|
||||
SOURCE_DIR / "pugixml-{}".format(base_deps_versions["pugixml"])
|
||||
]
|
||||
files_to_archive += HOME.glob(
|
||||
"BUILD_*/pugixml-{}".format(base_deps_versions["pugixml"])
|
||||
)
|
||||
if PLATFORM_TARGET.startswith("armhf"):
|
||||
files_to_archive += (SOURCE_DIR / "armhf").glob("*")
|
||||
toolchains_subdirs = HOME.glob("BUILD_*/TOOLCHAINS/*/*")
|
||||
for subdir in toolchains_subdirs:
|
||||
if not subdir.match("tools"):
|
||||
files_to_archive.append(subdir)
|
||||
|
||||
archive_file = TMP_DIR / archive_name
|
||||
with tarfile.open(str(archive_file), "w:xz") as tar:
|
||||
for name in set(files_to_archive):
|
||||
print(".{}".format(name), flush=True)
|
||||
tar.add(str(name), arcname=str(name.relative_to(relative_path)))
|
||||
|
||||
return archive_file
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import tarfile
|
||||
from urllib.request import urlopen
|
||||
from urllib.error import URLError
|
||||
|
||||
from kiwixbuild.versions import base_deps_meta_version
|
||||
|
||||
from common import (
|
||||
print_message,
|
||||
run_kiwix_build,
|
||||
upload,
|
||||
make_deps_archive,
|
||||
HOME,
|
||||
PLATFORM_TARGET,
|
||||
OS_NAME,
|
||||
MAKE_RELEASE,
|
||||
)
|
||||
|
||||
|
||||
def download_base_archive(base_name):
|
||||
url = "http://tmp.kiwix.org/ci/{}".format(base_name)
|
||||
file_path = str(HOME / base_name)
|
||||
batch_size = 1024 * 1024 * 8
|
||||
with urlopen(url) as resource, open(file_path, "wb") as file:
|
||||
while True:
|
||||
batch = resource.read(batch_size)
|
||||
if not batch:
|
||||
break
|
||||
print(".", end="", flush=True)
|
||||
file.write(batch)
|
||||
return file_path
|
||||
|
||||
|
||||
ARCHIVE_NAME_TEMPLATE = "base_deps2_{os}_{platform}_{version}_{release}.tar.xz"
|
||||
base_dep_archive_name = ARCHIVE_NAME_TEMPLATE.format(
|
||||
os=OS_NAME,
|
||||
platform=PLATFORM_TARGET,
|
||||
version=base_deps_meta_version,
|
||||
release="release" if MAKE_RELEASE else "debug",
|
||||
)
|
||||
|
||||
print_message("Getting archive {}", base_dep_archive_name)
|
||||
try:
|
||||
local_filename = download_base_archive(base_dep_archive_name)
|
||||
with tarfile.open(local_filename) as f:
|
||||
f.extractall(str(HOME))
|
||||
os.remove(str(local_filename))
|
||||
except URLError:
|
||||
print_message("Cannot get archive. Build dependencies")
|
||||
if PLATFORM_TARGET == "android":
|
||||
for arch in ("arm", "arm64", "x86", "x86_64"):
|
||||
archive_name = ARCHIVE_NAME_TEMPLATE.format(
|
||||
os=OS_NAME,
|
||||
platform="android_{}".format(arch),
|
||||
version=base_deps_meta_version,
|
||||
release="release" if MAKE_RELEASE else "debug",
|
||||
)
|
||||
print_message("Getting archive {}", archive_name)
|
||||
try:
|
||||
local_filename = download_base_archive(archive_name)
|
||||
with tarfile.open(local_filename) as f:
|
||||
f.extractall(str(HOME))
|
||||
os.remove(str(local_filename))
|
||||
except URLError:
|
||||
pass
|
||||
else:
|
||||
run_kiwix_build("alldependencies", platform=PLATFORM_TARGET)
|
||||
archive_file = make_deps_archive(name=base_dep_archive_name, full=True)
|
||||
upload(archive_file, "ci@tmp.kiwix.org", "/data/tmp/ci")
|
||||
os.remove(str(archive_file))
|
||||
|
|
@ -26,3 +26,61 @@ jobs:
|
|||
docker push ${FULLTAGNAME}
|
||||
fi
|
||||
|
||||
Linux:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
target:
|
||||
- native_static
|
||||
- native_dyn
|
||||
- android_arm
|
||||
- android_arm64
|
||||
- win32_static
|
||||
- win32_dyn
|
||||
include:
|
||||
- target: native_static
|
||||
image_variant: xenial
|
||||
lib_postfix: '/x86_64-linux-gnu'
|
||||
- target: native_dyn
|
||||
image_variant: xenial
|
||||
lib_postfix: '/x86_64-linux-gnu'
|
||||
- target: android_arm
|
||||
image_variant: xenial
|
||||
lib_postfix: '/x86_64-linux-gnu'
|
||||
- target: android_arm64
|
||||
image_variant: xenial
|
||||
lib_postfix: '/x86_64-linux-gnu'
|
||||
- target: win32_static
|
||||
image_variant: f30
|
||||
lib_postfix: '64'
|
||||
- target: win32_dyn
|
||||
image_variant: f30
|
||||
lib_postfix: '64'
|
||||
env:
|
||||
HOME: /home/runner
|
||||
SSH_KEY: /tmp/id_rsa
|
||||
runs-on: ubuntu-latest
|
||||
needs: Docker
|
||||
container:
|
||||
image: "kiwix/kiwix-build_ci:${{matrix.image_variant}}-25"
|
||||
steps:
|
||||
- name: Checkout code
|
||||
shell: bash
|
||||
run: |
|
||||
cd $HOME
|
||||
git clone https://github.com/${REP} --depth=1 --branch ${GITHUB_REF##*/}
|
||||
pip3 install --user --no-deps ./${REP##*/}
|
||||
env:
|
||||
REP: ${{github.repository}}
|
||||
- name: secret
|
||||
shell: bash
|
||||
run: |
|
||||
echo "${{secrets.ssh_key}}" > $SSH_KEY
|
||||
chmod 600 $SSH_KEY
|
||||
- name: Ensure base deps
|
||||
shell: bash
|
||||
run: |
|
||||
cd $HOME
|
||||
kiwix-build/.github/scripts/ensure_base_deps.py
|
||||
env:
|
||||
PLATFORM_TARGET: ${{matrix.target}}
|
||||
|
|
|
|||
Loading…
Reference in New Issue