smallsolar
/
kiwix-build
mirror of https://github.com/kiwix/kiwix-build.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #703 from kiwix/libzim_github_ci_windows 

Libzim GitHub CI windows
This commit is contained in:
Kelson 2024-06-15 16:43:10 +02:00 committed by GitHub
parent e8f873f106 05fc3b409c
commit e408b5b1ca
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
25 changed files with 437 additions and 232 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.github/scripts/build_definition.py vendored
View File

@ -21,6 +21,11 @@ BUILD_DEF = """
| manylinux | native_mixed | BP | | | | | linux-x86_64-manylinux | |
| manylinux | aarch64_mixed | BP | | | | | linux-aarch64-manylinux | |
----------------------------------------------------------------------------------------------------------------------------------------------
# On Windows, we build only libzim for now. And only native_mixed as xapian doesn't compile as dll
| windows | native_static | Bd | | | | | win-x86_64 | win-x86_64-static |
| windows | native_dyn | Bd | | | | | win-x86_64 | win-x86_64-dyn |
| windows | native_mixed | BPd | | | | | win-x86_64 | win-x86_64-mixed |
----------------------------------------------------------------------------------------------------------------------------------------------
# Osx builds, build binaries on native_dyn and native_static. On anyother things, build only the libraries
| macos | native_dyn | d | d | dB | B | | | macos-x86_64-dyn |
| macos | native_static | | | BP | BP | | macos-x86_64 | |

153
.github/scripts/common.py vendored
View File

@ -1,12 +1,13 @@
import os
from os import environ as _environ
from pathlib import Path
from pathlib import Path, PurePosixPath
from datetime import date
import tarfile
import zipfile
import subprocess
import re
import shutil
import platform
import requests
@ -27,7 +28,7 @@ def get_build_dir(config) -> Path:
command.append("--use-target-arch-name")
return Path(
subprocess.run(command, cwd=str(HOME), check=True, stdout=subprocess.PIPE)
.stdout[:-1]
.stdout.strip()
.decode("utf8")
)
@ -41,7 +42,8 @@ SOURCE_DIR = HOME / "SOURCE"
ARCHIVE_DIR = HOME / "ARCHIVE"
TOOLCHAIN_DIR = BASE_DIR / "TOOLCHAINS"
INSTALL_DIR = BASE_DIR / "INSTALL"
TMP_DIR = Path(os.getenv("TMP_DIR", "/tmp"))
default_tmp_dir = os.getenv("TEMP") if platform.system() == 'Windows' else "/tmp"
TMP_DIR = Path(os.getenv("TMP_DIR", default_tmp_dir))
KBUILD_SOURCE_DIR = HOME / "kiwix-build"
_ref = _environ.get("GITHUB_REF", "").split("/")[-1]
@ -195,55 +197,112 @@ def run_kiwix_build(
subprocess.check_call(command, cwd=str(HOME), env=env)
print_message("Build ended")
try:
import paramiko
def upload(file_to_upload, host, dest_path):
if not file_to_upload.exists():
print_message("No {} to upload!", file_to_upload)
return
def upload(file_to_upload, host, dest_path):
if not file_to_upload.exists():
print_message("No {} to upload!", file_to_upload)
return
if ":" in host:
host, port = host.split(":", 1)
else:
port = "22"
if ":" in host:
host, port = host.split(":", 1)
else:
port = "22"
if '@' in host:
user, host = host.split('@', 1)
else:
user = None
# sending SFTP mkdir command to the sftp interactive mode and not batch (-b) mode
# as the latter would exit on any mkdir error while it is most likely
# the first parts of the destination is already present and thus can't be created
sftp_commands = "\n".join(
[
f"mkdir {part}"
for part in list(reversed(Path(dest_path).parents)) + [dest_path]
from contextlib import contextmanager
@contextmanager
def get_client():
client = paramiko.client.SSHClient()
client.set_missing_host_key_policy(paramiko.client.WarningPolicy)
print_message(f"Connect to {host}:{port}")
client.connect(host, port=port, username=user, key_filename=_environ.get("SSH_KEY"), look_for_keys=False, compress=True)
try:
yield client
finally:
client.close()
@contextmanager
def get_sftp():
with get_client() as client:
sftp = client.open_sftp()
try:
yield sftp
finally:
sftp.close()
dest_path = PurePosixPath(dest_path)
remote_file = dest_path.joinpath(file_to_upload.name)
with get_sftp() as sftp:
for part in list(reversed(dest_path.parents)) + [dest_path]:
part = str(part)
try:
sftp.stat(part)
except FileNotFoundError:
sftp.mkdir(part)
print_message(f"Sending archive {file_to_upload} to {remote_file}")
sftp.put(str(file_to_upload), str(remote_file), confirm=True)
except ModuleNotFoundError:
# On old system (bionic) paramiko is really complex to install
# Keep the old implementaion on sush system.
def upload(file_to_upload, host, dest_path):
if not file_to_upload.exists():
print_message("No {} to upload!", file_to_upload)
return
if ":" in host:
host, port = host.split(":", 1)
else:
port = "22"
# sending SFTP mkdir command to the sftp interactive mode and not batch (-b) mode
# as the latter would exit on any mkdir error while it is most likely
# the first parts of the destination is already present and thus can't be created
sftp_commands = "\n".join(
[
f"mkdir {part}"
for part in list(reversed(Path(dest_path).parents)) + [dest_path]
]
)
command = [
"sftp",
"-i",
_environ.get("SSH_KEY"),
"-P",
port,
"-o",
"StrictHostKeyChecking=no",
host,
]
)
command = [
"sftp",
"-i",
_environ.get("SSH_KEY"),
"-P",
port,
"-o",
"StrictHostKeyChecking=no",
host,
]
print_message("Creating dest path {}", dest_path)
subprocess.run(command, input=sftp_commands.encode("utf-8"), check=True)
print_message("Creating dest path {}", dest_path)
subprocess.run(command, input=sftp_commands.encode("utf-8"), check=True)
command = [
"scp",
"-c",
"aes128-ctr",
"-rp",
"-P",
port,
"-i",
_environ.get("SSH_KEY"),
"-o",
"StrictHostKeyChecking=no",
str(file_to_upload),
"{}:{}".format(host, dest_path),
]
print_message("Sending archive with command {}", command)
subprocess.check_call(command)
command = [
"scp",
"-c",
"aes128-ctr",
"-rp",
"-P",
port,
"-i",
_environ.get("SSH_KEY"),
"-o",
"StrictHostKeyChecking=no",
str(file_to_upload),
"{}:{}".format(host, dest_path),
]
print_message("Sending archive with command {}", command)
subprocess.check_call(command)
def upload_archive(archive, project, make_release, dev_branch=None):

18
.github/scripts/upload_failure_logs.py vendored Executable file
View File

@ -0,0 +1,18 @@
#!/usr/bin/env python3
import tarfile
from pathlib import Path
from common import upload, OS_NAME, COMPILE_CONFIG, HOME
ARCHIVE_NAME = Path(f"fail_log_{OS_NAME}_{COMPILE_CONFIG}.tar.gz")
files_to_archive = []
files_to_archive += HOME.glob("BUILD_*")
files_to_archive += [HOME / "SOURCE", HOME / "LOGS", HOME / "TOOLCHAINS"]
with tarfile.open(ARCHIVE_NAME, "w:xz") as tar:
for name in set(files_to_archive):
tar.add(str(name))
upload(ARCHIVE_NAME, "ci@tmp.kiwix.org:30022", "/data/tmp/ci")

16
.github/scripts/upload_failure_logs.sh vendored
View File

@ -1,16 +0,0 @@
#!/usr/bin/env bash
set -e
cd $HOME
ARCHIVE_NAME=fail_log_${OS_NAME}_${COMPILE_CONFIG}.tar.gz
tar -czf ${ARCHIVE_NAME} $HOME/BUILD_* $HOME/SOURCE $HOME/LOGS $HOME/TOOLCHAINS
echo "Uploading archive $ARCHIVE_NAME"
scp -c aes128-ctr -P 30022 -p -i ${SSH_KEY} \
-o PasswordAuthentication=no \
-o StrictHostKeyChecking=no \
$ARCHIVE_NAME \
ci@tmp.kiwix.org:/data/tmp/ci

68
.github/workflows/ci.yml vendored
View File

@ -6,6 +6,61 @@ on:
- cron: '0 1 * * *'
jobs:
Windows:
strategy:
fail-fast: false
runs-on: windows-latest
env:
OS_NAME: windows
COMPILE_CONFIG: native_dyn
HOME: 'C:\\Users\\runneradmin'
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Setup python 3.8
uses: actions/setup-python@v3
with:
python-version: '3.8'
- name: Install packages
run: |
choco.exe install pkgconfiglite ninja
- name: Install python modules
shell: bash
run: |
pip3 install meson pytest requests distro paramiko
pip3 install --no-deps $GITHUB_WORKSPACE
- name: Setup MSVC compiler
uses: bus1/cabuild/action/msdevshell@v1
with:
architecture: x64
- name: secret
shell: bash
run: |
echo "${{secrets.ssh_key}}" > $SSH_KEY
env:
SSH_KEY: ${{ runner.temp }}/id_rsa
- name: Ensure base deps
run: |
python .github\\scripts\\ensure_base_deps.py
env:
SSH_KEY: ${{ runner.temp }}/id_rsa
- name: Compile all deps
run: |
python .github\\scripts\\compile_all_deps.py
env:
SSH_KEY: ${{ runner.temp }}/id_rsa
- name: Build projects
run: |
python .github\\scripts\\build_projects.py
env:
SSH_KEY: ${{ runner.temp }}/id_rsa
- name: Upload failure logs
if: failure()
run: |
python .github\\scripts\\upload_failure_logs.py
env:
SSH_KEY: ${{ runner.temp }}/id_rsa
Linux:
strategy:
fail-fast: false
@ -63,6 +118,10 @@ jobs:
pip3 install --user --no-deps .
env:
REP: ${{github.repository}}
- name: Install paramiko
if: ${{matrix.image_variant != 'bionic' }}
shell: bash
run: pip3 install --user paramiko
- name: secret
shell: bash
run: |
@ -91,7 +150,7 @@ jobs:
COMPILE_CONFIG: ${{matrix.config}}
- name: Upload failure logs
if: failure()
run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh
run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py
env:
COMPILE_CONFIG: ${{matrix.config}}
@ -112,6 +171,7 @@ jobs:
git clone https://github.com/${REP}
cd ./${REP##*/}
git checkout --force ${GITHUB_SHA}
pip3 install --user paramiko
pip3 install --user --no-deps .
env:
REP: ${{github.repository}}
@ -136,7 +196,7 @@ jobs:
kiwix-build/.github/scripts/build_projects.py
- name: Upload failure logs
if: failure()
run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.sh
run: $HOME/kiwix-build/.github/scripts/upload_failure_logs.py
Macos:
strategy:
@ -172,7 +232,7 @@ jobs:
brew install pkg-config ninja automake autoconf
- name: Install python modules
run: |
pip3 install meson pytest requests distro
pip3 install meson pytest requests distro paramiko
pip3 install --no-deps $GITHUB_WORKSPACE
- name: secret
shell: bash
@ -202,6 +262,6 @@ jobs:
COMPILE_CONFIG: ${{matrix.config}}
- name: Upload failure logs
if: failure()
run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.sh
run: $GITHUB_WORKSPACE/.github/scripts/upload_failure_logs.py
env:
COMPILE_CONFIG: ${{matrix.config}}

18
kiwixbuild/buildenv.py
View File

@ -37,15 +37,6 @@ class NeutralEnv:
def detect_platform(self):
_platform = platform.system()
self.distname = _platform
if _platform == "Windows":
print(
"ERROR: kiwix-build is not intented to run on Windows platform.\n"
"It should probably not work, but well, you still can have a try.",
file=sys.stderr,
)
cont = input("Do you want to continue ? [y/N]")
if cont.lower() != "y":
sys.exit(0)
if _platform == "Linux":
self.distname = distro.id()
if self.distname == "ubuntu":
@ -132,13 +123,12 @@ class BuildEnv:
def get_env(self, *, cross_comp_flags, cross_compilers, cross_path):
env = self.configInfo.get_env()
pkgconfig_path = pj(self.install_dir, self.libprefix, "pkgconfig")
env["PKG_CONFIG_PATH"] = ":".join([env["PKG_CONFIG_PATH"], pkgconfig_path])
env["PKG_CONFIG_PATH"].append(pkgconfig_path)
env["PATH"] = ":".join([escape_path(pj(self.install_dir, "bin")), env["PATH"]])
env["PATH"].insert(0, pj(self.install_dir, "bin"))
env["LD_LIBRARY_PATH"] = ":".join(
env["LD_LIBRARY_PATH"].extend(
[
env["LD_LIBRARY_PATH"],
pj(self.install_dir, "lib"),
pj(self.install_dir, self.libprefix),
]
@ -170,7 +160,7 @@ class BuildEnv:
if cross_compilers:
self.configInfo.set_compiler(env)
if cross_path:
env["PATH"] = ":".join(self.configInfo.get_bin_dir() + [env["PATH"]])
env["PATH"][0:0] = self.configInfo.get_bin_dir()
return env
@property

13
kiwixbuild/configs/armhf.py
View File

@ -76,18 +76,15 @@ class ArmConfigInfo(ConfigInfo):
def get_env(self):
env = super().get_env()
env["LD_LIBRARY_PATH"] = ":".join(
[
pj(self.root_path, self.arch_full, "lib64"),
pj(self.root_path, "lib"),
env["LD_LIBRARY_PATH"],
]
)
env["LD_LIBRARY_PATH"][0:0] = [
pj(self.root_path, self.arch_full, "lib64"),
pj(self.root_path, "lib"),
]
env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig")
env["QEMU_LD_PREFIX"] = pj(self.root_path, self.arch_full, "libc")
env["QEMU_SET_ENV"] = "LD_LIBRARY_PATH={}".format(
":".join(
[pj(self.root_path, self.arch_full, "lib"), env["LD_LIBRARY_PATH"]]
[pj(self.root_path, self.arch_full, "lib"), str(env["LD_LIBRARY_PATH"])]
)
)
return env

8
kiwixbuild/configs/base.py
View File

@ -161,18 +161,16 @@ def MixedMixin(static_name):
def get_env(self):
env = super().get_env()
env["PATH"] = ":".join(
[pj(self.static_buildEnv.install_dir, "bin")] + [env["PATH"]]
)
env["PATH"].insert(0, pj(self.static_buildEnv.install_dir, "bin"))
pkgconfig_path = pj(
self.static_buildEnv.install_dir,
self.static_buildEnv.libprefix,
"pkgconfig",
)
env["PKG_CONFIG_PATH"] = ":".join([env["PKG_CONFIG_PATH"], pkgconfig_path])
env["PKG_CONFIG_PATH"].append(pkgconfig_path)
env["CPPFLAGS"] = " ".join(
[
"-I" + pj(self.static_buildEnv.install_dir, "include"),
"-I" + pj(self.static_buildEnv.install_dir, "include"),
env["CPPFLAGS"],
]
)

17
kiwixbuild/configs/musl.py
View File

@ -73,27 +73,22 @@ class MuslConfigInfo(ConfigInfo):
def get_env(self):
env = super().get_env()
env["LD_LIBRARY_PATH"] = ":".join(
[
pj(self.root_path, self.arch_full, "lib64"),
pj(self.root_path, "lib"),
env["LD_LIBRARY_PATH"],
]
)
env["LD_LIBRARY_PATH"][0:0] = [
pj(self.root_path, self.arch_full, "lib64"),
pj(self.root_path, "lib"),
]
env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig")
env["QEMU_LD_PREFIX"] = pj(self.root_path, self.arch_full, "libc")
env["QEMU_SET_ENV"] = "LD_LIBRARY_PATH={}".format(
":".join(
[pj(self.root_path, self.arch_full, "lib"), env["LD_LIBRARY_PATH"]]
[pj(self.root_path, self.arch_full, "lib"), str(env["LD_LIBRARY_PATH"])]
)
)
return env
def set_comp_flags(self, env):
super().set_comp_flags(env)
env["LD_LIBRARY_PATH"] = ":".join(
[pj(self.root_path, self.arch_full, "lib"), env["LD_LIBRARY_PATH"]]
)
env["LD_LIBRARY_PATH"].insert(0, pj(self.root_path, self.arch_full, "lib"))
env["CFLAGS"] = (
" -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
+ env["CFLAGS"]

2
kiwixbuild/configs/native.py
View File

@ -29,7 +29,7 @@ class NativeConfigInfo(ConfigInfo):
class NativeDyn(NativeConfigInfo):
name = "native_dyn"
static = False
compatible_hosts = ["fedora", "debian", "Darwin", "almalinux"]
compatible_hosts = ["fedora", "debian", "Darwin", "almalinux", "Windows"]
class NativeStatic(NativeConfigInfo):

3
kiwixbuild/configs/wasm.py
View File

@ -79,9 +79,8 @@ class WasmConfigInfo(ConfigInfo):
def get_env(self):
env = super().get_env()
env["PATH"] = ":".join(
env["PATH"].extend(
[
env["PATH"],
self.install_path,
pj(self.install_path, "upstream", "emscripten"),
pj(self.install_path, "node", "14.18.2_64bit", "bin"),

2
kiwixbuild/dependencies/all_dependencies.py
View File

@ -13,6 +13,8 @@ class AllBaseDependencies(Dependency):
class Builder(NoopBuilder):
@classmethod
def get_dependencies(cls, configInfo, allDeps):
if neutralEnv("distname") == "Windows":
return ["zlib", "zstd", "icu4c", "zim-testing-suite"]
if configInfo.build == "wasm" or environ.get("OS_NAME") == "manylinux":
return ["zlib", "lzma", "zstd", "icu4c", "xapian-core"]

14
kiwixbuild/dependencies/base.py
View File

@ -131,8 +131,17 @@ class ReleaseDownload(Source):
def _download(self, context):
context.try_skip(neutralEnv("archive_dir"), self.full_name)
for archive in self.archives:
neutralEnv("download")(archive)
archive_iter = iter(self.archives)
archive = next(archive_iter, None)
while archive:
try:
neutralEnv("download")(archive)
except SkipCommand as e:
archive = next(archive_iter, None)
if not archive:
raise e
continue
archive = next(archive_iter, None)
def _extract(self, context):
context.try_skip(self.extract_path)
@ -544,6 +553,7 @@ class MesonBuilder(Builder):
cross_options += ["--cross-file", self.buildEnv.meson_crossfile]
command = [
*neutralEnv("meson_command"),
"setup",
".",
self.build_path,
f"--buildtype={self.build_type}",

110
kiwixbuild/dependencies/icu4c.py
View File

@ -1,9 +1,10 @@
from .base import Dependency, ReleaseDownload, MakeBuilder
from .base import Dependency, ReleaseDownload, MakeBuilder, MesonBuilder
from kiwixbuild.utils import pj, SkipCommand, Remotefile, extract_archive
from kiwixbuild._global import get_target_step, neutralEnv
import os, shutil
import fileinput
import platform
class Icu(Dependency):
@ -20,8 +21,13 @@ class Icu(Dependency):
"ca1ee076163b438461e484421a7679fc33a64cd0a54f9d4b401893fa1eb42701",
"https://github.com/unicode-org/icu/releases/download/release-73-2/icu4c-73_2-data.zip",
)
meson_patch = Remotefile(
"icu_73.2-2_patch.zip",
"218a5f20b58b6b2372e636c2eb2d611a898fdc11be17d6c4f35a3cd54d472010",
"https://wrapdb.mesonbuild.com/v2/icu_73.2-2/get_patch",
)
archives = [archive_src, archive_data]
archives = [archive_src, archive_data, meson_patch]
def _extract(self, context):
context.try_skip(self.extract_path)
@ -42,56 +48,72 @@ class Icu(Dependency):
topdir="data",
name="data",
)
extract_archive(
pj(neutralEnv("archive_dir"), self.meson_patch.name),
neutralEnv("source_dir"),
topdir="icu",
name=self.source_dir,
)
patches = [
"icu4c_fix_static_lib_name_mingw.patch",
# "icu4c_android_elf64_st_info.patch",
# "icu4c_custom_data.patch",
# "icu4c_noxlocale.patch",
"icu4c_rpath.patch",
# "icu4c_build_config.patch",
"icu4c_wasm.patch",
]
class Builder(MakeBuilder):
subsource_dir = "source"
make_install_targets = ["install"]
if platform.system() == "Windows":
@classmethod
def get_dependencies(cls, configInfo, allDeps):
plt = "native_static" if configInfo.static else "native_dyn"
return [(plt, "icu4c")]
@property
def configure_options(self):
yield "--disable-samples"
yield "--disable-tests"
yield "--disable-extras"
yield "--disable-dyload"
yield "--enable-rpath"
yield "--disable-icuio"
yield "--disable-layoutex"
configInfo = self.buildEnv.configInfo
if configInfo.build != "native":
icu_native_builder = get_target_step(
"icu4c", "native_static" if configInfo.static else "native_dyn"
class Builder(MesonBuilder):
def set_env(self, env):
env["ICU_DATA_FILTER_FILE"] = pj(
os.path.dirname(os.path.realpath(__file__)),
"icu4c_data_filter.json",
)
yield f"--with-cross-build={icu_native_builder.build_path}"
yield "--disable-tools"
if configInfo.build in ("android", "wasm"):
yield "--with-data-packaging=archive"
def set_env(self, env):
env["ICU_DATA_FILTER_FILE"] = pj(
os.path.dirname(os.path.realpath(__file__)), "icu4c_data_filter.json"
)
else:
def _post_configure_script(self, context):
if self.buildEnv.configInfo.build != "wasm":
context.skip()
context.try_skip(self.build_path)
for line in fileinput.input(pj(self.build_path, "Makefile"), inplace=True):
if line == "#DATASUBDIR = data\n":
print("DATASUBDIR = data")
else:
print(line, end="")
class Builder(MakeBuilder):
subsource_dir = "source"
make_install_targets = ["install"]
@classmethod
def get_dependencies(cls, configInfo, allDeps):
plt = "native_static" if configInfo.static else "native_dyn"
return [(plt, "icu4c")]
@property
def configure_options(self):
yield "--disable-samples"
yield "--disable-tests"
yield "--disable-extras"
yield "--disable-dyload"
yield "--enable-rpath"
yield "--disable-icuio"
yield "--disable-layoutex"
configInfo = self.buildEnv.configInfo
if configInfo.build != "native":
icu_native_builder = get_target_step(
"icu4c", "native_static" if configInfo.static else "native_dyn"
)
yield f"--with-cross-build={icu_native_builder.build_path}"
yield "--disable-tools"
if configInfo.build in ("android", "wasm"):
yield "--with-data-packaging=archive"
def set_env(self, env):
env["ICU_DATA_FILTER_FILE"] = pj(
os.path.dirname(os.path.realpath(__file__)),
"icu4c_data_filter.json",
)
def _post_configure_script(self, context):
if self.buildEnv.configInfo.build != "wasm":
context.skip()
context.try_skip(self.build_path)
for line in fileinput.input(
pj(self.build_path, "Makefile"), inplace=True
):
if line == "#DATASUBDIR = data\n":
print("DATASUBDIR = data")
else:
print(line, end="")

1
kiwixbuild/dependencies/libkiwix.py
View File

@ -18,7 +18,6 @@ class Libkiwix(Dependency):
"pugixml",
"libzim",
"zlib",
"lzma",
"libcurl",
"libmicrohttpd",
"icu4c",

4
kiwixbuild/dependencies/libmagic.py
View File

@ -46,7 +46,5 @@ class LibMagic(Dependency):
cross_comp_flags=True, cross_compilers=True, cross_path=True
)
libmagic_native_builder = get_target_step("libmagic", "native_static")
env["PATH"] = ":".join(
[pj(libmagic_native_builder.build_path, "src"), env["PATH"]]
)
env["PATH"].insert(0, pj(libmagic_native_builder.build_path, "src"))
run_command(command, self.build_path, context, env=env)

14
kiwixbuild/dependencies/libzim.py
View File

@ -1,5 +1,5 @@
from .base import Dependency, GitClone, MesonBuilder
from kiwixbuild._global import option, get_target_step
from kiwixbuild._global import option, get_target_step, neutralEnv
class Libzim(Dependency):
@ -10,6 +10,13 @@ class Libzim(Dependency):
git_remote = "https://github.com/openzim/libzim.git"
git_dir = "libzim"
@property
def git_ref(self):
if neutralEnv("distname") == "Windows":
return "libzim_github_ci_windows"
else:
return "main"
class Builder(MesonBuilder):
test_options = ["-t", "8"]
strip_options = []
@ -22,6 +29,8 @@ class Libzim(Dependency):
@classmethod
def get_dependencies(cls, configInfo, allDeps):
if neutralEnv("distname") == "Windows":
return ["zstd", "icu4c", "zim-testing-suite"]
deps = ["lzma", "zstd", "xapian-core", "icu4c"]
if configInfo.name not in ("flatpak", "wasm"):
deps.append("zim-testing-suite")
@ -30,6 +39,9 @@ class Libzim(Dependency):
@property
def configure_options(self):
configInfo = self.buildEnv.configInfo
if neutralEnv("distname") == "Windows":
yield "-Dwith_xapian=false"
yield "-Dwerror=false"
if configInfo.build == "android":
yield "-DUSE_BUFFER_HEADER=false"
yield "-Dstatic-linkage=true"

30
kiwixbuild/dependencies/lzma.py
View File

@ -1,28 +1,26 @@
from .base import Dependency, ReleaseDownload, MakeBuilder
from .base import (
Dependency,
ReleaseDownload,
MesonBuilder)
from kiwixbuild.utils import Remotefile
class lzma(Dependency):
name = "lzma"
name = 'lzma'
class Source(ReleaseDownload):
archive = Remotefile(
src_archive = Remotefile(
"xz-5.2.6.tar.gz",
"a2105abee17bcd2ebd15ced31b4f5eda6e17efd6b10f921a01cda4a44c91b3a0",
"https://altushost-swe.dl.sourceforge.net/project/lzmautils/xz-5.2.6.tar.gz",
)
meson_patch = Remotefile(
"liblzma_5.2.6-3_patch.zip",
"1c71536d364e1a3ce6bea61266576f89cc5cce4d3b9e11f3494417dafa29780b",
"https://wrapdb.mesonbuild.com/v2/liblzma_5.2.6-3/get_patch",
)
archives = [src_archive, meson_patch]
patches = ['lzma_meson_install.patch']
class Builder(MakeBuilder):
@property
def configure_options(self):
return [
"--disable-xz",
"--disable-xzdec",
"--disable-lzmadec",
"--disable-lzmainfo",
"--disable-lzma-links",
"--disable-scripts",
"--disable-doc",
# "--disable-symbol-versions"
]
Builder = MesonBuilder

2
kiwixbuild/dependencies/xapian.py
View File

@ -27,6 +27,8 @@ class Xapian(Dependency):
@classmethod
def get_dependencies(cls, configInfo, allDeps):
if neutralEnv("distname") == "Windows":
return ["zlib"]
deps = ["zlib", "lzma"]
if (
configInfo.build in ("win32", "wasm")

64
kiwixbuild/dependencies/zlib.py
View File

@ -1,63 +1,27 @@
import shutil
from .base import Dependency, ReleaseDownload, MakeBuilder
from .base import (
Dependency,
ReleaseDownload,
MesonBuilder)
from kiwixbuild.utils import Remotefile, pj, SkipCommand
from kiwixbuild._global import neutralEnv
class zlib(Dependency):
name = "zlib"
class Source(ReleaseDownload):
archive = Remotefile(
src_archive = Remotefile(
"zlib-1.2.12.tar.gz",
"91844808532e5ce316b3c010929493c0244f3d37593afd6de04f71821d5136d9",
"91844808532e5ce316b3c010929493c0244f3d37593afd6de04f71821d5136d9"
)
patches = ["zlib_std_libname.patch"]
meson_patch = Remotefile(
"zlib_1.2.12-1_patch.zip",
"8ec8344f3fe7b06ad4be768fd416694bc56cb4545ce78b0f1c18b3e72b3ec936",
"https://wrapdb.mesonbuild.com/v2/zlib_1.2.12-1/get_patch")
archives = [src_archive, meson_patch]
#patches = ['zlib_std_libname.patch']
class Builder(MakeBuilder):
dynamic_configure_options = ["--shared"]
static_configure_options = ["--static"]
make_install_targets = ["install"]
def _pre_build_script(self, context):
context.try_skip(self.build_path)
shutil.copytree(self.source_path, self.build_path)
def _configure(self, context):
if self.buildEnv.configInfo.build == "win32":
raise SkipCommand()
return super()._configure(context)
@property
def all_configure_options(self):
yield from self.configure_options
yield from self.static_configure_options if self.buildEnv.configInfo.static else self.dynamic_configure_options
yield from ("--prefix", self.buildEnv.install_dir)
yield from (
"--libdir",
pj(self.buildEnv.install_dir, self.buildEnv.libprefix),
)
@property
def make_options(self):
if self.buildEnv.configInfo.build != "win32":
return
yield "--makefile"
yield "win32/Makefile.gcc"
yield "PREFIX=i686-w64-mingw32-",
yield "SHARED_MODE={}".format(
"0" if self.buildEnv.configInfo.static else "1"
),
yield "INCLUDE_PATH={}".format(pj(self.buildEnv.install_dir, "include")),
yield "LIBRARY_PATH={}".format(
pj(self.buildEnv.install_dir, self.buildEnv.libprefix)
),
yield "BINARY_PATH={}".format(pj(self.buildEnv.install_dir, "bin"))
@property
def make_targets(self):
if self.buildEnv.configInfo.static:
return ["static"]
else:
return ["shared"]
Builder = MesonBuilder

6
kiwixbuild/dependencies/zstd.py
View File

@ -8,9 +8,9 @@ class zstd(Dependency):
class Source(ReleaseDownload):
archive = Remotefile(
"zstd-1.5.2.tar.gz",
"f7de13462f7a82c29ab865820149e778cbfe01087b3a55b5332707abf9db4a6e",
"https://github.com/facebook/zstd/archive/refs/tags/v1.5.2.tar.gz",
"zstd-1.5.5.tar.gz",
"98e9c3d949d1b924e28e01eccb7deed865eefebf25c2f21c702e5cd5b63b85e1",
"https://github.com/facebook/zstd/archive/refs/tags/v1.5.5.tar.gz",
)
class Builder(MesonBuilder):

54
kiwixbuild/patches/lzma_meson_install.patch Normal file
View File

@ -0,0 +1,54 @@
diff '--color=auto' -ur lzma-5.2.6_orig/src/liblzma/meson.build lzma-5.2.6/src/liblzma/meson.build
--- lzma-5.2.6_orig/src/liblzma/meson.build 2023-11-23 14:31:26.110195070 +0100
+++ lzma-5.2.6/src/liblzma/meson.build 2023-12-06 17:04:49.325148650 +0100
@@ -1,3 +1,5 @@
+pkg = import('pkgconfig')
+
lzma_sources = [
'../common/tuklib_physmem.c',
'common/common.c',
@@ -121,12 +123,44 @@
lzmainc = include_directories('api', 'common',
'check', 'lz', 'rangecoder', 'lzma', 'delta', 'simple', '../common')
+
+install_headers(
+ 'api/lzma.h',
+)
+
+install_headers(
+ 'api/lzma/version.h',
+ 'api/lzma/base.h',
+ 'api/lzma/vli.h',
+ 'api/lzma/check.h',
+ 'api/lzma/filter.h',
+ 'api/lzma/bcj.h',
+ 'api/lzma/delta.h',
+ 'api/lzma/lzma12.h',
+ 'api/lzma/container.h',
+ 'api/lzma/stream_flags.h',
+ 'api/lzma/block.h',
+ 'api/lzma/index.h',
+ 'api/lzma/index_hash.h',
+ 'api/lzma/hardware.h',
+ subdir: 'lzma'
+)
+
liblzma = library('lzma', lzma_sources,
main_dec_sources, main_enc_sources, check_sources,
simplefilter_sources, lzma1_sources,
lz_sources, delta_sources,
include_directories : [confinc, lzmainc],
c_args : ['-DHAVE_CONFIG_H', '-DTUKLIB_SYMBOL_PREFIX=lzma_'],
+ install: true
+)
+
+pkg.generate(liblzma,
+ name: 'liblzma',
+ filebase: 'liblzma',
+ description: 'The liblzma compression library',
+ version: meson.project_version(),
+ url: 'http://tukaani.org/xz/'
)
lzma_dep = declare_dependency(link_with : liblzma,

42
kiwixbuild/utils.py
View File

@ -63,9 +63,33 @@ class DefaultEnv(Defaultdict):
def __getitem__(self, name):
if name == b"PATH":
raise KeyError
if name in ["PATH", "PKG_CONFIG_PATH", "LD_LIBRARY_PATH"]:
item = super().__getitem__(name)
if isinstance(item, PathArray):
return item
else:
item = PathArray(item)
self[name] = item
return item
return super().__getitem__(name)
def get_separator():
return ";" if neutralEnv("distname") == "Windows" else ":"
class PathArray(list):
def __init__(self, value):
self.separator = get_separator()
if not value:
super().__init__([])
else:
super().__init__(value.split(self.separator))
def __str__(self):
return self.separator.join(self)
def remove_duplicates(iterable, key_function=None):
seen = set()
if key_function is None:
@ -273,9 +297,22 @@ def extract_archive(archive_path, dest_dir, topdir=None, name=None):
if isdir(member):
continue
perm = (member.external_attr >> 16) & 0x1FF
os.chmod(pj(tmpdir, getname(member)), perm)
if perm:
os.chmod(pj(tmpdir, getname(member)), perm)
name = name or topdir
os.rename(pj(tmpdir, topdir), pj(dest_dir, name))
shutil.copytree(
pj(tmpdir, topdir),
pj(dest_dir, name),
symlinks=True,
dirs_exist_ok=True,
)
# Be sure that all directory in tmpdir are writable to allow correct suppersion of it
for root, dirs, _files in os.walk(tmpdir):
for d in dirs:
os.chmod(
pj(root, d), stat.S_IWRITE | stat.S_IREAD | stat.S_IEXEC
)
else:
if name:
dest_dir = pj(dest_dir, name)
@ -297,6 +334,7 @@ def run_command(command, cwd, context, *, env=None, input=None):
print("run command '{}'".format(command), file=log)
print("current directory is '{}'".format(cwd), file=log)
print("env is :", file=log)
env = {k: str(v) for k, v in env.items()}
for k, v in env.items():
print(" {} : {!r}".format(k, v), file=log)

2
kiwixbuild/versions.py
View File

@ -39,7 +39,7 @@ release_versions = {
# This is the "version" of the whole base_deps_versions dict.
# Change this when you change base_deps_versions.
base_deps_meta_version = "99"
base_deps_meta_version = "00"
base_deps_versions = {
"zlib": "1.2.12",

3
setup.py
View File

@ -36,7 +36,8 @@ setup(
include_package_data=True,
install_requires=[
'meson',
'distro'
'distro',
'paramiko'
],
entry_points={
'console_scripts': [