smallsolar
/
kiwix-build
mirror of https://github.com/kiwix/kiwix-build.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Format our code with black

This commit is contained in:
Matthieu Gautier 2024-02-05 11:41:09 +01:00
parent 939f323709
commit 5a1175cf2d
51 changed files with 1916 additions and 1518 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

169
kiwixbuild/__init__.py
View File

@ -9,76 +9,141 @@ from .builder import Builder
from .flatpak_builder import FlatpakBuilder
from . import _global
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument('target', default='kiwix-tools', nargs='?', metavar='TARGET',
choices=Dependency.all_deps.keys())
parser.add_argument('--working-dir', default=".",
help=("Directory where kiwix-build puts all its files "
"(source, archive and build)\n"
"working-dir can be absolute path or a relative (to cwd) one."))
parser.add_argument('--build-dir', default=".",
help=("Directory where kiwix-build puts all build files.\n"
"build-dir can be absolute path or a relative (to working-dir) one."))
parser.add_argument('--libprefix', default=None)
parser.add_argument('--target-platform', choices=PlatformInfo.all_platforms)
parser.add_argument('--verbose', '-v', action="store_true",
help=("Print all logs on stdout instead of in specific"
" log files per commands"))
parser.add_argument('--hide-progress', action='store_false', dest='show_progress',
help="Hide intermediate progress information.")
parser.add_argument('--skip-source-prepare', action='store_true',
help="Skip the source download part")
parser.add_argument('--build-deps-only', action='store_true',
help="Build only the dependencies of the specified target.")
parser.add_argument('--build-nodeps', action='store_true',
help="Build only the target, not its dependencies.")
parser.add_argument('--make-dist', action='store_true',
help="Build distrubution (dist) source archive")
parser.add_argument('--make-release', action='store_true',
help="Build a release version")
subgroup = parser.add_argument_group('advanced')
subgroup.add_argument('--no-cert-check', action='store_true',
help="Skip SSL certificate verification during download")
subgroup.add_argument('--clean-at-end', action='store_true',
help="Clean all intermediate files after the (successfull) build")
subgroup.add_argument('--dont-install-packages', action='store_true',
help="Do not try to install packages before compiling")
subgroup.add_argument('--assume-packages-installed', action='store_true',
help="Assume the package to install to be aleady installed")
subgroup.add_argument('--android-arch', action='append',
help=("Specify the architecture to build for android application/libraries.\n"
"Can be specified several times to build for several architectures.\n"
"If not specified, all architectures will be build."))
subgroup.add_argument('--ios-arch', action='append',
help=("Specify the architecture to build for ios application/libraries.\n"
"Can be specified several times to build for several architectures.\n"
"If not specified, all architectures will be build."))
subgroup.add_argument('--fast-clone', action='store_true',
help=("Do not clone the whole repository.\n"
"This is useful for one shot build but it is not recommended if you want "
"to develop with the cloned sources."))
parser.add_argument(
"target",
default="kiwix-tools",
nargs="?",
metavar="TARGET",
choices=Dependency.all_deps.keys(),
)
parser.add_argument(
"--working-dir",
default=".",
help=(
"Directory where kiwix-build puts all its files "
"(source, archive and build)\n"
"working-dir can be absolute path or a relative (to cwd) one."
),
)
parser.add_argument(
"--build-dir",
default=".",
help=(
"Directory where kiwix-build puts all build files.\n"
"build-dir can be absolute path or a relative (to working-dir) one."
),
)
parser.add_argument("--libprefix", default=None)
parser.add_argument("--target-platform", choices=PlatformInfo.all_platforms)
parser.add_argument(
"--verbose",
"-v",
action="store_true",
help=(
"Print all logs on stdout instead of in specific" " log files per commands"
),
)
parser.add_argument(
"--hide-progress",
action="store_false",
dest="show_progress",
help="Hide intermediate progress information.",
)
parser.add_argument(
"--skip-source-prepare",
action="store_true",
help="Skip the source download part",
)
parser.add_argument(
"--build-deps-only",
action="store_true",
help="Build only the dependencies of the specified target.",
)
parser.add_argument(
"--build-nodeps",
action="store_true",
help="Build only the target, not its dependencies.",
)
parser.add_argument(
"--make-dist",
action="store_true",
help="Build distrubution (dist) source archive",
)
parser.add_argument(
"--make-release", action="store_true", help="Build a release version"
)
subgroup = parser.add_argument_group("advanced")
subgroup.add_argument(
"--no-cert-check",
action="store_true",
help="Skip SSL certificate verification during download",
)
subgroup.add_argument(
"--clean-at-end",
action="store_true",
help="Clean all intermediate files after the (successfull) build",
)
subgroup.add_argument(
"--dont-install-packages",
action="store_true",
help="Do not try to install packages before compiling",
)
subgroup.add_argument(
"--assume-packages-installed",
action="store_true",
help="Assume the package to install to be aleady installed",
)
subgroup.add_argument(
"--android-arch",
action="append",
help=(
"Specify the architecture to build for android application/libraries.\n"
"Can be specified several times to build for several architectures.\n"
"If not specified, all architectures will be build."
),
)
subgroup.add_argument(
"--ios-arch",
action="append",
help=(
"Specify the architecture to build for ios application/libraries.\n"
"Can be specified several times to build for several architectures.\n"
"If not specified, all architectures will be build."
),
)
subgroup.add_argument(
"--fast-clone",
action="store_true",
help=(
"Do not clone the whole repository.\n"
"This is useful for one shot build but it is not recommended if you want "
"to develop with the cloned sources."
),
)
options = parser.parse_args()
if not options.android_arch:
options.android_arch = ['arm', 'arm64', 'x86', 'x86_64']
options.android_arch = ["arm", "arm64", "x86", "x86_64"]
if not options.ios_arch:
options.ios_arch = ['arm64', 'x86_64']
options.ios_arch = ["arm64", "x86_64"]
if not options.target_platform:
options.target_platform = 'native_dyn'
options.target_platform = "native_dyn"
return options
def main():
options = parse_args()
options.working_dir = os.path.abspath(options.working_dir)
_global.set_options(options)
neutralEnv = buildenv.PlatformNeutralEnv()
_global.set_neutralEnv(neutralEnv)
if options.target_platform == 'flatpak':
if options.target_platform == "flatpak":
builder = FlatpakBuilder()
else:
builder = Builder()
builder.run()

18
kiwixbuild/_global.py
View File

@ -5,23 +5,29 @@ _neutralEnv = None
_options = None
_target_steps = _OrderedDict()
def set_neutralEnv(env):
global _neutralEnv
_neutralEnv = env
def neutralEnv(what):
return getattr(_neutralEnv, what)
def set_options(options):
global _options
_options = options
def option(what):
return getattr(_options, what)
def add_target_step(key, what):
_target_steps[key] = what
def get_target_step(key, default_context=None):
if isinstance(key, tuple):
context, target = key
@ -29,6 +35,7 @@ def get_target_step(key, default_context=None):
context, target = default_context, key
return _target_steps[(context, target)]
def target_steps():
return _target_steps
@ -39,14 +46,15 @@ def backend():
return _backend
_platform = platform.system()
if _platform == 'Windows':
print('ERROR: kiwix-build is not intented to run on Windows platform.\n'
'There is no backend for Windows, so we can\'t launch any commands.')
if _platform == "Windows":
print(
"ERROR: kiwix-build is not intented to run on Windows platform.\n"
"There is no backend for Windows, so we can't launch any commands."
)
sys.exit(0)
if _platform == 'Linux':
if _platform == "Linux":
_platform, _, _ = platform.linux_distribution()
_platform = _platform.lower()
_backend = backends.Linux()
return _backend

148
kiwixbuild/buildenv.py
View File

@ -1,4 +1,3 @@
import os, sys, shutil
import subprocess
import platform
@ -10,60 +9,58 @@ from ._global import neutralEnv, option
class PlatformNeutralEnv:
def __init__(self):
self.working_dir = option('working_dir')
self.working_dir = option("working_dir")
self.source_dir = pj(self.working_dir, "SOURCE")
self.archive_dir = pj(self.working_dir, "ARCHIVE")
self.toolchain_dir = pj(self.working_dir, "TOOLCHAINS")
self.log_dir = pj(self.working_dir, 'LOGS')
for d in (self.source_dir,
self.archive_dir,
self.toolchain_dir,
self.log_dir):
self.log_dir = pj(self.working_dir, "LOGS")
for d in (self.source_dir, self.archive_dir, self.toolchain_dir, self.log_dir):
os.makedirs(d, exist_ok=True)
self.detect_platform()
self.ninja_command = self._detect_command(
'ninja',
default=[['ninja'], ['ninja-build']])
"ninja", default=[["ninja"], ["ninja-build"]]
)
self.meson_command = self._detect_command(
'meson',
default= [['meson.py'], ['meson']])
"meson", default=[["meson.py"], ["meson"]]
)
self.mesontest_command = [*self.meson_command, "test"]
self.patch_command = self._detect_command('patch')
self.git_command = self._detect_command('git')
self.make_command = self._detect_command('make')
self.cmake_command = self._detect_command('cmake')
self.qmake_command = self._detect_command('qmake', required=False)
self.patch_command = self._detect_command("patch")
self.git_command = self._detect_command("git")
self.make_command = self._detect_command("make")
self.cmake_command = self._detect_command("cmake")
self.qmake_command = self._detect_command("qmake", required=False)
def detect_platform(self):
_platform = platform.system()
self.distname = _platform
if _platform == 'Windows':
print('ERROR: kiwix-build is not intented to run on Windows platform.\n'
'It should probably not work, but well, you still can have a try.')
cont = input('Do you want to continue ? [y/N]')
if cont.lower() != 'y':
if _platform == "Windows":
print(
"ERROR: kiwix-build is not intented to run on Windows platform.\n"
"It should probably not work, but well, you still can have a try."
)
cont = input("Do you want to continue ? [y/N]")
if cont.lower() != "y":
sys.exit(0)
if _platform == 'Linux':
if _platform == "Linux":
self.distname = distro.id()
if self.distname == 'ubuntu':
self.distname = 'debian'
if self.distname == "ubuntu":
self.distname = "debian"
def download(self, what, where=None):
where = where or self.archive_dir
download_remote(what, where)
def _detect_command(self, name, default=None, options=['--version'], required=True):
def _detect_command(self, name, default=None, options=["--version"], required=True):
if default is None:
default = [[name]]
env_key = 'KBUILD_{}_COMMAND'.format(name.upper())
env_key = "KBUILD_{}_COMMAND".format(name.upper())
if env_key in os.environ:
default = [os.environ[env_key].split()] + default
for command in default:
try:
retcode = subprocess.check_call(command + options,
stdout=subprocess.DEVNULL)
retcode = subprocess.check_call(
command + options, stdout=subprocess.DEVNULL
)
except (FileNotFoundError, PermissionError, OSError):
# Doesn't exist in PATH or isn't executable
continue
@ -76,22 +73,20 @@ class PlatformNeutralEnv:
print("WARNING: {} command not found".format(name))
return ["{}_NOT_FOUND".format(name.upper())]
class BuildEnv:
def __init__(self, platformInfo):
build_dir = "BUILD_{}".format(platformInfo.name)
self.platformInfo = platformInfo
self.base_build_dir = pj(option('working_dir'), option('build_dir'))
self.base_build_dir = pj(option("working_dir"), option("build_dir"))
self.build_dir = pj(self.base_build_dir, build_dir)
self.install_dir = pj(self.build_dir, "INSTALL")
self.toolchain_dir = pj(self.build_dir, "TOOLCHAINS")
self.log_dir = pj(self.build_dir, 'LOGS')
for d in (self.build_dir,
self.install_dir,
self.toolchain_dir,
self.log_dir):
self.log_dir = pj(self.build_dir, "LOGS")
for d in (self.build_dir, self.install_dir, self.toolchain_dir, self.log_dir):
os.makedirs(d, exist_ok=True)
self.libprefix = option('libprefix') or self._detect_libdir()
self.libprefix = option("libprefix") or self._detect_libdir()
def clean_intermediate_directories(self):
for subdir in os.listdir(self.build_dir):
@ -104,69 +99,72 @@ class BuildEnv:
os.remove(subpath)
def _is_debianlike(self):
return os.path.isfile('/etc/debian_version')
return os.path.isfile("/etc/debian_version")
def _detect_libdir(self):
if self.platformInfo.libdir is not None:
return self.platformInfo.libdir
if self._is_debianlike():
try:
pc = subprocess.Popen(['dpkg-architecture', '-qDEB_HOST_MULTIARCH'],
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL)
pc = subprocess.Popen(
["dpkg-architecture", "-qDEB_HOST_MULTIARCH"],
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
)
(stdo, _) = pc.communicate()
if pc.returncode == 0:
archpath = stdo.decode().strip()
return 'lib/' + archpath
return "lib/" + archpath
except Exception:
pass
if os.path.isdir('/usr/lib64') and not os.path.islink('/usr/lib64'):
return 'lib64'
return 'lib'
if os.path.isdir("/usr/lib64") and not os.path.islink("/usr/lib64"):
return "lib64"
return "lib"
def get_env(self, *, cross_comp_flags, cross_compilers, cross_path):
env = self.platformInfo.get_env()
pkgconfig_path = pj(self.install_dir, self.libprefix, 'pkgconfig')
env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path])
pkgconfig_path = pj(self.install_dir, self.libprefix, "pkgconfig")
env["PKG_CONFIG_PATH"] = ":".join([env["PKG_CONFIG_PATH"], pkgconfig_path])
env['PATH'] = ':'.join([
escape_path(pj(self.install_dir, 'bin')),
env['PATH']
])
env["PATH"] = ":".join([escape_path(pj(self.install_dir, "bin")), env["PATH"]])
env['LD_LIBRARY_PATH'] = ':'.join([env['LD_LIBRARY_PATH'],
pj(self.install_dir, 'lib'),
pj(self.install_dir, self.libprefix)
])
env["LD_LIBRARY_PATH"] = ":".join(
[
env["LD_LIBRARY_PATH"],
pj(self.install_dir, "lib"),
pj(self.install_dir, self.libprefix),
]
)
env['QMAKE_CXXFLAGS'] = " ".join([
escape_path('-I'+pj(self.install_dir, 'include')),
env['QMAKE_CXXFLAGS']
])
env['CPPFLAGS'] = " ".join([
escape_path('-I'+pj(self.install_dir, 'include')),
env['CPPFLAGS']
])
env['QMAKE_LFLAGS'] = " ".join([
escape_path('-L'+pj(self.install_dir, 'lib')),
escape_path('-L'+pj(self.install_dir, self.libprefix)),
env['QMAKE_LFLAGS']
])
env['LDFLAGS'] = " ".join([
escape_path('-L'+pj(self.install_dir, 'lib')),
escape_path('-L'+pj(self.install_dir, self.libprefix)),
env['LDFLAGS']
])
env["QMAKE_CXXFLAGS"] = " ".join(
[escape_path("-I" + pj(self.install_dir, "include")), env["QMAKE_CXXFLAGS"]]
)
env["CPPFLAGS"] = " ".join(
[escape_path("-I" + pj(self.install_dir, "include")), env["CPPFLAGS"]]
)
env["QMAKE_LFLAGS"] = " ".join(
[
escape_path("-L" + pj(self.install_dir, "lib")),
escape_path("-L" + pj(self.install_dir, self.libprefix)),
env["QMAKE_LFLAGS"],
]
)
env["LDFLAGS"] = " ".join(
[
escape_path("-L" + pj(self.install_dir, "lib")),
escape_path("-L" + pj(self.install_dir, self.libprefix)),
env["LDFLAGS"],
]
)
if cross_comp_flags:
self.platformInfo.set_comp_flags(env)
if cross_compilers:
self.platformInfo.set_compiler(env)
if cross_path:
env['PATH'] = ':'.join(self.platformInfo.get_bin_dir() + [env['PATH']])
env["PATH"] = ":".join(self.platformInfo.get_bin_dir() + [env["PATH"]])
return env
@property
def configure_wrapper(self):
try:

97
kiwixbuild/builder.py
View File

@ -1,4 +1,3 @@
import sys
from collections import OrderedDict
from .buildenv import *
@ -8,23 +7,32 @@ from .utils import remove_duplicates, StopBuild, colorize
from .dependencies import Dependency
from .packages import PACKAGE_NAME_MAPPERS
from ._global import (
neutralEnv, option,
add_target_step, get_target_step, target_steps,
backend)
neutralEnv,
option,
add_target_step,
get_target_step,
target_steps,
backend,
)
from . import _global
class Builder:
def __init__(self):
self._targets = {}
PlatformInfo.get_platform('neutral', self._targets)
PlatformInfo.get_platform("neutral", self._targets)
target_platform = option('target_platform')
target_platform = option("target_platform")
platform = PlatformInfo.get_platform(target_platform, self._targets)
if neutralEnv('distname') not in platform.compatible_hosts:
print((colorize('ERROR')+': The target platform {} cannot be build on host {}.\n'
'Select another target platform or change your host system.'
).format(platform.name, neutralEnv('distname')))
self.targetDefs = platform.add_targets(option('target'), self._targets)
if neutralEnv("distname") not in platform.compatible_hosts:
print(
(
colorize("ERROR")
+ ": The target platform {} cannot be build on host {}.\n"
"Select another target platform or change your host system."
).format(platform.name, neutralEnv("distname"))
)
self.targetDefs = platform.add_targets(option("target"), self._targets)
def finalize_target_steps(self):
steps = []
@ -32,19 +40,19 @@ class Builder:
steps += self.order_steps(targetDef)
steps = list(remove_duplicates(steps))
if option('build_nodeps'):
if option("build_nodeps"):
# add all platform steps
for dep in steps:
stepClass = Dependency.all_deps[dep[1]]
if stepClass.dont_skip:
add_target_step(dep, self._targets[dep])
src_targetDef = ('source', targetDef[1])
src_targetDef = ("source", targetDef[1])
add_target_step(src_targetDef, self._targets[src_targetDef])
add_target_step(targetDef, self._targets[targetDef])
else:
for dep in steps:
if option('build_deps_only') and dep[1] == targetDef[1]:
if option("build_deps_only") and dep[1] == targetDef[1]:
continue
add_target_step(dep, self._targets[dep])
self.instanciate_steps()
@ -54,14 +62,14 @@ class Builder:
plt = PlatformInfo.all_platforms[pltName]
for tlcName in plt.toolchain_names:
tlc = Dependency.all_deps[tlcName]
yield('source', tlcName)
yield('neutral' if tlc.neutral else pltName, tlcName)
_targets =dict(self._targets)
yield ("source", tlcName)
yield ("neutral" if tlc.neutral else pltName, tlcName)
_targets = dict(self._targets)
yield from self.order_dependencies(targetDef, _targets)
def order_dependencies(self, targetDef, targets):
targetPlatformName, targetName = targetDef
if targetPlatformName == 'source':
if targetPlatformName == "source":
# Do not try to order sources, they will be added as dep by the
# build step two lines later.
return
@ -75,38 +83,40 @@ class Builder:
depPlatform, depName = targetPlatform.get_fully_qualified_dep(dep)
if (depPlatform, depName) in targets:
yield from self.order_dependencies((depPlatform, depName), targets)
yield ('source', targetName)
yield ("source", targetName)
yield targetDef
def instanciate_steps(self):
for stepDef in list(target_steps()):
stepPlatform, stepName = stepDef
stepClass = Dependency.all_deps[stepName]
if stepPlatform == 'source':
if stepPlatform == "source":
source = get_target_step(stepDef)(stepClass)
add_target_step(stepDef, source)
else:
source = get_target_step(stepName, 'source')
source = get_target_step(stepName, "source")
env = PlatformInfo.get_platform(stepPlatform).buildEnv
builder = get_target_step(stepDef)(stepClass, source, env)
add_target_step(stepDef, builder)
def prepare_sources(self):
if option('skip_source_prepare'):
if option("skip_source_prepare"):
print(colorize("SKIP"))
return
sourceDefs = remove_duplicates(tDef for tDef in target_steps() if tDef[0]=='source')
sourceDefs = remove_duplicates(
tDef for tDef in target_steps() if tDef[0] == "source"
)
for sourceDef in sourceDefs:
print("prepare sources {} :".format(sourceDef[1]))
source = get_target_step(sourceDef)
source.prepare()
def build(self):
builderDefs = (tDef for tDef in target_steps() if tDef[0] != 'source')
builderDefs = (tDef for tDef in target_steps() if tDef[0] != "source")
for builderDef in builderDefs:
builder = get_target_step(builderDef)
if option('make_dist') and builderDef[1] == option('target'):
if option("make_dist") and builderDef[1] == option("target"):
print("make dist {} ({}):".format(builder.name, builderDef[0]))
builder.make_dist()
continue
@ -118,17 +128,17 @@ class Builder:
packages_list = []
for platform in PlatformInfo.all_running_platforms.values():
mapper_name = "{host}_{target}".format(
host=neutralEnv('distname'),
target=platform)
host=neutralEnv("distname"), target=platform
)
package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {})
packages_list += package_name_mapper.get('COMMON', [])
packages_list += package_name_mapper.get("COMMON", [])
to_drop = []
for builderDef in self._targets:
platformName, builderName = builderDef
mapper_name = "{host}_{target}".format(
host=neutralEnv('distname'),
target=platformName)
host=neutralEnv("distname"), target=platformName
)
package_name_mapper = PACKAGE_NAME_MAPPERS.get(mapper_name, {})
packages = package_name_mapper.get(builderName)
if packages:
@ -144,20 +154,20 @@ class Builder:
packages_to_have = self._get_packages()
packages_to_have = remove_duplicates(packages_to_have)
if option('assume_packages_installed'):
if option("assume_packages_installed"):
print(colorize("SKIP") + ", Assume package installed")
return
distname = neutralEnv('distname')
if distname in ('fedora', 'redhat', 'centos'):
package_installer = 'sudo dnf install {}'
package_checker = 'rpm -q --quiet {}'
elif distname in ('debian', 'Ubuntu'):
package_installer = 'sudo apt-get install {}'
distname = neutralEnv("distname")
if distname in ("fedora", "redhat", "centos"):
package_installer = "sudo dnf install {}"
package_checker = "rpm -q --quiet {}"
elif distname in ("debian", "Ubuntu"):
package_installer = "sudo apt-get install {}"
package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1'
elif distname == 'Darwin':
package_installer = 'brew install {}'
package_checker = 'brew ls --version {} > /dev/null'
elif distname == "Darwin":
package_installer = "brew install {}"
package_checker = "brew ls --version {} > /dev/null"
packages_to_install = []
for package in packages_to_have:
@ -176,12 +186,12 @@ class Builder:
print(command)
subprocess.check_call(command, shell=True)
else:
print(colorize("SKIP")+ ", No package to install.")
print(colorize("SKIP") + ", No package to install.")
def run(self):
try:
print("[INSTALL PACKAGES]")
if option('dont_install_packages'):
if option("dont_install_packages"):
print(colorize("SKIP"))
else:
self.install_packages()
@ -195,7 +205,7 @@ class Builder:
self.build()
# No error, clean intermediate file at end of build if needed.
print("[CLEAN]")
if option('clean_at_end'):
if option("clean_at_end"):
for platform in PlatformInfo.all_running_platforms.values():
platform.clean_intermediate_directories()
else:
@ -203,4 +213,3 @@ class Builder:
except StopBuild as e:
print(e)
sys.exit("Stopping build due to errors")

3
kiwixbuild/dependencies/__init__.py
View File

@ -1,4 +1,3 @@
from .base import *
from . import (
all_dependencies,
@ -28,5 +27,5 @@ from . import (
zim_tools,
zim_testing_suite,
zlib,
zstd
zstd,
)

39
kiwixbuild/dependencies/all_dependencies.py
View File

@ -1,32 +1,45 @@
from os import environ
from .base import (
Dependency,
NoopSource,
NoopBuilder)
from .base import Dependency, NoopSource, NoopBuilder
from kiwixbuild._global import neutralEnv
class AllBaseDependencies(Dependency):
name = "alldependencies"
Source = NoopSource
class Builder(NoopBuilder):
@classmethod
def get_dependencies(cls, platformInfo, allDeps):
if platformInfo.build == "wasm" or environ.get('OS_NAME') == 'bionic':
return ['zlib', 'lzma', 'zstd', 'icu4c', 'xapian-core']
if platformInfo.build == "wasm" or environ.get("OS_NAME") == "bionic":
return ["zlib", "lzma", "zstd", "icu4c", "xapian-core"]
base_deps = ['zlib', 'lzma', 'zstd', 'xapian-core', 'pugixml', 'libcurl', 'icu4c', 'mustache', 'libmicrohttpd', 'zim-testing-suite']
base_deps = [
"zlib",
"lzma",
"zstd",
"xapian-core",
"pugixml",
"libcurl",
"icu4c",
"mustache",
"libmicrohttpd",
"zim-testing-suite",
]
# Add specific dependencies depending of the platform
if platformInfo.build not in ('android', 'iOS'):
if platformInfo.build not in ("android", "iOS"):
# For zimtools
base_deps += ['docoptcpp']
if platformInfo.build != 'win32':
base_deps += ["docoptcpp"]
if platformInfo.build != "win32":
# zimwriterfs
base_deps += ['libmagic', 'gumbo']
if platformInfo.build == 'native' and neutralEnv('distname') != 'Darwin':
base_deps += ["libmagic", "gumbo"]
if (
platformInfo.build == "native"
and neutralEnv("distname") != "Darwin"
):
# We compile kiwix-desktop only on native and not on `Darwin`
# So we need aria2 only there
base_deps += ['aria2']
base_deps += ["aria2"]
return base_deps

23
kiwixbuild/dependencies/aria2.py
View File

@ -1,18 +1,17 @@
from .base import (
Dependency,
ReleaseDownload,
MakeBuilder
)
from .base import Dependency, ReleaseDownload, MakeBuilder
from kiwixbuild.utils import Remotefile, run_command
class Aria2(Dependency):
name = "aria2"
class Source(ReleaseDownload):
archive = Remotefile('aria2-1.37.0.tar.xz',
'60a420ad7085eb616cb6e2bdf0a7206d68ff3d37fb5a956dc44242eb2f79b66b',
'https://github.com/aria2/aria2/releases/download/release-1.37.0/aria2-1.37.0.tar.xz')
archive = Remotefile(
"aria2-1.37.0.tar.xz",
"60a420ad7085eb616cb6e2bdf0a7206d68ff3d37fb5a956dc44242eb2f79b66b",
"https://github.com/aria2/aria2/releases/download/release-1.37.0/aria2-1.37.0.tar.xz",
)
def _post_prepare_script(self, context):
context.try_skip(self.extract_path)
@ -20,5 +19,9 @@ class Aria2(Dependency):
run_command(command, self.extract_path, context)
class Builder(MakeBuilder):
dependencies = ['zlib']
configure_options = ["--disable-libaria2", "--disable-websocket", "--without-sqlite3"]
dependencies = ["zlib"]
configure_options = [
"--disable-libaria2",
"--disable-websocket",
"--without-sqlite3",
]

308
kiwixbuild/dependencies/base.py
View File

@ -3,7 +3,17 @@ import os
import shutil
import time
from kiwixbuild.utils import pj, Context, SkipCommand, WarningMessage, extract_archive, StopBuild, run_command, colorize, copy_tree
from kiwixbuild.utils import (
pj,
Context,
SkipCommand,
WarningMessage,
extract_archive,
StopBuild,
run_command,
colorize,
copy_tree,
)
from kiwixbuild.versions import main_project_versions, base_deps_versions
from kiwixbuild._global import neutralEnv, option, get_target_step
@ -13,8 +23,8 @@ SCRIPT_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
class _MetaDependency(type):
def __new__(cls, name, bases, dct):
_class = type.__new__(cls, name, bases, dct)
if name != 'Dependency':
dep_name = dct['name']
if name != "Dependency":
dep_name = dct["name"]
Dependency.all_deps[dep_name] = _class
return _class
@ -29,7 +39,7 @@ class Dependency(metaclass=_MetaDependency):
def version(cls):
if cls.name in base_deps_versions:
return base_deps_versions[cls.name]
elif option('make_release'):
elif option("make_release"):
return main_project_versions.get(cls.name, None)
return None
@ -42,8 +52,9 @@ class Dependency(metaclass=_MetaDependency):
class Source:
"""Base Class to the real preparator
A source preparator must install source in the self.source_dir attribute
inside the neutralEnv.source_dir."""
A source preparator must install source in the self.source_dir attribute
inside the neutralEnv.source_dir."""
def __init__(self, target):
self.target = target
@ -61,22 +72,22 @@ class Source:
@property
def source_path(self):
return pj(neutralEnv('source_dir'), self.source_dir)
return pj(neutralEnv("source_dir"), self.source_dir)
@property
def _log_dir(self):
return neutralEnv('log_dir')
return neutralEnv("log_dir")
def _patch(self, context):
context.try_skip(self.source_path)
for p in self.patches:
patch_file_path = pj(SCRIPT_DIR, 'patches', p)
patch_command = [*neutralEnv('patch_command'), "-p1", "-i", patch_file_path]
patch_file_path = pj(SCRIPT_DIR, "patches", p)
patch_command = [*neutralEnv("patch_command"), "-p1", "-i", patch_file_path]
run_command(patch_command, self.source_path, context)
def command(self, name, function, *args):
print(" {} {} : ".format(name, self.name), end="", flush=True)
log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
log = pj(self._log_dir, "cmd_{}_{}.log".format(name, self.name))
context = Context(name, log, True)
try:
start_time = time.time()
@ -92,7 +103,7 @@ class Source:
except subprocess.CalledProcessError:
print(colorize("ERROR"))
try:
with open(log, 'r') as f:
with open(log, "r") as f:
print(f.read())
except:
pass
@ -112,34 +123,36 @@ class ReleaseDownload(Source):
@property
def archives(self):
return (self.archive, )
return (self.archive,)
@property
def extract_path(self):
return pj(neutralEnv('source_dir'), self.source_dir)
return pj(neutralEnv("source_dir"), self.source_dir)
def _download(self, context):
context.try_skip(neutralEnv('archive_dir'), self.full_name)
context.try_skip(neutralEnv("archive_dir"), self.full_name)
for archive in self.archives:
neutralEnv('download')(archive)
neutralEnv("download")(archive)
def _extract(self, context):
context.try_skip(self.extract_path)
if os.path.exists(self.extract_path):
shutil.rmtree(self.extract_path)
for archive in self.archives:
extract_archive(pj(neutralEnv('archive_dir'), archive.name),
neutralEnv('source_dir'),
topdir=self.archive_top_dir,
name=self.source_dir)
extract_archive(
pj(neutralEnv("archive_dir"), archive.name),
neutralEnv("source_dir"),
topdir=self.archive_top_dir,
name=self.source_dir,
)
def prepare(self):
self.command('download', self._download)
self.command('extract', self._extract)
if hasattr(self, 'patches'):
self.command('patch', self._patch)
if hasattr(self, '_post_prepare_script'):
self.command('post_prepare_script', self._post_prepare_script)
self.command("download", self._download)
self.command("extract", self._extract)
if hasattr(self, "patches"):
self.command("patch", self._patch)
if hasattr(self, "_post_prepare_script"):
self.command("post_prepare_script", self._post_prepare_script)
class GitClone(Source):
@ -152,48 +165,66 @@ class GitClone(Source):
@property
def source_dir(self):
if option('make_release'):
if option("make_release"):
return "{}_release".format(self.git_dir)
else:
return self.git_dir
@property
def git_path(self):
return pj(neutralEnv('source_dir'), self.source_dir)
return pj(neutralEnv("source_dir"), self.source_dir)
@property
def git_ref(self):
if option('make_release'):
if option("make_release"):
return self.release_git_ref
else:
return self.base_git_ref
def _git_init(self, context):
if option('fast_clone') and self.force_full_clone == False:
command = [*neutralEnv('git_command'), "clone" , "--depth=1", "--branch", self.git_ref, self.git_remote, self.source_dir]
run_command(command, neutralEnv('source_dir'), context)
if option("fast_clone") and self.force_full_clone == False:
command = [
*neutralEnv("git_command"),
"clone",
"--depth=1",
"--branch",
self.git_ref,
self.git_remote,
self.source_dir,
]
run_command(command, neutralEnv("source_dir"), context)
else:
command = [*neutralEnv('git_command'), "clone", self.git_remote, self.source_dir]
run_command(command, neutralEnv('source_dir'), context)
command = [*neutralEnv('git_command'), "checkout", self.git_ref]
command = [
*neutralEnv("git_command"),
"clone",
self.git_remote,
self.source_dir,
]
run_command(command, neutralEnv("source_dir"), context)
command = [*neutralEnv("git_command"), "checkout", self.git_ref]
run_command(command, self.git_path, context)
def _git_update(self, context):
command = [*neutralEnv('git_command'), "fetch", "origin", self.git_ref]
command = [*neutralEnv("git_command"), "fetch", "origin", self.git_ref]
run_command(command, self.git_path, context)
try:
command = [*neutralEnv('git_command'), "merge", "--ff-only", f"origin/{self.git_ref}"]
command = [
*neutralEnv("git_command"),
"merge",
"--ff-only",
f"origin/{self.git_ref}",
]
run_command(command, self.git_path, context)
except subprocess.CalledProcessError:
raise WarningMessage("Cannot update, please check log for information")
def prepare(self):
if not os.path.exists(self.git_path):
self.command('gitinit', self._git_init)
self.command("gitinit", self._git_init)
else:
self.command('gitupdate', self._git_update)
if hasattr(self, '_post_prepare_script'):
self.command('post_prepare_script', self._post_prepare_script)
self.command("gitupdate", self._git_update)
if hasattr(self, "_post_prepare_script"):
self.command("post_prepare_script", self._post_prepare_script)
class Builder:
@ -230,7 +261,7 @@ class Builder:
def command(self, name, function, *args):
print(" {} {} : ".format(name, self.name), end="", flush=True)
log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
log = pj(self._log_dir, "cmd_{}_{}.log".format(name, self.name))
context = Context(name, log, self.target.force_native_build)
if self.target.force_build:
context.no_skip = True
@ -248,7 +279,7 @@ class Builder:
except subprocess.CalledProcessError:
print(colorize("ERROR"))
try:
with open(log, 'r') as f:
with open(log, "r") as f:
print(f.read())
except:
pass
@ -258,36 +289,40 @@ class Builder:
raise
def build(self):
if hasattr(self, '_pre_build_script'):
self.command('pre_build_script', self._pre_build_script)
self.command('configure', self._configure)
if hasattr(self, '_post_configure_script'):
self.command('post_configure_script', self._post_configure_script)
self.command('compile', self._compile)
if hasattr(self, '_test'):
self.command('test', self._test)
self.command('install', self._install)
if hasattr(self, '_post_build_script'):
self.command('post_build_script', self._post_build_script)
if hasattr(self, "_pre_build_script"):
self.command("pre_build_script", self._pre_build_script)
self.command("configure", self._configure)
if hasattr(self, "_post_configure_script"):
self.command("post_configure_script", self._post_configure_script)
self.command("compile", self._compile)
if hasattr(self, "_test"):
self.command("test", self._test)
self.command("install", self._install)
if hasattr(self, "_post_build_script"):
self.command("post_build_script", self._post_build_script)
def make_dist(self):
if hasattr(self, '_pre_build_script'):
self.command('pre_build_script', self._pre_build_script)
self.command('configure', self._configure)
self.command('make_dist', self._make_dist)
if hasattr(self, "_pre_build_script"):
self.command("pre_build_script", self._pre_build_script)
self.command("configure", self._configure)
self.command("make_dist", self._make_dist)
def set_flatpak_buildsystem(self, module):
if getattr(self, 'flatpak_buildsystem', None):
module['buildsystem'] = self.flatpak_buildsystem
if getattr(self, 'subsource_dir', None):
module['subdir'] = self.subsource_dir
if getattr(self, 'flatpack_build_options', None):
module['build-options'] = self.flatpack_build_options
if getattr(self, 'configure_option', ''):
module['config-opts'] = self.configure_option.split(' ')
if getattr(self, "flatpak_buildsystem", None):
module["buildsystem"] = self.flatpak_buildsystem
if getattr(self, "subsource_dir", None):
module["subdir"] = self.subsource_dir
if getattr(self, "flatpack_build_options", None):
module["build-options"] = self.flatpack_build_options
if getattr(self, "configure_option", ""):
module["config-opts"] = self.configure_option.split(" ")
def get_env(self, *, cross_comp_flags, cross_compilers, cross_path):
env = self.buildEnv.get_env(cross_comp_flags=cross_comp_flags, cross_compilers=cross_compilers, cross_path=cross_path)
env = self.buildEnv.get_env(
cross_comp_flags=cross_comp_flags,
cross_compilers=cross_compilers,
cross_path=cross_path,
)
for dep in self.get_dependencies(self.buildEnv.platformInfo, False):
try:
builder = get_target_step(dep, self.buildEnv.platformInfo.name)
@ -317,7 +352,7 @@ class TcCopyBuilder(Builder):
return pj(self.buildEnv.toolchain_dir, self.target.full_name())
def build(self):
self.command('copy', self._copy)
self.command("copy", self._copy)
def _copy(self, context):
context.try_skip(self.build_path)
@ -339,18 +374,18 @@ class MakeBuilder(Builder):
install_options = []
configure_script = "configure"
configure_env = {
'_format_CFLAGS' : '{env[CFLAGS]} -O3',
'_format_CXXFLAGS': '{env[CXXFLAGS]} -O3'
"_format_CFLAGS": "{env[CFLAGS]} -O3",
"_format_CXXFLAGS": "{env[CXXFLAGS]} -O3",
}
make_targets = []
flatpak_buildsystem = None
@property
def make_install_targets(self):
if self.buildEnv.platformInfo.build in ('iOS', "wasm"):
yield 'install'
if self.buildEnv.platformInfo.build in ("iOS", "wasm"):
yield "install"
else:
yield 'install-strip'
yield "install-strip"
@property
def all_configure_options(self):
@ -361,27 +396,26 @@ class MakeBuilder(Builder):
yield from self.dynamic_configure_options
if not self.target.force_native_build:
yield from self.buildEnv.platformInfo.configure_options
yield from ('--prefix', self.buildEnv.install_dir)
yield from ('--libdir', pj(self.buildEnv.install_dir, self.buildEnv.libprefix))
yield from ("--prefix", self.buildEnv.install_dir)
yield from ("--libdir", pj(self.buildEnv.install_dir, self.buildEnv.libprefix))
def set_configure_env(self, env):
dep_conf_env = self.configure_env
if not dep_conf_env:
return
for k, v in dep_conf_env.items():
if k.startswith('_format_'):
if k.startswith("_format_"):
v = v.format(buildEnv=self.buildEnv, env=env)
env[k[8:]] = v
else:
env[k] = v
def _configure(self, context):
context.try_skip(self.build_path)
command = [
*self.buildEnv.configure_wrapper,
pj(self.source_path, self.configure_script),
*self.all_configure_options
*self.all_configure_options,
]
env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
self.set_configure_env(env)
@ -391,10 +425,10 @@ class MakeBuilder(Builder):
context.try_skip(self.build_path)
command = [
*self.buildEnv.make_wrapper,
*neutralEnv('make_command'),
*neutralEnv("make_command"),
"-j4",
*self.make_targets,
*self.make_options
*self.make_options,
]
env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
run_command(command, self.build_path, context, env=env)
@ -403,26 +437,22 @@ class MakeBuilder(Builder):
context.try_skip(self.build_path)
command = [
*self.buildEnv.make_wrapper,
*neutralEnv('make_command'),
*neutralEnv("make_command"),
*self.make_install_targets,
*self.make_options
*self.make_options,
]
env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
run_command(command, self.build_path, context, env=env)
def _make_dist(self, context):
context.try_skip(self.build_path)
command = [
*self.buildEnv.make_wrapper,
*neutralEnv('make_command'),
"dist"
]
command = [*self.buildEnv.make_wrapper, *neutralEnv("make_command"), "dist"]
env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
run_command(command, self.build_path, context, env=env)
class CMakeBuilder(MakeBuilder):
flatpak_buildsystem = 'cmake'
flatpak_buildsystem = "cmake"
def _configure(self, context):
context.try_skip(self.build_path)
@ -430,74 +460,79 @@ class CMakeBuilder(MakeBuilder):
if not self.target.force_native_build and self.buildEnv.cmake_crossfile:
cross_options += [f"-DCMAKE_TOOLCHAIN_FILE={self.buildEnv.cmake_crossfile}"]
command = [
*neutralEnv('cmake_command'),
*neutralEnv("cmake_command"),
*self.configure_options,
"-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON",
f"-DCMAKE_INSTALL_PREFIX={self.buildEnv.install_dir}",
f"-DCMAKE_INSTALL_LIBDIR={self.buildEnv.libprefix}",
self.source_path,
*cross_options
*cross_options,
]
env = self.get_env(cross_comp_flags=True, cross_compilers=False, cross_path=True)
env = self.get_env(
cross_comp_flags=True, cross_compilers=False, cross_path=True
)
self.set_configure_env(env)
run_command(command, self.build_path, context, env=env)
def set_flatpak_buildsystem(self, module):
super().set_flatpak_buildsystem( module)
module['buildir'] = True
super().set_flatpak_buildsystem(module)
module["buildir"] = True
class QMakeBuilder(MakeBuilder):
qmake_targets = []
flatpak_buildsystem = 'qmake'
flatpak_buildsystem = "qmake"
@property
def env_options(self):
if 'QMAKE_CC' in os.environ:
if "QMAKE_CC" in os.environ:
yield f"QMAKE_CC={os.environ['QMAKE_CC']}"
if 'QMAKE_CXX' in os.environ:
if "QMAKE_CXX" in os.environ:
yield f"QMAKE_CXX={os.environ['QMAKE_CXX']}"
def _configure(self, context):
context.try_skip(self.build_path)
command = [
"qmake",
*self.configure_options,
*self.env_options,
self.source_path
*self.configure_options,
*self.env_options,
self.source_path,
]
env = self.get_env(cross_comp_flags=True, cross_compilers=False, cross_path=True)
env = self.get_env(
cross_comp_flags=True, cross_compilers=False, cross_path=True
)
self.set_configure_env(env)
run_command(command, self.build_path, context, env=env)
def _make_dist(self, context):
command = [
*neutralEnv('git_command'), "archive",
"-o", f"{self.build_path}/{self.target_full_name()}.tar.gz",
*neutralEnv("git_command"),
"archive",
"-o",
f"{self.build_path}/{self.target_full_name()}.tar.gz",
f"--prefix={self.target_full_name()}/",
"HEAD"
"HEAD",
]
run_command(command, self.source_path, context)
class MesonBuilder(Builder):
configure_options = []
test_options = []
flatpak_buildsystem = 'meson'
flatpak_buildsystem = "meson"
@property
def build_type(self):
return 'release' if option('make_release') else 'debug'
return "release" if option("make_release") else "debug"
@property
def strip_options(self):
if option('make_release'):
yield '--strip'
if option("make_release"):
yield "--strip"
@property
def library_type(self):
return 'static' if self.buildEnv.platformInfo.static else 'shared'
return "static" if self.buildEnv.platformInfo.static else "shared"
def _configure(self, context):
context.try_skip(self.build_path)
@ -508,47 +543,54 @@ class MesonBuilder(Builder):
if not self.target.force_native_build and self.buildEnv.meson_crossfile:
cross_options += ["--cross-file", self.buildEnv.meson_crossfile]
command = [
*neutralEnv('meson_command'),
'.', self.build_path,
f'--buildtype={self.build_type}',
*neutralEnv("meson_command"),
".",
self.build_path,
f"--buildtype={self.build_type}",
*self.strip_options,
f'--default-library={self.library_type}',
f"--default-library={self.library_type}",
*self.configure_options,
f'--prefix={self.buildEnv.install_dir}',
f'--libdir={self.buildEnv.libprefix}',
*cross_options
f"--prefix={self.buildEnv.install_dir}",
f"--libdir={self.buildEnv.libprefix}",
*cross_options,
]
env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True)
env = self.get_env(
cross_comp_flags=False, cross_compilers=False, cross_path=True
)
run_command(command, self.source_path, context, env=env)
def _compile(self, context):
context.try_skip(self.build_path)
command = [*neutralEnv('ninja_command'), "-v"]
env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True)
command = [*neutralEnv("ninja_command"), "-v"]
env = self.get_env(
cross_comp_flags=False, cross_compilers=False, cross_path=True
)
run_command(command, self.build_path, context, env=env)
def _test(self, context):
context.try_skip(self.build_path)
if ( self.buildEnv.platformInfo.build == 'android'
or (self.buildEnv.platformInfo.build != 'native'
and not self.buildEnv.platformInfo.static)
):
if self.buildEnv.platformInfo.build == "android" or (
self.buildEnv.platformInfo.build != "native"
and not self.buildEnv.platformInfo.static
):
raise SkipCommand()
command = [
*neutralEnv('mesontest_command'),
'--verbose',
*self.test_options
]
env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True)
command = [*neutralEnv("mesontest_command"), "--verbose", *self.test_options]
env = self.get_env(
cross_comp_flags=False, cross_compilers=False, cross_path=True
)
run_command(command, self.build_path, context, env=env)
def _install(self, context):
context.try_skip(self.build_path)
command = [*neutralEnv('ninja_command'), '-v', 'install']
env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True)
command = [*neutralEnv("ninja_command"), "-v", "install"]
env = self.get_env(
cross_comp_flags=False, cross_compilers=False, cross_path=True
)
run_command(command, self.build_path, context, env=env)
def _make_dist(self, context):
command = [*neutralEnv('ninja_command'), "-v", "dist"]
env = self.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=True)
command = [*neutralEnv("ninja_command"), "-v", "dist"]
env = self.get_env(
cross_comp_flags=False, cross_compilers=False, cross_path=True
)
run_command(command, self.build_path, context, env=env)

11
kiwixbuild/dependencies/docoptcpp.py
View File

@ -1,14 +1,10 @@
from .base import (
Dependency,
GitClone,
CMakeBuilder)
from .base import Dependency, GitClone, CMakeBuilder
from kiwixbuild.utils import Remotefile
class docoptcpp(Dependency):
name = 'docoptcpp'
name = "docoptcpp"
class Source(GitClone):
git_remote = "https://github.com/docopt/docopt.cpp.git"
@ -17,5 +13,4 @@ class docoptcpp(Dependency):
git_ref = "3dd23e3280f213bacefdf5fcb04857bf52e90917"
class Builder(CMakeBuilder):
make_install_targets = ['install']
make_install_targets = ["install"]

14
kiwixbuild/dependencies/gumbo.py
View File

@ -1,8 +1,4 @@
from .base import (
Dependency,
ReleaseDownload,
MakeBuilder
)
from .base import Dependency, ReleaseDownload, MakeBuilder
from kiwixbuild.utils import Remotefile, run_command
@ -11,9 +7,11 @@ class Gumbo(Dependency):
name = "gumbo"
class Source(ReleaseDownload):
archive = Remotefile('gumbo-0.10.1.tar.gz',
'28463053d44a5dfbc4b77bcf49c8cee119338ffa636cc17fc3378421d714efad',
'https://github.com/google/gumbo-parser/archive/v0.10.1.tar.gz')
archive = Remotefile(
"gumbo-0.10.1.tar.gz",
"28463053d44a5dfbc4b77bcf49c8cee119338ffa636cc17fc3378421d714efad",
"https://github.com/google/gumbo-parser/archive/v0.10.1.tar.gz",
)
def _post_prepare_script(self, context):
context.try_skip(self.extract_path)

78
kiwixbuild/dependencies/icu4c.py
View File

@ -1,25 +1,25 @@
from .base import (
Dependency,
ReleaseDownload,
MakeBuilder
)
from .base import Dependency, ReleaseDownload, MakeBuilder
from kiwixbuild.utils import pj, SkipCommand, Remotefile, extract_archive
from kiwixbuild._global import get_target_step, neutralEnv
import os, shutil
import fileinput
class Icu(Dependency):
name = "icu4c"
class Source(ReleaseDownload):
archive_src = Remotefile('icu4c-73_2-src.tgz',
'818a80712ed3caacd9b652305e01afc7fa167e6f2e94996da44b90c2ab604ce1',
'https://github.com/unicode-org/icu/releases/download/release-73-2/icu4c-73_2-src.tgz')
archive_data = Remotefile('icu4c-73_2-data.zip',
'ca1ee076163b438461e484421a7679fc33a64cd0a54f9d4b401893fa1eb42701',
'https://github.com/unicode-org/icu/releases/download/release-73-2/icu4c-73_2-data.zip')
archive_src = Remotefile(
"icu4c-73_2-src.tgz",
"818a80712ed3caacd9b652305e01afc7fa167e6f2e94996da44b90c2ab604ce1",
"https://github.com/unicode-org/icu/releases/download/release-73-2/icu4c-73_2-src.tgz",
)
archive_data = Remotefile(
"icu4c-73_2-data.zip",
"ca1ee076163b438461e484421a7679fc33a64cd0a54f9d4b401893fa1eb42701",
"https://github.com/unicode-org/icu/releases/download/release-73-2/icu4c-73_2-data.zip",
)
archives = [archive_src, archive_data]
@ -28,28 +28,30 @@ class Icu(Dependency):
if os.path.exists(self.extract_path):
shutil.rmtree(self.extract_path)
extract_archive(
pj(neutralEnv('archive_dir'), self.archive_src.name),
neutralEnv('source_dir'),
pj(neutralEnv("archive_dir"), self.archive_src.name),
neutralEnv("source_dir"),
topdir=None,
name=self.source_dir)
shutil.rmtree(pj(neutralEnv('source_dir'), self.source_dir, 'source', 'data'))
name=self.source_dir,
)
shutil.rmtree(
pj(neutralEnv("source_dir"), self.source_dir, "source", "data")
)
extract_archive(
pj(neutralEnv('archive_dir'), self.archive_data.name),
pj(neutralEnv('source_dir'), self.source_dir, 'source'),
topdir='data',
name='data'
pj(neutralEnv("archive_dir"), self.archive_data.name),
pj(neutralEnv("source_dir"), self.source_dir, "source"),
topdir="data",
name="data",
)
patches = [
"icu4c_fix_static_lib_name_mingw.patch",
# "icu4c_android_elf64_st_info.patch",
# "icu4c_custom_data.patch",
# "icu4c_noxlocale.patch",
"icu4c_rpath.patch",
# "icu4c_build_config.patch",
"icu4c_wasm.patch"
]
"icu4c_fix_static_lib_name_mingw.patch",
# "icu4c_android_elf64_st_info.patch",
# "icu4c_custom_data.patch",
# "icu4c_noxlocale.patch",
"icu4c_rpath.patch",
# "icu4c_build_config.patch",
"icu4c_wasm.patch",
]
class Builder(MakeBuilder):
subsource_dir = "source"
@ -57,8 +59,8 @@ class Icu(Dependency):
@classmethod
def get_dependencies(cls, platformInfo, allDeps):
plt = 'native_static' if platformInfo.static else 'native_dyn'
return [(plt, 'icu4c')]
plt = "native_static" if platformInfo.static else "native_dyn"
return [(plt, "icu4c")]
@property
def configure_options(self):
@ -70,23 +72,25 @@ class Icu(Dependency):
yield "--disable-icuio"
yield "--disable-layoutex"
platformInfo = self.buildEnv.platformInfo
if platformInfo.build != 'native':
if platformInfo.build != "native":
icu_native_builder = get_target_step(
'icu4c',
'native_static' if platformInfo.static else 'native_dyn')
"icu4c", "native_static" if platformInfo.static else "native_dyn"
)
yield f"--with-cross-build={icu_native_builder.build_path}"
yield "--disable-tools"
if platformInfo.build in ('android', 'wasm'):
if platformInfo.build in ("android", "wasm"):
yield "--with-data-packaging=archive"
def set_env(self, env):
env['ICU_DATA_FILTER_FILE'] = pj(os.path.dirname(os.path.realpath(__file__)), "icu4c_data_filter.json")
env["ICU_DATA_FILTER_FILE"] = pj(
os.path.dirname(os.path.realpath(__file__)), "icu4c_data_filter.json"
)
def _post_configure_script(self, context):
if self.buildEnv.platformInfo.build != "wasm":
context.skip()
context.try_skip(self.build_path)
for line in fileinput.input(pj(self.build_path, 'Makefile'), inplace=True):
for line in fileinput.input(pj(self.build_path, "Makefile"), inplace=True):
if line == "#DATASUBDIR = data\n":
print("DATASUBDIR = data")
else:

39
kiwixbuild/dependencies/ios_fat_lib.py
View File

@ -3,11 +3,7 @@ import os
from kiwixbuild.platforms import PlatformInfo
from kiwixbuild.utils import pj, copy_tree, run_command
from kiwixbuild._global import option
from .base import (
Dependency,
NoopSource,
Builder as BaseBuilder)
from .base import Dependency, NoopSource, Builder as BaseBuilder
class IOSFatLib(Dependency):
@ -16,38 +12,39 @@ class IOSFatLib(Dependency):
Source = NoopSource
class Builder(BaseBuilder):
@classmethod
def get_dependencies(self, platfomInfo, alldeps):
base_target = option('target')
return [('iOS_{}'.format(arch), base_target) for arch in option('ios_arch')]
base_target = option("target")
return [("iOS_{}".format(arch), base_target) for arch in option("ios_arch")]
def _copy_headers(self, context):
plt = PlatformInfo.get_platform('iOS_{}'.format(option('ios_arch')[0]))
include_src = pj(plt.buildEnv.install_dir, 'include')
include_dst = pj(self.buildEnv.install_dir, 'include')
plt = PlatformInfo.get_platform("iOS_{}".format(option("ios_arch")[0]))
include_src = pj(plt.buildEnv.install_dir, "include")
include_dst = pj(self.buildEnv.install_dir, "include")
copy_tree(include_src, include_dst)
def _merge_libs(self, context):
lib_dirs = []
for arch in option('ios_arch'):
plt = PlatformInfo.get_platform('iOS_{}'.format(arch))
lib_dirs.append(pj(plt.buildEnv.install_dir, 'lib'))
for arch in option("ios_arch"):
plt = PlatformInfo.get_platform("iOS_{}".format(arch))
lib_dirs.append(pj(plt.buildEnv.install_dir, "lib"))
libs = []
for f in os.listdir(lib_dirs[0]):
if os.path.islink(pj(lib_dirs[0], f)):
continue
if f.endswith('.a') or f.endswith('.dylib'):
if f.endswith(".a") or f.endswith(".dylib"):
libs.append(f)
os.makedirs(pj(self.buildEnv.install_dir, 'lib'), exist_ok=True)
os.makedirs(pj(self.buildEnv.install_dir, "lib"), exist_ok=True)
for l in libs:
command = [
'lipo',
'-create', *[pj(d, l) for d in lib_dirs],
'-output', pj(self.buildEnv.install_dir, 'lib', l)
"lipo",
"-create",
*[pj(d, l) for d in lib_dirs],
"-output",
pj(self.buildEnv.install_dir, "lib", l),
]
run_command(command, self.buildEnv.install_dir, context)
def build(self):
self.command('copy_headers', self._copy_headers)
self.command('merge_libs', self._merge_libs)
self.command("copy_headers", self._copy_headers)
self.command("merge_libs", self._merge_libs)

24
kiwixbuild/dependencies/kiwix_desktop.py
View File

@ -1,7 +1,5 @@
from .base import (
Dependency,
GitClone,
QMakeBuilder)
from .base import Dependency, GitClone, QMakeBuilder
class KiwixDesktop(Dependency):
name = "kiwix-desktop"
@ -13,22 +11,18 @@ class KiwixDesktop(Dependency):
class Builder(QMakeBuilder):
dependencies = ["qt", "qtwebengine", "libkiwix", "aria2"]
make_install_targets = ['install']
make_install_targets = ["install"]
configure_env = None
flatpack_build_options = {
"env": [
"QMAKEPATH=/app/lib"
]
}
flatpack_build_options = {"env": ["QMAKEPATH=/app/lib"]}
@property
def configure_options(self):
if self.buildEnv.platformInfo.name == 'flatpak':
yield 'QMAKE_INCDIR+=/app/include/QtWebEngine'
yield 'QMAKE_INCDIR+=/app/include/QtWebEngineCore'
yield 'QMAKE_INCDIR+=/app/include/QtWebEngineWidgets'
if self.buildEnv.platformInfo.name == "flatpak":
yield "QMAKE_INCDIR+=/app/include/QtWebEngine"
yield "QMAKE_INCDIR+=/app/include/QtWebEngineCore"
yield "QMAKE_INCDIR+=/app/include/QtWebEngineWidgets"
else:
yield f"PREFIX={self.buildEnv.install_dir}"
if self.buildEnv.platformInfo.static:
yield 'CONFIG+=static'
yield "CONFIG+=static"

6
kiwixbuild/dependencies/kiwix_tools.py
View File

@ -1,7 +1,5 @@
from .base import (
Dependency,
GitClone,
MesonBuilder)
from .base import Dependency, GitClone, MesonBuilder
class KiwixTools(Dependency):
name = "kiwix-tools"

48
kiwixbuild/dependencies/libcurl.py
View File

@ -9,22 +9,50 @@ from .base import (
from kiwixbuild.utils import Remotefile, pj, Defaultdict, SkipCommand, run_command
from kiwixbuild._global import get_target_step
class LibCurl(Dependency):
name = "libcurl"
class Source(ReleaseDownload):
name = "libcurl"
archive = Remotefile('curl-7.67.0.tar.xz',
'f5d2e7320379338c3952dcc7566a140abb49edb575f9f99272455785c40e536c',
'https://curl.haxx.se/download/curl-7.67.0.tar.xz')
archive = Remotefile(
"curl-7.67.0.tar.xz",
"f5d2e7320379338c3952dcc7566a140abb49edb575f9f99272455785c40e536c",
"https://curl.haxx.se/download/curl-7.67.0.tar.xz",
)
class Builder(MakeBuilder):
dependencies = ['zlib']
dependencies = ["zlib"]
configure_options = [
*[f"--without-{p}" for p in
('libssh2', 'ssl', 'libmetalink', 'librtmp', 'nghttp2', 'libidn2', 'brotli')
],
*[f"--disable-{p}" for p in
('ftp', 'file', 'ldap', 'ldaps', 'rtsp', 'dict', 'telnet',
'tftp', 'pop3', 'imap', 'smb', 'smtp', 'gopher', 'manual')]
*[
f"--without-{p}"
for p in (
"libssh2",
"ssl",
"libmetalink",
"librtmp",
"nghttp2",
"libidn2",
"brotli",
)
],
*[
f"--disable-{p}"
for p in (
"ftp",
"file",
"ldap",
"ldaps",
"rtsp",
"dict",
"telnet",
"tftp",
"pop3",
"imap",
"smb",
"smtp",
"gopher",
"manual",
)
],
]

38
kiwixbuild/dependencies/libkiwix.py
View File

@ -1,12 +1,10 @@
import shutil, os
from .base import (
Dependency,
GitClone,
MesonBuilder)
from .base import Dependency, GitClone, MesonBuilder
from kiwixbuild.utils import pj, copy_tree
from kiwixbuild._global import option, get_target_step, neutralEnv
class Libkiwix(Dependency):
name = "libkiwix"
force_build = True
@ -16,7 +14,17 @@ class Libkiwix(Dependency):
git_dir = "libkiwix"
class Builder(MesonBuilder):
dependencies = ["pugixml", "libzim", "zlib", "lzma", "libcurl", "libmicrohttpd", "icu4c", "mustache", "xapian-core"]
dependencies = [
"pugixml",
"libzim",
"zlib",
"lzma",
"libcurl",
"libmicrohttpd",
"icu4c",
"mustache",
"xapian-core",
]
strip_options = []
@property
@ -28,18 +36,18 @@ class Libkiwix(Dependency):
@property
def configure_options(self):
platformInfo = self.buildEnv.platformInfo
if platformInfo.build == 'android':
yield '-Dstatic-linkage=true'
yield '-Dwerror=false'
if platformInfo.build == 'iOS':
yield '-Db_bitcode=true'
if platformInfo.name == 'flatpak':
yield '--wrap-mode=nodownload'
if platformInfo.mixed and option('target') == 'libkiwix':
if platformInfo.build == "android":
yield "-Dstatic-linkage=true"
yield "-Dwerror=false"
if platformInfo.build == "iOS":
yield "-Db_bitcode=true"
if platformInfo.name == "flatpak":
yield "--wrap-mode=nodownload"
if platformInfo.mixed and option("target") == "libkiwix":
yield "-Dstatic-linkage=true"
@property
def library_type(self):
if self.buildEnv.platformInfo.build == 'android':
return 'shared'
if self.buildEnv.platformInfo.build == "android":
return "shared"
return super().library_type

32
kiwixbuild/dependencies/libmagic.py
View File

@ -9,15 +9,18 @@ from .base import (
from kiwixbuild.utils import Remotefile, pj, SkipCommand, run_command
from kiwixbuild._global import get_target_step
class LibMagic(Dependency):
name = "libmagic"
class Source(ReleaseDownload):
name = "libmagic"
source_dir = "libmagic"
archive_top_dir = 'file-5.44'
archive = Remotefile('file-5.44.tar.gz',
'3751c7fba8dbc831cb8d7cc8aff21035459b8ce5155ef8b0880a27d028475f3b')
archive_top_dir = "file-5.44"
archive = Remotefile(
"file-5.44.tar.gz",
"3751c7fba8dbc831cb8d7cc8aff21035459b8ce5155ef8b0880a27d028475f3b",
)
class Builder(MakeBuilder):
@property
@ -29,22 +32,21 @@ class LibMagic(Dependency):
@classmethod
def get_dependencies(cls, platformInfo, allDeps):
if platformInfo.build != 'native':
return [('native_static', 'libmagic')]
if platformInfo.build != "native":
return [("native_static", "libmagic")]
return []
def _compile(self, context):
platformInfo = self.buildEnv.platformInfo
if platformInfo.build == 'native':
if platformInfo.build == "native":
return super()._compile(context)
context.try_skip(self.build_path)
command = [
"make",
"-j4",
*self.make_targets,
*self.make_options
]
env = self.buildEnv.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
libmagic_native_builder = get_target_step('libmagic', 'native_static')
env['PATH'] = ':'.join([pj(libmagic_native_builder.build_path, 'src'), env['PATH']])
command = ["make", "-j4", *self.make_targets, *self.make_options]
env = self.buildEnv.get_env(
cross_comp_flags=True, cross_compilers=True, cross_path=True
)
libmagic_native_builder = get_target_step("libmagic", "native_static")
env["PATH"] = ":".join(
[pj(libmagic_native_builder.build_path, "src"), env["PATH"]]
)
run_command(command, self.build_path, context, env=env)

22
kiwixbuild/dependencies/libmicrohttpd.py
View File

@ -1,17 +1,23 @@
from .base import (
Dependency,
ReleaseDownload,
MakeBuilder)
from .base import Dependency, ReleaseDownload, MakeBuilder
from kiwixbuild.utils import Remotefile
class MicroHttpd(Dependency):
name = "libmicrohttpd"
class Source(ReleaseDownload):
archive = Remotefile('libmicrohttpd-0.9.76.tar.gz',
'f0b1547b5a42a6c0f724e8e1c1cb5ce9c4c35fb495e7d780b9930d35011ceb4c',
'https://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.76.tar.gz')
archive = Remotefile(
"libmicrohttpd-0.9.76.tar.gz",
"f0b1547b5a42a6c0f724e8e1c1cb5ce9c4c35fb495e7d780b9930d35011ceb4c",
"https://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.76.tar.gz",
)
class Builder(MakeBuilder):
configure_options = ["--disable-https", "--without-libgcrypt", "--without-libcurl", "--disable-doc", "--disable-examples"]
configure_options = [
"--disable-https",
"--without-libgcrypt",
"--without-libcurl",
"--disable-doc",
"--disable-examples",
]

24
kiwixbuild/dependencies/libzim.py
View File

@ -1,9 +1,7 @@
from .base import (
Dependency,
GitClone,
MesonBuilder)
from .base import Dependency, GitClone, MesonBuilder
from kiwixbuild._global import option, get_target_step
class Libzim(Dependency):
name = "libzim"
force_build = True
@ -24,18 +22,18 @@ class Libzim(Dependency):
@classmethod
def get_dependencies(cls, platformInfo, allDeps):
deps = ['lzma', 'zstd', 'xapian-core', 'icu4c']
if platformInfo.name not in ('flatpak', 'wasm'):
deps.append('zim-testing-suite')
deps = ["lzma", "zstd", "xapian-core", "icu4c"]
if platformInfo.name not in ("flatpak", "wasm"):
deps.append("zim-testing-suite")
return deps
@property
def configure_options(self):
platformInfo = self.buildEnv.platformInfo
if platformInfo.build == 'android':
if platformInfo.build == "android":
yield "-DUSE_BUFFER_HEADER=false"
yield "-Dstatic-linkage=true"
if platformInfo.mixed and option('target') == 'libzim':
if platformInfo.mixed and option("target") == "libzim":
yield "-Dstatic-linkage=true"
if platformInfo.name == "flatpak":
yield "--wrap-mode=nodownload"
@ -44,11 +42,11 @@ class Libzim(Dependency):
yield "-Dexamples=false"
yield "-DUSE_MMAP=false"
if platformInfo.name not in ("flatpak", "wasm"):
zim_testing_suite = get_target_step('zim-testing-suite', 'source')
yield '-Dtest_data_dir={}'.format(zim_testing_suite.source_path)
zim_testing_suite = get_target_step("zim-testing-suite", "source")
yield "-Dtest_data_dir={}".format(zim_testing_suite.source_path)
@property
def library_type(self):
if self.buildEnv.platformInfo.build == 'android':
return 'shared'
if self.buildEnv.platformInfo.build == "android":
return "shared"
return super().library_type

36
kiwixbuild/dependencies/lzma.py
View File

@ -1,28 +1,28 @@
from .base import (
Dependency,
ReleaseDownload,
MakeBuilder)
from .base import Dependency, ReleaseDownload, MakeBuilder
from kiwixbuild.utils import Remotefile
class lzma(Dependency):
name = 'lzma'
name = "lzma"
class Source(ReleaseDownload):
archive = Remotefile('xz-5.2.6.tar.gz',
'a2105abee17bcd2ebd15ced31b4f5eda6e17efd6b10f921a01cda4a44c91b3a0',
'https://altushost-swe.dl.sourceforge.net/project/lzmautils/xz-5.2.6.tar.gz'
)
archive = Remotefile(
"xz-5.2.6.tar.gz",
"a2105abee17bcd2ebd15ced31b4f5eda6e17efd6b10f921a01cda4a44c91b3a0",
"https://altushost-swe.dl.sourceforge.net/project/lzmautils/xz-5.2.6.tar.gz",
)
class Builder(MakeBuilder):
@property
def configure_options(self):
return ["--disable-xz",
"--disable-xzdec",
"--disable-lzmadec",
"--disable-lzmainfo",
"--disable-lzma-links",
"--disable-scripts",
"--disable-doc",
# "--disable-symbol-versions"
]
return [
"--disable-xz",
"--disable-xzdec",
"--disable-lzmadec",
"--disable-lzmainfo",
"--disable-lzma-links",
"--disable-scripts",
"--disable-doc",
# "--disable-symbol-versions"
]

26
kiwixbuild/dependencies/mustache.py
View File

@ -1,28 +1,30 @@
from .base import (
Dependency,
ReleaseDownload,
Builder as BaseBuilder)
from .base import Dependency, ReleaseDownload, Builder as BaseBuilder
from kiwixbuild.utils import Remotefile, pj
from shutil import copy2
class Mustache(Dependency):
name = "mustache"
class Source(ReleaseDownload):
archive = Remotefile('Mustache-4.1.tar.gz',
'acd66359feb4318b421f9574cfc5a511133a77d916d0b13c7caa3783c0bfe167',
'https://github.com/kainjow/Mustache/archive/v4.1.tar.gz')
archive = Remotefile(
"Mustache-4.1.tar.gz",
"acd66359feb4318b421f9574cfc5a511133a77d916d0b13c7caa3783c0bfe167",
"https://github.com/kainjow/Mustache/archive/v4.1.tar.gz",
)
class Builder(BaseBuilder):
def build(self):
self.command('copy_header', self._copy_header)
self.command("copy_header", self._copy_header)
def _copy_header(self, context):
context.try_skip(self.build_path)
copy2(pj(self.source_path, 'mustache.hpp'),
pj(self.buildEnv.install_dir, 'include'))
copy2(
pj(self.source_path, "mustache.hpp"),
pj(self.buildEnv.install_dir, "include"),
)
def set_flatpak_buildsystem(self, module):
module['buildsystem'] = 'simple'
module['build-commands'] = ['cp mustache.hpp /app/include']
module["buildsystem"] = "simple"
module["build-commands"] = ["cp mustache.hpp /app/include"]

14
kiwixbuild/dependencies/pugixml.py
View File

@ -1,19 +1,19 @@
from .base import (
Dependency,
ReleaseDownload,
MesonBuilder)
from .base import Dependency, ReleaseDownload, MesonBuilder
from kiwixbuild.utils import Remotefile
class Pugixml(Dependency):
name = "pugixml"
class Source(ReleaseDownload):
archive = Remotefile('pugixml-1.2.tar.gz',
'0f422dad86da0a2e56a37fb2a88376aae6e931f22cc8b956978460c9db06136b')
archive = Remotefile(
"pugixml-1.2.tar.gz",
"0f422dad86da0a2e56a37fb2a88376aae6e931f22cc8b956978460c9db06136b",
)
patches = ["pugixml_meson.patch"]
flatpak_dest = "src"
class Builder(MesonBuilder):
build_type = 'release'
build_type = "release"
strip_options = []

93
kiwixbuild/dependencies/qt.py
View File

@ -1,26 +1,24 @@
import shutil
from .base import (
Dependency,
ReleaseDownload,
MakeBuilder,
QMakeBuilder)
from .base import Dependency, ReleaseDownload, MakeBuilder, QMakeBuilder
from kiwixbuild.utils import Remotefile, pj, SkipCommand
class Qt(Dependency):
name = 'qt'
name = "qt"
class Source(ReleaseDownload):
name = "qt"
source_dir = "qt-5.10.1"
archive = Remotefile('qt-everywhere-src-5.10.1.tar.xz',
'',
'http://ftp.oregonstate.edu/.1/blfs/conglomeration/qt5/qt-everywhere-src-5.10.1.tar.xz')
archive = Remotefile(
"qt-everywhere-src-5.10.1.tar.xz",
"",
"http://ftp.oregonstate.edu/.1/blfs/conglomeration/qt5/qt-everywhere-src-5.10.1.tar.xz",
)
class Builder(MakeBuilder):
dependencies = ['icu4c', 'zlib']
dependencies = ["icu4c", "zlib"]
dynamic_configure_options = ["-shared"]
static_configure_options = ["-static"]
@ -33,47 +31,50 @@ class Qt(Dependency):
yield from self.dynamic_configure_options
if not self.target.force_native_build:
yield from self.buildEnv.platformInfo.configure_options
yield from ('-prefix', self.buildEnv.install_dir)
yield from ('-libdir', pj(self.buildEnv.install_dir, self.buildEnv.libprefix))
yield from ("-prefix", self.buildEnv.install_dir)
yield from (
"-libdir",
pj(self.buildEnv.install_dir, self.buildEnv.libprefix),
)
@property
def configure_options(self):
skip_modules = [
'qt3d',
'qtcanvas3d',
'qtcharts',
'qtconnectivity',
'qtdatavis3d',
# 'qtdeclarative',
'qtdoc',
'qtgamepad',
'qtgraphicaleffects',
'qtlocation',
'qtmultimedia',
'qtnetworkauth',
'qtpurchasing',
# 'qtquickcontrols',
'qtquickcontrols2',
'qtremoteobjects',
'qtscript',
'qtscxml',
'qtsensors',
'qtserialbus',
'qtserialport',
'qtspeech',
'qtvirtualkeyboard',
'qtwayland',
'qtwebglplugin',
'qtwebsockets',
# 'qtwebview',
"qt3d",
"qtcanvas3d",
"qtcharts",
"qtconnectivity",
"qtdatavis3d",
# 'qtdeclarative',
"qtdoc",
"qtgamepad",
"qtgraphicaleffects",
"qtlocation",
"qtmultimedia",
"qtnetworkauth",
"qtpurchasing",
# 'qtquickcontrols',
"qtquickcontrols2",
"qtremoteobjects",
"qtscript",
"qtscxml",
"qtsensors",
"qtserialbus",
"qtserialport",
"qtspeech",
"qtvirtualkeyboard",
"qtwayland",
"qtwebglplugin",
"qtwebsockets",
# 'qtwebview',
]
yield '-recheck'
yield '-opensource'
yield '-confirm-license'
yield '-ccache'
yield from ('-make', 'libs')
yield "-recheck"
yield "-opensource"
yield "-confirm-license"
yield "-ccache"
yield from ("-make", "libs")
for module in skip_modules:
yield from ('-skip', module)
yield from ("-skip", module)
class QtWebEngine(Dependency):
@ -82,5 +83,5 @@ class QtWebEngine(Dependency):
Source = Qt.Source
class Builder(QMakeBuilder):
dependencies = ['qt']
dependencies = ["qt"]
subsource_dir = "qtwebengine"

52
kiwixbuild/dependencies/tc_android_ndk.py
View File

@ -5,23 +5,25 @@ from kiwixbuild.utils import Remotefile, add_execution_right, run_command
pj = os.path.join
class android_ndk(Dependency):
dont_skip = True
neutral = False
name = 'android-ndk'
gccver = '4.9.x'
api = '24'
name = "android-ndk"
gccver = "4.9.x"
api = "24"
class Source(ReleaseDownload):
archive = Remotefile('android-ndk-r21e-linux-x86_64.zip',
'ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e',
'https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip')
archive = Remotefile(
"android-ndk-r21e-linux-x86_64.zip",
"ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e",
"https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip",
)
@property
def source_dir(self):
return self.target.full_name()
class Builder(Builder):
@property
def install_path(self):
@ -33,7 +35,7 @@ class android_ndk(Dependency):
@property
def platform(self):
return 'android-'+self.api
return "android-" + self.api
@property
def arch(self):
@ -45,24 +47,33 @@ class android_ndk(Dependency):
def _build_platform(self, context):
context.try_skip(self.build_path)
script = pj(self.source_path, 'build/tools/make_standalone_toolchain.py')
script = pj(self.source_path, "build/tools/make_standalone_toolchain.py")
add_execution_right(script)
command = [
script,
f'--arch={self.arch}',
f'--api={self.api}',
f'--install-dir={self.install_path}',
'--force'
f"--arch={self.arch}",
f"--api={self.api}",
f"--install-dir={self.install_path}",
"--force",
]
env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False)
env = self.buildEnv.get_env(
cross_comp_flags=False, cross_compilers=False, cross_path=False
)
run_command(command, self.build_path, context, env=env)
def _fix_permission_right(self, context):
context.try_skip(self.build_path)
bin_dirs = [pj(self.install_path, 'bin'),
pj(self.install_path, self.arch_full, 'bin'),
pj(self.install_path, 'libexec', 'gcc', self.arch_full, self.target.gccver)
]
bin_dirs = [
pj(self.install_path, "bin"),
pj(self.install_path, self.arch_full, "bin"),
pj(
self.install_path,
"libexec",
"gcc",
self.arch_full,
self.target.gccver,
),
]
for root, dirs, files in os.walk(self.install_path):
if not root in bin_dirs:
continue
@ -74,6 +85,5 @@ class android_ndk(Dependency):
add_execution_right(file_path)
def build(self):
self.command('build_platform', self._build_platform)
self.command('fix_permission_right', self._fix_permission_right)
self.command("build_platform", self._build_platform)
self.command("fix_permission_right", self._fix_permission_right)

37
kiwixbuild/dependencies/tc_armhf.py
View File

@ -4,21 +4,25 @@ from kiwixbuild.utils import Remotefile
# The arm toolchains
# This is based on toolchains published here : https://github.com/tttapa/docker-arm-cross-toolchain
base_url = "https://github.com/tttapa/docker-arm-cross-toolchain/releases/download/0.1.0/"
base_url = (
"https://github.com/tttapa/docker-arm-cross-toolchain/releases/download/0.1.0/"
)
aarch_base_url = 'https://master.dl.sourceforge.net/project/raspberry-pi-cross-compilers/Bonus%20Raspberry%20Pi%20GCC%2064-Bit%20Toolchains/Raspberry%20Pi%20GCC%2064-Bit%20Cross-Compiler%20Toolchains/Stretch/GCC%206.3.0/'
aarch_base_url = "https://master.dl.sourceforge.net/project/raspberry-pi-cross-compilers/Bonus%20Raspberry%20Pi%20GCC%2064-Bit%20Toolchains/Raspberry%20Pi%20GCC%2064-Bit%20Cross-Compiler%20Toolchains/Stretch/GCC%206.3.0/"
class armv6_toolchain(Dependency):
dont_skip = True
neutral = True
name = 'armv6'
name = "armv6"
class Source(ReleaseDownload):
archive = Remotefile('x-tools-armv6-rpi-linux-gnueabihf.tar.xz',
'4c371c4c5b55ebd1f3d7dd26b14703632d9ba47423f901bcd9303d83ad444434',
base_url + 'x-tools-armv6-rpi-linux-gnueabihf.tar.xz')
archive = Remotefile(
"x-tools-armv6-rpi-linux-gnueabihf.tar.xz",
"4c371c4c5b55ebd1f3d7dd26b14703632d9ba47423f901bcd9303d83ad444434",
base_url + "x-tools-armv6-rpi-linux-gnueabihf.tar.xz",
)
class Builder(TcCopyBuilder):
src_subdir = "armv6-rpi-linux-gnueabihf"
@ -27,24 +31,29 @@ class armv6_toolchain(Dependency):
class armv8_toolchain(Dependency):
dont_skip = True
neutral = True
name = 'armv8'
name = "armv8"
class Source(ReleaseDownload):
archive = Remotefile('x-tools-armv8-rpi-linux-gnueabihf.tar.xz',
'cc28f5c3f6a3e7d9985f98779c4e72224b4eb5a7e4dc2bcdefd90cb241fb94a5',
base_url + 'x-tools-armv8-rpi3-linux-gnueabihf.tar.xz')
archive = Remotefile(
"x-tools-armv8-rpi-linux-gnueabihf.tar.xz",
"cc28f5c3f6a3e7d9985f98779c4e72224b4eb5a7e4dc2bcdefd90cb241fb94a5",
base_url + "x-tools-armv8-rpi3-linux-gnueabihf.tar.xz",
)
class Builder(TcCopyBuilder):
src_subdir = "armv8-rpi3-linux-gnueabihf"
class aarch64_toolchain(Dependency):
dont_skip = True
neutral = True
name = "aarch64"
class Source(ReleaseDownload):
archive = Remotefile('cross-gcc-6.3.0-pi_64.tar.gz',
'1b048bb8886ad63d21797cd9129fc37b9ea0dfaac7e3c36f888aa16fbec1d320',
aarch_base_url + 'cross-gcc-6.3.0-pi_64.tar.gz')
archive = Remotefile(
"cross-gcc-6.3.0-pi_64.tar.gz",
"1b048bb8886ad63d21797cd9129fc37b9ea0dfaac7e3c36f888aa16fbec1d320",
aarch_base_url + "cross-gcc-6.3.0-pi_64.tar.gz",
)
Builder = TcCopyBuilder

20
kiwixbuild/dependencies/tc_emsdk.py
View File

@ -5,21 +5,23 @@ from kiwixbuild.utils import Remotefile, run_command, copy_tree
pj = os.path.join
class emsdk(Dependency):
dont_skip = True
neutral = False
name = 'emsdk'
name = "emsdk"
class Source(ReleaseDownload):
archive = Remotefile('emsdk-3.1.41.tar.gz',
'147a2d72df34227bdb4ffedc587a8cb674a42269c40458f3f69ae37e8966cdc6',
'https://codeload.github.com/emscripten-core/emsdk/tar.gz/refs/tags/3.1.41')
archive = Remotefile(
"emsdk-3.1.41.tar.gz",
"147a2d72df34227bdb4ffedc587a8cb674a42269c40458f3f69ae37e8966cdc6",
"https://codeload.github.com/emscripten-core/emsdk/tar.gz/refs/tags/3.1.41",
)
@property
def source_dir(self):
return self.target.full_name()
class Builder(Builder):
@property
def install_path(self):
@ -39,9 +41,7 @@ class emsdk(Dependency):
command = ["./emsdk", "activate", "3.1.24"]
run_command(command, self.install_path, context)
def build(self):
self.command('copy_source', self._copy_source)
self.command('install', self._install)
self.command('activate', self._activate)
self.command("copy_source", self._copy_source)
self.command("install", self._install)
self.command("activate", self._activate)

55
kiwixbuild/dependencies/tc_flatpak.py
View File

@ -5,9 +5,10 @@ from kiwixbuild.utils import Remotefile, add_execution_right, run_command
pj = os.path.join
class org_kde(Dependency):
neutral = False
name = 'org.kde'
name = "org.kde"
Source = NoopSource
@ -15,32 +16,42 @@ class org_kde(Dependency):
def _setup_remote(self, context):
command = [
"flatpak",
"--user", "remote-add", "--if-not-exists",
"--user",
"remote-add",
"--if-not-exists",
"flathub",
"https://flathub.org/repo/flathub.flatpakrepo"
"https://flathub.org/repo/flathub.flatpakrepo",
]
env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False)
env = self.buildEnv.get_env(
cross_comp_flags=False, cross_compilers=False, cross_path=False
)
run_command(command, self.buildEnv.build_dir, context, env=env)
def _install_sdk(self, context):
command = [
"flatpak",
"--user", "install", "--noninteractive", "--verbose", "-y",
"--user",
"install",
"--noninteractive",
"--verbose",
"-y",
"flathub",
f"{self.target.name}.Sdk//{self.target.version()}",
f"{self.target.name}.Platform//{self.target.version()}"
f"{self.target.name}.Platform//{self.target.version()}",
]
env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False)
env = self.buildEnv.get_env(
cross_comp_flags=False, cross_compilers=False, cross_path=False
)
run_command(command, self.buildEnv.build_dir, context, env=env)
def build(self):
self.command('setup_remote', self._setup_remote)
self.command('install_sdk', self._install_sdk)
self.command("setup_remote", self._setup_remote)
self.command("install_sdk", self._install_sdk)
class io_qt_qtwebengine(Dependency):
neutral = False
name = 'io.qt.qtwebengine'
name = "io.qt.qtwebengine"
Source = NoopSource
@ -48,23 +59,31 @@ class io_qt_qtwebengine(Dependency):
def _setup_remote(self, context):
command = [
"flatpak",
"--user", "remote-add", "--if-not-exists",
"--user",
"remote-add",
"--if-not-exists",
"flathub",
"https://flathub.org/repo/flathub.flatpakrepo"
"https://flathub.org/repo/flathub.flatpakrepo",
]
env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False)
env = self.buildEnv.get_env(
cross_comp_flags=False, cross_compilers=False, cross_path=False
)
run_command(command, self.buildEnv.build_dir, context, env=env)
def _install_sdk(self, context):
command = [
"flatpak",
"--user", "install", "-y",
"--user",
"install",
"-y",
"flathub",
f"{self.target.name}.BaseApp//{self.target.version()}"
f"{self.target.name}.BaseApp//{self.target.version()}",
]
env = self.buildEnv.get_env(cross_comp_flags=False, cross_compilers=False, cross_path=False)
env = self.buildEnv.get_env(
cross_comp_flags=False, cross_compilers=False, cross_path=False
)
run_command(command, self.buildEnv.build_dir, context, env=env)
def build(self):
self.command('setup_remote', self._setup_remote)
self.command('install_sdk', self._install_sdk)
self.command("setup_remote", self._setup_remote)
self.command("install_sdk", self._install_sdk)

17
kiwixbuild/dependencies/tc_musl.py
View File

@ -1,15 +1,18 @@
from .base import Dependency, ReleaseDownload, TcCopyBuilder
from kiwixbuild.utils import Remotefile
class aarch64_musl_toolchain(Dependency):
dont_skip = True
neutral = True
name = "aarch64_musl"
class Source(ReleaseDownload):
archive = Remotefile('aarch64-linux-musl-cross.tgz',
'0f18a885b161815520bbb5757a4b4ab40d0898c29bebee58d0cddd6112e59cc6',
'https://more.musl.cc/10/x86_64-linux-musl/aarch64-linux-musl-cross.tgz')
archive = Remotefile(
"aarch64-linux-musl-cross.tgz",
"0f18a885b161815520bbb5757a4b4ab40d0898c29bebee58d0cddd6112e59cc6",
"https://more.musl.cc/10/x86_64-linux-musl/aarch64-linux-musl-cross.tgz",
)
Builder = TcCopyBuilder
@ -20,8 +23,10 @@ class x86_64_musl_toolchain(Dependency):
name = "x86-64_musl"
class Source(ReleaseDownload):
archive = Remotefile('x86_64-linux-musl-cross.tgz',
'a3d55de8105739fcfb8b10eaa72cdb5d779319726bacff24149388d7608d1ed8',
'https://more.musl.cc/10/x86_64-linux-musl/x86_64-linux-musl-cross.tgz')
archive = Remotefile(
"x86_64-linux-musl-cross.tgz",
"a3d55de8105739fcfb8b10eaa72cdb5d779319726bacff24149388d7608d1ed8",
"https://more.musl.cc/10/x86_64-linux-musl/x86_64-linux-musl-cross.tgz",
)
Builder = TcCopyBuilder

40
kiwixbuild/dependencies/uuid.py
View File

@ -1,30 +1,30 @@
from .base import (
Dependency,
ReleaseDownload,
MakeBuilder
)
from .base import Dependency, ReleaseDownload, MakeBuilder
from kiwixbuild.utils import Remotefile
class UUID(Dependency):
name = 'uuid'
name = "uuid"
class Source(ReleaseDownload):
archive = Remotefile('e2fsprogs-libs-1.43.4.tar.gz',
'eed4516325768255c9745e7b82c9d7d0393abce302520a5b2cde693204b0e419',
'https://www.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v1.43.4/e2fsprogs-libs-1.43.4.tar.gz')
extract_dir = 'e2fsprogs-libs-1.43.4'
archive = Remotefile(
"e2fsprogs-libs-1.43.4.tar.gz",
"eed4516325768255c9745e7b82c9d7d0393abce302520a5b2cde693204b0e419",
"https://www.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/v1.43.4/e2fsprogs-libs-1.43.4.tar.gz",
)
extract_dir = "e2fsprogs-libs-1.43.4"
class Builder(MakeBuilder):
configure_options = ["--enable-libuuid",
"--disable-debugfs",
"--disable-imager",
"--disable-resizer",
"--disable-defrag",
"--enable-fsck",
"--disable-uuidd"]
configure_env = {'_format_CFLAGS': "{env.CFLAGS} -O3 -fPIC"}
configure_options = [
"--enable-libuuid",
"--disable-debugfs",
"--disable-imager",
"--disable-resizer",
"--disable-defrag",
"--enable-fsck",
"--disable-uuidd",
]
configure_env = {"_format_CFLAGS": "{env.CFLAGS} -O3 -fPIC"}
static_configure_options = dynamic_configure_options = []
make_targets = ['libs']
make_install_targets = ['install-libs']
make_targets = ["libs"]
make_install_targets = ["install-libs"]

32
kiwixbuild/dependencies/xapian.py
View File

@ -1,8 +1,4 @@
from .base import (
Dependency,
ReleaseDownload,
MakeBuilder
)
from .base import Dependency, ReleaseDownload, MakeBuilder
from kiwixbuild.utils import Remotefile
from kiwixbuild._global import neutralEnv
@ -12,23 +8,29 @@ class Xapian(Dependency):
name = "xapian-core"
class Source(ReleaseDownload):
archive = Remotefile('xapian-core-1.4.23.tar.xz',
'30d3518172084f310dab86d262b512718a7f9a13635aaa1a188e61dc26b2288c')
archive = Remotefile(
"xapian-core-1.4.23.tar.xz",
"30d3518172084f310dab86d262b512718a7f9a13635aaa1a188e61dc26b2288c",
)
class Builder(MakeBuilder):
configure_options = [
"--disable-sse",
"--disable-backend-chert",
"--disable-backend-remote",
"--disable-documentation"]
configure_env = {'_format_LDFLAGS': "{env.LDFLAGS} -L{buildEnv.install_dir}/{buildEnv.libprefix}",
'_format_CXXFLAGS': "{env.CXXFLAGS} -O3 -I{buildEnv.install_dir}/include"}
"--disable-documentation",
]
configure_env = {
"_format_LDFLAGS": "{env.LDFLAGS} -L{buildEnv.install_dir}/{buildEnv.libprefix}",
"_format_CXXFLAGS": "{env.CXXFLAGS} -O3 -I{buildEnv.install_dir}/include",
}
@classmethod
def get_dependencies(cls, platformInfo, allDeps):
deps = ['zlib', 'lzma']
if (platformInfo.build in ('win32', 'wasm')
or neutralEnv('distname') == 'Darwin'):
deps = ["zlib", "lzma"]
if (
platformInfo.build in ("win32", "wasm")
or neutralEnv("distname") == "Darwin"
):
return deps
return deps + ['uuid']
return deps + ["uuid"]

14
kiwixbuild/dependencies/zim_testing_suite.py
View File

@ -1,8 +1,4 @@
from .base import (
Dependency,
ReleaseDownload,
NoopBuilder
)
from .base import Dependency, ReleaseDownload, NoopBuilder
from kiwixbuild.utils import Remotefile
@ -12,8 +8,10 @@ class ZimTestingSuite(Dependency):
dont_skip = True
class Source(ReleaseDownload):
archive = Remotefile('zim-testing-suite-0.3.tar.gz',
'cd7d1ccc48af3783af9156cb6bf3c18d9a3319a73fdeefe65f0b4cae402d3d66',
'https://github.com/openzim/zim-testing-suite/releases/download/v0.3/zim-testing-suite-0.3.tar.gz')
archive = Remotefile(
"zim-testing-suite-0.3.tar.gz",
"cd7d1ccc48af3783af9156cb6bf3c18d9a3319a73fdeefe65f0b4cae402d3d66",
"https://github.com/openzim/zim-testing-suite/releases/download/v0.3/zim-testing-suite-0.3.tar.gz",
)
Builder = NoopBuilder

14
kiwixbuild/dependencies/zim_tools.py
View File

@ -1,7 +1,5 @@
from .base import (
Dependency,
GitClone,
MesonBuilder)
from .base import Dependency, GitClone, MesonBuilder
class ZimTools(Dependency):
name = "zim-tools"
@ -14,15 +12,15 @@ class ZimTools(Dependency):
class Builder(MesonBuilder):
@classmethod
def get_dependencies(cls, platformInfo, allDeps):
base_deps = ['libzim', 'docoptcpp', 'mustache']
if platformInfo.build != 'win32':
base_deps += ['libmagic', 'gumbo']
base_deps = ["libzim", "docoptcpp", "mustache"]
if platformInfo.build != "win32":
base_deps += ["libmagic", "gumbo"]
return base_deps
@property
def configure_options(self):
# We don't build zimwriterfs on win32, and so we don't have magic
if self.buildEnv.platformInfo.build != 'win32':
if self.buildEnv.platformInfo.build != "win32":
yield f"-Dmagic-install-prefix={self.buildEnv.install_dir}"
if self.buildEnv.platformInfo.static:
yield "-Dstatic-linkage=true"

42
kiwixbuild/dependencies/zlib.py
View File

@ -1,34 +1,31 @@
import shutil
from .base import (
Dependency,
ReleaseDownload,
MakeBuilder)
from .base import Dependency, ReleaseDownload, MakeBuilder
from kiwixbuild.utils import Remotefile, pj, SkipCommand
class zlib(Dependency):
name = 'zlib'
name = "zlib"
class Source(ReleaseDownload):
archive = Remotefile('zlib-1.2.12.tar.gz',
'91844808532e5ce316b3c010929493c0244f3d37593afd6de04f71821d5136d9')
patches = ['zlib_std_libname.patch']
archive = Remotefile(
"zlib-1.2.12.tar.gz",
"91844808532e5ce316b3c010929493c0244f3d37593afd6de04f71821d5136d9",
)
patches = ["zlib_std_libname.patch"]
class Builder(MakeBuilder):
dynamic_configure_options = ["--shared"]
static_configure_options = ["--static"]
make_install_targets = ['install']
make_install_targets = ["install"]
def _pre_build_script(self, context):
context.try_skip(self.build_path)
shutil.copytree(self.source_path, self.build_path)
def _configure(self, context):
if self.buildEnv.platformInfo.build == 'win32':
if self.buildEnv.platformInfo.build == "win32":
raise SkipCommand()
return super()._configure(context)
@ -36,20 +33,27 @@ class zlib(Dependency):
def all_configure_options(self):
yield from self.configure_options
yield from self.static_configure_options if self.buildEnv.platformInfo.static else self.dynamic_configure_options
yield from ('--prefix', self.buildEnv.install_dir)
yield from ('--libdir', pj(self.buildEnv.install_dir, self.buildEnv.libprefix))
yield from ("--prefix", self.buildEnv.install_dir)
yield from (
"--libdir",
pj(self.buildEnv.install_dir, self.buildEnv.libprefix),
)
@property
def make_options(self):
if self.buildEnv.platformInfo.build != 'win32':
if self.buildEnv.platformInfo.build != "win32":
return
yield "--makefile"
yield "win32/Makefile.gcc"
yield "PREFIX=i686-w64-mingw32-",
yield "SHARED_MODE={}".format("0" if self.buildEnv.platformInfo.static else "1"),
yield "INCLUDE_PATH={}".format(pj(self.buildEnv.install_dir, 'include')),
yield "LIBRARY_PATH={}".format(pj(self.buildEnv.install_dir, self.buildEnv.libprefix)),
yield "BINARY_PATH={}".format(pj(self.buildEnv.install_dir, 'bin'))
yield "SHARED_MODE={}".format(
"0" if self.buildEnv.platformInfo.static else "1"
),
yield "INCLUDE_PATH={}".format(pj(self.buildEnv.install_dir, "include")),
yield "LIBRARY_PATH={}".format(
pj(self.buildEnv.install_dir, self.buildEnv.libprefix)
),
yield "BINARY_PATH={}".format(pj(self.buildEnv.install_dir, "bin"))
@property
def make_targets(self):

22
kiwixbuild/dependencies/zstd.py
View File

@ -1,22 +1,20 @@
from .base import (
Dependency,
ReleaseDownload,
MesonBuilder)
from .base import Dependency, ReleaseDownload, MesonBuilder
from kiwixbuild.utils import Remotefile
class zstd(Dependency):
name = 'zstd'
name = "zstd"
class Source(ReleaseDownload):
archive = Remotefile('zstd-1.5.2.tar.gz',
'f7de13462f7a82c29ab865820149e778cbfe01087b3a55b5332707abf9db4a6e',
'https://github.com/facebook/zstd/archive/refs/tags/v1.5.2.tar.gz')
archive = Remotefile(
"zstd-1.5.2.tar.gz",
"f7de13462f7a82c29ab865820149e778cbfe01087b3a55b5332707abf9db4a6e",
"https://github.com/facebook/zstd/archive/refs/tags/v1.5.2.tar.gz",
)
class Builder(MesonBuilder):
subsource_dir = 'build/meson'
build_type = 'release'
subsource_dir = "build/meson"
build_type = "release"
strip_options = []
configure_options = ['-Dbin_programs=false', '-Dbin_contrib=false']
configure_options = ["-Dbin_programs=false", "-Dbin_contrib=false"]

242
kiwixbuild/flatpak_builder.py
View File

@ -1,4 +1,3 @@
import sys
from collections import OrderedDict
from .buildenv import *
@ -9,9 +8,13 @@ from .dependencies import Dependency
from .packages import PACKAGE_NAME_MAPPERS
from .versions import base_deps_versions
from ._global import (
neutralEnv, option,
add_target_step, get_target_step, target_steps,
backend)
neutralEnv,
option,
add_target_step,
get_target_step,
target_steps,
backend,
)
from . import _global
from .dependencies.base import (
Source,
@ -22,7 +25,8 @@ from .dependencies.base import (
CMakeBuilder,
QMakeBuilder,
MakeBuilder,
SCRIPT_DIR)
SCRIPT_DIR,
)
import json
from shutil import copyfile
from urllib.parse import urlparse
@ -30,54 +34,60 @@ from urllib.request import urlopen
import json
MANIFEST = {
'app-id': 'org.kiwix.desktop',
'runtime': 'org.kde.Platform',
'runtime-version': base_deps_versions['org.kde'],
'base': 'io.qt.qtwebengine.BaseApp',
'base-version': base_deps_versions['org.kde'], # keep BaseApp (qwebengine) in sync with org.kde
'sdk': 'org.kde.Sdk',
'command': 'kiwix-desktop',
'rename-icon': 'kiwix-desktop',
'finish-args': [
'--socket=wayland',
'--socket=x11',
'--share=network',
'--share=ipc',
'--device=dri',
'--socket=pulseaudio',
"app-id": "org.kiwix.desktop",
"runtime": "org.kde.Platform",
"runtime-version": base_deps_versions["org.kde"],
"base": "io.qt.qtwebengine.BaseApp",
"base-version": base_deps_versions[
"org.kde"
], # keep BaseApp (qwebengine) in sync with org.kde
"sdk": "org.kde.Sdk",
"command": "kiwix-desktop",
"rename-icon": "kiwix-desktop",
"finish-args": [
"--socket=wayland",
"--socket=x11",
"--share=network",
"--share=ipc",
"--device=dri",
"--socket=pulseaudio",
],
"cleanup": [
"/include",
"/lib/pkgconfig",
"/lib/cmake",
"/lib/*.la",
"/bin/curl",
"/bin/copydatabase",
"/bin/kiwix-compile-resources",
"/bin/kiwix-manage",
"/bin/kiwix-read",
"/bin/kiwix-search",
"/bin/quest",
"/bin/simple*",
"/bin/xapian-*",
"/share/aclocal",
"/share/doc",
"/share/man",
],
'cleanup': [
'/include',
'/lib/pkgconfig',
'/lib/cmake',
'/lib/*.la',
'/bin/curl',
'/bin/copydatabase',
'/bin/kiwix-compile-resources',
'/bin/kiwix-manage',
'/bin/kiwix-read',
'/bin/kiwix-search',
'/bin/quest',
'/bin/simple*',
'/bin/xapian-*',
'/share/aclocal',
'/share/doc',
'/share/man'
]
}
GET_REF_URL_API_TEMPLATE = 'https://api.github.com/repos{repo}/git/refs/tags/{ref}'
GET_REF_URL_API_TEMPLATE = "https://api.github.com/repos{repo}/git/refs/tags/{ref}"
class FlatpakBuilder:
def __init__(self):
self._targets = {}
PlatformInfo.get_platform('neutral', self._targets)
self.platform = PlatformInfo.get_platform('flatpak', self._targets)
if neutralEnv('distname') not in self.platform.compatible_hosts:
print(('ERROR: The target platform {} cannot be build on host {}.\n'
'Select another target platform or change your host system.'
).format(self.platform.name, neutralEnv('distname')))
self.targetDefs = self.platform.add_targets(option('target'), self._targets)
PlatformInfo.get_platform("neutral", self._targets)
self.platform = PlatformInfo.get_platform("flatpak", self._targets)
if neutralEnv("distname") not in self.platform.compatible_hosts:
print(
(
"ERROR: The target platform {} cannot be build on host {}.\n"
"Select another target platform or change your host system."
).format(self.platform.name, neutralEnv("distname"))
)
self.targetDefs = self.platform.add_targets(option("target"), self._targets)
def finalize_target_steps(self):
steps = []
@ -89,9 +99,9 @@ class FlatpakBuilder:
plt = PlatformInfo.all_platforms[pltName]
for tlcName in plt.toolchain_names:
tlc = Dependency.all_deps[tlcName]
src_plt_step = ('source', tlcName)
src_plt_step = ("source", tlcName)
add_target_step(src_plt_step, self._targets[src_plt_step])
blt_plt_step = ('neutral' if tlc.neutral else pltName, tlcName)
blt_plt_step = ("neutral" if tlc.neutral else pltName, tlcName)
add_target_step(blt_plt_step, self._targets[blt_plt_step])
for dep in steps:
@ -104,7 +114,7 @@ class FlatpakBuilder:
def order_dependencies(self, targetDef, targets):
targetPlatformName, targetName = targetDef
if targetPlatformName == 'source':
if targetPlatformName == "source":
# Do not try to order sources, they will be added as dep by the
# build step two lines later.
return
@ -121,18 +131,18 @@ class FlatpakBuilder:
depPlatform, depName = targetPlatformName, dep
if (depPlatform, depName) in targets:
yield from self.order_dependencies((depPlatform, depName), targets)
yield ('source', targetName)
yield ("source", targetName)
yield targetDef
def instanciate_steps(self):
for stepDef in list(target_steps()):
stepPlatform, stepName = stepDef
stepClass = Dependency.all_deps[stepName]
if stepPlatform == 'source':
if stepPlatform == "source":
source = get_target_step(stepDef)(stepClass)
add_target_step(stepDef, source)
else:
source = get_target_step(stepName, 'source')
source = get_target_step(stepName, "source")
env = PlatformInfo.get_platform(stepPlatform).buildEnv
builder = get_target_step(stepDef)(stepClass, source, env)
add_target_step(stepDef, builder)
@ -142,121 +152,126 @@ class FlatpakBuilder:
modules = OrderedDict()
for stepDef in steps:
module = modules.setdefault(stepDef[1], {})
module['name'] = stepDef[1]
if stepDef[0] == 'source':
module["name"] = stepDef[1]
if stepDef[0] == "source":
source = get_target_step(stepDef)
if getattr(source, 'flatpak_no_autogen', False):
module['no-autogen'] = True
module_sources = module.setdefault('sources', [])
if getattr(source, "flatpak_no_autogen", False):
module["no-autogen"] = True
module_sources = module.setdefault("sources", [])
if isinstance(source, ReleaseDownload):
src = {
'type': 'archive',
'sha256': source.archive.sha256,
'url': source.archive.url
"type": "archive",
"sha256": source.archive.sha256,
"url": source.archive.url,
}
if hasattr(source, 'flatpak_dest'):
src['dest'] = source.flatpak_dest
if hasattr(source, "flatpak_dest"):
src["dest"] = source.flatpak_dest
module_sources.append(src)
elif isinstance(source, GitClone):
src = {
'type': 'git',
'url': source.git_remote,
'tag': source.git_ref
"type": "git",
"url": source.git_remote,
"tag": source.git_ref,
}
try:
parsed = urlparse(source.git_remote)
if parsed.hostname == 'github.com':
if parsed.hostname == "github.com":
repo = parsed.path[:-4]
api_url = GET_REF_URL_API_TEMPLATE.format(
repo = repo,
ref = source.git_ref)
repo=repo, ref=source.git_ref
)
with urlopen(api_url) as r:
ret = json.loads(r.read())
src['commit'] = ret['object']['sha']
src["commit"] = ret["object"]["sha"]
except:
pass
module_sources.append(src)
for p in getattr(source, 'patches', []):
patch = {
'type': 'patch',
'path': 'patches/' + p
}
for p in getattr(source, "patches", []):
patch = {"type": "patch", "path": "patches/" + p}
module_sources.append(patch)
if hasattr(source, 'flatpak_command'):
command = {
'type': 'shell',
'commands': [source.flatpak_command]
}
if hasattr(source, "flatpak_command"):
command = {"type": "shell", "commands": [source.flatpak_command]}
module_sources.append(command)
else:
builder = get_target_step(stepDef)
builder.set_flatpak_buildsystem(module)
print(module['name'])
print(module["name"])
manifest = MANIFEST.copy()
modules = [m for m in modules.values() if m.get('sources')]
modules = [m for m in modules.values() if m.get("sources")]
for m in modules:
temp = m['sources']
del m['sources']
m['sources'] = temp
manifest['modules'] = modules
manifest_name = "{}.json".format(MANIFEST['app-id'])
temp = m["sources"]
del m["sources"]
m["sources"] = temp
manifest["modules"] = modules
manifest_name = "{}.json".format(MANIFEST["app-id"])
manifest_path = pj(self.platform.buildEnv.build_dir, manifest_name)
with open(manifest_path, 'w') as f:
with open(manifest_path, "w") as f:
f.write(json.dumps(manifest, indent=4))
def copy_patches(self):
sourceDefs = (tDef for tDef in target_steps() if tDef[0] == 'source')
sourceDefs = (tDef for tDef in target_steps() if tDef[0] == "source")
for sourceDef in sourceDefs:
source = get_target_step(sourceDef)
if not hasattr(source, 'patches'):
if not hasattr(source, "patches"):
continue
for p in source.patches:
path = pj(SCRIPT_DIR, 'patches', p)
os.makedirs(pj(self.platform.buildEnv.build_dir, 'patches'), exist_ok=True)
dest = pj(self.platform.buildEnv.build_dir, 'patches', p)
path = pj(SCRIPT_DIR, "patches", p)
os.makedirs(
pj(self.platform.buildEnv.build_dir, "patches"), exist_ok=True
)
dest = pj(self.platform.buildEnv.build_dir, "patches", p)
copyfile(path, dest)
def build(self):
log = pj(self.platform.buildEnv.log_dir, 'cmd_build_flatpak.log')
context = Context('build', log, False)
log = pj(self.platform.buildEnv.log_dir, "cmd_build_flatpak.log")
context = Context("build", log, False)
command = [
"flatpak-builder",
"--user", "--ccache", "--force-clean", "--keep-build-dirs",
"--disable-rofiles-fuse", "--repo=repo", "builddir",
f"{MANIFEST['app-id']}.json"
"--user",
"--ccache",
"--force-clean",
"--keep-build-dirs",
"--disable-rofiles-fuse",
"--repo=repo",
"builddir",
f"{MANIFEST['app-id']}.json",
]
try:
run_command(command, self.platform.buildEnv.build_dir, context, env=self.platform.get_env())
run_command(
command,
self.platform.buildEnv.build_dir,
context,
env=self.platform.get_env(),
)
context._finalise()
except subprocess.CalledProcessError:
with open(log, 'r') as f:
with open(log, "r") as f:
print(f.read())
raise StopBuild()
def bundle(self):
log = pj(self.platform.buildEnv.log_dir, 'cmd_bundle_flatpak.log')
context = Context('bundle', log, False)
app_id = MANIFEST['app-id']
command = [
"flatpak", "build-bundle", "repo",
f"{app_id}.flatpak",
app_id
]
log = pj(self.platform.buildEnv.log_dir, "cmd_bundle_flatpak.log")
context = Context("bundle", log, False)
app_id = MANIFEST["app-id"]
command = ["flatpak", "build-bundle", "repo", f"{app_id}.flatpak", app_id]
try:
run_command(command, self.platform.buildEnv.build_dir, context, env=self.platform.get_env())
run_command(
command,
self.platform.buildEnv.build_dir,
context,
env=self.platform.get_env(),
)
context._finalise()
except subprocess.CalledProcessError:
with open(log, 'r') as f:
with open(log, "r") as f:
print(f.read())
raise StopBuild()
def _get_packages(self):
package_name_mapper = PACKAGE_NAME_MAPPERS.get('flatpak', {})
package_name_mapper = PACKAGE_NAME_MAPPERS.get("flatpak", {})
to_drop = []
for builderDef in self._targets:
@ -295,11 +310,10 @@ class FlatpakBuilder:
self.bundle()
# No error, clean intermediate file at end of build if needed.
print("[CLEAN]")
if option('clean_at_end'):
if option("clean_at_end"):
for platform in PlatformInfo.all_running_platforms.values():
platform.clean_intermediate_directories()
else:
print("SKIP")
except StopBuild:
sys.exit("Stopping build due to errors")

212
kiwixbuild/packages.py
View File

@ -1,107 +1,147 @@
_fedora_common = ['automake', 'libtool', 'cmake', 'git', 'subversion', 'ccache', 'pkgconf-pkg-config', 'gcc-c++', 'gettext-devel']
_debian_common = ['automake', 'libtool', 'cmake', 'git', 'subversion', 'ccache', 'pkg-config', 'gcc', 'autopoint']
_fedora_common = [
"automake",
"libtool",
"cmake",
"git",
"subversion",
"ccache",
"pkgconf-pkg-config",
"gcc-c++",
"gettext-devel",
]
_debian_common = [
"automake",
"libtool",
"cmake",
"git",
"subversion",
"ccache",
"pkg-config",
"gcc",
"autopoint",
]
PACKAGE_NAME_MAPPERS = {
'flatpak': {
'zlib': True,
'lzma': True,
'icu4c': True,
'qt': True,
'qtwebengine': True,
'uuid': True,
'libxml2': True,
'libssl': True,
'libcurl': True,
"flatpak": {
"zlib": True,
"lzma": True,
"icu4c": True,
"qt": True,
"qtwebengine": True,
"uuid": True,
"libxml2": True,
"libssl": True,
"libcurl": True,
},
'fedora_native_dyn': {
'COMMON': _fedora_common,
'uuid': ['libuuid-devel'],
'xapian-core': None, # Not the right version on fedora 25
'pugixml': None, # ['pugixml-devel'] but package doesn't provide pkg-config file
'libmicrohttpd': ['libmicrohttpd-devel'],
'zlib': ['zlib-devel'],
'lzma': ['xz-devel'],
'icu4c': None,
'zimlib': None,
'file' : ['file-devel'],
'gumbo' : ['gumbo-parser-devel'],
'aria2': ['aria2'],
'qt': ['qt5-qtbase-devel', 'qt5-qtsvg'],
'qtwebengine': ['qt5-qtwebengine-devel']
"fedora_native_dyn": {
"COMMON": _fedora_common,
"uuid": ["libuuid-devel"],
"xapian-core": None, # Not the right version on fedora 25
"pugixml": None, # ['pugixml-devel'] but package doesn't provide pkg-config file
"libmicrohttpd": ["libmicrohttpd-devel"],
"zlib": ["zlib-devel"],
"lzma": ["xz-devel"],
"icu4c": None,
"zimlib": None,
"file": ["file-devel"],
"gumbo": ["gumbo-parser-devel"],
"aria2": ["aria2"],
"qt": ["qt5-qtbase-devel", "qt5-qtsvg"],
"qtwebengine": ["qt5-qtwebengine-devel"],
},
'fedora_native_static': {
'COMMON': _fedora_common + ['glibc-static', 'libstdc++-static'],
'lzma': ['xz-devel', 'xz-static']
"fedora_native_static": {
"COMMON": _fedora_common + ["glibc-static", "libstdc++-static"],
"lzma": ["xz-devel", "xz-static"]
# Either there is no packages, or no static or too old
},
'fedora_i586_dyn': {
'COMMON': _fedora_common + ['glibc-devel.i686', 'libstdc++-devel.i686'],
"fedora_i586_dyn": {
"COMMON": _fedora_common + ["glibc-devel.i686", "libstdc++-devel.i686"],
},
'fedora_i586_static': {
'COMMON': _fedora_common + ['glibc-devel.i686'],
"fedora_i586_static": {
"COMMON": _fedora_common + ["glibc-devel.i686"],
},
'fedora_win32_dyn': {
'COMMON': _fedora_common + ['mingw32-gcc-c++', 'mingw32-bzip2', 'mingw32-win-iconv', 'mingw32-winpthreads', 'wine'],
'zlib': ['mingw32-zlib'],
'lzma': ['mingw32-xz-libs'],
'libmicrohttpd': ['mingw32-libmicrohttpd'],
"fedora_win32_dyn": {
"COMMON": _fedora_common
+ [
"mingw32-gcc-c++",
"mingw32-bzip2",
"mingw32-win-iconv",
"mingw32-winpthreads",
"wine",
],
"zlib": ["mingw32-zlib"],
"lzma": ["mingw32-xz-libs"],
"libmicrohttpd": ["mingw32-libmicrohttpd"],
},
'fedora_win32_static': {
'COMMON': _fedora_common + ['mingw32-gcc-c++', 'mingw32-bzip2-static', 'mingw32-win-iconv-static', 'mingw32-winpthreads-static', 'wine'],
'zlib': ['mingw32-zlib-static'],
'lzma': ['mingw32-xz-libs-static'],
'libmicrohttpd': None, # ['mingw32-libmicrohttpd-static'] packaging dependecy seems buggy, and some static lib are name libfoo.dll.a and
# gcc cannot found them.
"fedora_win32_static": {
"COMMON": _fedora_common
+ [
"mingw32-gcc-c++",
"mingw32-bzip2-static",
"mingw32-win-iconv-static",
"mingw32-winpthreads-static",
"wine",
],
"zlib": ["mingw32-zlib-static"],
"lzma": ["mingw32-xz-libs-static"],
"libmicrohttpd": None, # ['mingw32-libmicrohttpd-static'] packaging dependecy seems buggy, and some static lib are name libfoo.dll.a and
# gcc cannot found them.
},
'fedora_armhf_static': {
'COMMON': _fedora_common
"fedora_armhf_static": {"COMMON": _fedora_common},
"fedora_armhf_dyn": {"COMMON": _fedora_common},
"fedora_android": {"COMMON": _fedora_common},
"debian_native_dyn": {
"COMMON": _debian_common + ["libbz2-dev", "libmagic-dev"],
"zlib": ["zlib1g-dev"],
"uuid": ["uuid-dev"],
"libmicrohttpd": ["libmicrohttpd-dev", "ccache"],
"qt": ["libqt5gui5", "qtbase5-dev", "qt5-default"],
"qtwebengine": ["qtwebengine5-dev"],
"aria2": ["aria2"],
},
'fedora_armhf_dyn': {
'COMMON': _fedora_common
"debian_native_static": {
"COMMON": _debian_common + ["libbz2-dev", "libmagic-dev"],
},
'fedora_android': {
'COMMON': _fedora_common
"debian_i586_dyn": {
"COMMON": _debian_common
+ ["libc6-dev-i386", "lib32stdc++6", "gcc-multilib", "g++-multilib"],
},
'debian_native_dyn': {
'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'],
'zlib': ['zlib1g-dev'],
'uuid': ['uuid-dev'],
'libmicrohttpd': ['libmicrohttpd-dev', 'ccache'],
'qt' : ['libqt5gui5', 'qtbase5-dev', 'qt5-default'],
'qtwebengine' : ['qtwebengine5-dev'],
'aria2': ['aria2'],
"debian_i586_static": {
"COMMON": _debian_common
+ ["libc6-dev-i386", "lib32stdc++6", "gcc-multilib", "g++-multilib"],
},
'debian_native_static': {
'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'],
"debian_win32_dyn": {
"COMMON": _debian_common
+ [
"g++-mingw-w64-i686",
"gcc-mingw-w64-i686",
"gcc-mingw-w64-base",
"mingw-w64-tools",
],
},
'debian_i586_dyn': {
'COMMON': _debian_common + ['libc6-dev-i386', 'lib32stdc++6', 'gcc-multilib', 'g++-multilib'],
"debian_win32_static": {
"COMMON": _debian_common
+ [
"g++-mingw-w64-i686",
"gcc-mingw-w64-i686",
"gcc-mingw-w64-base",
"mingw-w64-tools",
],
},
'debian_i586_static': {
'COMMON': _debian_common + ['libc6-dev-i386', 'lib32stdc++6', 'gcc-multilib', 'g++-multilib'],
"debian_armhf_static": {
"COMMON": _debian_common,
},
'debian_win32_dyn': {
'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'],
"debian_armhf_dyn": {
"COMMON": _debian_common,
},
'debian_win32_static': {
'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'],
"debian_android": {
"COMMON": _debian_common,
},
'debian_armhf_static': {
'COMMON': _debian_common,
"Darwin_native_dyn": {
"COMMON": ["autoconf", "automake", "libtool", "cmake", "pkg-config"],
"file": ["libmagic"],
},
'debian_armhf_dyn': {
'COMMON': _debian_common,
},
'debian_android': {
'COMMON': _debian_common,
},
'Darwin_native_dyn': {
'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'],
'file': ['libmagic']
},
'Darwin_iOS': {
'COMMON': ['autoconf', 'automake', 'libtool', 'cmake', 'pkg-config'],
'file': ['libmagic']
"Darwin_iOS": {
"COMMON": ["autoconf", "automake", "libtool", "cmake", "pkg-config"],
"file": ["libmagic"],
},
}

14
kiwixbuild/platforms/__init__.py
View File

@ -1,15 +1,3 @@
from .base import *
from . import (
android,
armhf,
musl,
flatpak,
i586,
ios,
native,
neutral,
win32,
wasm
)
from . import android, armhf, musl, flatpak, i586, ios, native, neutral, win32, wasm

145
kiwixbuild/platforms/android.py
View File

@ -4,137 +4,152 @@ from kiwixbuild._global import get_target_step, option
class AndroidPlatformInfo(PlatformInfo):
build = 'android'
build = "android"
static = True
toolchain_names = ['android-ndk']
compatible_hosts = ['fedora', 'debian']
toolchain_names = ["android-ndk"]
compatible_hosts = ["fedora", "debian"]
def __str__(self):
return "android"
@property
def libdir(self):
return 'lib/{}'.format(self.arch_full)
return "lib/{}".format(self.arch_full)
@property
def binaries_name(self):
arch_full = self.arch_full
return {
'CC': '{}-{}'.format(arch_full, 'clang'),
'CXX': '{}-{}'.format(arch_full, 'clang++'),
'AR': '{}-{}'.format(arch_full, 'ar'),
'STRIP': '{}-{}'.format(arch_full, 'strip'),
'RANLIB': '{}-{}'.format(arch_full, 'ranlib'),
'LD': '{}-{}'.format(arch_full, 'ld')
"CC": "{}-{}".format(arch_full, "clang"),
"CXX": "{}-{}".format(arch_full, "clang++"),
"AR": "{}-{}".format(arch_full, "ar"),
"STRIP": "{}-{}".format(arch_full, "strip"),
"RANLIB": "{}-{}".format(arch_full, "ranlib"),
"LD": "{}-{}".format(arch_full, "ld"),
}
def binaries(self):
install_path = self.install_path
binaries = {k:pj(install_path, 'bin', v)
for k,v in self.binaries_name.items()}
binaries['PKGCONFIG'] = 'pkg-config'
binaries = {
k: pj(install_path, "bin", v) for k, v in self.binaries_name.items()
}
binaries["PKGCONFIG"] = "pkg-config"
return binaries
@property
def ndk_builder(self):
return get_target_step('android-ndk', self.name)
return get_target_step("android-ndk", self.name)
@property
def install_path(self):
return self.ndk_builder.install_path
def get_cross_config(self):
extra_libs = ['-llog']
extra_cflags = ['-I{}'.format(include_dir) for include_dir in self.get_include_dirs()]
if hasattr(self, 'march'):
extra_libs.append('-march={}'.format(self.march))
extra_cflags.append('-march={}'.format(self.march))
extra_libs = ["-llog"]
extra_cflags = [
"-I{}".format(include_dir) for include_dir in self.get_include_dirs()
]
if hasattr(self, "march"):
extra_libs.append("-march={}".format(self.march))
extra_cflags.append("-march={}".format(self.march))
return {
'exe_wrapper_def': '',
'install_path': self.install_path,
'binaries': self.binaries(),
'root_path': pj(self.install_path, 'sysroot'),
'extra_libs': extra_libs,
'extra_cflags': extra_cflags,
'host_machine': {
'system': 'Android',
'lsystem': 'android',
'cpu_family': self.arch,
'cpu': self.cpu,
'endian': 'little',
'abi': self.abi
}
"exe_wrapper_def": "",
"install_path": self.install_path,
"binaries": self.binaries(),
"root_path": pj(self.install_path, "sysroot"),
"extra_libs": extra_libs,
"extra_cflags": extra_cflags,
"host_machine": {
"system": "Android",
"lsystem": "android",
"cpu_family": self.arch,
"cpu": self.cpu,
"endian": "little",
"abi": self.abi,
},
}
def get_env(self):
env = super().get_env()
root_path = pj(self.install_path, 'sysroot')
env['PKG_CONFIG_LIBDIR'] = pj(root_path, 'lib', 'pkgconfig')
env['NDK_DEBUG'] = '0'
root_path = pj(self.install_path, "sysroot")
env["PKG_CONFIG_LIBDIR"] = pj(root_path, "lib", "pkgconfig")
env["NDK_DEBUG"] = "0"
return env
def get_bin_dir(self):
return [pj(self.install_path, 'bin')]
return [pj(self.install_path, "bin")]
def set_comp_flags(self, env):
super().set_comp_flags(env)
root_path = pj(self.install_path, 'sysroot')
march = '-march={}'.format(self.march) if hasattr(self,'march') else ''
env['CFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} {} '.format(root_path, march) + env['CFLAGS']
env['CXXFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} {} '.format(root_path, march) + env['CXXFLAGS']
env['LDFLAGS'] = '--sysroot={} {} '.format(root_path, march) + env['LDFLAGS']
root_path = pj(self.install_path, "sysroot")
march = "-march={}".format(self.march) if hasattr(self, "march") else ""
env["CFLAGS"] = (
"-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} {} ".format(
root_path, march
)
+ env["CFLAGS"]
)
env["CXXFLAGS"] = (
"-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} {} ".format(
root_path, march
)
+ env["CXXFLAGS"]
)
env["LDFLAGS"] = "--sysroot={} {} ".format(root_path, march) + env["LDFLAGS"]
def set_compiler(self, env):
binaries = self.binaries()
for k,v in binaries.items():
for k, v in binaries.items():
env[k] = v
@property
def configure_options(self):
yield '--host={}'.format(self.arch_full)
yield "--host={}".format(self.arch_full)
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_android_cross_file.txt', 'cmake_cross_file.txt')
self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
self.buildEnv.cmake_crossfile = self._gen_crossfile(
"cmake_android_cross_file.txt", "cmake_cross_file.txt"
)
self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
class AndroidArm(AndroidPlatformInfo):
name = 'android_arm'
arch = cpu = 'arm'
arch_full = 'arm-linux-androideabi'
abi = 'armeabi-v7a'
march = 'armv7-a'
name = "android_arm"
arch = cpu = "arm"
arch_full = "arm-linux-androideabi"
abi = "armeabi-v7a"
march = "armv7-a"
class AndroidArm64(AndroidPlatformInfo):
name = 'android_arm64'
arch = 'arm64'
arch_full = 'aarch64-linux-android'
cpu = 'aarch64'
abi = 'arm64-v8a'
name = "android_arm64"
arch = "arm64"
arch_full = "aarch64-linux-android"
cpu = "aarch64"
abi = "arm64-v8a"
class AndroidX86(AndroidPlatformInfo):
name = 'android_x86'
arch = abi = 'x86'
arch_full = 'i686-linux-android'
cpu = 'i686'
name = "android_x86"
arch = abi = "x86"
arch_full = "i686-linux-android"
cpu = "i686"
class AndroidX8664(AndroidPlatformInfo):
name = 'android_x86_64'
arch = cpu = abi = 'x86_64'
arch_full = 'x86_64-linux-android'
name = "android_x86_64"
arch = cpu = abi = "x86_64"
arch_full = "x86_64-linux-android"
class Android(MetaPlatformInfo):
name = "android"
compatible_hosts = ['fedora', 'debian']
compatible_hosts = ["fedora", "debian"]
@property
def subPlatformNames(self):
return ['android_{}'.format(arch) for arch in option('android_arch')]
return ["android_{}".format(arch) for arch in option("android_arch")]
def add_targets(self, targetName, targets):
return super().add_targets(targetName, targets)

162
kiwixbuild/platforms/armhf.py
View File

@ -6,23 +6,25 @@ from kiwixbuild._global import get_target_step
# Base platform
class ArmPlatformInfo(PlatformInfo):
compatible_hosts = ['fedora', 'debian']
compatible_hosts = ["fedora", "debian"]
def get_cross_config(self):
return {
'binaries': self.binaries,
'exe_wrapper_def': '',
'root_path': self.root_path,
'extra_libs': [],
'extra_cflags': ['-I{}'.format(include_dir) for include_dir in self.get_include_dirs()],
'host_machine': {
'system': 'linux',
'lsystem': 'linux',
'cpu_family': self.cpu_family,
'cpu': self.cpu,
'endian': 'little',
'abi': ''
}
"binaries": self.binaries,
"exe_wrapper_def": "",
"root_path": self.root_path,
"extra_libs": [],
"extra_cflags": [
"-I{}".format(include_dir) for include_dir in self.get_include_dirs()
],
"host_machine": {
"system": "linux",
"lsystem": "linux",
"cpu_family": self.cpu_family,
"cpu": self.cpu,
"endian": "little",
"abi": "",
},
}
@property
@ -31,7 +33,7 @@ class ArmPlatformInfo(PlatformInfo):
@property
def toolchain(self):
return get_target_step(self.build, 'neutral')
return get_target_step(self.build, "neutral")
@property
def root_path(self):
@ -39,26 +41,27 @@ class ArmPlatformInfo(PlatformInfo):
@property
def binaries(self):
binaries = ((k,'{}-{}'.format(self.arch_full, v))
for k, v in (('CC', 'gcc'),
('CXX', 'g++'),
('AR', 'ar'),
('STRIP', 'strip'),
('WINDRES', 'windres'),
('RANLIB', 'ranlib'),
('LD', 'ld'),
('LDSHARED', 'g++ -shared')
)
)
binaries = {k:pj(self.root_path, 'bin', v)
for k,v in binaries}
binaries['PKGCONFIG'] = 'pkg-config'
binaries = (
(k, "{}-{}".format(self.arch_full, v))
for k, v in (
("CC", "gcc"),
("CXX", "g++"),
("AR", "ar"),
("STRIP", "strip"),
("WINDRES", "windres"),
("RANLIB", "ranlib"),
("LD", "ld"),
("LDSHARED", "g++ -shared"),
)
)
binaries = {k: pj(self.root_path, "bin", v) for k, v in binaries}
binaries["PKGCONFIG"] = "pkg-config"
return binaries
@property
def exe_wrapper_def(self):
try:
which('qemu-arm')
which("qemu-arm")
except subprocess.CalledProcessError:
return ""
else:
@ -66,31 +69,39 @@ class ArmPlatformInfo(PlatformInfo):
@property
def configure_options(self):
yield '--host={}'.format(self.arch_full)
yield "--host={}".format(self.arch_full)
def get_bin_dir(self):
return [pj(self.root_path, 'bin')]
return [pj(self.root_path, "bin")]
def get_env(self):
env = super().get_env()
env['LD_LIBRARY_PATH'] = ':'.join([
pj(self.root_path, self.arch_full, 'lib64'),
pj(self.root_path, 'lib'),
env['LD_LIBRARY_PATH']
])
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
env['QEMU_LD_PREFIX'] = pj(self.root_path, self.arch_full, "libc")
env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format(
':'.join([
pj(self.root_path, self.arch_full, "lib"),
env['LD_LIBRARY_PATH']
]))
env["LD_LIBRARY_PATH"] = ":".join(
[
pj(self.root_path, self.arch_full, "lib64"),
pj(self.root_path, "lib"),
env["LD_LIBRARY_PATH"],
]
)
env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig")
env["QEMU_LD_PREFIX"] = pj(self.root_path, self.arch_full, "libc")
env["QEMU_SET_ENV"] = "LD_LIBRARY_PATH={}".format(
":".join(
[pj(self.root_path, self.arch_full, "lib"), env["LD_LIBRARY_PATH"]]
)
)
return env
def set_comp_flags(self, env):
super().set_comp_flags(env)
env['CFLAGS'] = " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS']
env['CXXFLAGS'] = " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
env["CFLAGS"] = (
" -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
+ env["CFLAGS"]
)
env["CXXFLAGS"] = (
" -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
+ env["CXXFLAGS"]
)
def set_compiler(self, env):
for k, v in self.binaries.items():
@ -98,69 +109,78 @@ class ArmPlatformInfo(PlatformInfo):
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt")
self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
class Armv6(ArmPlatformInfo):
build = "armv6"
arch_full = "armv6-rpi-linux-gnueabihf"
toolchain_names = ['armv6']
cpu_family = 'arm'
cpu = 'armv6'
toolchain_names = ["armv6"]
cpu_family = "arm"
cpu = "armv6"
class Armv6Dyn(Armv6):
name = 'armv6_dyn'
name = "armv6_dyn"
static = False
class Armv6Static(Armv6):
name = 'armv6_static'
name = "armv6_static"
static = True
class Armv6Mixed(MixedMixin('armv6_static'), Armv6):
name = 'armv6_mixed'
class Armv6Mixed(MixedMixin("armv6_static"), Armv6):
name = "armv6_mixed"
static = False
class Armv8(ArmPlatformInfo):
build = "armv8"
arch_full = "armv8-rpi3-linux-gnueabihf"
toolchain_names = ['armv8']
cpu_family = 'arm'
cpu = 'armv8'
toolchain_names = ["armv8"]
cpu_family = "arm"
cpu = "armv8"
class Armv8Dyn(Armv8):
name = 'armv8_dyn'
name = "armv8_dyn"
static = False
class Armv8Static(Armv8):
name = 'armv8_static'
name = "armv8_static"
static = True
class Armv8Mixed(MixedMixin('armv8_static'), Armv8):
name = 'armv8_mixed'
class Armv8Mixed(MixedMixin("armv8_static"), Armv8):
name = "armv8_mixed"
static = False
class Aarch64(ArmPlatformInfo):
build = 'aarch64'
arch_full = 'aarch64-linux-gnu'
toolchain_names = ['aarch64']
cpu_family = 'aarch64'
cpu = 'aarch64'
build = "aarch64"
arch_full = "aarch64-linux-gnu"
toolchain_names = ["aarch64"]
cpu_family = "aarch64"
cpu = "aarch64"
@property
def root_path(self):
return self.toolchain.build_path
class Aarch64Dyn(Aarch64):
name = 'aarch64_dyn'
name = "aarch64_dyn"
static = False
class Aarch64Static(Aarch64):
name = 'aarch64_static'
name = "aarch64_static"
static = True
class Aarch64Mixed(MixedMixin('aarch64_static'), Aarch64):
name = 'aarch64_mixed'
class Aarch64Mixed(MixedMixin("aarch64_static"), Aarch64):
name = "aarch64_mixed"
static = False

68
kiwixbuild/platforms/base.py
View File

@ -1,4 +1,3 @@
import os, sys
import subprocess
@ -8,13 +7,14 @@ from kiwixbuild.buildenv import BuildEnv
from kiwixbuild._global import neutralEnv, option, target_steps
_SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
TEMPLATES_DIR = pj(os.path.dirname(_SCRIPT_DIR), 'templates')
TEMPLATES_DIR = pj(os.path.dirname(_SCRIPT_DIR), "templates")
class _MetaPlatform(type):
def __new__(cls, name, bases, dct):
_class = type.__new__(cls, name, bases, dct)
if name not in ('PlatformInfo', 'MetaPlatformInfo') and 'name' in dct:
dep_name = dct['name']
if name not in ("PlatformInfo", "MetaPlatformInfo") and "name" in dct:
dep_name = dct["name"]
PlatformInfo.all_platforms[dep_name] = _class
return _class
@ -43,20 +43,20 @@ class PlatformInfo(metaclass=_MetaPlatform):
self.setup_toolchains(targets)
def __str__(self):
return "{}_{}".format(self.build, 'static' if self.static else 'dyn')
return "{}_{}".format(self.build, "static" if self.static else "dyn")
def setup_toolchains(self, targets):
for tlc_name in self.toolchain_names:
ToolchainClass = Dependency.all_deps[tlc_name]
targets[('source', tlc_name)] = ToolchainClass.Source
plt_name = 'neutral' if ToolchainClass.neutral else self.name
targets[("source", tlc_name)] = ToolchainClass.Source
plt_name = "neutral" if ToolchainClass.neutral else self.name
targets[(plt_name, tlc_name)] = ToolchainClass.Builder
def add_targets(self, targetName, targets):
if (self.name, targetName) in targets:
return []
targetClass = Dependency.all_deps[targetName]
targets[('source', targetName)] = targetClass.Source
targets[("source", targetName)] = targetClass.Source
targets[(self.name, targetName)] = targetClass.Builder
for dep in targetClass.Builder.get_dependencies(self, False):
if isinstance(dep, tuple):
@ -68,47 +68,40 @@ class PlatformInfo(metaclass=_MetaPlatform):
return [(self.name, targetName)]
def get_fully_qualified_dep(self, dep):
if isinstance(dep, tuple):
if isinstance(dep, tuple):
return dep
else:
else:
return self.name, dep
def get_cross_config(self):
return {}
def get_include_dirs(self):
return [pj(self.buildEnv.install_dir, 'include')]
return [pj(self.buildEnv.install_dir, "include")]
def get_env(self):
return DefaultEnv()
def get_bin_dir(self):
return []
def set_compiler(self, env):
pass
def set_comp_flags(self, env):
if self.static:
env['CFLAGS'] = env['CFLAGS'] + ' -fPIC'
env['CXXFLAGS'] = env['CXXFLAGS'] + ' -fPIC'
env["CFLAGS"] = env["CFLAGS"] + " -fPIC"
env["CXXFLAGS"] = env["CXXFLAGS"] + " -fPIC"
def _gen_crossfile(self, name, outname=None):
if outname is None:
outname = name
crossfile = pj(self.buildEnv.build_dir, outname)
template_file = pj(TEMPLATES_DIR, name)
with open(template_file, 'r') as f:
with open(template_file, "r") as f:
template = f.read()
content = template.format(
**self.get_cross_config()
)
with open(crossfile, 'w') as outfile:
content = template.format(**self.get_cross_config())
with open(crossfile, "w") as outfile:
outfile.write(content)
return crossfile
@ -121,7 +114,6 @@ class PlatformInfo(metaclass=_MetaPlatform):
self.buildEnv.clean_intermediate_directories()
class MetaPlatformInfo(PlatformInfo):
subPlatformNames = []
@ -133,7 +125,6 @@ class MetaPlatformInfo(PlatformInfo):
return targetDefs
def MixedMixin(static_name):
class MixedMixinClass:
mixed = True
@ -141,7 +132,7 @@ def MixedMixin(static_name):
def add_targets(self, targetName, targets):
print(targetName)
if option('target') == targetName:
if option("target") == targetName:
return super().add_targets(targetName, targets)
else:
static_platform = self.get_platform(static_name, targets)
@ -150,7 +141,7 @@ def MixedMixin(static_name):
def get_fully_qualified_dep(self, dep):
if isinstance(dep, tuple):
return dep
if option('target') == dep:
if option("target") == dep:
return self.name, dep
return static_name, dep
@ -161,16 +152,27 @@ def MixedMixin(static_name):
def get_include_dirs(self):
return [
pj(self.buildEnv.install_dir, 'include'),
pj(self.static_buildEnv.install_dir, 'include')
pj(self.buildEnv.install_dir, "include"),
pj(self.static_buildEnv.install_dir, "include"),
]
def get_env(self):
env = super().get_env()
env['PATH'] = ':'.join([pj(self.static_buildEnv.install_dir, 'bin')] + [env['PATH']])
pkgconfig_path = pj(self.static_buildEnv.install_dir, self.static_buildEnv.libprefix, 'pkgconfig')
env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path])
env['CPPFLAGS'] = " ".join(['-I'+pj(self.static_buildEnv.install_dir, 'include'), env['CPPFLAGS']])
env["PATH"] = ":".join(
[pj(self.static_buildEnv.install_dir, "bin")] + [env["PATH"]]
)
pkgconfig_path = pj(
self.static_buildEnv.install_dir,
self.static_buildEnv.libprefix,
"pkgconfig",
)
env["PKG_CONFIG_PATH"] = ":".join([env["PKG_CONFIG_PATH"], pkgconfig_path])
env["CPPFLAGS"] = " ".join(
[
"-I" + pj(self.static_buildEnv.install_dir, "include"),
env["CPPFLAGS"],
]
)
return env
return MixedMixinClass

14
kiwixbuild/platforms/flatpak.py
View File

@ -1,18 +1,18 @@
from .base import PlatformInfo
from kiwixbuild._global import option, neutralEnv
class FlatpakPlatformInfo(PlatformInfo):
name = 'flatpak'
build = 'flatpak'
static = ''
toolchain_names = ['org.kde', 'io.qt.qtwebengine']
compatible_hosts = ['debian', 'fedora']
name = "flatpak"
build = "flatpak"
static = ""
toolchain_names = ["org.kde", "io.qt.qtwebengine"]
compatible_hosts = ["debian", "fedora"]
def __str__(self):
return "flatpak"
def get_env(self):
env = super().get_env()
env['FLATPAK_USER_DIR'] = self.buildEnv.build_dir
env["FLATPAK_USER_DIR"] = self.buildEnv.build_dir
return env

83
kiwixbuild/platforms/i586.py
View File

@ -5,66 +5,75 @@ from kiwixbuild.utils import which, pj
class I586PlatformInfo(PlatformInfo):
build = 'i586'
arch_full = 'i586-linux-gnu'
compatible_hosts = ['fedora', 'debian']
build = "i586"
arch_full = "i586-linux-gnu"
compatible_hosts = ["fedora", "debian"]
def get_cross_config(self):
return {
'binaries': self.binaries,
'exe_wrapper_def': '',
'extra_libs': ['-m32', '-march=i586', '-mno-sse'],
'extra_cflags': [
'-m32',
'-march=i586',
'-mno-sse',
*('-I{}'.format(include_dir) for include_dir in self.get_include_dirs())
"binaries": self.binaries,
"exe_wrapper_def": "",
"extra_libs": ["-m32", "-march=i586", "-mno-sse"],
"extra_cflags": [
"-m32",
"-march=i586",
"-mno-sse",
*(
"-I{}".format(include_dir)
for include_dir in self.get_include_dirs()
),
],
'host_machine': {
'system': 'linux',
'lsystem': 'linux',
'cpu_family': 'x86',
'cpu': 'i586',
'endian': 'little',
'abi': ''
}
"host_machine": {
"system": "linux",
"lsystem": "linux",
"cpu_family": "x86",
"cpu": "i586",
"endian": "little",
"abi": "",
},
}
@property
def configure_options(self):
yield f'--host={self.arch_full}'
yield f"--host={self.arch_full}"
@property
def binaries(self):
return {k:which(v)
for k, v in (('CC', os.environ.get('CC', 'gcc')),
('CXX', os.environ.get('CXX', 'g++')),
('AR', 'ar'),
('STRIP', 'strip'),
('RANLIB', 'ranlib'),
('LD', 'ld'),
('PKGCONFIG', 'pkg-config'))
}
return {
k: which(v)
for k, v in (
("CC", os.environ.get("CC", "gcc")),
("CXX", os.environ.get("CXX", "g++")),
("AR", "ar"),
("STRIP", "strip"),
("RANLIB", "ranlib"),
("LD", "ld"),
("PKGCONFIG", "pkg-config"),
)
}
def set_comp_flags(self, env):
super().set_comp_flags(env)
env['CFLAGS'] = "-m32 -march=i586 -mno-sse "+env['CFLAGS']
env['CXXFLAGS'] = "-m32 -march=i586 -mno-sse "+env['CXXFLAGS']
env['LDFLAGS'] = "-m32 -march=i586 -mno-sse "+env['LDFLAGS']
env["CFLAGS"] = "-m32 -march=i586 -mno-sse " + env["CFLAGS"]
env["CXXFLAGS"] = "-m32 -march=i586 -mno-sse " + env["CXXFLAGS"]
env["LDFLAGS"] = "-m32 -march=i586 -mno-sse " + env["LDFLAGS"]
def get_bin_dir(self):
return []
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_i586_cross_file.txt', 'cmake_cross_file.txt')
self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
self.buildEnv.cmake_crossfile = self._gen_crossfile(
"cmake_i586_cross_file.txt", "cmake_cross_file.txt"
)
self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
class I586Dyn(I586PlatformInfo):
name = 'i586_dyn'
name = "i586_dyn"
static = False
class I586Static(I586PlatformInfo):
name = 'i586_static'
name = "i586_static"
static = True

219
kiwixbuild/platforms/ios.py
View File

@ -6,12 +6,13 @@ from .base import PlatformInfo, MetaPlatformInfo, MixedMixin
from kiwixbuild.dependencies.apple_xcframework import AppleXCFramework
MIN_MACOS_VERSION = '12.0'
MIN_MACOS_VERSION = "12.0"
class ApplePlatformInfo(PlatformInfo):
build = 'iOS'
build = "iOS"
static = True
compatible_hosts = ['Darwin']
compatible_hosts = ["Darwin"]
arch = None
host = None
target = None
@ -35,168 +36,194 @@ class ApplePlatformInfo(PlatformInfo):
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_ios_cross_file.txt', 'cmake_cross_file.txt')
self.buildEnv.meson_crossfile = self._gen_crossfile('meson_ios_cross_file.txt', 'meson_cross_file.txt')
self.buildEnv.cmake_crossfile = self._gen_crossfile(
"cmake_ios_cross_file.txt", "cmake_cross_file.txt"
)
self.buildEnv.meson_crossfile = self._gen_crossfile(
"meson_ios_cross_file.txt", "meson_cross_file.txt"
)
def get_cross_config(self):
config = {
'root_path': self.root_path,
'binaries': self.binaries,
'exe_wrapper_def': '',
'extra_libs': [
'-isysroot', self.root_path,
'-arch', self.arch,
'-target', self.target,
"root_path": self.root_path,
"binaries": self.binaries,
"exe_wrapper_def": "",
"extra_libs": [
"-isysroot",
self.root_path,
"-arch",
self.arch,
"-target",
self.target,
],
'extra_cflags': [
'-isysroot', self.root_path,
'-arch', self.arch,
'-target', self.target,
*('-I{}'.format(include_dir) for include_dir in self.get_include_dirs())
"extra_cflags": [
"-isysroot",
self.root_path,
"-arch",
self.arch,
"-target",
self.target,
*(
"-I{}".format(include_dir)
for include_dir in self.get_include_dirs()
),
],
'host_machine': {
'system': 'Darwin',
'lsystem': 'darwin',
'cpu_family': self.arch,
'cpu': self.cpu,
'endian': '',
'abi': ''
}
"host_machine": {
"system": "Darwin",
"lsystem": "darwin",
"cpu_family": self.arch,
"cpu": self.cpu,
"endian": "",
"abi": "",
},
}
if self.min_iphoneos_version:
config['extra_libs'].append('-miphoneos-version-min={}'.format(self.min_iphoneos_version))
config['extra_cflags'].append('-miphoneos-version-min={}'.format(self.min_iphoneos_version))
config["extra_libs"].append(
"-miphoneos-version-min={}".format(self.min_iphoneos_version)
)
config["extra_cflags"].append(
"-miphoneos-version-min={}".format(self.min_iphoneos_version)
)
if self.min_macos_version:
config['extra_libs'].append('-mmacosx-version-min={}'.format(self.min_macos_version))
config['extra_cflags'].append('-mmacosx-version-min={}'.format(self.min_macos_version))
config["extra_libs"].append(
"-mmacosx-version-min={}".format(self.min_macos_version)
)
config["extra_cflags"].append(
"-mmacosx-version-min={}".format(self.min_macos_version)
)
return config
def get_env(self):
env = super().get_env()
cflags = [env['CFLAGS']]
cflags = [env["CFLAGS"]]
if self.min_iphoneos_version:
cflags.append('-miphoneos-version-min={}'.format(self.min_iphoneos_version))
cflags.append("-miphoneos-version-min={}".format(self.min_iphoneos_version))
if self.min_macos_version:
cflags.append('-mmacosx-version-min={}'.format(self.min_macos_version))
env['CFLAGS'] = ' '.join(cflags)
cflags.append("-mmacosx-version-min={}".format(self.min_macos_version))
env["CFLAGS"] = " ".join(cflags)
return env
def set_comp_flags(self, env):
super().set_comp_flags(env)
cflags = [
'-isysroot {}'.format(self.root_path),
'-arch {}'.format(self.arch),
'-target {}'.format(self.target),
env['CFLAGS'],
"-isysroot {}".format(self.root_path),
"-arch {}".format(self.arch),
"-target {}".format(self.target),
env["CFLAGS"],
]
if self.min_iphoneos_version:
cflags.append('-miphoneos-version-min={}'.format(self.min_iphoneos_version))
env['CFLAGS'] = ' '.join(cflags)
env['CXXFLAGS'] = ' '.join([
env['CFLAGS'],
'-std=c++11',
env['CXXFLAGS'],
])
env['LDFLAGS'] = ' '.join([
' -arch {}'.format(self.arch),
'-isysroot {}'.format(self.root_path),
])
cflags.append("-miphoneos-version-min={}".format(self.min_iphoneos_version))
env["CFLAGS"] = " ".join(cflags)
env["CXXFLAGS"] = " ".join(
[
env["CFLAGS"],
"-std=c++11",
env["CXXFLAGS"],
]
)
env["LDFLAGS"] = " ".join(
[
" -arch {}".format(self.arch),
"-isysroot {}".format(self.root_path),
]
)
def get_bin_dir(self):
return [pj(self.root_path, 'bin')]
return [pj(self.root_path, "bin")]
@property
def binaries(self):
return {
'CC': xrun_find('clang'),
'CXX': xrun_find('clang++'),
'AR': xrun_find('ar'),
'STRIP': xrun_find('strip'),
'RANLIB': xrun_find('ranlib'),
'LD': xrun_find('ld'),
'PKGCONFIG': 'pkg-config',
"CC": xrun_find("clang"),
"CXX": xrun_find("clang++"),
"AR": xrun_find("ar"),
"STRIP": xrun_find("strip"),
"RANLIB": xrun_find("ranlib"),
"LD": xrun_find("ld"),
"PKGCONFIG": "pkg-config",
}
@property
def configure_options(self):
yield f'--host={self.host}'
yield f"--host={self.host}"
class iOSArm64(ApplePlatformInfo):
name = 'iOS_arm64'
arch = cpu = 'arm64'
host = 'arm-apple-darwin'
target = 'aarch64-apple-ios'
sdk_name = 'iphoneos'
min_iphoneos_version = '15.0'
name = "iOS_arm64"
arch = cpu = "arm64"
host = "arm-apple-darwin"
target = "aarch64-apple-ios"
sdk_name = "iphoneos"
min_iphoneos_version = "15.0"
class iOSx64Simulator(ApplePlatformInfo):
name = 'iOSSimulator_x86_64'
arch = cpu = 'x86_64'
host = 'x86_64-apple-darwin'
target = 'x86-apple-ios-simulator'
sdk_name = 'iphonesimulator'
min_iphoneos_version = '15.0'
name = "iOSSimulator_x86_64"
arch = cpu = "x86_64"
host = "x86_64-apple-darwin"
target = "x86-apple-ios-simulator"
sdk_name = "iphonesimulator"
min_iphoneos_version = "15.0"
class iOSArm64Simulator(ApplePlatformInfo):
name = 'iOSSimulator_arm64'
arch = cpu = 'arm64'
host = 'arm-apple-darwin'
target = 'aarch64-apple-ios-simulator'
sdk_name = 'iphonesimulator'
min_iphoneos_version = '15.0'
name = "iOSSimulator_arm64"
arch = cpu = "arm64"
host = "arm-apple-darwin"
target = "aarch64-apple-ios-simulator"
sdk_name = "iphonesimulator"
min_iphoneos_version = "15.0"
class macOSArm64(ApplePlatformInfo):
name = 'macOS_arm64_static'
arch = cpu = 'arm64'
host = 'aarch64-apple-darwin'
target = 'arm64-apple-macos'
sdk_name = 'macosx'
name = "macOS_arm64_static"
arch = cpu = "arm64"
host = "aarch64-apple-darwin"
target = "arm64-apple-macos"
sdk_name = "macosx"
min_iphoneos_version = None
min_macos_version = MIN_MACOS_VERSION
class macOSArm64Mixed(MixedMixin('macOS_arm64_static'), ApplePlatformInfo):
name = 'macOS_arm64_mixed'
arch = cpu = 'arm64'
host = 'aarch64-apple-darwin'
target = 'arm64-apple-macos'
sdk_name = 'macosx'
class macOSArm64Mixed(MixedMixin("macOS_arm64_static"), ApplePlatformInfo):
name = "macOS_arm64_mixed"
arch = cpu = "arm64"
host = "aarch64-apple-darwin"
target = "arm64-apple-macos"
sdk_name = "macosx"
min_iphoneos_version = None
min_macos_version = MIN_MACOS_VERSION
class macOSx64(ApplePlatformInfo):
name = 'macOS_x86_64'
arch = cpu = 'x86_64'
host = 'x86_64-apple-darwin'
target = 'x86_64-apple-macos'
sdk_name = 'macosx'
name = "macOS_x86_64"
arch = cpu = "x86_64"
host = "x86_64-apple-darwin"
target = "x86_64-apple-macos"
sdk_name = "macosx"
min_iphoneos_version = None
min_macos_version = MIN_MACOS_VERSION
class IOS(MetaPlatformInfo):
name = "iOS_multi"
compatible_hosts = ['Darwin']
compatible_hosts = ["Darwin"]
@property
def subPlatformNames(self):
return ['iOS_{}'.format(arch) for arch in option('ios_arch')]
return ["iOS_{}".format(arch) for arch in option("ios_arch")]
def add_targets(self, targetName, targets):
super().add_targets(targetName, targets)
return PlatformInfo.add_targets(self, '_ios_fat_lib', targets)
return PlatformInfo.add_targets(self, "_ios_fat_lib", targets)
def __str__(self):
return self.name
class AppleStaticAll(MetaPlatformInfo):
name = "apple_all_static"
compatible_hosts = ['Darwin']
compatible_hosts = ["Darwin"]
@property
def subPlatformNames(self):
@ -204,7 +231,7 @@ class AppleStaticAll(MetaPlatformInfo):
def add_targets(self, targetName, targets):
super().add_targets(targetName, targets)
return PlatformInfo.add_targets(self, 'apple_xcframework', targets)
return PlatformInfo.add_targets(self, "apple_xcframework", targets)
def __str__(self):
return self.name

154
kiwixbuild/platforms/musl.py
View File

@ -5,28 +5,30 @@ from kiwixbuild._global import get_target_step
class MuslPlatformInfo(PlatformInfo):
compatible_hosts = ['fedora', 'debian']
compatible_hosts = ["fedora", "debian"]
def get_cross_config(self):
return {
'binaries': self.binaries,
'exe_wrapper_def': '',
'root_path': self.root_path,
'extra_libs': [],
'extra_cflags': ['-I{}'.format(include_dir) for include_dir in self.get_include_dirs()],
'host_machine': {
'system': 'linux',
'lsystem': 'linux',
'cpu_family': self.cpu_family,
'cpu': self.cpu,
'endian': 'little',
'abi': ''
}
"binaries": self.binaries,
"exe_wrapper_def": "",
"root_path": self.root_path,
"extra_libs": [],
"extra_cflags": [
"-I{}".format(include_dir) for include_dir in self.get_include_dirs()
],
"host_machine": {
"system": "linux",
"lsystem": "linux",
"cpu_family": self.cpu_family,
"cpu": self.cpu,
"endian": "little",
"abi": "",
},
}
@property
def toolchain(self):
return get_target_step(self.build, 'neutral')
return get_target_step(self.build, "neutral")
@property
def root_path(self):
@ -34,20 +36,21 @@ class MuslPlatformInfo(PlatformInfo):
@property
def binaries(self):
binaries = ((k,'{}-{}'.format(self.arch_full, v))
for k, v in (('CC', 'gcc'),
('CXX', 'g++'),
('AR', 'ar'),
('STRIP', 'strip'),
('WINDRES', 'windres'),
('RANLIB', 'ranlib'),
('LD', 'ld'),
('LDSHARED', 'g++ -shared')
)
)
binaries = {k:pj(self.root_path, 'bin', v)
for k,v in binaries}
binaries['PKGCONFIG'] = 'pkg-config'
binaries = (
(k, "{}-{}".format(self.arch_full, v))
for k, v in (
("CC", "gcc"),
("CXX", "g++"),
("AR", "ar"),
("STRIP", "strip"),
("WINDRES", "windres"),
("RANLIB", "ranlib"),
("LD", "ld"),
("LDSHARED", "g++ -shared"),
)
)
binaries = {k: pj(self.root_path, "bin", v) for k, v in binaries}
binaries["PKGCONFIG"] = "pkg-config"
return binaries
@property
@ -63,35 +66,42 @@ class MuslPlatformInfo(PlatformInfo):
@property
def configure_options(self):
return [f'--host={self.arch_full}']
return [f"--host={self.arch_full}"]
def get_bin_dir(self):
return [pj(self.root_path, 'bin')]
return [pj(self.root_path, "bin")]
def get_env(self):
env = super().get_env()
env['LD_LIBRARY_PATH'] = ':'.join([
pj(self.root_path, self.arch_full, 'lib64'),
pj(self.root_path, 'lib'),
env['LD_LIBRARY_PATH']
])
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
env['QEMU_LD_PREFIX'] = pj(self.root_path, self.arch_full, "libc")
env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format(
':'.join([
pj(self.root_path, self.arch_full, "lib"),
env['LD_LIBRARY_PATH']
]))
env["LD_LIBRARY_PATH"] = ":".join(
[
pj(self.root_path, self.arch_full, "lib64"),
pj(self.root_path, "lib"),
env["LD_LIBRARY_PATH"],
]
)
env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig")
env["QEMU_LD_PREFIX"] = pj(self.root_path, self.arch_full, "libc")
env["QEMU_SET_ENV"] = "LD_LIBRARY_PATH={}".format(
":".join(
[pj(self.root_path, self.arch_full, "lib"), env["LD_LIBRARY_PATH"]]
)
)
return env
def set_comp_flags(self, env):
super().set_comp_flags(env)
env['LD_LIBRARY_PATH'] = ':'.join([
pj(self.root_path, self.arch_full, 'lib'),
env['LD_LIBRARY_PATH']
])
env['CFLAGS'] = " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS']
env['CXXFLAGS'] = " -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
env["LD_LIBRARY_PATH"] = ":".join(
[pj(self.root_path, self.arch_full, "lib"), env["LD_LIBRARY_PATH"]]
)
env["CFLAGS"] = (
" -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
+ env["CFLAGS"]
)
env["CXXFLAGS"] = (
" -fPIC -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
+ env["CXXFLAGS"]
)
def set_compiler(self, env):
for k, v in self.binaries.items():
@ -99,50 +109,54 @@ class MuslPlatformInfo(PlatformInfo):
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt")
self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
class Aarch64MuslPlatformInfo(MuslPlatformInfo):
build = 'aarch64_musl'
arch_full = 'aarch64-linux-musl'
toolchain_names = ['aarch64_musl']
build = "aarch64_musl"
arch_full = "aarch64-linux-musl"
toolchain_names = ["aarch64_musl"]
libdir = "lib/aarch64-linux-musl"
cpu_family = 'arm'
cpu = 'armhf'
qemu = 'qemu-arm'
cpu_family = "arm"
cpu = "armhf"
qemu = "qemu-arm"
class Aarch64MuslDyn(Aarch64MuslPlatformInfo):
name = 'aarch64_musl_dyn'
name = "aarch64_musl_dyn"
static = False
class Aarch64MuslStatic(Aarch64MuslPlatformInfo):
name = 'aarch64_musl_static'
name = "aarch64_musl_static"
static = True
class Aarch64MuslMixed(MixedMixin('aarch64_musl_static'), Aarch64MuslPlatformInfo):
name = 'aarch64_musl_mixed'
class Aarch64MuslMixed(MixedMixin("aarch64_musl_static"), Aarch64MuslPlatformInfo):
name = "aarch64_musl_mixed"
static = False
class X86_64MuslPlatformInfo(MuslPlatformInfo):
build = 'x86-64_musl'
arch_full = 'x86_64-linux-musl'
toolchain_names = ['x86-64_musl']
build = "x86-64_musl"
arch_full = "x86_64-linux-musl"
toolchain_names = ["x86-64_musl"]
libdir = "lib/x86_64-linux-musl"
cpu_family = 'x86_64'
cpu = 'x86_64'
cpu_family = "x86_64"
cpu = "x86_64"
class X86_64MuslDyn(X86_64MuslPlatformInfo):
name = 'x86-64_musl_dyn'
name = "x86-64_musl_dyn"
static = False
class X86_64MuslStatic(X86_64MuslPlatformInfo):
name = 'x86-64_musl_static'
name = "x86-64_musl_static"
static = True
class x86_64MuslMixed(MixedMixin('x86-64_musl_static'), X86_64MuslPlatformInfo):
name = 'x86-64_musl_mixed'
class x86_64MuslMixed(MixedMixin("x86-64_musl_static"), X86_64MuslPlatformInfo):
name = "x86-64_musl_mixed"
static = False

27
kiwixbuild/platforms/native.py
View File

@ -4,29 +4,32 @@ from kiwixbuild.utils import pj
from kiwixbuild._global import option, neutralEnv
from kiwixbuild.platforms.ios import MIN_MACOS_VERSION
class NativePlatformInfo(PlatformInfo):
build = 'native'
build = "native"
def get_env(self):
env = super().get_env()
if neutralEnv('distname') == 'fedora':
env['QT_SELECT'] = "5-64"
if neutralEnv('distname') == 'Darwin':
env['CFLAGS'] += f'-mmacosx-version-min={MIN_MACOS_VERSION}'
if neutralEnv("distname") == "fedora":
env["QT_SELECT"] = "5-64"
if neutralEnv("distname") == "Darwin":
env["CFLAGS"] += f"-mmacosx-version-min={MIN_MACOS_VERSION}"
return env
class NativeDyn(NativePlatformInfo):
name = 'native_dyn'
name = "native_dyn"
static = False
compatible_hosts = ['fedora', 'debian', 'Darwin']
compatible_hosts = ["fedora", "debian", "Darwin"]
class NativeStatic(NativePlatformInfo):
name = 'native_static'
name = "native_static"
static = True
compatible_hosts = ['fedora', 'debian']
compatible_hosts = ["fedora", "debian"]
class NativeMixed(MixedMixin('native_static'), NativePlatformInfo):
name = 'native_mixed'
class NativeMixed(MixedMixin("native_static"), NativePlatformInfo):
name = "native_mixed"
static = False
compatible_hosts = ['fedora', 'debian', 'Darwin']
compatible_hosts = ["fedora", "debian", "Darwin"]

7
kiwixbuild/platforms/neutral.py
View File

@ -1,9 +1,10 @@
from .base import PlatformInfo
class NeutralPlatformInfo(PlatformInfo):
name = 'neutral'
static = ''
compatible_hosts = ['fedora', 'debian', 'Darwin']
name = "neutral"
static = ""
compatible_hosts = ["fedora", "debian", "Darwin"]
def __str__(self):
return "neutral"

103
kiwixbuild/platforms/wasm.py
View File

@ -5,36 +5,36 @@ from kiwixbuild._global import get_target_step
class WasmPlatformInfo(PlatformInfo):
name = 'wasm'
name = "wasm"
static = True
build = 'wasm'
arch_full = 'wasm64-emscripten'
build = "wasm"
arch_full = "wasm64-emscripten"
libdir = "lib"
#arch_full = 'wasm64-linux'
toolchain_names = ['emsdk']
compatible_hosts = ['fedora', 'debian']
# arch_full = 'wasm64-linux'
toolchain_names = ["emsdk"]
compatible_hosts = ["fedora", "debian"]
exe_wrapper_def = ""
def get_cross_config(self):
return {
'binaries': self.binaries,
'exe_wrapper_def': '',
'root_path': self.root_path,
'extra_libs': [],
'extra_cflags': [],
'host_machine': {
'system': 'emscripten',
'lsystem': 'emscripten',
'cpu_family': 'wasm64',
'cpu': 'wasm64',
'endian': 'little',
'abi': ''
}
"binaries": self.binaries,
"exe_wrapper_def": "",
"root_path": self.root_path,
"extra_libs": [],
"extra_cflags": [],
"host_machine": {
"system": "emscripten",
"lsystem": "emscripten",
"cpu_family": "wasm64",
"cpu": "wasm64",
"endian": "little",
"abi": "",
},
}
@property
def wasm_sdk(self):
return get_target_step('emsdk', self.name)
return get_target_step("emsdk", self.name)
@property
def install_path(self):
@ -46,22 +46,25 @@ class WasmPlatformInfo(PlatformInfo):
@property
def binaries(self):
binaries = (('CC', 'emcc'),
('CXX', 'em++'),
('AR', 'emar'),
('STRIP', 'emstrip'),
('WINDRES', 'windres'),
('RANLIB', 'emranlib'),
('LD', 'wasm-ld'))
binaries = {k:pj(self.install_path, 'upstream', 'emscripten', v)
for k,v in binaries}
binaries['PKGCONFIG'] = 'pkg-config'
binaries = (
("CC", "emcc"),
("CXX", "em++"),
("AR", "emar"),
("STRIP", "emstrip"),
("WINDRES", "windres"),
("RANLIB", "emranlib"),
("LD", "wasm-ld"),
)
binaries = {
k: pj(self.install_path, "upstream", "emscripten", v) for k, v in binaries
}
binaries["PKGCONFIG"] = "pkg-config"
return binaries
@property
def configure_options(self):
#return ""
return [f'--host={self.arch_full}']
# return ""
return [f"--host={self.arch_full}"]
@property
def configure_wrapper(self):
@ -72,24 +75,34 @@ class WasmPlatformInfo(PlatformInfo):
return "emmake"
def get_bin_dir(self):
return [pj(self.install_path, 'bin')]
return [pj(self.install_path, "bin")]
def get_env(self):
env = super().get_env()
env['PATH'] = ':'.join([
env['PATH'],
self.install_path,
pj(self.install_path, 'upstream', 'emscripten'),
pj(self.install_path, 'node', '14.18.2_64bit', 'bin')
])
env['EMSDK'] = self.install_path
env['EMSDK_NODE'] = pj(self.install_path, 'node', '14.18.2_64bit', 'bin', 'node')
env["PATH"] = ":".join(
[
env["PATH"],
self.install_path,
pj(self.install_path, "upstream", "emscripten"),
pj(self.install_path, "node", "14.18.2_64bit", "bin"),
]
)
env["EMSDK"] = self.install_path
env["EMSDK_NODE"] = pj(
self.install_path, "node", "14.18.2_64bit", "bin", "node"
)
return env
def set_comp_flags(self, env):
super().set_comp_flags(env)
env['CFLAGS'] = " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS']
env['CXXFLAGS'] = " -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
env["CFLAGS"] = (
" -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
+ env["CFLAGS"]
)
env["CXXFLAGS"] = (
" -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "
+ env["CXXFLAGS"]
)
def set_compiler(self, env):
for k, v in self.binaries.items():
@ -97,5 +110,5 @@ class WasmPlatformInfo(PlatformInfo):
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt")
self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")

87
kiwixbuild/platforms/win32.py
View File

@ -6,57 +6,66 @@ from kiwixbuild._global import neutralEnv
class Win32PlatformInfo(PlatformInfo):
build = 'win32'
compatible_hosts = ['fedora', 'debian']
arch_full = 'i686-w64-mingw32'
extra_libs = ['-lwinmm', '-lshlwapi', '-lws2_32', '-lssp']
build = "win32"
compatible_hosts = ["fedora", "debian"]
arch_full = "i686-w64-mingw32"
extra_libs = ["-lwinmm", "-lshlwapi", "-lws2_32", "-lssp"]
def get_cross_config(self):
return {
'exe_wrapper_def': self.exe_wrapper_def,
'binaries': self.binaries,
'root_path': self.root_path,
'extra_libs': self.extra_libs,
'extra_cflags': ['-DWIN32', *('-I{}'.format(include_dir) for include_dir in self.get_include_dirs())],
'host_machine': {
'system': 'Windows',
'lsystem': 'windows',
'cpu_family': 'x86',
'cpu': 'i686',
'endian': 'little',
'abi': ''
}
"exe_wrapper_def": self.exe_wrapper_def,
"binaries": self.binaries,
"root_path": self.root_path,
"extra_libs": self.extra_libs,
"extra_cflags": [
"-DWIN32",
*(
"-I{}".format(include_dir)
for include_dir in self.get_include_dirs()
),
],
"host_machine": {
"system": "Windows",
"lsystem": "windows",
"cpu_family": "x86",
"cpu": "i686",
"endian": "little",
"abi": "",
},
}
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt")
self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
@property
def root_path(self):
root_paths = {
'fedora': '/usr/i686-w64-mingw32/sys-root/mingw',
'debian': '/usr/i686-w64-mingw32'
"fedora": "/usr/i686-w64-mingw32/sys-root/mingw",
"debian": "/usr/i686-w64-mingw32",
}
return root_paths[neutralEnv('distname')]
return root_paths[neutralEnv("distname")]
@property
def binaries(self):
return {k:which('{}-{}'.format(self.arch_full, v))
for k, v in (('CC', 'gcc'),
('CXX', 'g++'),
('AR', 'ar'),
('STRIP', 'strip'),
('WINDRES', 'windres'),
('RANLIB', 'ranlib'),
('PKGCONFIG', 'pkg-config'))
}
return {
k: which("{}-{}".format(self.arch_full, v))
for k, v in (
("CC", "gcc"),
("CXX", "g++"),
("AR", "ar"),
("STRIP", "strip"),
("WINDRES", "windres"),
("RANLIB", "ranlib"),
("PKGCONFIG", "pkg-config"),
)
}
@property
def exe_wrapper_def(self):
try:
which('wine')
which("wine")
except subprocess.CalledProcessError:
return ""
else:
@ -64,25 +73,27 @@ class Win32PlatformInfo(PlatformInfo):
@property
def configure_options(self):
yield f'--host={self.arch_full}'
yield f"--host={self.arch_full}"
def set_compiler(self, env):
for k, v in self.binaries.items():
env[k] = v
def get_bin_dir(self):
return [pj(self.root_path, 'bin')]
return [pj(self.root_path, "bin")]
def get_env(self):
env = super().get_env()
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
env['LIBS'] = " ".join(self.extra_libs) + " " +env['LIBS']
env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig")
env["LIBS"] = " ".join(self.extra_libs) + " " + env["LIBS"]
return env
class Win32Dyn(Win32PlatformInfo):
name = 'win32_dyn'
name = "win32_dyn"
static = False
class Win32Static(Win32PlatformInfo):
name = 'win32_static'
name = "win32_static"
static = True

91
kiwixbuild/platforms/win64.py
View File

@ -6,57 +6,70 @@ from kiwixbuild._global import neutralEnv
class Win64PlatformInfo(PlatformInfo):
extra_libs = ['-lmingw32', '-lwinmm', '-lws2_32', '-lshlwapi', '-lrpcrt4', '-lmsvcr100', '-liphlpapi', '-lshell32', '-lkernel32']
build = 'win64'
compatible_hosts = ['fedora', 'debian']
arch_full = 'x86_64-w64-mingw32'
extra_libs = [
"-lmingw32",
"-lwinmm",
"-lws2_32",
"-lshlwapi",
"-lrpcrt4",
"-lmsvcr100",
"-liphlpapi",
"-lshell32",
"-lkernel32",
]
build = "win64"
compatible_hosts = ["fedora", "debian"]
arch_full = "x86_64-w64-mingw32"
def get_cross_config(self):
return {
'exe_wrapper_def': self.exe_wrapper_def,
'binaries': self.binaries,
'root_path': self.root_path,
'extra_libs': self.extra_libs,
'extra_cflags': ['-DWIN32'],
'host_machine': {
'system': 'Windows',
'lsystem': 'windows',
'cpu_family': 'x86_64',
'cpu': 'x86_64',
'endian': 'little',
'abi': ''
}
"exe_wrapper_def": self.exe_wrapper_def,
"binaries": self.binaries,
"root_path": self.root_path,
"extra_libs": self.extra_libs,
"extra_cflags": ["-DWIN32"],
"host_machine": {
"system": "Windows",
"lsystem": "windows",
"cpu_family": "x86_64",
"cpu": "x86_64",
"endian": "little",
"abi": "",
},
}
def finalize_setup(self):
super().finalize_setup()
self.buildEnv.cmake_crossfile = self._gen_crossfile('cmake_cross_file.txt')
self.buildEnv.meson_crossfile = self._gen_crossfile('meson_cross_file.txt')
self.buildEnv.cmake_crossfile = self._gen_crossfile("cmake_cross_file.txt")
self.buildEnv.meson_crossfile = self._gen_crossfile("meson_cross_file.txt")
@property
def root_path(self):
root_paths = {
'fedora': '/usr/x86_64-w64-mingw32/sys-root/mingw',
'debian': '/usr/x86_64-w64-mingw32'
"fedora": "/usr/x86_64-w64-mingw32/sys-root/mingw",
"debian": "/usr/x86_64-w64-mingw32",
}
return root_paths[neutralEnv('distname')]
return root_paths[neutralEnv("distname")]
@property
def binaries(self):
return {k:which('{}-{}'.format(self.arch_full, v))
for k, v in (('CC', 'gcc'),
('CXX', 'g++'),
('AR', 'ar'),
('STRIP', 'strip'),
('WINDRES', 'windres'),
('RANLIB', 'ranlib'),
('PKGCONFIG', 'pkg-config'))
}
return {
k: which("{}-{}".format(self.arch_full, v))
for k, v in (
("CC", "gcc"),
("CXX", "g++"),
("AR", "ar"),
("STRIP", "strip"),
("WINDRES", "windres"),
("RANLIB", "ranlib"),
("PKGCONFIG", "pkg-config"),
)
}
@property
def exe_wrapper_def(self):
try:
which('wine')
which("wine")
except subprocess.CalledProcessError:
return ""
else:
@ -64,25 +77,27 @@ class Win64PlatformInfo(PlatformInfo):
@property
def configure_options(self):
return [f'--host={self.arch_full}']
return [f"--host={self.arch_full}"]
def set_compiler(self, env):
for k, v in self.binaries.items():
env[k] = v
def get_bin_dir(self):
return [pj(self.root_path, 'bin')]
return [pj(self.root_path, "bin")]
def get_env(self):
env = super().get_env()
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
env['LIBS'] = " ".join(self.extra_libs) + " " +env['LIBS']
env["PKG_CONFIG_LIBDIR"] = pj(self.root_path, "lib", "pkgconfig")
env["LIBS"] = " ".join(self.extra_libs) + " " + env["LIBS"]
return env
class Win64Dyn(Win64PlatformInfo):
name = 'win64_dyn'
name = "win64_dyn"
static = False
class Win64Static(Win64PlatformInfo):
name = 'win64_static'
name = "win64_static"
static = True

88
kiwixbuild/utils.py
View File

@ -17,16 +17,16 @@ pj = os.path.join
COLORS = {
'OK': '\033[92m',
'WARNING': '\033[93m',
'NEEDED': '\033[93m',
'SKIP': '\033[34m',
'ERROR': '\033[91m',
'': '\033[0m',
"OK": "\033[92m",
"WARNING": "\033[93m",
"NEEDED": "\033[93m",
"SKIP": "\033[34m",
"ERROR": "\033[91m",
"": "\033[0m",
}
REMOTE_PREFIX = 'http://mirror.download.kiwix.org/dev/kiwix-build/'
REMOTE_PREFIX = "http://mirror.download.kiwix.org/dev/kiwix-build/"
def which(name):
@ -34,16 +34,19 @@ def which(name):
output = subprocess.check_output(command, shell=True)
return output[:-1].decode()
def xrun_find(name):
command = "xcrun -find {}".format(name)
output = subprocess.check_output(command, shell=True)
return output[:-1].decode()
regex_space = re.compile(r'((?<!\\) )')
regex_space = re.compile(r"((?<!\\) )")
def escape_path(path):
path = str(path)
return regex_space.sub(r'\ ', path)
return regex_space.sub(r"\ ", path)
class Defaultdict(defaultdict):
@ -56,7 +59,7 @@ class DefaultEnv(Defaultdict):
super().__init__(str, os.environ)
def __getitem__(self, name):
if name == b'PATH':
if name == b"PATH":
raise KeyError
return super().__getitem__(name)
@ -78,25 +81,25 @@ def get_sha256(path):
current = 0
batch_size = 1024 * 8
sha256 = hashlib.sha256()
with open(path, 'br') as f:
with open(path, "br") as f:
while True:
batch = f.read(batch_size)
if not batch:
break
sha256.update(batch)
print_progress(progress_chars[current])
current = (current+1)%4
current = (current + 1) % 4
return sha256.hexdigest()
def colorize(text, color=None):
if color is None:
color = text
return "{}{}{}".format(COLORS[color], text, COLORS[''])
return "{}{}{}".format(COLORS[color], text, COLORS[""])
def print_progress(progress):
if option('show_progress'):
if option("show_progress"):
text = "{}\033[{}D".format(progress, len(progress))
print(text, end="")
@ -126,18 +129,20 @@ def download_remote(what, where):
raise SkipCommand()
os.remove(file_path)
if option('no_cert_check'):
if option("no_cert_check"):
context = ssl.create_default_context()
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
else:
context = None
batch_size = 1024 * 8
extra_args = {'context':context} if sys.version_info >= (3, 4, 3) else {}
extra_args = {"context": context} if sys.version_info >= (3, 4, 3) else {}
progress_chars = "/-\|"
try:
with urllib.request.urlopen(what.url, **extra_args) as resource, open(file_path, 'wb') as file:
tsize = resource.info().get('Content-Length', None)
with urllib.request.urlopen(what.url, **extra_args) as resource, open(
file_path, "wb"
) as file:
tsize = resource.info().get("Content-Length", None)
if tsize is not None:
tsize = int(tsize)
current = 0
@ -147,17 +152,17 @@ def download_remote(what, where):
break
if tsize:
current += batch_size
print_progress("{:.2%}".format(current/tsize))
print_progress("{:.2%}".format(current / tsize))
else:
print_progress(progress_chars[current])
current = (current+1)%4
print_progress(progress_chars[current])
current = (current + 1) % 4
file.write(batch)
except urllib.error.URLError as e:
print("Cannot download url {}:\n{}".format(what.url, e.reason))
raise StopBuild()
if not what.sha256:
print('Sha256 for {} not set, do no verify download'.format(what.name))
print("Sha256 for {} not set, do no verify download".format(what.name))
elif what.sha256 != get_sha256(file_path):
os.remove(file_path)
raise StopBuild("Sha 256 doesn't correspond")
@ -187,7 +192,7 @@ class StopBuild(BaseCommandResult):
pass
class Remotefile(namedtuple('Remotefile', ('name', 'sha256', 'url'))):
class Remotefile(namedtuple("Remotefile", ("name", "sha256", "url"))):
def __new__(cls, name, sha256, url=None):
if url is None:
url = REMOTE_PREFIX + name
@ -217,28 +222,28 @@ class Context:
def _finalise(self):
if self.autoskip_file is not None:
os.makedirs(os.path.dirname(self.autoskip_file), exist_ok=True)
with open(self.autoskip_file, 'w'):
with open(self.autoskip_file, "w"):
pass
def extract_archive(archive_path, dest_dir, topdir=None, name=None):
is_zip_archive = archive_path.endswith('.zip')
is_zip_archive = archive_path.endswith(".zip")
archive = None
try:
if is_zip_archive:
archive = zipfile.ZipFile(archive_path)
members = archive.infolist()
getname = lambda info : info.filename
isdir = lambda info: info.filename.endswith('/')
getname = lambda info: info.filename
isdir = lambda info: info.filename.endswith("/")
else:
archive = tarfile.open(archive_path)
members = archive.getmembers()
getname = lambda info : getattr(info, 'name')
getname = lambda info: getattr(info, "name")
isdir = lambda info: info.isdir()
if not topdir:
for d in (m for m in members if isdir(m)):
_name = getname(d)
if _name.endswith('/'):
if _name.endswith("/"):
_name = _name[:-1]
if not os.path.dirname(_name):
if topdir:
@ -249,9 +254,13 @@ def extract_archive(archive_path, dest_dir, topdir=None, name=None):
break
topdir = _name
if topdir:
members_to_extract = [m for m in members if getname(m).startswith(topdir+'/')]
members_to_extract = [
m for m in members if getname(m).startswith(topdir + "/")
]
os.makedirs(dest_dir, exist_ok=True)
with tempfile.TemporaryDirectory(prefix=os.path.basename(archive_path), dir=dest_dir) as tmpdir:
with tempfile.TemporaryDirectory(
prefix=os.path.basename(archive_path), dir=dest_dir
) as tmpdir:
if is_zip_archive:
_members_to_extract = [getname(m) for m in members_to_extract]
else:
@ -281,8 +290,8 @@ def run_command(command, cwd, context, *, env=None, input=None):
env = DefaultEnv()
log = None
try:
if not option('verbose'):
log = open(context.log_file, 'w')
if not option("verbose"):
log = open(context.log_file, "w")
print("run command '{}'".format(command), file=log)
print("current directory is '{}'".format(cwd), file=log)
print("env is :", file=log)
@ -293,8 +302,15 @@ def run_command(command, cwd, context, *, env=None, input=None):
log.flush()
kwargs = dict()
if input:
kwargs['stdin'] = subprocess.PIPE
process = subprocess.Popen(command, cwd=cwd, env=env, stdout=log or sys.stdout, stderr=subprocess.STDOUT, **kwargs)
kwargs["stdin"] = subprocess.PIPE
process = subprocess.Popen(
command,
cwd=cwd,
env=env,
stdout=log or sys.stdout,
stderr=subprocess.STDOUT,
**kwargs
)
if input:
input = input.encode()
while True:
@ -308,7 +324,7 @@ def run_command(command, cwd, context, *, env=None, input=None):
# `communicate` timeout (and we must set `input` to None
# to not communicate again).
input = None
print('.', end='', flush=True)
print(".", end="", flush=True)
else:
break
if process.returncode:

63
kiwixbuild/versions.py
View File

@ -1,12 +1,11 @@
# This file reference all the versions of the depedencies we use in kiwix-build.
main_project_versions = {
'libzim': '9.1.0',
'libkiwix': '13.0.0',
'kiwix-tools': '3.6.0',
'zim-tools': '3.4.0',
'kiwix-desktop': '2.3.1' # Also change KIWIX_DESKTOP_VERSION and KIWIX_DESKTOP_RELEASE in appveyor.yml
"libzim": "9.1.0",
"libkiwix": "13.0.0",
"kiwix-tools": "3.6.0",
"zim-tools": "3.4.0",
"kiwix-desktop": "2.3.1", # Also change KIWIX_DESKTOP_VERSION and KIWIX_DESKTOP_RELEASE in appveyor.yml
}
# This dictionnary specify what we need to build at each release process.
@ -30,37 +29,37 @@ main_project_versions = {
# - set KIWIX_DESKTOP_RELEASE to 0
release_versions = {
'libzim': None, # Depends of base deps (was 0)
'libkiwix': None, # Depends of libzim (was 1)
'kiwix-tools': None, # Depends of libkiwix and libzim (was 0)
'zim-tools': 0, # Depends of libzim (was None)
'kiwix-desktop': None # Depends of libkiwix and libzim (was 4)
"libzim": None, # Depends of base deps (was 0)
"libkiwix": None, # Depends of libzim (was 1)
"kiwix-tools": None, # Depends of libkiwix and libzim (was 0)
"zim-tools": 0, # Depends of libzim (was None)
"kiwix-desktop": None, # Depends of libkiwix and libzim (was 4)
}
# This is the "version" of the whole base_deps_versions dict.
# Change this when you change base_deps_versions.
base_deps_meta_version = '96'
base_deps_meta_version = "96"
base_deps_versions = {
'zlib' : '1.2.12',
'lzma' : '5.2.6',
'zstd' : '1.5.2',
'docoptcpp' : '0.6.2',
'uuid' : '1.43.4',
'xapian-core' : '1.4.23',
'mustache' : '4.1',
'pugixml' : '1.2',
'libmicrohttpd' : '0.9.76',
'gumbo' : '0.10.1',
'icu4c' : '73.2',
'libaria2' : '1.37.0',
'libmagic' : '5.44',
'android-ndk' : 'r21e',
'qt' : '5.10.1',
'qtwebengine' : '5.10.1',
'org.kde' : '5.15-21.08',
'io.qt.qtwebengine' : '5.15-21.08',
'zim-testing-suite': '0.3',
'emsdk': '3.1.41',
"zlib": "1.2.12",
"lzma": "5.2.6",
"zstd": "1.5.2",
"docoptcpp": "0.6.2",
"uuid": "1.43.4",
"xapian-core": "1.4.23",
"mustache": "4.1",
"pugixml": "1.2",
"libmicrohttpd": "0.9.76",
"gumbo": "0.10.1",
"icu4c": "73.2",
"libaria2": "1.37.0",
"libmagic": "5.44",
"android-ndk": "r21e",
"qt": "5.10.1",
"qtwebengine": "5.10.1",
"org.kde": "5.15-21.08",
"io.qt.qtwebengine": "5.15-21.08",
"zim-testing-suite": "0.3",
"emsdk": "3.1.41",
}