From 8648e8bc27922e63e49896e6965bbeaee850ba7f Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 17 Jun 2024 12:09:12 +0200 Subject: [PATCH 01/10] Revert "Do not build libzim with xapian dependency on Windows." This reverts commit 6bfe9ec8b159fa463fa46ccecff8197727e03830. --- kiwixbuild/dependencies/libzim.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kiwixbuild/dependencies/libzim.py b/kiwixbuild/dependencies/libzim.py index 4928c94..a85215c 100644 --- a/kiwixbuild/dependencies/libzim.py +++ b/kiwixbuild/dependencies/libzim.py @@ -23,7 +23,7 @@ class Libzim(Dependency): @classmethod def get_dependencies(cls, configInfo, allDeps): if neutralEnv("distname") == "Windows": - return ["zstd", "icu4c", "zim-testing-suite"] + return ["zstd", "xapian-core", "icu4c", "zim-testing-suite"] deps = ["lzma", "zstd", "xapian-core", "icu4c"] if configInfo.name not in ("flatpak", "wasm"): deps.append("zim-testing-suite") @@ -33,7 +33,6 @@ class Libzim(Dependency): def configure_options(self): configInfo = self.buildEnv.configInfo if neutralEnv("distname") == "Windows": - yield "-Dwith_xapian=false" yield "-Dwerror=false" if configInfo.build == "android": yield "-DUSE_BUFFER_HEADER=false" From f9b890c58df1ff5aeb4f9e6d568809c53415a346 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 27 Nov 2023 16:15:41 +0100 Subject: [PATCH 02/10] Build xapian using meson build system. --- kiwixbuild/dependencies/xapian.py | 38 +++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/kiwixbuild/dependencies/xapian.py b/kiwixbuild/dependencies/xapian.py index 252d2c2..fb61613 100644 --- a/kiwixbuild/dependencies/xapian.py +++ b/kiwixbuild/dependencies/xapian.py @@ -1,4 +1,8 @@ -from .base import Dependency, ReleaseDownload, MakeBuilder +from .base import ( + Dependency, + ReleaseDownload, + MakeBuilder, MesonBuilder +) from kiwixbuild.utils import Remotefile from kiwixbuild._global import neutralEnv @@ -8,22 +12,32 @@ class Xapian(Dependency): name = "xapian-core" class Source(ReleaseDownload): - archive = Remotefile( + src_archive = Remotefile( "xapian-core-1.4.23.tar.xz", - "30d3518172084f310dab86d262b512718a7f9a13635aaa1a188e61dc26b2288c", + "30d3518172084f310dab86d262b512718a7f9a13635aaa1a188e61dc26b2288c" ) + meson_patch = Remotefile( + "xapian-core-1.4.23-1_patch.zip", + "", + "https://public.kymeria.fr/KIWIX/xapian-core-1.4.23-1_patch.zip" + ) + archives = [src_archive, meson_patch] - class Builder(MakeBuilder): +# class Builder(MakeBuilder): +# configure_options = [ +# "--disable-sse", +# "--disable-backend-chert", +# "--disable-backend-remote", +# "--disable-documentation"] +# configure_env = {'_format_LDFLAGS': "{env.LDFLAGS} -L{buildEnv.install_dir}/{buildEnv.libprefix}", +# '_format_CXXFLAGS': "{env.CXXFLAGS} -I{buildEnv.install_dir}/include"} + + class Builder(MesonBuilder): configure_options = [ - "--disable-sse", - "--disable-backend-chert", - "--disable-backend-remote", - "--disable-documentation", + "-Denable-sse=false", + "-Denable-backend-chert=false", + "-Denable-backend-remote=false" ] - configure_env = { - "_format_LDFLAGS": "{env.LDFLAGS} -L{buildEnv.install_dir}/{buildEnv.libprefix}", - "_format_CXXFLAGS": "{env.CXXFLAGS} -O3 -I{buildEnv.install_dir}/include", - } @classmethod def get_dependencies(cls, configInfo, allDeps): From b58949607602ecdcb575ac0c3ed805517af74d1e Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Fri, 19 Jul 2024 17:46:54 +0200 Subject: [PATCH 03/10] Do not use custom data for icu4c on Windows. Meson build system of ICU seems to not support custom data on Windows. So let's use the default data on Windows. --- kiwixbuild/dependencies/icu4c.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/kiwixbuild/dependencies/icu4c.py b/kiwixbuild/dependencies/icu4c.py index 689d415..b3dfaaa 100644 --- a/kiwixbuild/dependencies/icu4c.py +++ b/kiwixbuild/dependencies/icu4c.py @@ -39,15 +39,16 @@ class Icu(Dependency): topdir=None, name=self.source_dir, ) - shutil.rmtree( - pj(neutralEnv("source_dir"), self.source_dir, "source", "data") - ) - extract_archive( - pj(neutralEnv("archive_dir"), self.archive_data.name), - pj(neutralEnv("source_dir"), self.source_dir, "source"), - topdir="data", - name="data", - ) + if platform.system() != "Windows": + shutil.rmtree( + pj(neutralEnv("source_dir"), self.source_dir, "source", "data") + ) + extract_archive( + pj(neutralEnv("archive_dir"), self.archive_data.name), + pj(neutralEnv("source_dir"), self.source_dir, "source"), + topdir="data", + name="data", + ) extract_archive( pj(neutralEnv("archive_dir"), self.meson_patch.name), neutralEnv("source_dir"), From 942dba4d9c6d9ca447afc8ce4f3154f070fe6407 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 22 Jul 2024 12:15:53 +0200 Subject: [PATCH 04/10] Use git source for xapian. We have move the meson build system (and various fixes) in a git repository. --- kiwixbuild/dependencies/xapian.py | 49 ++++++++++++++++--------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/kiwixbuild/dependencies/xapian.py b/kiwixbuild/dependencies/xapian.py index fb61613..0ba92f0 100644 --- a/kiwixbuild/dependencies/xapian.py +++ b/kiwixbuild/dependencies/xapian.py @@ -1,8 +1,4 @@ -from .base import ( - Dependency, - ReleaseDownload, - MakeBuilder, MesonBuilder -) +from .base import Dependency, GitClone, ReleaseDownload, MakeBuilder, MesonBuilder from kiwixbuild.utils import Remotefile from kiwixbuild._global import neutralEnv @@ -11,26 +7,30 @@ from kiwixbuild._global import neutralEnv class Xapian(Dependency): name = "xapian-core" - class Source(ReleaseDownload): - src_archive = Remotefile( - "xapian-core-1.4.23.tar.xz", - "30d3518172084f310dab86d262b512718a7f9a13635aaa1a188e61dc26b2288c" - ) - meson_patch = Remotefile( - "xapian-core-1.4.23-1_patch.zip", - "", - "https://public.kymeria.fr/KIWIX/xapian-core-1.4.23-1_patch.zip" - ) - archives = [src_archive, meson_patch] + class Source(GitClone): + git_remote = "https://github.com/openzim/xapian-meson.git" + git_dir = "xapian-core" -# class Builder(MakeBuilder): -# configure_options = [ -# "--disable-sse", -# "--disable-backend-chert", -# "--disable-backend-remote", -# "--disable-documentation"] -# configure_env = {'_format_LDFLAGS': "{env.LDFLAGS} -L{buildEnv.install_dir}/{buildEnv.libprefix}", -# '_format_CXXFLAGS': "{env.CXXFLAGS} -I{buildEnv.install_dir}/include"} + # class Source(ReleaseDownload): + # src_archive = Remotefile( + # "xapian-core-1.4.23.tar.xz", + # "30d3518172084f310dab86d262b512718a7f9a13635aaa1a188e61dc26b2288c" + # ) + # meson_patch = Remotefile( + # "xapian-core-1.4.23-1_patch.zip", + # "", + # "https://public.kymeria.fr/KIWIX/xapian-core-1.4.23-1_patch.zip" + # ) + # archives = [src_archive, meson_patch] + + # class Builder(MakeBuilder): + # configure_options = [ + # "--disable-sse", + # "--disable-backend-chert", + # "--disable-backend-remote", + # "--disable-documentation"] + # configure_env = {'_format_LDFLAGS': "{env.LDFLAGS} -L{buildEnv.install_dir}/{buildEnv.libprefix}", + # '_format_CXXFLAGS': "{env.CXXFLAGS} -I{buildEnv.install_dir}/include"} class Builder(MesonBuilder): configure_options = [ @@ -38,6 +38,7 @@ class Xapian(Dependency): "-Denable-backend-chert=false", "-Denable-backend-remote=false" ] + subsource_dir = "xapian-core" @classmethod def get_dependencies(cls, configInfo, allDeps): From b026e7e06cf6caf9df8231b0a5f513eaf09be1b2 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 12 Aug 2024 14:55:19 +0200 Subject: [PATCH 05/10] Bump base_deps_meta_version to rebuild xapian-core --- kiwixbuild/versions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kiwixbuild/versions.py b/kiwixbuild/versions.py index 0fdc9bd..d78bb2a 100644 --- a/kiwixbuild/versions.py +++ b/kiwixbuild/versions.py @@ -39,7 +39,7 @@ release_versions = { # This is the "version" of the whole base_deps_versions dict. # Change this when you change base_deps_versions. -base_deps_meta_version = "01" +base_deps_meta_version = "02" base_deps_versions = { "zlib": "1.2.12", @@ -47,7 +47,7 @@ base_deps_versions = { "zstd": "1.5.2", "docoptcpp": "0.6.2", "uuid": "1.43.4", - "xapian-core": "1.4.23", + "xapian-core": "1.4.26", "mustache": "4.1", "pugixml": "1.2", "libmicrohttpd": "0.9.76", From ef6f654e7bf4c6325c622c30f06b0b80120c1998 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 12 Aug 2024 15:55:09 +0200 Subject: [PATCH 06/10] Build native_mixed on windows Xapian build with meson is static only. This is mainly due to missing "ddl export" or ".def" file. Autotool buildsystem seems to handle that automatically but not meson. See https://github.com/mesonbuild/meson/issues/2132 about meson supporting this. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9303d68..75db2da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: runs-on: windows-latest env: OS_NAME: windows - COMPILE_CONFIG: native_dyn + COMPILE_CONFIG: native_mixed HOME: 'C:\\Users\\runneradmin' steps: - name: Checkout code From 16654f956329b9f8ef8e68264363ba85d3715723 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Mon, 12 Aug 2024 16:14:35 +0200 Subject: [PATCH 07/10] All native config are compatible with windows. --- kiwixbuild/configs/native.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kiwixbuild/configs/native.py b/kiwixbuild/configs/native.py index 20c55a5..3da055a 100644 --- a/kiwixbuild/configs/native.py +++ b/kiwixbuild/configs/native.py @@ -35,10 +35,10 @@ class NativeDyn(NativeConfigInfo): class NativeStatic(NativeConfigInfo): name = "native_static" static = True - compatible_hosts = ["fedora", "debian", "Darwin", "almalinux"] + compatible_hosts = ["fedora", "debian", "Darwin", "almalinux", "Windows"] class NativeMixed(MixedMixin("native_static"), NativeConfigInfo): name = "native_mixed" static = False - compatible_hosts = ["fedora", "debian", "Darwin", "almalinux"] + compatible_hosts = ["fedora", "debian", "Darwin", "almalinux", "Windows"] From 752bdd18acf0d7177dc1084effa01fc1147f6b26 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 14 Aug 2024 10:24:57 +0200 Subject: [PATCH 08/10] Build xapian with classic autotools on other platform than Windows. The meson build fails on cross compiliation `win32_*` and on `native_dyn` on Macos (but doesn't fail on `native_static` or `native_mixed`...) On top of that, the fact that xapian-meson build system is fluky on Windows doesn't make me confident. As I will be out for few months, I prefer keep the autotool build system on already working configs. --- kiwixbuild/dependencies/xapian.py | 85 ++++++++++++++++--------------- 1 file changed, 45 insertions(+), 40 deletions(-) diff --git a/kiwixbuild/dependencies/xapian.py b/kiwixbuild/dependencies/xapian.py index 0ba92f0..e49289f 100644 --- a/kiwixbuild/dependencies/xapian.py +++ b/kiwixbuild/dependencies/xapian.py @@ -3,51 +3,56 @@ from .base import Dependency, GitClone, ReleaseDownload, MakeBuilder, MesonBuild from kiwixbuild.utils import Remotefile from kiwixbuild._global import neutralEnv +import platform + class Xapian(Dependency): name = "xapian-core" - class Source(GitClone): - git_remote = "https://github.com/openzim/xapian-meson.git" - git_dir = "xapian-core" + if platform.system() == "Windows": - # class Source(ReleaseDownload): - # src_archive = Remotefile( - # "xapian-core-1.4.23.tar.xz", - # "30d3518172084f310dab86d262b512718a7f9a13635aaa1a188e61dc26b2288c" - # ) - # meson_patch = Remotefile( - # "xapian-core-1.4.23-1_patch.zip", - # "", - # "https://public.kymeria.fr/KIWIX/xapian-core-1.4.23-1_patch.zip" - # ) - # archives = [src_archive, meson_patch] + class Source(GitClone): + git_remote = "https://github.com/openzim/xapian-meson.git" + git_dir = "xapian-core" - # class Builder(MakeBuilder): - # configure_options = [ - # "--disable-sse", - # "--disable-backend-chert", - # "--disable-backend-remote", - # "--disable-documentation"] - # configure_env = {'_format_LDFLAGS': "{env.LDFLAGS} -L{buildEnv.install_dir}/{buildEnv.libprefix}", - # '_format_CXXFLAGS': "{env.CXXFLAGS} -I{buildEnv.install_dir}/include"} + class Builder(MesonBuilder): + configure_options = [ + "-Denable-sse=false", + "-Denable-backend-chert=false", + "-Denable-backend-remote=false", + ] + subsource_dir = "xapian-core" - class Builder(MesonBuilder): - configure_options = [ - "-Denable-sse=false", - "-Denable-backend-chert=false", - "-Denable-backend-remote=false" - ] - subsource_dir = "xapian-core" - - @classmethod - def get_dependencies(cls, configInfo, allDeps): - if neutralEnv("distname") == "Windows": + @classmethod + def get_dependencies(cls, configInfo, allDeps): return ["zlib"] - deps = ["zlib", "lzma"] - if ( - configInfo.build in ("win32", "wasm") - or neutralEnv("distname") == "Darwin" - ): - return deps - return deps + ["uuid"] + + else: + + class Source(ReleaseDownload): + archive = Remotefile( + "xapian-core-1.4.23.tar.xz", + "30d3518172084f310dab86d262b512718a7f9a13635aaa1a188e61dc26b2288c", + ) + + class Builder(MakeBuilder): + configure_options = [ + "--disable-sse", + "--disable-backend-chert", + "--disable-backend-remote", + "--disable-documentation", + ] + configure_env = { + "_format_LDFLAGS": "{env.LDFLAGS} -L{buildEnv.install_dir}/{buildEnv.libprefix}", + "_format_CXXFLAGS": "{env.CXXFLAGS} -I{buildEnv.install_dir}/include", + } + + @classmethod + def get_dependencies(cls, configInfo, allDeps): + deps = ["zlib", "lzma"] + if ( + configInfo.build in ("win32", "win64", "wasm") + or neutralEnv("distname") == "Darwin" + ): + return deps + return deps + ["uuid"] From 28d02bfd7ad13e567231cc886fa2145ac9d94397 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 14 Aug 2024 12:05:45 +0200 Subject: [PATCH 09/10] Deactivate xapian fuller compact on libzim. This flag used on xapian-meson build on Windows ends in a crash in libzim when compacting the database. To be fixed at a moment but let's not use this flag for now on Windows --- kiwixbuild/dependencies/libzim.py | 1 + 1 file changed, 1 insertion(+) diff --git a/kiwixbuild/dependencies/libzim.py b/kiwixbuild/dependencies/libzim.py index a85215c..cb5adcf 100644 --- a/kiwixbuild/dependencies/libzim.py +++ b/kiwixbuild/dependencies/libzim.py @@ -33,6 +33,7 @@ class Libzim(Dependency): def configure_options(self): configInfo = self.buildEnv.configInfo if neutralEnv("distname") == "Windows": + yield "-Dwith_xapian_fuller=false" yield "-Dwerror=false" if configInfo.build == "android": yield "-DUSE_BUFFER_HEADER=false" From be232d790e4f7899362cc939db01014f7b2052db Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 14 Aug 2024 14:27:06 +0200 Subject: [PATCH 10/10] Add xapian-core to base dependencies on Windows. --- kiwixbuild/dependencies/all_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kiwixbuild/dependencies/all_dependencies.py b/kiwixbuild/dependencies/all_dependencies.py index 1894f57..25dba4f 100644 --- a/kiwixbuild/dependencies/all_dependencies.py +++ b/kiwixbuild/dependencies/all_dependencies.py @@ -14,7 +14,7 @@ class AllBaseDependencies(Dependency): @classmethod def get_dependencies(cls, configInfo, allDeps): if neutralEnv("distname") == "Windows": - return ["zlib", "zstd", "icu4c", "zim-testing-suite"] + return ["zlib", "zstd", "icu4c", "zim-testing-suite", "xapian-core"] if configInfo.build == "wasm" or environ.get("OS_NAME") == "manylinux": return ["zlib", "lzma", "zstd", "icu4c", "xapian-core"]