From a417b044f86faa37f875fdeacd04b70a9ff07c6c Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 9 May 2018 10:36:28 +0200 Subject: [PATCH 1/4] Add libmagic dependency. --- kiwixbuild/dependencies.py | 56 ++++++++++++++++++++++++++++--- kiwixbuild/dependency_versions.py | 5 +-- kiwixbuild/utils.py | 2 +- 3 files changed, 55 insertions(+), 8 deletions(-) diff --git a/kiwixbuild/dependencies.py b/kiwixbuild/dependencies.py index d23ca51..4c6a27d 100644 --- a/kiwixbuild/dependencies.py +++ b/kiwixbuild/dependencies.py @@ -14,7 +14,7 @@ from kiwixbuild.dependency_utils import ( NoopBuilder, Builder as BaseBuilder) -from kiwixbuild.utils import Remotefile, pj, SkipCommand, copy_tree, add_execution_right +from kiwixbuild.utils import Remotefile, pj, SkipCommand, copy_tree, add_execution_right, Defaultdict # ************************************* # Missing dependencies @@ -323,15 +323,14 @@ class ZimTools(Dependency): class Zimwriterfs(Dependency): name = "zimwriterfs" - extra_packages = ['file'] @property def dependencies(self): base_dependencies = ['libzim', 'zlib', 'lzma', 'xapian-core', 'gumbo'] if self.buildEnv.platform_info.build != 'native': - return base_dependencies + ["icu4c_cross-compile"] + return base_dependencies + ["icu4c_cross-compile", "libmagic_cross-compile"] else: - return base_dependencies + ["icu4c"] + return base_dependencies + ["icu4c", "libmagic"] class Source(GitClone): git_remote = "https://github.com/openzim/zimwriterfs.git" @@ -417,6 +416,51 @@ class Gradle(Dependency): pj(self.buildEnv.install_dir, "lib")) +class LibMagicBase(Dependency): + name = "libmagic" + + class Source(ReleaseDownload): + name = "libmagic" + source_dir = "libmagic" + archive = Remotefile('file-5.33.tar.gz', + '1c52c8c3d271cd898d5511c36a68059cda94036111ab293f01f83c3525b737c6', + 'ftp://ftp.astron.com/pub/file/file-5.33.tar.gz') + + Builder = MakeBuilder + + +class LibMagic_native(LibMagicBase): + name = "libmagic_native" + force_native_build = True + + class Builder(LibMagicBase.Builder): + static_configure_option = dynamic_configure_option = "--disable-shared --enable-static" + + @property + def build_path(self): + return super().build_path+"_native" + + def _install(self, context): + raise SkipCommand() + + +class LibMagic_cross_compile(LibMagicBase): + name = "libmagic_cross-compile" + dependencies = ['libmagic_native'] + + class Builder(LibMagicBase.Builder): + def _compile(self, context): + context.try_skip(self.build_path) + command = "make -j4 {make_target} {make_option}".format( + make_target=self.make_target, + make_option=self.make_option + ) + libmagic_native_dep = self.buildEnv.targetsDict['libmagic_native'] + env = Defaultdict(str, os.environ) + env['PATH'] = ':'.join([pj(libmagic_native_dep.builder.build_path, 'src'), env['PATH']]) + self.buildEnv.run_command(command, self.build_path, context, env=env) + + class AllBaseDependencies(Dependency): name = "alldependencies" @@ -425,8 +469,10 @@ class AllBaseDependencies(Dependency): base_deps = ['zlib', 'lzma', 'xapian-core', 'gumbo', 'pugixml', 'libmicrohttpd', 'libaria2'] if self.buildEnv.platform_info.build != 'native': base_deps += ["icu4c_cross-compile"] + if self.buildEnv.platform_info.build != 'win32': + base_deps += ["libmagic_cross-compile"] else: - base_deps += ["icu4c"] + base_deps += ["icu4c", "libmagic"] if ( self.buildEnv.platform_info.build != 'android' and self.buildEnv.distname != 'Darwin'): base_deps += ['ctpp2c', 'ctpp2'] diff --git a/kiwixbuild/dependency_versions.py b/kiwixbuild/dependency_versions.py index 1c18381..e2253ec 100644 --- a/kiwixbuild/dependency_versions.py +++ b/kiwixbuild/dependency_versions.py @@ -10,7 +10,7 @@ main_project_versions = { # This is the "version" of the whole base_deps_versions dict. # Change this when you change base_deps_versions. -base_deps_meta_version = '2' +base_deps_meta_version = '3' base_deps_versions = { @@ -24,5 +24,6 @@ base_deps_versions = { 'gumbo' : '0.10.1', 'icu4c' : '58.2', 'Gradle' : '4.6', - 'libaria2' : '1.33.1' + 'libaria2' : '1.33.1', + 'libmagic' : '5.33' } diff --git a/kiwixbuild/utils.py b/kiwixbuild/utils.py index 2803de6..a194da7 100644 --- a/kiwixbuild/utils.py +++ b/kiwixbuild/utils.py @@ -103,7 +103,7 @@ def download_remote(what, where, check_certificate=True): progress_chars = "/-\|" try: with urllib.request.urlopen(file_url, **extra_args) as resource, open(file_path, 'wb') as file: - tsize = resource.getheader('Content-Length', None) + tsize = resource.info().get('Content-Length', None) if tsize is not None: tsize = int(tsize) current = 0 From dac4fcd1f879b350ac3ab7bccee2c8d520813231 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 9 May 2018 10:38:24 +0200 Subject: [PATCH 2/4] Also try to cross-compile zim-tools in travis cron. --- travis/compile_all.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis/compile_all.py b/travis/compile_all.py index dbc24bb..60d0ef9 100755 --- a/travis/compile_all.py +++ b/travis/compile_all.py @@ -227,7 +227,7 @@ elif PLATFORM.startswith('native_'): else: TARGETS = ('libzim', 'zimwriterfs', 'zim-tools', 'kiwix-lib', 'kiwix-tools') else: - TARGETS = ('libzim', 'kiwix-lib', 'kiwix-tools') + TARGETS = ('libzim', 'zim-tools', 'kiwix-lib', 'kiwix-tools') for target in TARGETS: if environ['TRAVIS_EVENT_TYPE'] == 'cron': From ee5a8eda96dc1c615dde9fb249bea2a055cb606c Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 9 May 2018 10:38:49 +0200 Subject: [PATCH 3/4] Build zimwriterfs using meson. --- kiwixbuild/dependencies.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/kiwixbuild/dependencies.py b/kiwixbuild/dependencies.py index 4c6a27d..b1a1f05 100644 --- a/kiwixbuild/dependencies.py +++ b/kiwixbuild/dependencies.py @@ -326,7 +326,7 @@ class Zimwriterfs(Dependency): @property def dependencies(self): - base_dependencies = ['libzim', 'zlib', 'lzma', 'xapian-core', 'gumbo'] + base_dependencies = ['libzim', 'zlib', 'xapian-core', 'gumbo'] if self.buildEnv.platform_info.build != 'native': return base_dependencies + ["icu4c_cross-compile", "libmagic_cross-compile"] else: @@ -337,12 +337,14 @@ class Zimwriterfs(Dependency): git_dir = "zimwriterfs" release_git_ref = "1.1" - def _post_prepare_script(self, context): - context.try_skip(self.git_path) - command = "./autogen.sh" - self.buildEnv.run_command(command, self.git_path, context) + class Builder(MesonBuilder): + @property + def configure_option(self): + base_option = "-Dmagic-install-prefix={buildEnv.install_dir}" + if self.buildEnv.platform_info.static: + base_option += " -Dstatic-linkage=true" + return base_option - Builder = MakeBuilder class Kiwixlib(Dependency): From d001372a8b4f549dc2af4f94c5c5eda70eadbc0e Mon Sep 17 00:00:00 2001 From: Emmanuel Engelhart Date: Wed, 9 May 2018 16:10:41 +0200 Subject: [PATCH 4/4] Use alternative (HTTPS) file source tarball URL --- kiwixbuild/dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kiwixbuild/dependencies.py b/kiwixbuild/dependencies.py index b1a1f05..21c594e 100644 --- a/kiwixbuild/dependencies.py +++ b/kiwixbuild/dependencies.py @@ -426,7 +426,7 @@ class LibMagicBase(Dependency): source_dir = "libmagic" archive = Remotefile('file-5.33.tar.gz', '1c52c8c3d271cd898d5511c36a68059cda94036111ab293f01f83c3525b737c6', - 'ftp://ftp.astron.com/pub/file/file-5.33.tar.gz') + 'https://fossies.org/linux/misc/file-5.33.tar.gz') Builder = MakeBuilder