Merge pull request #167 from kiwix/zimwriterfs_meson

Zimwriterfs meson

kiwixbuild/dependencies.py

@@ -14,7 +14,7 @@ from kiwixbuild.dependency_utils import (
     NoopBuilder,
     Builder as BaseBuilder)
 
-from kiwixbuild.utils import Remotefile, pj, SkipCommand, copy_tree, add_execution_right
+from kiwixbuild.utils import Remotefile, pj, SkipCommand, copy_tree, add_execution_right, Defaultdict
 
 # *************************************
 # Missing dependencies
@@ -323,27 +323,28 @@ class ZimTools(Dependency):
 
 class Zimwriterfs(Dependency):
     name = "zimwriterfs"
-    extra_packages = ['file']
 
     @property
     def dependencies(self):
-        base_dependencies = ['libzim', 'zlib', 'lzma', 'xapian-core', 'gumbo']
+        base_dependencies = ['libzim', 'zlib', 'xapian-core', 'gumbo']
         if self.buildEnv.platform_info.build != 'native':
-            return base_dependencies + ["icu4c_cross-compile"]
+            return base_dependencies + ["icu4c_cross-compile", "libmagic_cross-compile"]
         else:
-            return base_dependencies + ["icu4c"]
+            return base_dependencies + ["icu4c", "libmagic"]
 
     class Source(GitClone):
         git_remote = "https://github.com/openzim/zimwriterfs.git"
         git_dir = "zimwriterfs"
         release_git_ref = "1.1"
 
-        def _post_prepare_script(self, context):
-            context.try_skip(self.git_path)
-            command = "./autogen.sh"
-            self.buildEnv.run_command(command, self.git_path, context)
+    class Builder(MesonBuilder):
+        @property
+        def configure_option(self):
+            base_option = "-Dmagic-install-prefix={buildEnv.install_dir}"
+            if self.buildEnv.platform_info.static:
+                base_option += " -Dstatic-linkage=true"
+            return base_option
 
-    Builder = MakeBuilder
 
 
 class Kiwixlib(Dependency):
@@ -417,6 +418,51 @@ class Gradle(Dependency):
                 pj(self.buildEnv.install_dir, "lib"))
 
 
+class LibMagicBase(Dependency):
+    name = "libmagic"
+
+    class Source(ReleaseDownload):
+        name = "libmagic"
+        source_dir = "libmagic"
+        archive = Remotefile('file-5.33.tar.gz',
+                             '1c52c8c3d271cd898d5511c36a68059cda94036111ab293f01f83c3525b737c6',
+                             'https://fossies.org/linux/misc/file-5.33.tar.gz')
+
+    Builder = MakeBuilder
+
+
+class LibMagic_native(LibMagicBase):
+    name = "libmagic_native"
+    force_native_build = True
+
+    class Builder(LibMagicBase.Builder):
+        static_configure_option = dynamic_configure_option = "--disable-shared --enable-static"
+
+        @property
+        def build_path(self):
+            return super().build_path+"_native"
+
+        def _install(self, context):
+            raise SkipCommand()
+
+
+class LibMagic_cross_compile(LibMagicBase):
+    name = "libmagic_cross-compile"
+    dependencies = ['libmagic_native']
+
+    class Builder(LibMagicBase.Builder):
+        def _compile(self, context):
+            context.try_skip(self.build_path)
+            command = "make -j4 {make_target} {make_option}".format(
+                make_target=self.make_target,
+                make_option=self.make_option
+            )
+            libmagic_native_dep = self.buildEnv.targetsDict['libmagic_native']
+            env = Defaultdict(str, os.environ)
+            env['PATH'] = ':'.join([pj(libmagic_native_dep.builder.build_path, 'src'), env['PATH']])
+            self.buildEnv.run_command(command, self.build_path, context, env=env)
+
+
 class AllBaseDependencies(Dependency):
     name = "alldependencies"
 
@@ -425,8 +471,10 @@ class AllBaseDependencies(Dependency):
         base_deps = ['zlib', 'lzma', 'xapian-core', 'gumbo', 'pugixml', 'libmicrohttpd', 'libaria2']
         if self.buildEnv.platform_info.build != 'native':
             base_deps += ["icu4c_cross-compile"]
+            if self.buildEnv.platform_info.build != 'win32':
+                base_deps += ["libmagic_cross-compile"]
         else:
-            base_deps += ["icu4c"]
+            base_deps += ["icu4c", "libmagic"]
         if ( self.buildEnv.platform_info.build != 'android'
            and self.buildEnv.distname != 'Darwin'):
             base_deps += ['ctpp2c', 'ctpp2']

kiwixbuild/dependency_versions.py

@@ -10,7 +10,7 @@ main_project_versions = {
 
 # This is the "version" of the whole base_deps_versions dict.
 # Change this when you change base_deps_versions.
-base_deps_meta_version = '2'
+base_deps_meta_version = '3'
 
 
 base_deps_versions = {
@@ -24,5 +24,6 @@ base_deps_versions = {
   'gumbo' : '0.10.1',
   'icu4c' : '58.2',
   'Gradle' : '4.6',
-  'libaria2' : '1.33.1'
+  'libaria2' : '1.33.1',
+  'libmagic' : '5.33'
 }

kiwixbuild/utils.py

@@ -103,7 +103,7 @@ def download_remote(what, where, check_certificate=True):
     progress_chars = "/-\|"
     try:
         with urllib.request.urlopen(file_url, **extra_args) as resource, open(file_path, 'wb') as file:
-            tsize = resource.getheader('Content-Length', None)
+            tsize = resource.info().get('Content-Length', None)
             if tsize is not None:
                 tsize = int(tsize)
             current = 0

travis/compile_all.py

@@ -227,7 +227,7 @@ elif PLATFORM.startswith('native_'):
     else:
         TARGETS = ('libzim', 'zimwriterfs', 'zim-tools', 'kiwix-lib', 'kiwix-tools')
 else:
-    TARGETS = ('libzim', 'kiwix-lib', 'kiwix-tools')
+    TARGETS = ('libzim', 'zim-tools', 'kiwix-lib', 'kiwix-tools')
 
 for target in TARGETS:
     if environ['TRAVIS_EVENT_TYPE'] == 'cron':