Merge pull request #423 from kiwix/zstd

Added zstd dependency (fixes #422)

appveyor.yml

@@ -3,7 +3,7 @@ version: '0.1.{build}'
 environment:
   EXTRA_DIR: 'C:\extra'
   MINGW64_EXTRA_DIR: '/c/extra'
-  PATH: '%PATH%;%EXTRA_DIR%\bin;C:\\Python36\\Scripts;C:\\Program Files (x86)\\Windows Kits\\10\\bin\\%platform%'
+  PATH: 'C:\\Python36\\Scripts;%PATH%;%EXTRA_DIR%\bin;C:\\Program Files (x86)\\Windows Kits\\10\\bin\\%platform%'
   PKG_CONFIG_PATH: '%EXTRA_DIR%\lib\pkgconfig'
   MSYS2_PATH_TYPE: 'inherit'
   MSYS2_ARG_CONV_EXCL: '-Tp'
@@ -49,6 +49,7 @@ build_script:
   - IF "%USE_CACHE%" EQU "1" appveyor\setup_from_cache.cmd
   - IF "%USE_CACHE%" NEQ "1" appveyor\install_zlib.cmd
   - IF "%USE_CACHE%" NEQ "1" appveyor\install_lzma.cmd
+  - IF "%USE_CACHE%" NEQ "1" appveyor\install_zstd.cmd
   - IF "%USE_CACHE%" NEQ "1" appveyor\install_libcurl.cmd
   - IF "%USE_CACHE%" NEQ "1" appveyor\install_pthread.cmd
   - IF "%USE_CACHE%" NEQ "1" appveyor\install_dirent.cmd

appveyor/apply_patch.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+mydir=$(dirname "$0")
+patch -p1 < "$mydir"/../kiwixbuild/patches/"$1"

appveyor/install_xapian.cmd

@@ -2,6 +2,7 @@ REM ========================================================
 REM Install xapian
 curl -fsSL -O http://download.kiwix.org/dev/xapian-core-1.4.14.zip || exit /b 1
 7z x xapian-core-1.4.14.zip || exit /b 1
+%MINGW64_RUN% "cd /c/Projects/kiwix-build/xapian-core-1.4.14 && ../appveyor/apply_patch.sh xapian_remote.patch" || exit /b 1
 cd xapian-core-1.4.14
 mkdir build
 cd build

appveyor/install_zstd.cmd

@@ -0,0 +1,12 @@
+REM ========================================================
+REM Install zstd
+curl -fsSL -o zstd-v1.4.4.zip https://github.com/facebook/zstd/archive/v1.4.4.zip || exit /b 1
+7z x zstd-v1.4.4.zip || exit /b 1
+REM Fixing https://github.com/facebook/zstd/issues/2073
+%MINGW64_RUN% "cd /c/projects/kiwix-build/zstd-1.4.4 && ../appveyor/apply_patch.sh zstd_meson.patch" || exit /b 1
+cd zstd-1.4.4/build/meson
+meson . builddir --prefix %EXTRA_DIR% --default-library static --buildtype release -Dbin_programs=false -Dbin_contrib=false || exit /b 1
+cd builddir
+ninja || exit /b 1
+ninja install || exit /b 1
+cd ..\..\..\..

kiwixbuild/dependencies/__init__.py

@@ -25,5 +25,6 @@ from . import (
     xapian,
     zim_tools,
     zimwriterfs,
-    zlib
+    zlib,
+    zstd
 )

kiwixbuild/dependencies/all_dependencies.py

@@ -12,7 +12,7 @@ class AllBaseDependencies(Dependency):
     class Builder(NoopBuilder):
         @classmethod
         def get_dependencies(cls, platformInfo, allDeps):
-            base_deps = ['zlib', 'lzma', 'xapian-core', 'pugixml', 'libcurl', 'icu4c', 'mustache', 'libmicrohttpd']
+            base_deps = ['zlib', 'lzma', 'zstd', 'xapian-core', 'pugixml', 'libcurl', 'icu4c', 'mustache', 'libmicrohttpd']
             # zimwriterfs
             if platformInfo.build not in ('android', 'win32'):
                 base_deps += ['libmagic', 'gumbo']

kiwixbuild/dependencies/libzim.py

@@ -14,7 +14,7 @@ class Libzim(Dependency):
 
     class Builder(MesonBuilder):
         test_option = "-t 8"
-        dependencies = ['zlib', 'lzma', 'xapian-core', 'icu4c']
+        dependencies = ['zlib', 'lzma', 'zstd', 'xapian-core', 'icu4c']
         strip_option = ''
 
         @property

kiwixbuild/dependencies/zstd.py

@@ -0,0 +1,23 @@
+from .base import (
+    Dependency,
+    ReleaseDownload,
+    MesonBuilder)
+
+from kiwixbuild.utils import Remotefile
+
+
+
+class zstd(Dependency):
+    name = 'zstd'
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('zstd-1.4.4.tar.gz',
+                             '59ef70ebb757ffe74a7b3fe9c305e2ba3350021a918d168a046c6300aeea9315',
+                             'https://github.com/facebook/zstd/releases/download/v1.4.4/zstd-1.4.4.tar.gz')
+        patches = ['zstd_meson.patch']
+
+    class Builder(MesonBuilder):
+        subsource_dir = 'build/meson'
+        build_type = 'release'
+        strip_option = ''
+        configure_option = '-Dbin_programs=false -Dbin_contrib=false'

kiwixbuild/patches/zstd_meson.patch

@@ -0,0 +1,20 @@
+diff -ur zstd-1.4.4/build/meson/meson.build zstd-1.4.4.patched/build/meson/meson.build
+--- zstd-1.4.4/build/meson/meson.build  2019-11-04 21:54:32.000000000 +0400
++++ zstd-1.4.4.patched/build/meson/meson.build  2020-04-07 13:59:26.012106549 +0400
+@@ -68,6 +68,7 @@
+ # Built-in options
+ use_debug = get_option('debug')
+ buildtype = get_option('buildtype')
++default_library_type = get_option('default_library')
+
+ # Custom options
+ debug_level = get_option('debug_level')
+@@ -121,7 +122,7 @@
+   if use_multi_thread
+     msvc_compile_flags += '/MP'
+   endif
+-  if enable_static_runtime
++  if use_static_runtime
+     msvc_compile_flags += '/MT'
+   endif
+   add_project_arguments(msvc_compile_flags, language: ['c', 'cpp'])

kiwixbuild/versions.py

@@ -42,11 +42,12 @@ release_versions = {
 
 # This is the "version" of the whole base_deps_versions dict.
 # Change this when you change base_deps_versions.
-base_deps_meta_version = '64'
+base_deps_meta_version = '65'
 
 base_deps_versions = {
   'zlib' : '1.2.8',
   'lzma' : '5.2.4',
+  'zstd' : '1.4.4',
   'uuid' : '1.43.4',
   'xapian-core' : '1.4.14',
   'mustache' : '3.2',