Merge pull request #262 from kiwix/build_libcurl

Force the build of libcurl instead of using system one.

.travis.yml

@@ -92,7 +92,6 @@ matrix:
             - zlib1g-dev
             - uuid-dev
             - libctpp2-dev
-            - libcurl4-openssl-dev
     - env: PLATFORM="win32_dyn"
       addons: &win32_addons
         apt:
@@ -115,7 +114,6 @@ matrix:
             - g++-multilib
             - libc6-dev:i386
             - libstdc++-6-dev:i386
-            - libcurl4-openssl-dev:i386
     - env: PLATFORM="i586_static"
       addons: *i586_addons
     - env: PLATFORM="android"

appveyor.yml

@@ -35,22 +35,29 @@ install:
      REM Install openssh (scp) for deployement
      choco install openssh openssl.light
 
+  - cmd: openssl enc -d -aes-256-cbc -md sha256 -pass pass:%ENC_PASSWD% -in appveyor\nightlybot_id_key.enc -out appveyor\nightlybot_id_key
+
+
+
 build_script:
-  - appveyor\install_zlib.cmd
+  - IF "%USE_CACHE%" EQU "1" appveyor\setup_from_cache.cmd
-  - appveyor\install_lzma.cmd
+  - IF "%USE_CACHE%" NEQ "1" appveyor\install_zlib.cmd
-  - appveyor\install_pthread.cmd
+  - IF "%USE_CACHE%" NEQ "1" appveyor\install_lzma.cmd
-  - appveyor\install_dirent.cmd
+  - IF "%USE_CACHE%" NEQ "1" appveyor\install_libcurl.cmd
-  - appveyor\install_xapian.cmd
+  - IF "%USE_CACHE%" NEQ "1" appveyor\install_pthread.cmd
-  - appveyor\install_icu.cmd
+  - IF "%USE_CACHE%" NEQ "1" appveyor\install_dirent.cmd
+  - IF "%USE_CACHE%" NEQ "1" appveyor\install_xapian.cmd
+  - IF "%USE_CACHE%" NEQ "1" appveyor\install_icu.cmd
+  - IF "%USE_CACHE%" NEQ "1" appveyor\install_pugixml.cmd
+  - IF "%USE_CACHE%" NEQ "1" appveyor\install_aria2.cmd
+  - IF "%USE_CACHE%" NEQ "1" %MINGW64_RUN% "cd /c/projects/kiwix-build && appveyor/create_cache.sh"
   - appveyor\install_libzim.cmd
-  - appveyor\install_pugixml.cmd
   - appveyor\install_kiwix-lib.cmd
   - appveyor\install_kiwix-desktop.cmd
 
 before_deploy:
   - cmd: openssl enc -d -aes-256-cbc -md sha256 -pass pass:%win_certificate_secret% -in appveyor\kiwix.p12.enc -out appveyor\kiwix.pfx
   - cmd: '%MINGW64_RUN% "cd /c/projects/kiwix-build && appveyor/package_kiwix-desktop.sh"'
-  - cmd: openssl enc -d -aes-256-cbc -md sha256 -pass pass:%ENC_PASSWD% -in appveyor\nightlybot_id_key.enc -out appveyor\nightlybot_id_key
 
 deploy_script:
   - cmd: '%MINGW64_RUN% "cd /c/projects/kiwix-build && appveyor/deploy.sh"'

appveyor/create_cache.sh

@@ -0,0 +1,11 @@
+
+set -e
+
+ARCHIVE_NAME="deps_windows_windows.zip"
+SSH_KEY=C:\\projects\\kiwix-build\\appveyor\\nightlybot_id_key
+
+7z a -tzip $ARCHIVE_NAME $MINGW64_EXTRA_DIR
+scp -vrp -i ${SSH_KEY} -o StrictHostKeyChecking=no \
+  ${ARCHIVE_NAME} \
+  nightlybot@download.kiwix.org:/var/www/tmp.kiwix.org/ci
+

appveyor/install_aria2.cmd

@@ -0,0 +1,4 @@
+REM ========================================================
+REM Install aria2
+curl -fsSL -O https://github.com/aria2/aria2/releases/download/release-1.34.0/aria2-1.34.0-win-64bit-build1.zip || exit /b 1
+7z e aria2-1.34.0-win-64bit-build1.zip -o%EXTRA_DIR% aria2-1.34.0-win-64bit-build1\aria2c.exe || exit /b 1

appveyor/install_kiwix-desktop.cmd

@@ -5,7 +5,7 @@ cd kiwix-desktop
 echo "Getting fix_desktop"
 curl -fsSL -O http://public.kymeria.fr/KIWIX/windows/fix_desktop_makefile.py_ || exit /b 1
 echo "Running qmake"
-C:\Qt\5.11\msvc2017_64\bin\qmake.exe || exit /b 1
+C:\Qt\5.11\msvc2017_64\bin\qmake.exe "CONFIG+=static" || exit /b 1
 echo "Running fix_desktop"
 C:\Python36\Python fix_desktop_makefile.py_ Makefile.Release || exit /b 1
 echo "Running nmake"

appveyor/install_libcurl.cmd

@@ -0,0 +1,14 @@
+REM ========================================================
+REM Install libcurl
+curl -fsSL -O https://curl.haxx.se/download/curl-7.61.1.zip || exit /b 1
+7z x curl-7.61.1.zip || exit /b 1
+cd curl-7.61.1\winbuild
+nmake /f Makefile.vc mode=static MACHINE=x64 DEBUG=no VC=15 ENABLE_IDN=no || exit /b 1
+mkdir %EXTRA_DIR%\include\curl
+copy ..\builds\libcurl-vc15-x64-release-static-ipv6-sspi-winssl\include\curl\*.h %EXTRA_DIR%\include\curl
+copy ..\builds\libcurl-vc15-x64-release-static-ipv6-sspi-winssl\lib\libcurl_a.lib %EXTRA_DIR%\lib
+move %EXTRA_DIR%\lib\libcurl_a.lib %EXTRA_DIR%\lib\libcurl.a
+dir %EXTRA_DIR%\include\curl
+dir %EXTRA_DIR%\lib
+curl -fsSl -o%PKG_CONFIG_PATH%\libcurl.pc http://public.kymeria.fr/KIWIX/windows/libcurl.pc || exit /b 1
+cd ..\..

appveyor/package_kiwix-desktop.sh

@@ -12,6 +12,7 @@ mkdir -p NIGHTLY_KIWIX_ARCHIVES_DIR
 cp /c/projects/kiwix-build/kiwix-desktop/Release/kiwix-desktop.exe $KIWIX_DIR
 /c/Qt/5.11/msvc2017_64/bin/windeployqt --compiler-runtime $KIWIX_DIR
 
+cp $MINGW64_EXTRA_DIR/aria2c.exe $KIWIX_DIR
 cp $MINGW64_EXTRA_DIR/bin/*.dll $KIWIX_DIR
 
 /c/Program\ Files\ \(x86\)/Windows\ Kits/10/bin/x64/signtool.exe sign -f appveyor/kiwix.pfx -p $win_certificate_password -t http://timestamp.verisign.com/scripts/timestamp.dll -d "Kiwix-desktop application" $KIWIX_DIR/kiwix-desktop.exe

appveyor/setup_from_cache.cmd

@@ -0,0 +1,4 @@
+REM ========================================================
+REM Setup from cache
+curl -fsSL -O http://tmp.kiwix.org/ci/deps_windows_windows.zip || exit /b 1
+7z x deps_windows_windows.zip -oc: -aoa || exit /b 1

kiwixbuild/dependencies/libcurl.py

@@ -18,4 +18,12 @@ class LibCurl(Dependency):
                              'ef6e55192d04713673b4409ccbcb4cb6cd723137d6e10ca45b0c593a454e1720',
                              'https://curl.haxx.se/download/curl-7.61.0.tar.xz')
 
-    Builder = MakeBuilder
+    class Builder(MakeBuilder):
+        dependencies = ['zlib']
+        configure_option = " ".join(
+            ["--without-{}".format(p)
+                for p in ('libssh2', 'ssl', 'libmetalink', 'librtmp')] +
+            ["--disable-{}".format(p)
+                for p in ('ftp', 'file', 'ldap', 'ldaps', 'rtsp', 'dict',
+                          'telnet', 'tftp', 'pop3', 'imap', 'smb', 'smtp',
+                          'gopher', 'manual')])

kiwixbuild/packages.py

@@ -16,7 +16,6 @@ PACKAGE_NAME_MAPPERS = {
         'zimlib': None,
         'file' : ['file-devel'],
         'gumbo' : ['gumbo-parser-devel'],
-        'libcurl' : ['libcurl-devel'],
     },
     'fedora_native_static': {
         'COMMON': _fedora_common + ['glibc-static', 'libstdc++-static'],
@@ -26,7 +25,6 @@ PACKAGE_NAME_MAPPERS = {
     },
     'fedora_i586_dyn': {
         'COMMON': _fedora_common + ['glibc-devel.i686', 'libstdc++-devel.i686'],
-        'libcurl':  ['libcurl-devel.i686'],
     },
     'fedora_i586_static': {
         'COMMON': _fedora_common + ['glibc-devel.i686'],
@@ -36,7 +34,6 @@ PACKAGE_NAME_MAPPERS = {
         'zlib': ['mingw32-zlib'],
         'lzma': ['mingw32-xz-libs'],
         'libmicrohttpd': ['mingw32-libmicrohttpd'],
-        'libcurl': ['mingw32-curl'],
     },
     'fedora_win32_static': {
         'COMMON': _fedora_common + ['mingw32-gcc-c++', 'mingw32-bzip2-static', 'mingw32-win-iconv-static', 'mingw32-winpthreads-static', 'wine'],
@@ -44,7 +41,6 @@ PACKAGE_NAME_MAPPERS = {
         'lzma': ['mingw32-xz-libs-static'],
         'libmicrohttpd': None, # ['mingw32-libmicrohttpd-static'] packaging dependecy seems buggy, and some static lib are name libfoo.dll.a and
                                # gcc cannot found them.
-        'libcurl': ['mingw32-curl-static'],
     },
     'fedora_armhf_static': {
         'COMMON': _fedora_common
@@ -64,7 +60,6 @@ PACKAGE_NAME_MAPPERS = {
         'libmicrohttpd': ['libmicrohttpd-dev', 'ccache'],
         'qt' : ['libqt5gui5', 'qtbase5-dev', 'qt5-default'],
         'qtwebengine' : ['qtwebengine5-dev'],
-        'libcurl': ['libcurl4-openssl-dev'],
     },
     'debian_native_static': {
         'COMMON': _debian_common + ['libbz2-dev', 'libmagic-dev'],
@@ -72,15 +67,12 @@ PACKAGE_NAME_MAPPERS = {
         'uuid': ['uuid-dev'],
         'ctpp2': ['libctpp2-dev'],
         'ctpp2c': ['ctpp2-utils'],
-        'libcurl': ['libcurl4-openssl-dev'],
     },
     'debian_i586_dyn': {
         'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib'],
-        'libcurl': ['libcurl4-openssl-dev:i386'],
     },
     'debian_i586_static': {
         'COMMON': _debian_common + ['libc6-dev:i386', 'libstdc++-6-dev:i386', 'gcc-multilib', 'g++-multilib'],
-        'libcurl': ['libcurl4-openssl-dev:i386'],
     },
     'debian_win32_dyn': {
         'COMMON': _debian_common + ['g++-mingw-w64-i686', 'gcc-mingw-w64-i686', 'gcc-mingw-w64-base', 'mingw-w64-tools'],

travis/Dockerfile

@@ -33,7 +33,6 @@ RUN \
     ctpp2-utils \
     libctpp2-dev \
     libmicrohttpd-dev \
-    libcurl4-openssl-dev \
 # Qt packages
     libqt5gui5 \
     qtbase5-dev \