From 0a93cb0872a1c1a37c5f5ed29900046e8e2846db Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 29 Aug 2018 12:06:52 +0200 Subject: [PATCH] Add aria2 downloader using subprocess aria2c. --- meson.build | 5 +- src/aria2.cpp | 132 ++++++++++++++++++++++++++++++++++++++++++++++++ src/aria2.h | 43 ++++++++++++++++ src/meson.build | 1 + 4 files changed, 179 insertions(+), 2 deletions(-) create mode 100644 src/aria2.cpp create mode 100644 src/aria2.h diff --git a/meson.build b/meson.build index 22e2bed8b..096b68943 100644 --- a/meson.build +++ b/meson.build @@ -17,6 +17,7 @@ thread_dep = dependency('threads') libicu_dep = dependency('icu-i18n', static:static_deps) libzim_dep = dependency('libzim', version : '>=4.0.0', static:static_deps) pugixml_dep = dependency('pugixml', static:static_deps) +libcurl_dep = dependency('libcurl', static:static_deps) libaria2_dep = dependency('libaria2', static:static_deps, required:false) ctpp2_include_path = '' @@ -78,7 +79,7 @@ endif xapian_dep = dependency('xapian-core', required:false, static:static_deps) -all_deps = [thread_dep, libicu_dep, libzim_dep, xapian_dep, pugixml_dep, libaria2_dep] +all_deps = [thread_dep, libicu_dep, libzim_dep, xapian_dep, pugixml_dep, libcurl_dep, libaria2_dep] if has_ctpp2_dep all_deps += [ctpp2_dep] endif @@ -102,7 +103,7 @@ subdir('static') subdir('src') subdir('test') -pkg_requires = ['libzim', 'icu-i18n', 'pugixml'] +pkg_requires = ['libzim', 'icu-i18n', 'pugixml', 'libcurl'] if libaria2_dep.found() pkg_requires += ['libaria2'] endif diff --git a/src/aria2.cpp b/src/aria2.cpp new file mode 100644 index 000000000..0b403e091 --- /dev/null +++ b/src/aria2.cpp @@ -0,0 +1,132 @@ + + +#include "aria2.h" +#include "xmlrpc.h" +#include +#include +#include +#include +#include + +namespace kiwix { + +Aria2::Aria2(): + mp_aria(nullptr), + m_port(42042), + m_secret("kiwixariarpc"), + mp_curl(nullptr), + m_lock(PTHREAD_MUTEX_INITIALIZER) +{ + m_downloadDir = getDataDirectory(); + std::vector callCmd; + + std::string rpc_port = "--rpc-listen-port=" + std::to_string(m_port); + std::string download_dir = "--dir=" + getDataDirectory(); +// std::string log_dir = "--log=\"" + logDir + "\""; +#ifdef _WIN32 + int pid = GetCurrentProcessId(); +#else + pid_t pid = getpid(); +#endif + std::string stop_with_pid = "--stop-with-process=" + std::to_string(pid); + std::string rpc_secret = "--rpc-secret=" + m_secret; + m_secret = "token:"+m_secret; + + callCmd.push_back("aria2c"); + callCmd.push_back("--enable-rpc"); + callCmd.push_back(rpc_port.c_str()); + callCmd.push_back(download_dir.c_str()); +// callCmd.push_back(log_dir.c_str()); + callCmd.push_back(stop_with_pid.c_str()); + callCmd.push_back("--allow-overwrite=true"); + callCmd.push_back("--dht-entry-point=router.bittorrent.com:6881"); + callCmd.push_back("--dht-entry-point6=router.bittorrent.com:6881"); + callCmd.push_back("--quiet=true"); + callCmd.push_back("--bt-enable-lpd=true"); + callCmd.push_back("--always-resume=true"); + callCmd.push_back("--max-concurrent-downloads=42"); + callCmd.push_back("--rpc-max-request-size=6M"); + callCmd.push_back("--file-allocation=none"); + mp_aria = Subprocess::run(callCmd); + mp_curl = curl_easy_init(); + curl_easy_setopt(mp_curl, CURLOPT_URL, "http://localhost/rpc"); + curl_easy_setopt(mp_curl, CURLOPT_PORT, m_port); + curl_easy_setopt(mp_curl, CURLOPT_POST, 1L); + + while(true) { + std::this_thread::sleep_for(std::chrono::microseconds(100)); + auto res = curl_easy_perform(mp_curl); + if (res == CURLE_OK) { + break; + } + } +} + +Aria2::~Aria2() +{ + curl_easy_cleanup(mp_curl); +}; + +size_t write_callback_to_iss(char* ptr, size_t size, size_t nmemb, void* userdata) +{ + auto str = static_cast(userdata); + str->write(ptr, nmemb); + return nmemb; +} + +std::string Aria2::doRequest(const MethodCall& methodCall) +{ + pthread_mutex_lock(&m_lock); + auto requestContent = methodCall.toString(); + std::stringstream stringstream; + CURLcode res; + curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDSIZE, requestContent.size()); + curl_easy_setopt(mp_curl, CURLOPT_POSTFIELDS, requestContent.c_str()); + curl_easy_setopt(mp_curl, CURLOPT_WRITEFUNCTION, &write_callback_to_iss); + curl_easy_setopt(mp_curl, CURLOPT_WRITEDATA, &stringstream); + res = curl_easy_perform(mp_curl); + if (res == CURLE_OK) { + long response_code; + curl_easy_getinfo(mp_curl, CURLINFO_RESPONSE_CODE, &response_code); + pthread_mutex_unlock(&m_lock); + if (response_code == 200) { + return stringstream.str(); + } + } + pthread_mutex_unlock(&m_lock); + return ""; +} + +std::string Aria2::addUri(const std::vector& uris) +{ + MethodCall methodCall("aria2.addUri"); + methodCall.getParams().addParam().getValue().set(m_secret); + auto uriParams = methodCall.getParams().addParam().getValue().getArray(); + for (auto& uri : uris) { + uriParams.addValue().set(uri); + } + auto ret = doRequest(methodCall); + MethodResponse response(ret); + try { + return response.getParams().getParam(0).getValue().getAsS(); + } catch (InvalidRPCNode& err) { + std::cerr << response.getFault().getFaultString(); + } + return ""; +} + +std::string Aria2::tellStatus(const std::string& gid, const std::vector& statusKey) +{ + MethodCall methodCall("aria2.tellStatus"); + methodCall.getParams().addParam().getValue().set(m_secret); + methodCall.getParams().addParam().getValue().set(gid); + if (!statusKey.empty()) { + auto statusArray = methodCall.getParams().addParam().getValue().getArray(); + for (auto& key : statusKey) { + statusArray.addValue().set(key); + } + } + return doRequest(methodCall); +} + +} // end namespace kiwix diff --git a/src/aria2.h b/src/aria2.h new file mode 100644 index 000000000..d94a62508 --- /dev/null +++ b/src/aria2.h @@ -0,0 +1,43 @@ + + +#ifndef KIWIXLIB_ARIA2_H_ +#define KIWIXLIB_ARIA2_H_ + +#ifdef _WIN32 +// winsock2.h need to be included before windows.h (included by curl.h) +# include +#endif + +#include "subprocess.h" +#include "xmlrpc.h" + +#include +#include + +namespace kiwix { + +class Aria2 +{ + private: + std::unique_ptr mp_aria; + int m_port; + std::string m_secret; + std::string m_downloadDir; + CURL* mp_curl; + pthread_mutex_t m_lock; + + std::string doRequest(const MethodCall& methodCall); + + public: + Aria2(); + virtual ~Aria2(); + + std::string addUri(const std::vector& uri); + std::string tellStatus(const std::string& gid, const std::vector& statusKey); + + +}; + +}; //end namespace kiwix + +#endif // KIWIXLIB_ARIA2_H_ diff --git a/src/meson.build b/src/meson.build index ab0760099..ed0036e7f 100644 --- a/src/meson.build +++ b/src/meson.build @@ -7,6 +7,7 @@ kiwix_sources = [ 'entry.cpp', 'searcher.cpp', 'subprocess.cpp', + 'aria2.cpp', 'common/base64.cpp', 'common/pathTools.cpp', 'common/regexTools.cpp',