From a819d9e3e004a267d59517c98b14c5021cc9c360 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 28 Jul 2020 15:43:16 +0200 Subject: [PATCH 01/16] Make the server handle pointer to response instead of plain response. This is a preparatory work. We will specialize the response and so we need a pointer to response instead of plain response. --- src/server.cpp | 167 +++++++++++++++++++++++++------------------------ 1 file changed, 86 insertions(+), 81 deletions(-) diff --git a/src/server.cpp b/src/server.cpp index 5d5b53d89..02665af0e 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -112,24 +112,24 @@ class InternalServer { void stop(); private: // functions - Response handle_request(const RequestContext& request); - Response build_500(const std::string& msg); - Response build_404(const RequestContext& request, const std::string& zimName); - Response build_304(const RequestContext& request, const ETag& etag) const; - Response build_redirect(const std::string& bookName, const kiwix::Entry& entry) const; - Response build_homepage(const RequestContext& request); - Response handle_skin(const RequestContext& request); - Response handle_catalog(const RequestContext& request); - Response handle_meta(const RequestContext& request); - Response handle_search(const RequestContext& request); - Response handle_suggest(const RequestContext& request); - Response handle_random(const RequestContext& request); - Response handle_captured_external(const RequestContext& request); - Response handle_content(const RequestContext& request); + std::unique_ptr handle_request(const RequestContext& request); + std::unique_ptr build_500(const std::string& msg); + std::unique_ptr build_404(const RequestContext& request, const std::string& zimName); + std::unique_ptr build_304(const RequestContext& request, const ETag& etag) const; + std::unique_ptr build_redirect(const std::string& bookName, const kiwix::Entry& entry) const; + std::unique_ptr build_homepage(const RequestContext& request); + std::unique_ptr handle_skin(const RequestContext& request); + std::unique_ptr handle_catalog(const RequestContext& request); + std::unique_ptr handle_meta(const RequestContext& request); + std::unique_ptr handle_search(const RequestContext& request); + std::unique_ptr handle_suggest(const RequestContext& request); + std::unique_ptr handle_random(const RequestContext& request); + std::unique_ptr handle_captured_external(const RequestContext& request); + std::unique_ptr handle_content(const RequestContext& request); MustacheData get_default_data() const; MustacheData homepage_data() const; - Response get_default_response() const; + std::unique_ptr get_default_response() const; std::shared_ptr get_reader(const std::string& bookName) const; bool etag_not_needed(const RequestContext& r) const; @@ -317,7 +317,7 @@ MHD_Result InternalServer::handlerCallback(struct MHD_Connection* connection, auto response = handle_request(request); - if (response.getReturnCode() == MHD_HTTP_INTERNAL_SERVER_ERROR) { + if (response->getReturnCode() == MHD_HTTP_INTERNAL_SERVER_ERROR) { printf("========== INTERNAL ERROR !! ============\n"); if (!m_verbose.load()) { printf("Requesting : \n"); @@ -326,10 +326,10 @@ MHD_Result InternalServer::handlerCallback(struct MHD_Connection* connection, } } - if (response.getReturnCode() == MHD_HTTP_OK && !etag_not_needed(request)) - response.set_server_id(m_server_id); + if (response->getReturnCode() == MHD_HTTP_OK && !etag_not_needed(request)) + response->set_server_id(m_server_id); - auto ret = response.send(request, connection); + auto ret = response->send(request, connection); auto end_time = std::chrono::steady_clock::now(); auto time_span = std::chrono::duration_cast>(end_time - start_time); if (m_verbose.load()) { @@ -339,16 +339,16 @@ MHD_Result InternalServer::handlerCallback(struct MHD_Connection* connection, return ret; } -Response InternalServer::build_304(const RequestContext& request, const ETag& etag) const +std::unique_ptr InternalServer::build_304(const RequestContext& request, const ETag& etag) const { auto response = get_default_response(); - response.set_code(MHD_HTTP_NOT_MODIFIED); - response.set_etag(etag); - response.set_content(""); + response->set_code(MHD_HTTP_NOT_MODIFIED); + response->set_etag(etag); + response->set_content(""); return response; } -Response InternalServer::handle_request(const RequestContext& request) +std::unique_ptr InternalServer::handle_request(const RequestContext& request) { try { if (! request.is_valid_url()) @@ -396,36 +396,42 @@ MustacheData InternalServer::get_default_data() const return data; } -Response InternalServer::get_default_response() const +std::unique_ptr InternalServer::get_default_response() const { - return Response(m_root, m_verbose.load(), m_withTaskbar, m_withLibraryButton, m_blockExternalLinks); + return std::unique_ptr(new Response( + m_root, + m_verbose.load(), + m_withTaskbar, + m_withLibraryButton, + m_blockExternalLinks)); } -Response InternalServer::build_404(const RequestContext& request, +std::unique_ptr InternalServer::build_404(const RequestContext& request, const std::string& bookName) { MustacheData results; results.set("url", request.get_full_url()); auto response = get_default_response(); - response.set_template(RESOURCE::templates::_404_html, results); - response.set_mimeType("text/html"); - response.set_code(MHD_HTTP_NOT_FOUND); - response.set_compress(true); - response.set_taskbar(bookName, ""); + response->set_template(RESOURCE::templates::_404_html, results); + response->set_mimeType("text/html"); + response->set_code(MHD_HTTP_NOT_FOUND); + response->set_compress(true); + response->set_taskbar(bookName, ""); return response; } -Response InternalServer::build_500(const std::string& msg) +std::unique_ptr InternalServer::build_500(const std::string& msg) { MustacheData data; data.set("error", msg); - Response response(m_root, true, false, false, false); - response.set_template(RESOURCE::templates::_500_html, data); - response.set_mimeType("text/html"); - response.set_code(MHD_HTTP_INTERNAL_SERVER_ERROR); + std::unique_ptr response( + new Response(m_root, true, false, false, false)); + response->set_template(RESOURCE::templates::_500_html, data); + response->set_mimeType("text/html"); + response->set_code(MHD_HTTP_INTERNAL_SERVER_ERROR); return response; } @@ -471,16 +477,16 @@ InternalServer::get_matching_if_none_match_etag(const RequestContext& r) const } } -Response InternalServer::build_homepage(const RequestContext& request) +std::unique_ptr InternalServer::build_homepage(const RequestContext& request) { auto response = get_default_response(); - response.set_template(RESOURCE::templates::index_html, homepage_data()); - response.set_mimeType("text/html; charset=utf-8"); - response.set_compress(true); + response->set_template(RESOURCE::templates::index_html, homepage_data()); + response->set_mimeType("text/html; charset=utf-8"); + response->set_compress(true); return response; } -Response InternalServer::handle_meta(const RequestContext& request) +std::unique_ptr InternalServer::handle_meta(const RequestContext& request) { std::string bookName; std::string bookId; @@ -525,14 +531,14 @@ Response InternalServer::handle_meta(const RequestContext& request) } auto response = get_default_response(); - response.set_content(content); - response.set_mimeType(mimeType); - response.set_compress(false); - response.set_cacheable(); + response->set_content(content); + response->set_mimeType(mimeType); + response->set_compress(false); + response->set_cacheable(); return response; } -Response InternalServer::handle_suggest(const RequestContext& request) +std::unique_ptr InternalServer::handle_suggest(const RequestContext& request) { if (m_verbose.load()) { printf("** running handle_suggest\n"); @@ -591,13 +597,13 @@ Response InternalServer::handle_suggest(const RequestContext& request) data.set("suggestions", results); auto response = get_default_response(); - response.set_template(RESOURCE::templates::suggestion_json, data); - response.set_mimeType("application/json; charset=utf-8"); - response.set_compress(true); + response->set_template(RESOURCE::templates::suggestion_json, data); + response->set_mimeType("application/json; charset=utf-8"); + response->set_compress(true); return response; } -Response InternalServer::handle_skin(const RequestContext& request) +std::unique_ptr InternalServer::handle_skin(const RequestContext& request) { if (m_verbose.load()) { printf("** running handle_skin\n"); @@ -606,17 +612,17 @@ Response InternalServer::handle_skin(const RequestContext& request) auto response = get_default_response(); auto resourceName = request.get_url().substr(1); try { - response.set_content(getResource(resourceName)); + response->set_content(getResource(resourceName)); } catch (const ResourceNotFound& e) { return build_404(request, ""); } - response.set_mimeType(getMimeTypeForFile(resourceName)); - response.set_compress(true); - response.set_cacheable(); + response->set_mimeType(getMimeTypeForFile(resourceName)); + response->set_compress(true); + response->set_cacheable(); return response; } -Response InternalServer::handle_search(const RequestContext& request) +std::unique_ptr InternalServer::handle_search(const RequestContext& request) { if (m_verbose.load()) { printf("** running handle_search\n"); @@ -672,23 +678,23 @@ Response InternalServer::handle_search(const RequestContext& request) /* If article found then redirect directly to it */ if (!patternCorrespondingUrl.empty()) { auto response = get_default_response(); - response.set_redirection(m_root + "/" + bookName + "/" + patternCorrespondingUrl); + response->set_redirection(m_root + "/" + bookName + "/" + patternCorrespondingUrl); return response; } } /* Make the search */ auto response = get_default_response(); - response.set_mimeType("text/html; charset=utf-8"); - response.set_taskbar(bookName, reader ? reader->getTitle() : ""); - response.set_compress(true); + response->set_mimeType("text/html; charset=utf-8"); + response->set_taskbar(bookName, reader ? reader->getTitle() : ""); + response->set_compress(true); if ( (!reader && !bookName.empty()) || (patternString.empty() && ! has_geo_query) ) { auto data = get_default_data(); data.set("pattern", encodeDiples(patternString)); - response.set_template(RESOURCE::templates::no_search_result_html, data); - response.set_code(MHD_HTTP_NOT_FOUND); + response->set_template(RESOURCE::templates::no_search_result_html, data); + response->set_code(MHD_HTTP_NOT_FOUND); return response; } @@ -738,7 +744,7 @@ Response InternalServer::handle_search(const RequestContext& request) renderer.setProtocolPrefix(m_root + "/"); renderer.setSearchProtocolPrefix(m_root + "/search?"); renderer.setPageLength(pageLength); - response.set_content(renderer.getHtml()); + response->set_content(renderer.getHtml()); } catch (const std::exception& e) { std::cerr << e.what() << std::endl; } @@ -752,7 +758,7 @@ Response InternalServer::handle_search(const RequestContext& request) return response; } -Response InternalServer::handle_random(const RequestContext& request) +std::unique_ptr InternalServer::handle_random(const RequestContext& request) { if (m_verbose.load()) { printf("** running handle_random\n"); @@ -781,7 +787,7 @@ Response InternalServer::handle_random(const RequestContext& request) } } -Response InternalServer::handle_captured_external(const RequestContext& request) +std::unique_ptr InternalServer::handle_captured_external(const RequestContext& request) { std::string source = ""; try { @@ -794,13 +800,13 @@ Response InternalServer::handle_captured_external(const RequestContext& request) auto data = get_default_data(); data.set("source", source); auto response = get_default_response(); - response.set_template(RESOURCE::templates::captured_external_html, data); - response.set_mimeType("text/html; charset=utf-8"); - response.set_compress(true); + response->set_template(RESOURCE::templates::captured_external_html, data); + response->set_mimeType("text/html; charset=utf-8"); + response->set_compress(true); return response; } -Response InternalServer::handle_catalog(const RequestContext& request) +std::unique_ptr InternalServer::handle_catalog(const RequestContext& request) { if (m_verbose.load()) { printf("** running handle_catalog"); @@ -820,10 +826,10 @@ Response InternalServer::handle_catalog(const RequestContext& request) } auto response = get_default_response(); - response.set_compress(true); + response->set_compress(true); if (url == "searchdescription.xml") { - response.set_template(RESOURCE::opensearchdescription_xml, get_default_data()); - response.set_mimeType("application/opensearchdescription+xml"); + response->set_template(RESOURCE::opensearchdescription_xml, get_default_data()); + response->set_mimeType("application/opensearchdescription+xml"); return response; } @@ -832,7 +838,7 @@ Response InternalServer::handle_catalog(const RequestContext& request) opdsDumper.setRootLocation(m_root); opdsDumper.setSearchDescriptionUrl("catalog/searchdescription.xml"); opdsDumper.setLibrary(mp_library); - response.set_mimeType("application/atom+xml; profile=opds-catalog; kind=acquisition; charset=utf-8"); + response->set_mimeType("application/atom+xml; profile=opds-catalog; kind=acquisition; charset=utf-8"); std::vector bookIdsToDump; if (url == "root.xml") { opdsDumper.setTitle("All zims"); @@ -880,7 +886,7 @@ Response InternalServer::handle_catalog(const RequestContext& request) } opdsDumper.setId(kiwix::to_string(uuid)); - response.set_content(opdsDumper.dumpOPDSFeed(bookIdsToDump)); + response->set_content(opdsDumper.dumpOPDSFeed(bookIdsToDump)); return response; } @@ -910,16 +916,16 @@ InternalServer::get_reader(const std::string& bookName) const return reader; } -Response +std::unique_ptr InternalServer::build_redirect(const std::string& bookName, const kiwix::Entry& entry) const { auto response = get_default_response(); - response.set_redirection(m_root + "/" + bookName + "/" + + response->set_redirection(m_root + "/" + bookName + "/" + kiwix::urlEncode(entry.getPath())); return response; } -Response InternalServer::handle_content(const RequestContext& request) +std::unique_ptr InternalServer::handle_content(const RequestContext& request) { if (m_verbose.load()) { printf("** running handle_content\n"); @@ -957,15 +963,14 @@ Response InternalServer::handle_content(const RequestContext& request) auto response = get_default_response(); - response.set_entry(entry, request); - response.set_taskbar(bookName, reader->getTitle()); + response->set_entry(entry, request); + response->set_taskbar(bookName, reader->getTitle()); if (m_verbose.load()) { printf("Found %s\n", entry.getPath().c_str()); - printf("mimeType: %s\n", response.get_mimeType().c_str()); + printf("mimeType: %s\n", response->get_mimeType().c_str()); } - return response; } From a0bdc0821c496d29e916abfb06024d23c43962d8 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Tue, 28 Jul 2020 18:06:10 +0200 Subject: [PATCH 02/16] Move internalServer code into its own source files. --- src/meson.build | 3 +- src/server.cpp | 908 +--------------------------------- src/server/internalServer.cpp | 867 ++++++++++++++++++++++++++++++++ src/server/internalServer.h | 112 +++++ 4 files changed, 982 insertions(+), 908 deletions(-) create mode 100644 src/server/internalServer.cpp create mode 100644 src/server/internalServer.h diff --git a/src/meson.build b/src/meson.build index fb2ab4595..333c3a22f 100644 --- a/src/meson.build +++ b/src/meson.build @@ -24,7 +24,8 @@ kiwix_sources = [ 'server/byte_range.cpp', 'server/etag.cpp', 'server/request_context.cpp', - 'server/response.cpp' + 'server/response.cpp', + 'server/internalServer.cpp' ] kiwix_sources += lib_resources diff --git a/src/server.cpp b/src/server.cpp index 02665af0e..c72bd77ec 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -19,140 +19,15 @@ #include "server.h" -#ifdef _WIN32 -# if !defined(__MINGW32__) && (_MSC_VER < 1600) -# include "stdint4win.h" -# endif -# include -# include -# ifdef __GNUC__ - // inet_pton is not declared in mingw, even if the function exists. - extern "C" { - WINSOCK_API_LINKAGE INT WSAAPI inet_pton( INT Family, PCSTR pszAddrString, PVOID pAddrBuf); - } -# endif - typedef UINT64 uint64_t; - typedef UINT16 uint16_t; -#endif - -extern "C" { -#include "microhttpd_wrapper.h" -} - -#include "tools/otherTools.h" -#include "tools/pathTools.h" -#include "tools/regexTools.h" -#include "tools/stringTools.h" #include "library.h" #include "name_mapper.h" -#include "entry.h" -#include "searcher.h" -#include "search_renderer.h" -#include "opds_dumper.h" -#include - -#include - -#include -#include #include -#include -#include -#include "kiwixlib-resources.h" -#ifndef _WIN32 -# include -#endif - -#include "server/request_context.h" -#include "server/response.h" - -#define MAX_SEARCH_LEN 140 -#define KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE 100 +#include "server/internalServer.h" namespace kiwix { -static IdNameMapper defaultNameMapper; - -typedef kainjow::mustache::data MustacheData; - -static MHD_Result staticHandlerCallback(void* cls, - struct MHD_Connection* connection, - const char* url, - const char* method, - const char* version, - const char* upload_data, - size_t* upload_data_size, - void** cont_cls); - - -class InternalServer { - public: - InternalServer(Library* library, - NameMapper* nameMapper, - std::string addr, - int port, - std::string root, - int nbThreads, - bool verbose, - bool withTaskbar, - bool withLibraryButton, - bool blockExternalLinks); - virtual ~InternalServer() = default; - - MHD_Result handlerCallback(struct MHD_Connection* connection, - const char* url, - const char* method, - const char* version, - const char* upload_data, - size_t* upload_data_size, - void** cont_cls); - bool start(); - void stop(); - - private: // functions - std::unique_ptr handle_request(const RequestContext& request); - std::unique_ptr build_500(const std::string& msg); - std::unique_ptr build_404(const RequestContext& request, const std::string& zimName); - std::unique_ptr build_304(const RequestContext& request, const ETag& etag) const; - std::unique_ptr build_redirect(const std::string& bookName, const kiwix::Entry& entry) const; - std::unique_ptr build_homepage(const RequestContext& request); - std::unique_ptr handle_skin(const RequestContext& request); - std::unique_ptr handle_catalog(const RequestContext& request); - std::unique_ptr handle_meta(const RequestContext& request); - std::unique_ptr handle_search(const RequestContext& request); - std::unique_ptr handle_suggest(const RequestContext& request); - std::unique_ptr handle_random(const RequestContext& request); - std::unique_ptr handle_captured_external(const RequestContext& request); - std::unique_ptr handle_content(const RequestContext& request); - - MustacheData get_default_data() const; - MustacheData homepage_data() const; - std::unique_ptr get_default_response() const; - - std::shared_ptr get_reader(const std::string& bookName) const; - bool etag_not_needed(const RequestContext& r) const; - ETag get_matching_if_none_match_etag(const RequestContext& request) const; - - private: // data - std::string m_addr; - int m_port; - std::string m_root; - int m_nbThreads; - std::atomic_bool m_verbose; - bool m_withTaskbar; - bool m_withLibraryButton; - bool m_blockExternalLinks; - struct MHD_Daemon* mp_daemon; - - Library* mp_library; - NameMapper* mp_nameMapper; - - std::string m_server_id; -}; - - Server::Server(Library* library, NameMapper* nameMapper) : mp_library(library), mp_nameMapper(nameMapper), @@ -193,785 +68,4 @@ void Server::setRoot(const std::string& root) } } - -InternalServer::InternalServer(Library* library, - NameMapper* nameMapper, - std::string addr, - int port, - std::string root, - int nbThreads, - bool verbose, - bool withTaskbar, - bool withLibraryButton, - bool blockExternalLinks) : - m_addr(addr), - m_port(port), - m_root(root), - m_nbThreads(nbThreads), - m_verbose(verbose), - m_withTaskbar(withTaskbar), - m_withLibraryButton(withLibraryButton), - m_blockExternalLinks(blockExternalLinks), - mp_daemon(nullptr), - mp_library(library), - mp_nameMapper(nameMapper ? nameMapper : &defaultNameMapper) -{} - -bool InternalServer::start() { -#ifdef _WIN32 - int flags = MHD_USE_SELECT_INTERNALLY; -#else - int flags = MHD_USE_POLL_INTERNALLY; -#endif - if (m_verbose.load()) - flags |= MHD_USE_DEBUG; - - - struct sockaddr_in sockAddr; - memset(&sockAddr, 0, sizeof(sockAddr)); - sockAddr.sin_family = AF_INET; - sockAddr.sin_port = htons(m_port); - if (m_addr.empty()) { - if (0 != INADDR_ANY) - sockAddr.sin_addr.s_addr = htonl(INADDR_ANY); - } else { - if (inet_pton(AF_INET, m_addr.c_str(), &(sockAddr.sin_addr.s_addr)) == 0) { - std::cerr << "Ip address " << m_addr << " is not a valid ip address" << std::endl; - return false; - } - } - - mp_daemon = MHD_start_daemon(flags, - m_port, - NULL, - NULL, - &staticHandlerCallback, - this, - MHD_OPTION_SOCK_ADDR, &sockAddr, - MHD_OPTION_THREAD_POOL_SIZE, m_nbThreads, - MHD_OPTION_END); - if (mp_daemon == nullptr) { - std::cerr << "Unable to instantiate the HTTP daemon. The port " << m_port - << " is maybe already occupied or need more permissions to be open. " - "Please try as root or with a port number higher or equal to 1024." - << std::endl; - return false; - } - auto server_start_time = std::chrono::system_clock::now().time_since_epoch(); - m_server_id = kiwix::to_string(server_start_time.count()); - return true; -} - -void InternalServer::stop() -{ - MHD_stop_daemon(mp_daemon); -} - -static MHD_Result staticHandlerCallback(void* cls, - struct MHD_Connection* connection, - const char* url, - const char* method, - const char* version, - const char* upload_data, - size_t* upload_data_size, - void** cont_cls) -{ - InternalServer* _this = static_cast(cls); - - return _this->handlerCallback(connection, - url, - method, - version, - upload_data, - upload_data_size, - cont_cls); -} - -MHD_Result InternalServer::handlerCallback(struct MHD_Connection* connection, - const char* url, - const char* method, - const char* version, - const char* upload_data, - size_t* upload_data_size, - void** cont_cls) -{ - auto start_time = std::chrono::steady_clock::now(); - if (m_verbose.load() ) { - printf("======================\n"); - printf("Requesting : \n"); - printf("full_url : %s\n", url); - } - RequestContext request(connection, m_root, url, method, version); - - if (m_verbose.load() ) { - request.print_debug_info(); - } - /* Unexpected method */ - if (request.get_method() != RequestMethod::GET - && request.get_method() != RequestMethod::POST - && request.get_method() != RequestMethod::HEAD) { - printf("Reject request because of unhandled request method.\n"); - printf("----------------------\n"); - return MHD_NO; - } - - auto response = handle_request(request); - - if (response->getReturnCode() == MHD_HTTP_INTERNAL_SERVER_ERROR) { - printf("========== INTERNAL ERROR !! ============\n"); - if (!m_verbose.load()) { - printf("Requesting : \n"); - printf("full_url : %s\n", url); - request.print_debug_info(); - } - } - - if (response->getReturnCode() == MHD_HTTP_OK && !etag_not_needed(request)) - response->set_server_id(m_server_id); - - auto ret = response->send(request, connection); - auto end_time = std::chrono::steady_clock::now(); - auto time_span = std::chrono::duration_cast>(end_time - start_time); - if (m_verbose.load()) { - printf("Request time : %fs\n", time_span.count()); - printf("----------------------\n"); - } - return ret; -} - -std::unique_ptr InternalServer::build_304(const RequestContext& request, const ETag& etag) const -{ - auto response = get_default_response(); - response->set_code(MHD_HTTP_NOT_MODIFIED); - response->set_etag(etag); - response->set_content(""); - return response; -} - -std::unique_ptr InternalServer::handle_request(const RequestContext& request) -{ - try { - if (! request.is_valid_url()) - return build_404(request, ""); - - const ETag etag = get_matching_if_none_match_etag(request); - if ( etag ) - return build_304(request, etag); - - if (kiwix::startsWith(request.get_url(), "/skin/")) - return handle_skin(request); - - if (startsWith(request.get_url(), "/catalog")) - return handle_catalog(request); - - if (request.get_url() == "/meta") - return handle_meta(request); - - if (request.get_url() == "/search") - return handle_search(request); - - if (request.get_url() == "/suggest") - return handle_suggest(request); - - if (request.get_url() == "/random") - return handle_random(request); - - if (request.get_url() == "/catch/external") - return handle_captured_external(request); - - return handle_content(request); - } catch (std::exception& e) { - fprintf(stderr, "===== Unhandled error : %s\n", e.what()); - return build_500(e.what()); - } catch (...) { - fprintf(stderr, "===== Unhandled unknown error\n"); - return build_500("Unknown error"); - } -} - -MustacheData InternalServer::get_default_data() const -{ - MustacheData data; - data.set("root", m_root); - return data; -} - -std::unique_ptr InternalServer::get_default_response() const -{ - return std::unique_ptr(new Response( - m_root, - m_verbose.load(), - m_withTaskbar, - m_withLibraryButton, - m_blockExternalLinks)); -} - - -std::unique_ptr InternalServer::build_404(const RequestContext& request, - const std::string& bookName) -{ - MustacheData results; - results.set("url", request.get_full_url()); - - auto response = get_default_response(); - response->set_template(RESOURCE::templates::_404_html, results); - response->set_mimeType("text/html"); - response->set_code(MHD_HTTP_NOT_FOUND); - response->set_compress(true); - response->set_taskbar(bookName, ""); - - return response; -} - -std::unique_ptr InternalServer::build_500(const std::string& msg) -{ - MustacheData data; - data.set("error", msg); - std::unique_ptr response( - new Response(m_root, true, false, false, false)); - response->set_template(RESOURCE::templates::_500_html, data); - response->set_mimeType("text/html"); - response->set_code(MHD_HTTP_INTERNAL_SERVER_ERROR); - return response; -} - -MustacheData InternalServer::homepage_data() const -{ - auto data = get_default_data(); - - MustacheData books{MustacheData::type::list}; - for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) { - auto& currentBook = mp_library->getBookById(bookId); - - MustacheData book; - book.set("name", mp_nameMapper->getNameForId(bookId)); - book.set("title", currentBook.getTitle()); - book.set("description", currentBook.getDescription()); - book.set("articleCount", beautifyInteger(currentBook.getArticleCount())); - book.set("mediaCount", beautifyInteger(currentBook.getMediaCount())); - books.push_back(book); - } - - data.set("books", books); - return data; -} - -bool InternalServer::etag_not_needed(const RequestContext& request) const -{ - const std::string url = request.get_url(); - return kiwix::startsWith(url, "/catalog") - || url == "/search" - || url == "/suggest" - || url == "/random" - || url == "/catch/external"; -} - -ETag -InternalServer::get_matching_if_none_match_etag(const RequestContext& r) const -{ - try { - const std::string etag_list = r.get_header(MHD_HTTP_HEADER_IF_NONE_MATCH); - return ETag::match(etag_list, m_server_id); - } catch (const std::out_of_range&) { - return ETag(); - } -} - -std::unique_ptr InternalServer::build_homepage(const RequestContext& request) -{ - auto response = get_default_response(); - response->set_template(RESOURCE::templates::index_html, homepage_data()); - response->set_mimeType("text/html; charset=utf-8"); - response->set_compress(true); - return response; -} - -std::unique_ptr InternalServer::handle_meta(const RequestContext& request) -{ - std::string bookName; - std::string bookId; - std::string meta_name; - std::shared_ptr reader; - try { - bookName = request.get_argument("content"); - bookId = mp_nameMapper->getIdForName(bookName); - meta_name = request.get_argument("name"); - reader = mp_library->getReaderById(bookId); - } catch (const std::out_of_range& e) { - return build_404(request, bookName); - } - - if (reader == nullptr) { - return build_404(request, bookName); - } - - std::string content; - std::string mimeType = "text"; - - if (meta_name == "title") { - content = reader->getTitle(); - } else if (meta_name == "description") { - content = reader->getDescription(); - } else if (meta_name == "language") { - content = reader->getLanguage(); - } else if (meta_name == "name") { - content = reader->getName(); - } else if (meta_name == "tags") { - content = reader->getTags(); - } else if (meta_name == "date") { - content = reader->getDate(); - } else if (meta_name == "creator") { - content = reader->getCreator(); - } else if (meta_name == "publisher") { - content = reader->getPublisher(); - } else if (meta_name == "favicon") { - reader->getFavicon(content, mimeType); - } else { - return build_404(request, bookName); - } - - auto response = get_default_response(); - response->set_content(content); - response->set_mimeType(mimeType); - response->set_compress(false); - response->set_cacheable(); - return response; -} - -std::unique_ptr InternalServer::handle_suggest(const RequestContext& request) -{ - if (m_verbose.load()) { - printf("** running handle_suggest\n"); - } - - std::string content; - std::string mimeType; - unsigned int maxSuggestionCount = 10; - unsigned int suggestionCount = 0; - - std::string bookName; - std::string bookId; - std::string term; - std::shared_ptr reader; - try { - bookName = request.get_argument("content"); - bookId = mp_nameMapper->getIdForName(bookName); - term = request.get_argument("term"); - reader = mp_library->getReaderById(bookId); - } catch (const std::out_of_range&) { - return build_404(request, bookName); - } - - if (m_verbose.load()) { - printf("Searching suggestions for: \"%s\"\n", term.c_str()); - } - - MustacheData results{MustacheData::type::list}; - - bool first = true; - if (reader != nullptr) { - /* Get the suggestions */ - SuggestionsList_t suggestions; - reader->searchSuggestionsSmart(term, maxSuggestionCount, suggestions); - for(auto& suggestion:suggestions) { - MustacheData result; - result.set("label", suggestion[0]); - result.set("value", suggestion[0]); - result.set("first", first); - first = false; - results.push_back(result); - suggestionCount++; - } - } - - /* Propose the fulltext search if possible */ - if (reader->hasFulltextIndex()) { - MustacheData result; - result.set("label", "containing '" + term + "'..."); - result.set("value", term + " "); - result.set("first", first); - results.push_back(result); - } - - auto data = get_default_data(); - data.set("suggestions", results); - - auto response = get_default_response(); - response->set_template(RESOURCE::templates::suggestion_json, data); - response->set_mimeType("application/json; charset=utf-8"); - response->set_compress(true); - return response; -} - -std::unique_ptr InternalServer::handle_skin(const RequestContext& request) -{ - if (m_verbose.load()) { - printf("** running handle_skin\n"); - } - - auto response = get_default_response(); - auto resourceName = request.get_url().substr(1); - try { - response->set_content(getResource(resourceName)); - } catch (const ResourceNotFound& e) { - return build_404(request, ""); - } - response->set_mimeType(getMimeTypeForFile(resourceName)); - response->set_compress(true); - response->set_cacheable(); - return response; -} - -std::unique_ptr InternalServer::handle_search(const RequestContext& request) -{ - if (m_verbose.load()) { - printf("** running handle_search\n"); - } - - std::string bookName; - std::string bookId; - try { - bookName = request.get_argument("content"); - bookId = mp_nameMapper->getIdForName(bookName); - } catch (const std::out_of_range&) {} - - std::string patternString; - try { - patternString = request.get_argument("pattern"); - } catch (const std::out_of_range&) {} - - /* Retrive geo search */ - bool has_geo_query = false; - float latitude = 0; - float longitude = 0; - float distance = 0; - try { - latitude = request.get_argument("latitude"); - longitude = request.get_argument("longitude"); - distance = request.get_argument("distance"); - has_geo_query = true; - } catch(const std::out_of_range&) {} - catch(const std::invalid_argument&) {} - - std::shared_ptr reader(nullptr); - try { - reader = mp_library->getReaderById(bookId); - } catch (const std::out_of_range&) {} - - /* Try first to load directly the article */ - if (reader != nullptr && !patternString.empty()) { - std::string patternCorrespondingUrl; - auto variants = reader->getTitleVariants(patternString); - auto variantsItr = variants.begin(); - - while (patternCorrespondingUrl.empty() && variantsItr != variants.end()) { - try { - auto entry = reader->getEntryFromTitle(*variantsItr); - entry = entry.getFinalEntry(); - patternCorrespondingUrl = entry.getPath(); - break; - } catch(kiwix::NoEntry& e) { - variantsItr++; - } - } - - /* If article found then redirect directly to it */ - if (!patternCorrespondingUrl.empty()) { - auto response = get_default_response(); - response->set_redirection(m_root + "/" + bookName + "/" + patternCorrespondingUrl); - return response; - } - } - - /* Make the search */ - auto response = get_default_response(); - response->set_mimeType("text/html; charset=utf-8"); - response->set_taskbar(bookName, reader ? reader->getTitle() : ""); - response->set_compress(true); - - if ( (!reader && !bookName.empty()) - || (patternString.empty() && ! has_geo_query) ) { - auto data = get_default_data(); - data.set("pattern", encodeDiples(patternString)); - response->set_template(RESOURCE::templates::no_search_result_html, data); - response->set_code(MHD_HTTP_NOT_FOUND); - return response; - } - - Searcher searcher; - if (reader) { - searcher.add_reader(reader.get()); - } else { - for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) { - auto currentReader = mp_library->getReaderById(bookId); - if (currentReader) { - searcher.add_reader(currentReader.get()); - } - } - } - - auto start = 0; - try { - start = request.get_argument("start"); - } catch (const std::exception&) {} - auto pageLength=25; - try{ - pageLength=request.get_argument("pageLength"); - }catch(const std::exception&){} - if (pageLength > MAX_SEARCH_LEN) { - pageLength = MAX_SEARCH_LEN; - } - - if(pageLength==0) - { - pageLength=25; - } - - auto end=start+pageLength; - - /* Get the results */ - try { - if (patternString.empty()) { - searcher.geo_search(latitude, longitude, distance, - start, end , m_verbose.load()); - } else { - searcher.search(patternString, - start, end , m_verbose.load()); - } - SearchRenderer renderer(&searcher, mp_nameMapper); - renderer.setSearchPattern(patternString); - renderer.setSearchContent(bookName); - renderer.setProtocolPrefix(m_root + "/"); - renderer.setSearchProtocolPrefix(m_root + "/search?"); - renderer.setPageLength(pageLength); - response->set_content(renderer.getHtml()); - } catch (const std::exception& e) { - std::cerr << e.what() << std::endl; - } - - //changing status code if no result obtained - if(searcher.getEstimatedResultCount() == 0) - { - response.set_code(MHD_HTTP_NO_CONTENT); - } - - return response; -} - -std::unique_ptr InternalServer::handle_random(const RequestContext& request) -{ - if (m_verbose.load()) { - printf("** running handle_random\n"); - } - - std::string bookName; - std::string bookId; - std::shared_ptr reader; - try { - bookName = request.get_argument("content"); - bookId = mp_nameMapper->getIdForName(bookName); - reader = mp_library->getReaderById(bookId); - } catch (const std::out_of_range&) { - return build_404(request, bookName); - } - - if (reader == nullptr) { - return build_404(request, bookName); - } - - try { - auto entry = reader->getRandomPage(); - return build_redirect(bookName, entry.getFinalEntry()); - } catch(kiwix::NoEntry& e) { - return build_404(request, bookName); - } -} - -std::unique_ptr InternalServer::handle_captured_external(const RequestContext& request) -{ - std::string source = ""; - try { - source = kiwix::urlDecode(request.get_argument("source")); - } catch (const std::out_of_range& e) {} - - if (source.empty()) - return build_404(request, ""); - - auto data = get_default_data(); - data.set("source", source); - auto response = get_default_response(); - response->set_template(RESOURCE::templates::captured_external_html, data); - response->set_mimeType("text/html; charset=utf-8"); - response->set_compress(true); - return response; -} - -std::unique_ptr InternalServer::handle_catalog(const RequestContext& request) -{ - if (m_verbose.load()) { - printf("** running handle_catalog"); - } - - std::string host; - std::string url; - try { - host = request.get_header("Host"); - url = request.get_url_part(1); - } catch (const std::out_of_range&) { - return build_404(request, ""); - } - - if (url != "searchdescription.xml" && url != "root.xml" && url != "search") { - return build_404(request, ""); - } - - auto response = get_default_response(); - response->set_compress(true); - if (url == "searchdescription.xml") { - response->set_template(RESOURCE::opensearchdescription_xml, get_default_data()); - response->set_mimeType("application/opensearchdescription+xml"); - return response; - } - - zim::Uuid uuid; - kiwix::OPDSDumper opdsDumper; - opdsDumper.setRootLocation(m_root); - opdsDumper.setSearchDescriptionUrl("catalog/searchdescription.xml"); - opdsDumper.setLibrary(mp_library); - response->set_mimeType("application/atom+xml; profile=opds-catalog; kind=acquisition; charset=utf-8"); - std::vector bookIdsToDump; - if (url == "root.xml") { - opdsDumper.setTitle("All zims"); - uuid = zim::Uuid::generate(host); - bookIdsToDump = mp_library->filter(kiwix::Filter().valid(true).local(true).remote(true)); - } else if (url == "search") { - auto filter = kiwix::Filter().valid(true).local(true).remote(true); - string query(""); - size_t count(10); - size_t startIndex(0); - try { - query = request.get_argument("q"); - filter.query(query); - } catch (const std::out_of_range&) {} - try { - filter.maxSize(extractFromString(request.get_argument("maxsize"))); - } catch (...) {} - try { - filter.name(request.get_argument("name")); - } catch (const std::out_of_range&) {} - try { - filter.lang(request.get_argument("lang")); - } catch (const std::out_of_range&) {} - try { - count = extractFromString(request.get_argument("count")); - } catch (...) {} - try { - startIndex = extractFromString(request.get_argument("start")); - } catch (...) {} - try { - filter.acceptTags(kiwix::split(request.get_argument("tag"), ";")); - } catch (...) {} - try { - filter.rejectTags(kiwix::split(request.get_argument("notag"), ";")); - } catch (...) {} - opdsDumper.setTitle("Search result for " + query); - uuid = zim::Uuid::generate(); - bookIdsToDump = mp_library->filter(filter); - auto totalResults = bookIdsToDump.size(); - bookIdsToDump.erase(bookIdsToDump.begin(), bookIdsToDump.begin()+startIndex); - if (count>0 && bookIdsToDump.size() > count) { - bookIdsToDump.resize(count); - } - opdsDumper.setOpenSearchInfo(totalResults, startIndex, bookIdsToDump.size()); - } - - opdsDumper.setId(kiwix::to_string(uuid)); - response->set_content(opdsDumper.dumpOPDSFeed(bookIdsToDump)); - return response; -} - -namespace -{ - -std::string get_book_name(const RequestContext& request) -{ - try { - return request.get_url_part(0); - } catch (const std::out_of_range& e) { - return std::string(); - } -} - -} // unnamed namespace - -std::shared_ptr -InternalServer::get_reader(const std::string& bookName) const -{ - std::shared_ptr reader; - try { - const std::string bookId = mp_nameMapper->getIdForName(bookName); - reader = mp_library->getReaderById(bookId); - } catch (const std::out_of_range& e) { - } - return reader; -} - -std::unique_ptr -InternalServer::build_redirect(const std::string& bookName, const kiwix::Entry& entry) const -{ - auto response = get_default_response(); - response->set_redirection(m_root + "/" + bookName + "/" + - kiwix::urlEncode(entry.getPath())); - return response; -} - -std::unique_ptr InternalServer::handle_content(const RequestContext& request) -{ - if (m_verbose.load()) { - printf("** running handle_content\n"); - } - - const std::string bookName = get_book_name(request); - if (bookName.empty()) - return build_homepage(request); - - const std::shared_ptr reader = get_reader(bookName); - if (reader == nullptr) { - return build_404(request, bookName); - } - - auto urlStr = request.get_url().substr(bookName.size()+1); - if (urlStr[0] == '/') { - urlStr = urlStr.substr(1); - } - - kiwix::Entry entry; - - try { - entry = reader->getEntryFromPath(urlStr); - if (entry.isRedirect() || urlStr.empty()) { - // If urlStr is empty, we want to mainPage. - // We must do a redirection to the real page. - return build_redirect(bookName, entry.getFinalEntry()); - } - } catch(kiwix::NoEntry& e) { - if (m_verbose.load()) - printf("Failed to find %s\n", urlStr.c_str()); - - return build_404(request, bookName); - } - - auto response = get_default_response(); - - response->set_entry(entry, request); - response->set_taskbar(bookName, reader->getTitle()); - - if (m_verbose.load()) { - printf("Found %s\n", entry.getPath().c_str()); - printf("mimeType: %s\n", response->get_mimeType().c_str()); - } - - return response; -} - } diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp new file mode 100644 index 000000000..018379bcc --- /dev/null +++ b/src/server/internalServer.cpp @@ -0,0 +1,867 @@ +/* + * Copyright 2019 Matthieu Gautier + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301, USA. + */ + +#include "internalServer.h" + +#ifdef _WIN32 +# if !defined(__MINGW32__) && (_MSC_VER < 1600) +# include "stdint4win.h" +# endif +# include +# include +# ifdef __GNUC__ + // inet_pton is not declared in mingw, even if the function exists. + extern "C" { + WINSOCK_API_LINKAGE INT WSAAPI inet_pton( INT Family, PCSTR pszAddrString, PVOID pAddrBuf); + } +# endif + typedef UINT64 uint64_t; + typedef UINT16 uint16_t; +#endif + +extern "C" { +#include "microhttpd_wrapper.h" +} + +#include "tools/otherTools.h" +#include "tools/pathTools.h" +#include "tools/regexTools.h" +#include "tools/stringTools.h" +#include "library.h" +#include "name_mapper.h" +#include "entry.h" +#include "searcher.h" +#include "search_renderer.h" +#include "opds_dumper.h" + +#include + +#include + +#include +#include +#include +#include +#include +#include "kiwixlib-resources.h" + +#ifndef _WIN32 +# include +#endif + +#include "request_context.h" +#include "response.h" + +#define MAX_SEARCH_LEN 140 +#define KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE 100 + +namespace kiwix { + +static IdNameMapper defaultNameMapper; + +static MHD_Result staticHandlerCallback(void* cls, + struct MHD_Connection* connection, + const char* url, + const char* method, + const char* version, + const char* upload_data, + size_t* upload_data_size, + void** cont_cls); + + +InternalServer::InternalServer(Library* library, + NameMapper* nameMapper, + std::string addr, + int port, + std::string root, + int nbThreads, + bool verbose, + bool withTaskbar, + bool withLibraryButton, + bool blockExternalLinks) : + m_addr(addr), + m_port(port), + m_root(root), + m_nbThreads(nbThreads), + m_verbose(verbose), + m_withTaskbar(withTaskbar), + m_withLibraryButton(withLibraryButton), + m_blockExternalLinks(blockExternalLinks), + mp_daemon(nullptr), + mp_library(library), + mp_nameMapper(nameMapper ? nameMapper : &defaultNameMapper) +{} + +bool InternalServer::start() { +#ifdef _WIN32 + int flags = MHD_USE_SELECT_INTERNALLY; +#else + int flags = MHD_USE_POLL_INTERNALLY; +#endif + if (m_verbose.load()) + flags |= MHD_USE_DEBUG; + + + struct sockaddr_in sockAddr; + memset(&sockAddr, 0, sizeof(sockAddr)); + sockAddr.sin_family = AF_INET; + sockAddr.sin_port = htons(m_port); + if (m_addr.empty()) { + if (0 != INADDR_ANY) + sockAddr.sin_addr.s_addr = htonl(INADDR_ANY); + } else { + if (inet_pton(AF_INET, m_addr.c_str(), &(sockAddr.sin_addr.s_addr)) == 0) { + std::cerr << "Ip address " << m_addr << " is not a valid ip address" << std::endl; + return false; + } + } + + mp_daemon = MHD_start_daemon(flags, + m_port, + NULL, + NULL, + &staticHandlerCallback, + this, + MHD_OPTION_SOCK_ADDR, &sockAddr, + MHD_OPTION_THREAD_POOL_SIZE, m_nbThreads, + MHD_OPTION_END); + if (mp_daemon == nullptr) { + std::cerr << "Unable to instantiate the HTTP daemon. The port " << m_port + << " is maybe already occupied or need more permissions to be open. " + "Please try as root or with a port number higher or equal to 1024." + << std::endl; + return false; + } + auto server_start_time = std::chrono::system_clock::now().time_since_epoch(); + m_server_id = kiwix::to_string(server_start_time.count()); + return true; +} + +void InternalServer::stop() +{ + MHD_stop_daemon(mp_daemon); +} + +static MHD_Result staticHandlerCallback(void* cls, + struct MHD_Connection* connection, + const char* url, + const char* method, + const char* version, + const char* upload_data, + size_t* upload_data_size, + void** cont_cls) +{ + InternalServer* _this = static_cast(cls); + + return _this->handlerCallback(connection, + url, + method, + version, + upload_data, + upload_data_size, + cont_cls); +} + +MHD_Result InternalServer::handlerCallback(struct MHD_Connection* connection, + const char* url, + const char* method, + const char* version, + const char* upload_data, + size_t* upload_data_size, + void** cont_cls) +{ + auto start_time = std::chrono::steady_clock::now(); + if (m_verbose.load() ) { + printf("======================\n"); + printf("Requesting : \n"); + printf("full_url : %s\n", url); + } + RequestContext request(connection, m_root, url, method, version); + + if (m_verbose.load() ) { + request.print_debug_info(); + } + /* Unexpected method */ + if (request.get_method() != RequestMethod::GET + && request.get_method() != RequestMethod::POST + && request.get_method() != RequestMethod::HEAD) { + printf("Reject request because of unhandled request method.\n"); + printf("----------------------\n"); + return MHD_NO; + } + + auto response = handle_request(request); + + if (response->getReturnCode() == MHD_HTTP_INTERNAL_SERVER_ERROR) { + printf("========== INTERNAL ERROR !! ============\n"); + if (!m_verbose.load()) { + printf("Requesting : \n"); + printf("full_url : %s\n", url); + request.print_debug_info(); + } + } + + if (response->getReturnCode() == MHD_HTTP_OK && !etag_not_needed(request)) + response->set_server_id(m_server_id); + + auto ret = response->send(request, connection); + auto end_time = std::chrono::steady_clock::now(); + auto time_span = std::chrono::duration_cast>(end_time - start_time); + if (m_verbose.load()) { + printf("Request time : %fs\n", time_span.count()); + printf("----------------------\n"); + } + return ret; +} + +std::unique_ptr InternalServer::build_304(const RequestContext& request, const ETag& etag) const +{ + auto response = get_default_response(); + response->set_code(MHD_HTTP_NOT_MODIFIED); + response->set_etag(etag); + response->set_content(""); + return response; +} + +std::unique_ptr InternalServer::handle_request(const RequestContext& request) +{ + try { + if (! request.is_valid_url()) + return build_404(request, ""); + + const ETag etag = get_matching_if_none_match_etag(request); + if ( etag ) + return build_304(request, etag); + + if (kiwix::startsWith(request.get_url(), "/skin/")) + return handle_skin(request); + + if (startsWith(request.get_url(), "/catalog")) + return handle_catalog(request); + + if (request.get_url() == "/meta") + return handle_meta(request); + + if (request.get_url() == "/search") + return handle_search(request); + + if (request.get_url() == "/suggest") + return handle_suggest(request); + + if (request.get_url() == "/random") + return handle_random(request); + + if (request.get_url() == "/catch/external") + return handle_captured_external(request); + + return handle_content(request); + } catch (std::exception& e) { + fprintf(stderr, "===== Unhandled error : %s\n", e.what()); + return build_500(e.what()); + } catch (...) { + fprintf(stderr, "===== Unhandled unknown error\n"); + return build_500("Unknown error"); + } +} + +MustacheData InternalServer::get_default_data() const +{ + MustacheData data; + data.set("root", m_root); + return data; +} + +std::unique_ptr InternalServer::get_default_response() const +{ + return std::unique_ptr(new Response( + m_root, + m_verbose.load(), + m_withTaskbar, + m_withLibraryButton, + m_blockExternalLinks)); +} + + +std::unique_ptr InternalServer::build_404(const RequestContext& request, + const std::string& bookName) +{ + MustacheData results; + results.set("url", request.get_full_url()); + + auto response = get_default_response(); + response->set_template(RESOURCE::templates::_404_html, results); + response->set_mimeType("text/html"); + response->set_code(MHD_HTTP_NOT_FOUND); + response->set_compress(true); + response->set_taskbar(bookName, ""); + + return response; +} + +std::unique_ptr InternalServer::build_500(const std::string& msg) +{ + MustacheData data; + data.set("error", msg); + std::unique_ptr response( + new Response(m_root, true, false, false, false)); + response->set_template(RESOURCE::templates::_500_html, data); + response->set_mimeType("text/html"); + response->set_code(MHD_HTTP_INTERNAL_SERVER_ERROR); + return response; +} + +MustacheData InternalServer::homepage_data() const +{ + auto data = get_default_data(); + + MustacheData books{MustacheData::type::list}; + for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) { + auto& currentBook = mp_library->getBookById(bookId); + + MustacheData book; + book.set("name", mp_nameMapper->getNameForId(bookId)); + book.set("title", currentBook.getTitle()); + book.set("description", currentBook.getDescription()); + book.set("articleCount", beautifyInteger(currentBook.getArticleCount())); + book.set("mediaCount", beautifyInteger(currentBook.getMediaCount())); + books.push_back(book); + } + + data.set("books", books); + return data; +} + +bool InternalServer::etag_not_needed(const RequestContext& request) const +{ + const std::string url = request.get_url(); + return kiwix::startsWith(url, "/catalog") + || url == "/search" + || url == "/suggest" + || url == "/random" + || url == "/catch/external"; +} + +ETag +InternalServer::get_matching_if_none_match_etag(const RequestContext& r) const +{ + try { + const std::string etag_list = r.get_header(MHD_HTTP_HEADER_IF_NONE_MATCH); + return ETag::match(etag_list, m_server_id); + } catch (const std::out_of_range&) { + return ETag(); + } +} + +std::unique_ptr InternalServer::build_homepage(const RequestContext& request) +{ + auto response = get_default_response(); + response->set_template(RESOURCE::templates::index_html, homepage_data()); + response->set_mimeType("text/html; charset=utf-8"); + response->set_compress(true); + return response; +} + +std::unique_ptr InternalServer::handle_meta(const RequestContext& request) +{ + std::string bookName; + std::string bookId; + std::string meta_name; + std::shared_ptr reader; + try { + bookName = request.get_argument("content"); + bookId = mp_nameMapper->getIdForName(bookName); + meta_name = request.get_argument("name"); + reader = mp_library->getReaderById(bookId); + } catch (const std::out_of_range& e) { + return build_404(request, bookName); + } + + if (reader == nullptr) { + return build_404(request, bookName); + } + + std::string content; + std::string mimeType = "text"; + + if (meta_name == "title") { + content = reader->getTitle(); + } else if (meta_name == "description") { + content = reader->getDescription(); + } else if (meta_name == "language") { + content = reader->getLanguage(); + } else if (meta_name == "name") { + content = reader->getName(); + } else if (meta_name == "tags") { + content = reader->getTags(); + } else if (meta_name == "date") { + content = reader->getDate(); + } else if (meta_name == "creator") { + content = reader->getCreator(); + } else if (meta_name == "publisher") { + content = reader->getPublisher(); + } else if (meta_name == "favicon") { + reader->getFavicon(content, mimeType); + } else { + return build_404(request, bookName); + } + + auto response = get_default_response(); + response->set_content(content); + response->set_mimeType(mimeType); + response->set_compress(false); + response->set_cacheable(); + return response; +} + +std::unique_ptr InternalServer::handle_suggest(const RequestContext& request) +{ + if (m_verbose.load()) { + printf("** running handle_suggest\n"); + } + + std::string content; + std::string mimeType; + unsigned int maxSuggestionCount = 10; + unsigned int suggestionCount = 0; + + std::string bookName; + std::string bookId; + std::string term; + std::shared_ptr reader; + try { + bookName = request.get_argument("content"); + bookId = mp_nameMapper->getIdForName(bookName); + term = request.get_argument("term"); + reader = mp_library->getReaderById(bookId); + } catch (const std::out_of_range&) { + return build_404(request, bookName); + } + + if (m_verbose.load()) { + printf("Searching suggestions for: \"%s\"\n", term.c_str()); + } + + MustacheData results{MustacheData::type::list}; + + bool first = true; + if (reader != nullptr) { + /* Get the suggestions */ + SuggestionsList_t suggestions; + reader->searchSuggestionsSmart(term, maxSuggestionCount, suggestions); + for(auto& suggestion:suggestions) { + MustacheData result; + result.set("label", suggestion[0]); + result.set("value", suggestion[0]); + result.set("first", first); + first = false; + results.push_back(result); + suggestionCount++; + } + } + + /* Propose the fulltext search if possible */ + if (reader->hasFulltextIndex()) { + MustacheData result; + result.set("label", "containing '" + term + "'..."); + result.set("value", term + " "); + result.set("first", first); + results.push_back(result); + } + + auto data = get_default_data(); + data.set("suggestions", results); + + auto response = get_default_response(); + response->set_template(RESOURCE::templates::suggestion_json, data); + response->set_mimeType("application/json; charset=utf-8"); + response->set_compress(true); + return response; +} + +std::unique_ptr InternalServer::handle_skin(const RequestContext& request) +{ + if (m_verbose.load()) { + printf("** running handle_skin\n"); + } + + auto response = get_default_response(); + auto resourceName = request.get_url().substr(1); + try { + response->set_content(getResource(resourceName)); + } catch (const ResourceNotFound& e) { + return build_404(request, ""); + } + response->set_mimeType(getMimeTypeForFile(resourceName)); + response->set_compress(true); + response->set_cacheable(); + return response; +} + +std::unique_ptr InternalServer::handle_search(const RequestContext& request) +{ + if (m_verbose.load()) { + printf("** running handle_search\n"); + } + + std::string bookName; + std::string bookId; + try { + bookName = request.get_argument("content"); + bookId = mp_nameMapper->getIdForName(bookName); + } catch (const std::out_of_range&) {} + + std::string patternString; + try { + patternString = request.get_argument("pattern"); + } catch (const std::out_of_range&) {} + + /* Retrive geo search */ + bool has_geo_query = false; + float latitude = 0; + float longitude = 0; + float distance = 0; + try { + latitude = request.get_argument("latitude"); + longitude = request.get_argument("longitude"); + distance = request.get_argument("distance"); + has_geo_query = true; + } catch(const std::out_of_range&) {} + catch(const std::invalid_argument&) {} + + std::shared_ptr reader(nullptr); + try { + reader = mp_library->getReaderById(bookId); + } catch (const std::out_of_range&) {} + + /* Try first to load directly the article */ + if (reader != nullptr && !patternString.empty()) { + std::string patternCorrespondingUrl; + auto variants = reader->getTitleVariants(patternString); + auto variantsItr = variants.begin(); + + while (patternCorrespondingUrl.empty() && variantsItr != variants.end()) { + try { + auto entry = reader->getEntryFromTitle(*variantsItr); + entry = entry.getFinalEntry(); + patternCorrespondingUrl = entry.getPath(); + break; + } catch(kiwix::NoEntry& e) { + variantsItr++; + } + } + + /* If article found then redirect directly to it */ + if (!patternCorrespondingUrl.empty()) { + auto response = get_default_response(); + response->set_redirection(m_root + "/" + bookName + "/" + patternCorrespondingUrl); + return response; + } + } + + /* Make the search */ + auto response = get_default_response(); + response->set_mimeType("text/html; charset=utf-8"); + response->set_taskbar(bookName, reader ? reader->getTitle() : ""); + response->set_compress(true); + + if ( (!reader && !bookName.empty()) + || (patternString.empty() && ! has_geo_query) ) { + auto data = get_default_data(); + data.set("pattern", encodeDiples(patternString)); + response->set_template(RESOURCE::templates::no_search_result_html, data); + response->set_code(MHD_HTTP_NOT_FOUND); + return response; + } + + Searcher searcher; + if (reader) { + searcher.add_reader(reader.get()); + } else { + for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) { + auto currentReader = mp_library->getReaderById(bookId); + if (currentReader) { + searcher.add_reader(currentReader.get()); + } + } + } + + auto start = 0; + try { + start = request.get_argument("start"); + } catch (const std::exception&) {} + + auto pageLength = 25; + try { + pageLength = request.get_argument("pageLength"); + } catch (const std::exception&) {} + if (pageLength > MAX_SEARCH_LEN) { + pageLength = MAX_SEARCH_LEN; + } + if (pageLength == 0) { + pageLength = 25; + } + + auto end = start + pageLength; + + /* Get the results */ + try { + if (patternString.empty()) { + searcher.geo_search(latitude, longitude, distance, + start, end, m_verbose.load()); + } else { + searcher.search(patternString, + start, end, m_verbose.load()); + } + SearchRenderer renderer(&searcher, mp_nameMapper); + renderer.setSearchPattern(patternString); + renderer.setSearchContent(bookName); + renderer.setProtocolPrefix(m_root + "/"); + renderer.setSearchProtocolPrefix(m_root + "/search?"); + renderer.setPageLength(pageLength); + response->set_content(renderer.getHtml()); + } catch (const std::exception& e) { + std::cerr << e.what() << std::endl; + } + + //changing status code if no result obtained + if(searcher.getEstimatedResultCount() == 0) + { + response.set_code(MHD_HTTP_NO_CONTENT); + } + + return response; +} + +std::unique_ptr InternalServer::handle_random(const RequestContext& request) +{ + if (m_verbose.load()) { + printf("** running handle_random\n"); + } + + std::string bookName; + std::string bookId; + std::shared_ptr reader; + try { + bookName = request.get_argument("content"); + bookId = mp_nameMapper->getIdForName(bookName); + reader = mp_library->getReaderById(bookId); + } catch (const std::out_of_range&) { + return build_404(request, bookName); + } + + if (reader == nullptr) { + return build_404(request, bookName); + } + + try { + auto entry = reader->getRandomPage(); + return build_redirect(bookName, entry.getFinalEntry()); + } catch(kiwix::NoEntry& e) { + return build_404(request, bookName); + } +} + +std::unique_ptr InternalServer::handle_captured_external(const RequestContext& request) +{ + std::string source = ""; + try { + source = kiwix::urlDecode(request.get_argument("source")); + } catch (const std::out_of_range& e) {} + + if (source.empty()) + return build_404(request, ""); + + auto data = get_default_data(); + data.set("source", source); + auto response = get_default_response(); + response->set_template(RESOURCE::templates::captured_external_html, data); + response->set_mimeType("text/html; charset=utf-8"); + response->set_compress(true); + return response; +} + +std::unique_ptr InternalServer::handle_catalog(const RequestContext& request) +{ + if (m_verbose.load()) { + printf("** running handle_catalog"); + } + + std::string host; + std::string url; + try { + host = request.get_header("Host"); + url = request.get_url_part(1); + } catch (const std::out_of_range&) { + return build_404(request, ""); + } + + if (url != "searchdescription.xml" && url != "root.xml" && url != "search") { + return build_404(request, ""); + } + + auto response = get_default_response(); + response->set_compress(true); + if (url == "searchdescription.xml") { + response->set_template(RESOURCE::opensearchdescription_xml, get_default_data()); + response->set_mimeType("application/opensearchdescription+xml"); + return response; + } + + zim::Uuid uuid; + kiwix::OPDSDumper opdsDumper; + opdsDumper.setRootLocation(m_root); + opdsDumper.setSearchDescriptionUrl("catalog/searchdescription.xml"); + opdsDumper.setLibrary(mp_library); + response->set_mimeType("application/atom+xml; profile=opds-catalog; kind=acquisition; charset=utf-8"); + std::vector bookIdsToDump; + if (url == "root.xml") { + opdsDumper.setTitle("All zims"); + uuid = zim::Uuid::generate(host); + bookIdsToDump = mp_library->filter(kiwix::Filter().valid(true).local(true).remote(true)); + } else if (url == "search") { + auto filter = kiwix::Filter().valid(true).local(true).remote(true); + string query(""); + size_t count(10); + size_t startIndex(0); + try { + query = request.get_argument("q"); + filter.query(query); + } catch (const std::out_of_range&) {} + try { + filter.maxSize(extractFromString(request.get_argument("maxsize"))); + } catch (...) {} + try { + filter.name(request.get_argument("name")); + } catch (const std::out_of_range&) {} + try { + filter.lang(request.get_argument("lang")); + } catch (const std::out_of_range&) {} + try { + count = extractFromString(request.get_argument("count")); + } catch (...) {} + try { + startIndex = extractFromString(request.get_argument("start")); + } catch (...) {} + try { + filter.acceptTags(kiwix::split(request.get_argument("tag"), ";")); + } catch (...) {} + try { + filter.rejectTags(kiwix::split(request.get_argument("notag"), ";")); + } catch (...) {} + opdsDumper.setTitle("Search result for " + query); + uuid = zim::Uuid::generate(); + bookIdsToDump = mp_library->filter(filter); + auto totalResults = bookIdsToDump.size(); + bookIdsToDump.erase(bookIdsToDump.begin(), bookIdsToDump.begin()+startIndex); + if (count>0 && bookIdsToDump.size() > count) { + bookIdsToDump.resize(count); + } + opdsDumper.setOpenSearchInfo(totalResults, startIndex, bookIdsToDump.size()); + } + + opdsDumper.setId(kiwix::to_string(uuid)); + response->set_content(opdsDumper.dumpOPDSFeed(bookIdsToDump)); + return response; +} + +namespace +{ + +std::string get_book_name(const RequestContext& request) +{ + try { + return request.get_url_part(0); + } catch (const std::out_of_range& e) { + return std::string(); + } +} + +} // unnamed namespace + +std::shared_ptr +InternalServer::get_reader(const std::string& bookName) const +{ + std::shared_ptr reader; + try { + const std::string bookId = mp_nameMapper->getIdForName(bookName); + reader = mp_library->getReaderById(bookId); + } catch (const std::out_of_range& e) { + } + return reader; +} + +std::unique_ptr +InternalServer::build_redirect(const std::string& bookName, const kiwix::Entry& entry) const +{ + auto response = get_default_response(); + response->set_redirection(m_root + "/" + bookName + "/" + + kiwix::urlEncode(entry.getPath())); + return response; +} + +std::unique_ptr InternalServer::handle_content(const RequestContext& request) +{ + if (m_verbose.load()) { + printf("** running handle_content\n"); + } + + const std::string bookName = get_book_name(request); + if (bookName.empty()) + return build_homepage(request); + + const std::shared_ptr reader = get_reader(bookName); + if (reader == nullptr) { + return build_404(request, bookName); + } + + auto urlStr = request.get_url().substr(bookName.size()+1); + if (urlStr[0] == '/') { + urlStr = urlStr.substr(1); + } + + kiwix::Entry entry; + + try { + entry = reader->getEntryFromPath(urlStr); + if (entry.isRedirect() || urlStr.empty()) { + // If urlStr is empty, we want to mainPage. + // We must do a redirection to the real page. + return build_redirect(bookName, entry.getFinalEntry()); + } + } catch(kiwix::NoEntry& e) { + if (m_verbose.load()) + printf("Failed to find %s\n", urlStr.c_str()); + + return build_404(request, bookName); + } + + auto response = get_default_response(); + + response->set_entry(entry, request); + response->set_taskbar(bookName, reader->getTitle()); + + if (m_verbose.load()) { + printf("Found %s\n", entry.getPath().c_str()); + printf("mimeType: %s\n", response->get_mimeType().c_str()); + } + + return response; +} + +} diff --git a/src/server/internalServer.h b/src/server/internalServer.h new file mode 100644 index 000000000..e97b29011 --- /dev/null +++ b/src/server/internalServer.h @@ -0,0 +1,112 @@ +/* + * Copyright 2019 Matthieu Gautier + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301, USA. + */ + +#ifndef KIWIXLIB_SERVER_INTERNALSERVER_H +#define KIWIXLIB_SERVER_INTERNALSERVER_H + + +extern "C" { +#include "microhttpd_wrapper.h" +} + +#include "library.h" +#include "name_mapper.h" + +#include + +#include +#include + +#include "server/request_context.h" +#include "server/response.h" + +namespace kiwix { + +typedef kainjow::mustache::data MustacheData; + +class Entry; + +class InternalServer { + public: + InternalServer(Library* library, + NameMapper* nameMapper, + std::string addr, + int port, + std::string root, + int nbThreads, + bool verbose, + bool withTaskbar, + bool withLibraryButton, + bool blockExternalLinks); + virtual ~InternalServer() = default; + + MHD_Result handlerCallback(struct MHD_Connection* connection, + const char* url, + const char* method, + const char* version, + const char* upload_data, + size_t* upload_data_size, + void** cont_cls); + bool start(); + void stop(); + + private: // functions + std::unique_ptr handle_request(const RequestContext& request); + std::unique_ptr build_500(const std::string& msg); + std::unique_ptr build_404(const RequestContext& request, const std::string& zimName); + std::unique_ptr build_304(const RequestContext& request, const ETag& etag) const; + std::unique_ptr build_redirect(const std::string& bookName, const kiwix::Entry& entry) const; + std::unique_ptr build_homepage(const RequestContext& request); + std::unique_ptr handle_skin(const RequestContext& request); + std::unique_ptr handle_catalog(const RequestContext& request); + std::unique_ptr handle_meta(const RequestContext& request); + std::unique_ptr handle_search(const RequestContext& request); + std::unique_ptr handle_suggest(const RequestContext& request); + std::unique_ptr handle_random(const RequestContext& request); + std::unique_ptr handle_captured_external(const RequestContext& request); + std::unique_ptr handle_content(const RequestContext& request); + + MustacheData get_default_data() const; + MustacheData homepage_data() const; + std::unique_ptr get_default_response() const; + + std::shared_ptr get_reader(const std::string& bookName) const; + bool etag_not_needed(const RequestContext& r) const; + ETag get_matching_if_none_match_etag(const RequestContext& request) const; + + private: // data + std::string m_addr; + int m_port; + std::string m_root; + int m_nbThreads; + std::atomic_bool m_verbose; + bool m_withTaskbar; + bool m_withLibraryButton; + bool m_blockExternalLinks; + struct MHD_Daemon* mp_daemon; + + Library* mp_library; + NameMapper* mp_nameMapper; + + std::string m_server_id; +}; + +} + +#endif //KIWIXLIB_SERVER_INTERNALSERVER_H From 9e351b279e26988c75576a6e98f1ace0eb4e1d55 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 29 Jul 2020 10:35:27 +0200 Subject: [PATCH 03/16] Remove `get_default_response` in favor of a static Response method. We want to build different kind of response depending of the context. --- src/server/internalServer.cpp | 35 ++++++++++++----------------------- src/server/internalServer.h | 3 ++- src/server/response.cpp | 11 +++++++++++ src/server/response.h | 3 +++ 4 files changed, 28 insertions(+), 24 deletions(-) diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index 018379bcc..6583d1935 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -232,7 +232,7 @@ MHD_Result InternalServer::handlerCallback(struct MHD_Connection* connection, std::unique_ptr InternalServer::build_304(const RequestContext& request, const ETag& etag) const { - auto response = get_default_response(); + auto response = Response::build(*this); response->set_code(MHD_HTTP_NOT_MODIFIED); response->set_etag(etag); response->set_content(""); @@ -287,24 +287,13 @@ MustacheData InternalServer::get_default_data() const return data; } -std::unique_ptr InternalServer::get_default_response() const -{ - return std::unique_ptr(new Response( - m_root, - m_verbose.load(), - m_withTaskbar, - m_withLibraryButton, - m_blockExternalLinks)); -} - - std::unique_ptr InternalServer::build_404(const RequestContext& request, const std::string& bookName) { MustacheData results; results.set("url", request.get_full_url()); - auto response = get_default_response(); + auto response = Response::build(*this); response->set_template(RESOURCE::templates::_404_html, results); response->set_mimeType("text/html"); response->set_code(MHD_HTTP_NOT_FOUND); @@ -370,7 +359,7 @@ InternalServer::get_matching_if_none_match_etag(const RequestContext& r) const std::unique_ptr InternalServer::build_homepage(const RequestContext& request) { - auto response = get_default_response(); + auto response = Response::build(*this); response->set_template(RESOURCE::templates::index_html, homepage_data()); response->set_mimeType("text/html; charset=utf-8"); response->set_compress(true); @@ -421,7 +410,7 @@ std::unique_ptr InternalServer::handle_meta(const RequestContext& requ return build_404(request, bookName); } - auto response = get_default_response(); + auto response = Response::build(*this); response->set_content(content); response->set_mimeType(mimeType); response->set_compress(false); @@ -487,7 +476,7 @@ std::unique_ptr InternalServer::handle_suggest(const RequestContext& r auto data = get_default_data(); data.set("suggestions", results); - auto response = get_default_response(); + auto response = Response::build(*this); response->set_template(RESOURCE::templates::suggestion_json, data); response->set_mimeType("application/json; charset=utf-8"); response->set_compress(true); @@ -500,7 +489,7 @@ std::unique_ptr InternalServer::handle_skin(const RequestContext& requ printf("** running handle_skin\n"); } - auto response = get_default_response(); + auto response = Response::build(*this); auto resourceName = request.get_url().substr(1); try { response->set_content(getResource(resourceName)); @@ -568,14 +557,14 @@ std::unique_ptr InternalServer::handle_search(const RequestContext& re /* If article found then redirect directly to it */ if (!patternCorrespondingUrl.empty()) { - auto response = get_default_response(); + auto response = Response::build(*this); response->set_redirection(m_root + "/" + bookName + "/" + patternCorrespondingUrl); return response; } } /* Make the search */ - auto response = get_default_response(); + auto response = Response::build(*this); response->set_mimeType("text/html; charset=utf-8"); response->set_taskbar(bookName, reader ? reader->getTitle() : ""); response->set_compress(true); @@ -689,7 +678,7 @@ std::unique_ptr InternalServer::handle_captured_external(const Request auto data = get_default_data(); data.set("source", source); - auto response = get_default_response(); + auto response = Response::build(*this); response->set_template(RESOURCE::templates::captured_external_html, data); response->set_mimeType("text/html; charset=utf-8"); response->set_compress(true); @@ -715,7 +704,7 @@ std::unique_ptr InternalServer::handle_catalog(const RequestContext& r return build_404(request, ""); } - auto response = get_default_response(); + auto response = Response::build(*this); response->set_compress(true); if (url == "searchdescription.xml") { response->set_template(RESOURCE::opensearchdescription_xml, get_default_data()); @@ -809,7 +798,7 @@ InternalServer::get_reader(const std::string& bookName) const std::unique_ptr InternalServer::build_redirect(const std::string& bookName, const kiwix::Entry& entry) const { - auto response = get_default_response(); + auto response = Response::build(*this); response->set_redirection(m_root + "/" + bookName + "/" + kiwix::urlEncode(entry.getPath())); return response; @@ -851,7 +840,7 @@ std::unique_ptr InternalServer::handle_content(const RequestContext& r return build_404(request, bookName); } - auto response = get_default_response(); + auto response = Response::build(*this); response->set_entry(entry, request); response->set_taskbar(bookName, reader->getTitle()); diff --git a/src/server/internalServer.h b/src/server/internalServer.h index e97b29011..b8d8f89b3 100644 --- a/src/server/internalServer.h +++ b/src/server/internalServer.h @@ -84,7 +84,6 @@ class InternalServer { MustacheData get_default_data() const; MustacheData homepage_data() const; - std::unique_ptr get_default_response() const; std::shared_ptr get_reader(const std::string& bookName) const; bool etag_not_needed(const RequestContext& r) const; @@ -105,6 +104,8 @@ class InternalServer { NameMapper* mp_nameMapper; std::string m_server_id; + + friend std::unique_ptr Response::build(const InternalServer& server); }; } diff --git a/src/server/response.cpp b/src/server/response.cpp index 3a3fb89a2..b06a5d811 100644 --- a/src/server/response.cpp +++ b/src/server/response.cpp @@ -3,6 +3,7 @@ #include "response.h" #include "request_context.h" +#include "internalServer.h" #include "kiwixlib-resources.h" #include "tools/regexTools.h" @@ -56,6 +57,16 @@ Response::Response(const std::string& root, bool verbose, bool withTaskbar, bool { } +std::unique_ptr Response::build(const InternalServer& server) +{ + return std::unique_ptr(new Response( + server.m_root, + server.m_verbose.load(), + server.m_withTaskbar, + server.m_withLibraryButton, + server.m_blockExternalLinks)); +} + static MHD_Result print_key_value (void *cls, enum MHD_ValueKind kind, const char *key, const char *value) diff --git a/src/server/response.h b/src/server/response.h index f3373a977..c8930dbf1 100644 --- a/src/server/response.h +++ b/src/server/response.h @@ -41,6 +41,7 @@ enum class ResponseMode { ENTRY }; +class InternalServer; class RequestContext; class Response { @@ -48,6 +49,8 @@ class Response { Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks); ~Response() = default; + static std::unique_ptr build(const InternalServer& server); + MHD_Result send(const RequestContext& request, MHD_Connection* connection); void set_template(const std::string& template_str, kainjow::mustache::data data); From 1011d1ff0bcc3ae2e67007a6387e1640b6e982aa Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 29 Jul 2020 10:59:39 +0200 Subject: [PATCH 04/16] Move the redirection response in its own class. The redirection is the easiest to move, let's start with this one. --- src/server/internalServer.cpp | 11 ++++----- src/server/internalServer.h | 1 + src/server/response.cpp | 43 ++++++++++++++++++++++------------- src/server/response.h | 23 ++++++++++++++----- 4 files changed, 49 insertions(+), 29 deletions(-) diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index 6583d1935..bd0af5a9d 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -557,9 +557,8 @@ std::unique_ptr InternalServer::handle_search(const RequestContext& re /* If article found then redirect directly to it */ if (!patternCorrespondingUrl.empty()) { - auto response = Response::build(*this); - response->set_redirection(m_root + "/" + bookName + "/" + patternCorrespondingUrl); - return response; + auto redirectUrl = m_root + "/" + bookName + "/" + patternCorrespondingUrl; + return RedirectionResponse::build(*this, redirectUrl); } } @@ -798,10 +797,8 @@ InternalServer::get_reader(const std::string& bookName) const std::unique_ptr InternalServer::build_redirect(const std::string& bookName, const kiwix::Entry& entry) const { - auto response = Response::build(*this); - response->set_redirection(m_root + "/" + bookName + "/" + - kiwix::urlEncode(entry.getPath())); - return response; + auto redirectUrl = m_root + "/" + bookName + "/" + kiwix::urlEncode(entry.getPath()); + return RedirectionResponse::build(*this, redirectUrl); } std::unique_ptr InternalServer::handle_content(const RequestContext& request) diff --git a/src/server/internalServer.h b/src/server/internalServer.h index b8d8f89b3..9ac9f4aae 100644 --- a/src/server/internalServer.h +++ b/src/server/internalServer.h @@ -106,6 +106,7 @@ class InternalServer { std::string m_server_id; friend std::unique_ptr Response::build(const InternalServer& server); + friend std::unique_ptr RedirectionResponse::build(const InternalServer& server, const std::string& redirectionUrl); }; } diff --git a/src/server/response.cpp b/src/server/response.cpp index b06a5d811..f40321aae 100644 --- a/src/server/response.cpp +++ b/src/server/response.cpp @@ -252,13 +252,6 @@ Response::create_raw_content_mhd_response(const RequestContext& request) return response; } -MHD_Response* -Response::create_redirection_mhd_response() const -{ - MHD_Response* response = MHD_create_response_from_buffer(0, nullptr, MHD_RESPMEM_MUST_COPY); - MHD_add_response_header(response, MHD_HTTP_HEADER_LOCATION, m_content.c_str()); - return response; -} MHD_Response* Response::create_entry_mhd_response() const @@ -296,9 +289,6 @@ Response::create_mhd_response(const RequestContext& request) case ResponseMode::RAW_CONTENT : return create_raw_content_mhd_response(request); - case ResponseMode::REDIRECTION : - return create_redirection_mhd_response(); - case ResponseMode::ENTRY : return create_entry_mhd_response(); } @@ -338,12 +328,6 @@ void Response::set_content(const std::string& content) { m_mode = ResponseMode::RAW_CONTENT; } -void Response::set_redirection(const std::string& url) { - m_content = url; - m_mode = ResponseMode::REDIRECTION; - m_returnCode = MHD_HTTP_FOUND; -} - void Response::set_entry(const Entry& entry, const RequestContext& request) { m_entry = entry; m_mode = ResponseMode::ENTRY; @@ -374,4 +358,31 @@ void Response::set_taskbar(const std::string& bookName, const std::string& bookT } +RedirectionResponse::RedirectionResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& redirectionUrl) : + Response(root, verbose, withTaskbar, withLibraryButton, blockExternalLinks), + m_redirectionUrl(redirectionUrl) +{ + m_returnCode = MHD_HTTP_FOUND; +} + +std::unique_ptr RedirectionResponse::build(const InternalServer& server, const std::string& redirectionUrl) +{ + return std::unique_ptr(new RedirectionResponse( + server.m_root, + server.m_verbose.load(), + server.m_withTaskbar, + server.m_withLibraryButton, + server.m_blockExternalLinks, + redirectionUrl)); +} + +MHD_Response* RedirectionResponse::create_mhd_response(const RequestContext& request) +{ + MHD_Response* response = MHD_create_response_from_buffer(0, nullptr, MHD_RESPMEM_MUST_COPY); + MHD_add_response_header(response, MHD_HTTP_HEADER_LOCATION, m_redirectionUrl.c_str()); + return response; +} + + + } diff --git a/src/server/response.h b/src/server/response.h index c8930dbf1..6b46b3be4 100644 --- a/src/server/response.h +++ b/src/server/response.h @@ -37,7 +37,6 @@ namespace kiwix { enum class ResponseMode { ERROR_RESPONSE, RAW_CONTENT, - REDIRECTION, ENTRY }; @@ -47,7 +46,7 @@ class RequestContext; class Response { public: Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks); - ~Response() = default; + virtual ~Response() = default; static std::unique_ptr build(const InternalServer& server); @@ -55,7 +54,6 @@ class Response { void set_template(const std::string& template_str, kainjow::mustache::data data); void set_content(const std::string& content); - void set_redirection(const std::string& url); void set_entry(const Entry& entry, const RequestContext& request); @@ -77,13 +75,12 @@ class Response { bool contentDecorationAllowed() const; private: // functions - MHD_Response* create_mhd_response(const RequestContext& request); + virtual MHD_Response* create_mhd_response(const RequestContext& request); MHD_Response* create_error_response(const RequestContext& request) const; MHD_Response* create_raw_content_mhd_response(const RequestContext& request); - MHD_Response* create_redirection_mhd_response() const; MHD_Response* create_entry_mhd_response() const; - private: // data + protected: // data bool m_verbose; ResponseMode m_mode; std::string m_root; @@ -101,6 +98,20 @@ class Response { ETag m_etag; }; + +class RedirectionResponse : public Response { + public: + RedirectionResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& redirectionUrl); + + static std::unique_ptr build(const InternalServer& server, const std::string& redirectionUrl); + + + private: + MHD_Response* create_mhd_response(const RequestContext& request); + + std::string m_redirectionUrl; +}; + } #endif //KIWIXLIB_SERVER_RESPONSE_H From f014fb289585bd196aae1473e943870d2414dfae Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 29 Jul 2020 11:31:17 +0200 Subject: [PATCH 05/16] Introduce a ContentResponse. This is only an "interface" for now as other type of response (entry) may be "transformed" to a ContentResponse. We cannot move all the code in the class. --- src/server/internalServer.cpp | 61 ++++++++++++++++++----------------- src/server/internalServer.h | 1 + src/server/response.cpp | 31 ++++++++++++++---- src/server/response.h | 8 ++++- 4 files changed, 64 insertions(+), 37 deletions(-) diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index bd0af5a9d..7c7d23386 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -232,10 +232,10 @@ MHD_Result InternalServer::handlerCallback(struct MHD_Connection* connection, std::unique_ptr InternalServer::build_304(const RequestContext& request, const ETag& etag) const { - auto response = Response::build(*this); + // This is just to set the m_mode. + auto response = ContentResponse::build(*this, "", ""); response->set_code(MHD_HTTP_NOT_MODIFIED); response->set_etag(etag); - response->set_content(""); return response; } @@ -410,9 +410,7 @@ std::unique_ptr InternalServer::handle_meta(const RequestContext& requ return build_404(request, bookName); } - auto response = Response::build(*this); - response->set_content(content); - response->set_mimeType(mimeType); + auto response = ContentResponse::build(*this, content, mimeType); response->set_compress(false); response->set_cacheable(); return response; @@ -489,17 +487,18 @@ std::unique_ptr InternalServer::handle_skin(const RequestContext& requ printf("** running handle_skin\n"); } - auto response = Response::build(*this); auto resourceName = request.get_url().substr(1); try { - response->set_content(getResource(resourceName)); + auto response = ContentResponse::build( + *this, + getResource(resourceName), + getMimeTypeForFile(resourceName)); + response->set_compress(true); + response->set_cacheable(); + return response; } catch (const ResourceNotFound& e) { return build_404(request, ""); } - response->set_mimeType(getMimeTypeForFile(resourceName)); - response->set_compress(true); - response->set_cacheable(); - return response; } std::unique_ptr InternalServer::handle_search(const RequestContext& request) @@ -563,15 +562,14 @@ std::unique_ptr InternalServer::handle_search(const RequestContext& re } /* Make the search */ - auto response = Response::build(*this); - response->set_mimeType("text/html; charset=utf-8"); - response->set_taskbar(bookName, reader ? reader->getTitle() : ""); - response->set_compress(true); - if ( (!reader && !bookName.empty()) || (patternString.empty() && ! has_geo_query) ) { auto data = get_default_data(); data.set("pattern", encodeDiples(patternString)); + auto response = Response::build(*this); + response->set_mimeType("text/html; charset=utf-8"); + response->set_taskbar(bookName, reader ? reader->getTitle() : ""); + response->set_compress(true); response->set_template(RESOURCE::templates::no_search_result_html, data); response->set_code(MHD_HTTP_NOT_FOUND); return response; @@ -622,18 +620,20 @@ std::unique_ptr InternalServer::handle_search(const RequestContext& re renderer.setProtocolPrefix(m_root + "/"); renderer.setSearchProtocolPrefix(m_root + "/search?"); renderer.setPageLength(pageLength); - response->set_content(renderer.getHtml()); + auto response = ContentResponse::build(*this, renderer.getHtml(), "text/html; charset=utf-8"); + response->set_taskbar(bookName, reader ? reader->getTitle() : ""); + response->set_compress(true); + //changing status code if no result obtained + if(searcher.getEstimatedResultCount() == 0) + { + response->set_code(MHD_HTTP_NO_CONTENT); + } + + return response; } catch (const std::exception& e) { std::cerr << e.what() << std::endl; + return build_500(e.what()); } - - //changing status code if no result obtained - if(searcher.getEstimatedResultCount() == 0) - { - response.set_code(MHD_HTTP_NO_CONTENT); - } - - return response; } std::unique_ptr InternalServer::handle_random(const RequestContext& request) @@ -703,9 +703,9 @@ std::unique_ptr InternalServer::handle_catalog(const RequestContext& r return build_404(request, ""); } - auto response = Response::build(*this); - response->set_compress(true); if (url == "searchdescription.xml") { + auto response = Response::build(*this); + response->set_compress(true); response->set_template(RESOURCE::opensearchdescription_xml, get_default_data()); response->set_mimeType("application/opensearchdescription+xml"); return response; @@ -716,7 +716,6 @@ std::unique_ptr InternalServer::handle_catalog(const RequestContext& r opdsDumper.setRootLocation(m_root); opdsDumper.setSearchDescriptionUrl("catalog/searchdescription.xml"); opdsDumper.setLibrary(mp_library); - response->set_mimeType("application/atom+xml; profile=opds-catalog; kind=acquisition; charset=utf-8"); std::vector bookIdsToDump; if (url == "root.xml") { opdsDumper.setTitle("All zims"); @@ -764,7 +763,11 @@ std::unique_ptr InternalServer::handle_catalog(const RequestContext& r } opdsDumper.setId(kiwix::to_string(uuid)); - response->set_content(opdsDumper.dumpOPDSFeed(bookIdsToDump)); + auto response = ContentResponse::build( + *this, + opdsDumper.dumpOPDSFeed(bookIdsToDump), + "application/atom+xml; profile=opds-catalog; kind=acquisition; charset=utf-8"); + response->set_compress(true); return response; } diff --git a/src/server/internalServer.h b/src/server/internalServer.h index 9ac9f4aae..ba34fd3d2 100644 --- a/src/server/internalServer.h +++ b/src/server/internalServer.h @@ -107,6 +107,7 @@ class InternalServer { friend std::unique_ptr Response::build(const InternalServer& server); friend std::unique_ptr RedirectionResponse::build(const InternalServer& server, const std::string& redirectionUrl); + friend std::unique_ptr ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype); }; } diff --git a/src/server/response.cpp b/src/server/response.cpp index f40321aae..9fac355b2 100644 --- a/src/server/response.cpp +++ b/src/server/response.cpp @@ -320,11 +320,7 @@ MHD_Result Response::send(const RequestContext& request, MHD_Connection* connect } void Response::set_template(const std::string& template_str, kainjow::mustache::data data) { - set_content(render_template(template_str, data)); -} - -void Response::set_content(const std::string& content) { - m_content = content; + m_content = render_template(template_str, data); m_mode = ResponseMode::RAW_CONTENT; } @@ -342,11 +338,12 @@ void Response::set_entry(const Entry& entry, const RequestContext& request) { zim::Blob raw_content = entry.getBlob(); const std::string content = string(raw_content.data(), raw_content.size()); - set_content(content); + m_content = content; + m_mode = ResponseMode::RAW_CONTENT; set_compress(true); } else if ( m_byteRange.kind() == ByteRange::RESOLVED_UNSATISFIABLE ) { set_code(416); - set_content(""); + m_content = ""; m_mode = ResponseMode::ERROR_RESPONSE; } } @@ -383,6 +380,26 @@ MHD_Response* RedirectionResponse::create_mhd_response(const RequestContext& req return response; } +ContentResponse::ContentResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype) : + Response(root, verbose, withTaskbar, withLibraryButton, blockExternalLinks) +{ + m_content = content; + m_mimeType = mimetype; + m_mode = ResponseMode::RAW_CONTENT; +} + +std::unique_ptr ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype) +{ + return std::unique_ptr(new ContentResponse( + server.m_root, + server.m_verbose.load(), + server.m_withTaskbar, + server.m_withLibraryButton, + server.m_blockExternalLinks, + content, + mimetype)); +} + } diff --git a/src/server/response.h b/src/server/response.h index 6b46b3be4..837de396b 100644 --- a/src/server/response.h +++ b/src/server/response.h @@ -53,7 +53,6 @@ class Response { MHD_Result send(const RequestContext& request, MHD_Connection* connection); void set_template(const std::string& template_str, kainjow::mustache::data data); - void set_content(const std::string& content); void set_entry(const Entry& entry, const RequestContext& request); @@ -112,6 +111,13 @@ class RedirectionResponse : public Response { std::string m_redirectionUrl; }; +class ContentResponse : public Response { + public: + ContentResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype); + + static std::unique_ptr build(const InternalServer& server, const std::string& content, const std::string& mimetype); +}; + } #endif //KIWIXLIB_SERVER_RESPONSE_H From 7b2ee3743755614ff84e2b6434edc6a71b70cdb4 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 29 Jul 2020 14:03:03 +0200 Subject: [PATCH 06/16] Move the entry response to its own class. --- src/server/internalServer.cpp | 4 +- src/server/internalServer.h | 2 + src/server/response.cpp | 123 ++++++++++++++++++++-------------- src/server/response.h | 22 ++++-- 4 files changed, 91 insertions(+), 60 deletions(-) diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index 7c7d23386..4f11b3d77 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -840,9 +840,7 @@ std::unique_ptr InternalServer::handle_content(const RequestContext& r return build_404(request, bookName); } - auto response = Response::build(*this); - - response->set_entry(entry, request); + auto response = EntryResponse::build(*this, request, entry); response->set_taskbar(bookName, reader->getTitle()); if (m_verbose.load()) { diff --git a/src/server/internalServer.h b/src/server/internalServer.h index ba34fd3d2..7e27962fe 100644 --- a/src/server/internalServer.h +++ b/src/server/internalServer.h @@ -108,6 +108,8 @@ class InternalServer { friend std::unique_ptr Response::build(const InternalServer& server); friend std::unique_ptr RedirectionResponse::build(const InternalServer& server, const std::string& redirectionUrl); friend std::unique_ptr ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype); + friend std::unique_ptr EntryResponse::build(const InternalServer& server, const RequestContext& request, const Entry& entry); + }; } diff --git a/src/server/response.cpp b/src/server/response.cpp index 9fac355b2..f1293f059 100644 --- a/src/server/response.cpp +++ b/src/server/response.cpp @@ -253,31 +253,6 @@ Response::create_raw_content_mhd_response(const RequestContext& request) } -MHD_Response* -Response::create_entry_mhd_response() const -{ - const auto content_length = m_byteRange.length(); - MHD_Response* response = MHD_create_response_from_callback(content_length, - 16384, - callback_reader_from_entry, - new RunningResponse(m_entry, m_byteRange.first()), - callback_free_response); - MHD_add_response_header(response, - MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str()); - MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes"); - if ( m_byteRange.kind() == ByteRange::RESOLVED_PARTIAL_CONTENT ) { - std::ostringstream oss; - oss << "bytes " << m_byteRange.first() << "-" << m_byteRange.last() - << "/" << m_entry.getSize(); - - MHD_add_response_header(response, - MHD_HTTP_HEADER_CONTENT_RANGE, oss.str().c_str()); - } - - MHD_add_response_header(response, - MHD_HTTP_HEADER_CONTENT_LENGTH, kiwix::to_string(content_length).c_str()); - return response; -} MHD_Response* Response::create_mhd_response(const RequestContext& request) @@ -288,9 +263,6 @@ Response::create_mhd_response(const RequestContext& request) case ResponseMode::RAW_CONTENT : return create_raw_content_mhd_response(request); - - case ResponseMode::ENTRY : - return create_entry_mhd_response(); } return nullptr; } @@ -324,30 +296,6 @@ void Response::set_template(const std::string& template_str, kainjow::mustache:: m_mode = ResponseMode::RAW_CONTENT; } -void Response::set_entry(const Entry& entry, const RequestContext& request) { - m_entry = entry; - m_mode = ResponseMode::ENTRY; - - const std::string mimeType = get_mime_type(entry); - set_mimeType(mimeType); - set_cacheable(); - - m_byteRange = request.get_range().resolve(entry.getSize()); - const bool noRange = m_byteRange.kind() == ByteRange::RESOLVED_FULL_CONTENT; - if ( noRange && is_compressible_mime_type(mimeType) ) { - zim::Blob raw_content = entry.getBlob(); - const std::string content = string(raw_content.data(), raw_content.size()); - - m_content = content; - m_mode = ResponseMode::RAW_CONTENT; - set_compress(true); - } else if ( m_byteRange.kind() == ByteRange::RESOLVED_UNSATISFIABLE ) { - set_code(416); - m_content = ""; - m_mode = ResponseMode::ERROR_RESPONSE; - } -} - void Response::set_taskbar(const std::string& bookName, const std::string& bookTitle) { m_bookName = bookName; @@ -400,6 +348,77 @@ std::unique_ptr ContentResponse::build(const InternalServer& server, c mimetype)); } +EntryResponse::EntryResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const Entry& entry, const std::string& mimetype, const ByteRange& byterange) : + Response(root, verbose, withTaskbar, withLibraryButton, blockExternalLinks), + m_entry(entry) +{ + m_mimeType = mimetype; + m_byteRange = byterange; + m_mode = ResponseMode::RAW_CONTENT; // We don't care about raw, but we must init it to something else than error. + set_cacheable(); +} + +std::unique_ptr EntryResponse::build(const InternalServer& server, const RequestContext& request, const Entry& entry) +{ + const std::string mimetype = get_mime_type(entry); + auto byteRange = request.get_range().resolve(entry.getSize()); + const bool noRange = byteRange.kind() == ByteRange::RESOLVED_FULL_CONTENT; + if (noRange && is_compressible_mime_type(mimetype)) { + // Return a contentResponse + zim::Blob raw_content = entry.getBlob(); + const std::string content = string(raw_content.data(), raw_content.size()); + auto response = ContentResponse::build(server, content, mimetype); + response->set_cacheable(); + response->set_compress(true); + response->m_byteRange = byteRange; + return response; + } + + if (byteRange.kind() == ByteRange::RESOLVED_UNSATISFIABLE) { + auto response = ContentResponse::build(server, "", mimetype); + response->set_cacheable(); + response->set_code(416); + response->m_byteRange = byteRange; + response->m_mode = ResponseMode::ERROR_RESPONSE; + return response; + } + + return std::unique_ptr(new EntryResponse( + server.m_root, + server.m_verbose.load(), + server.m_withTaskbar, + server.m_withLibraryButton, + server.m_blockExternalLinks, + entry, + mimetype, + byteRange)); +} + +MHD_Response* +EntryResponse::create_mhd_response(const RequestContext& request) +{ + const auto content_length = m_byteRange.length(); + MHD_Response* response = MHD_create_response_from_callback(content_length, + 16384, + callback_reader_from_entry, + new RunningResponse(m_entry, m_byteRange.first()), + callback_free_response); + MHD_add_response_header(response, + MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str()); + MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes"); + if ( m_byteRange.kind() == ByteRange::RESOLVED_PARTIAL_CONTENT ) { + std::ostringstream oss; + oss << "bytes " << m_byteRange.first() << "-" << m_byteRange.last() + << "/" << m_entry.getSize(); + + MHD_add_response_header(response, + MHD_HTTP_HEADER_CONTENT_RANGE, oss.str().c_str()); + } + + MHD_add_response_header(response, + MHD_HTTP_HEADER_CONTENT_LENGTH, kiwix::to_string(content_length).c_str()); + return response; +} } diff --git a/src/server/response.h b/src/server/response.h index 837de396b..9134e85ba 100644 --- a/src/server/response.h +++ b/src/server/response.h @@ -37,12 +37,13 @@ namespace kiwix { enum class ResponseMode { ERROR_RESPONSE, RAW_CONTENT, - ENTRY }; class InternalServer; class RequestContext; +class EntryResponse; + class Response { public: Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks); @@ -53,8 +54,6 @@ class Response { MHD_Result send(const RequestContext& request, MHD_Connection* connection); void set_template(const std::string& template_str, kainjow::mustache::data data); - void set_entry(const Entry& entry, const RequestContext& request); - void set_mimeType(const std::string& mimeType) { m_mimeType = mimeType; } void set_code(int code) { m_returnCode = code; } @@ -77,14 +76,12 @@ class Response { virtual MHD_Response* create_mhd_response(const RequestContext& request); MHD_Response* create_error_response(const RequestContext& request) const; MHD_Response* create_raw_content_mhd_response(const RequestContext& request); - MHD_Response* create_entry_mhd_response() const; protected: // data bool m_verbose; ResponseMode m_mode; std::string m_root; std::string m_content; - Entry m_entry; std::string m_mimeType; int m_returnCode; bool m_withTaskbar; @@ -95,6 +92,8 @@ class Response { std::string m_bookTitle; ByteRange m_byteRange; ETag m_etag; + + friend class EntryResponse; // temporary to allow the builder to change m_mode }; @@ -118,6 +117,19 @@ class ContentResponse : public Response { static std::unique_ptr build(const InternalServer& server, const std::string& content, const std::string& mimetype); }; +class EntryResponse : public Response { + public: + EntryResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const Entry& entry, const std::string& mimetype, const ByteRange& byterange); + + static std::unique_ptr build(const InternalServer& server, const RequestContext& request, const Entry& entry); + + private: + MHD_Response* create_mhd_response(const RequestContext& request); + + Entry m_entry; + +}; + } #endif //KIWIXLIB_SERVER_RESPONSE_H From eee621d15ba3f41fc57bf5a804ee2f5d321bedb5 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 29 Jul 2020 15:38:19 +0200 Subject: [PATCH 07/16] Move small utilities method to create response in Response class. --- src/server/internalServer.cpp | 93 +++++++++-------------------------- src/server/internalServer.h | 4 +- src/server/response.cpp | 66 +++++++++++++++++++------ src/server/response.h | 11 ++--- 4 files changed, 79 insertions(+), 95 deletions(-) diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index 4f11b3d77..140dc1e94 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -230,24 +230,15 @@ MHD_Result InternalServer::handlerCallback(struct MHD_Connection* connection, return ret; } -std::unique_ptr InternalServer::build_304(const RequestContext& request, const ETag& etag) const -{ - // This is just to set the m_mode. - auto response = ContentResponse::build(*this, "", ""); - response->set_code(MHD_HTTP_NOT_MODIFIED); - response->set_etag(etag); - return response; -} - std::unique_ptr InternalServer::handle_request(const RequestContext& request) { try { if (! request.is_valid_url()) - return build_404(request, ""); + return Response::build_404(*this, request, ""); const ETag etag = get_matching_if_none_match_etag(request); if ( etag ) - return build_304(request, etag); + return Response::build_304(*this, etag); if (kiwix::startsWith(request.get_url(), "/skin/")) return handle_skin(request); @@ -273,10 +264,10 @@ std::unique_ptr InternalServer::handle_request(const RequestContext& r return handle_content(request); } catch (std::exception& e) { fprintf(stderr, "===== Unhandled error : %s\n", e.what()); - return build_500(e.what()); + return Response::build_500(*this, e.what()); } catch (...) { fprintf(stderr, "===== Unhandled unknown error\n"); - return build_500("Unknown error"); + return Response::build_500(*this, "Unknown error"); } } @@ -287,34 +278,6 @@ MustacheData InternalServer::get_default_data() const return data; } -std::unique_ptr InternalServer::build_404(const RequestContext& request, - const std::string& bookName) -{ - MustacheData results; - results.set("url", request.get_full_url()); - - auto response = Response::build(*this); - response->set_template(RESOURCE::templates::_404_html, results); - response->set_mimeType("text/html"); - response->set_code(MHD_HTTP_NOT_FOUND); - response->set_compress(true); - response->set_taskbar(bookName, ""); - - return response; -} - -std::unique_ptr InternalServer::build_500(const std::string& msg) -{ - MustacheData data; - data.set("error", msg); - std::unique_ptr response( - new Response(m_root, true, false, false, false)); - response->set_template(RESOURCE::templates::_500_html, data); - response->set_mimeType("text/html"); - response->set_code(MHD_HTTP_INTERNAL_SERVER_ERROR); - return response; -} - MustacheData InternalServer::homepage_data() const { auto data = get_default_data(); @@ -359,9 +322,7 @@ InternalServer::get_matching_if_none_match_etag(const RequestContext& r) const std::unique_ptr InternalServer::build_homepage(const RequestContext& request) { - auto response = Response::build(*this); - response->set_template(RESOURCE::templates::index_html, homepage_data()); - response->set_mimeType("text/html; charset=utf-8"); + auto response = ContentResponse::build(*this, RESOURCE::templates::index_html, homepage_data(), "text/html; charset=utf-8"); response->set_compress(true); return response; } @@ -378,11 +339,11 @@ std::unique_ptr InternalServer::handle_meta(const RequestContext& requ meta_name = request.get_argument("name"); reader = mp_library->getReaderById(bookId); } catch (const std::out_of_range& e) { - return build_404(request, bookName); + return Response::build_404(*this, request, bookName); } if (reader == nullptr) { - return build_404(request, bookName); + return Response::build_404(*this, request, bookName); } std::string content; @@ -407,7 +368,7 @@ std::unique_ptr InternalServer::handle_meta(const RequestContext& requ } else if (meta_name == "favicon") { reader->getFavicon(content, mimeType); } else { - return build_404(request, bookName); + return Response::build_404(*this, request, bookName); } auto response = ContentResponse::build(*this, content, mimeType); @@ -437,7 +398,7 @@ std::unique_ptr InternalServer::handle_suggest(const RequestContext& r term = request.get_argument("term"); reader = mp_library->getReaderById(bookId); } catch (const std::out_of_range&) { - return build_404(request, bookName); + return Response::build_404(*this, request, bookName); } if (m_verbose.load()) { @@ -474,9 +435,7 @@ std::unique_ptr InternalServer::handle_suggest(const RequestContext& r auto data = get_default_data(); data.set("suggestions", results); - auto response = Response::build(*this); - response->set_template(RESOURCE::templates::suggestion_json, data); - response->set_mimeType("application/json; charset=utf-8"); + auto response = ContentResponse::build(*this, RESOURCE::templates::suggestion_json, data, "application/json; charset=utf-8"); response->set_compress(true); return response; } @@ -497,7 +456,7 @@ std::unique_ptr InternalServer::handle_skin(const RequestContext& requ response->set_cacheable(); return response; } catch (const ResourceNotFound& e) { - return build_404(request, ""); + return Response::build_404(*this, request, ""); } } @@ -566,11 +525,9 @@ std::unique_ptr InternalServer::handle_search(const RequestContext& re || (patternString.empty() && ! has_geo_query) ) { auto data = get_default_data(); data.set("pattern", encodeDiples(patternString)); - auto response = Response::build(*this); - response->set_mimeType("text/html; charset=utf-8"); + auto response = ContentResponse::build(*this, RESOURCE::templates::no_search_result_html, data, "text/html; charset=utf-8"); response->set_taskbar(bookName, reader ? reader->getTitle() : ""); response->set_compress(true); - response->set_template(RESOURCE::templates::no_search_result_html, data); response->set_code(MHD_HTTP_NOT_FOUND); return response; } @@ -632,7 +589,7 @@ std::unique_ptr InternalServer::handle_search(const RequestContext& re return response; } catch (const std::exception& e) { std::cerr << e.what() << std::endl; - return build_500(e.what()); + return Response::build_500(*this, e.what()); } } @@ -650,18 +607,18 @@ std::unique_ptr InternalServer::handle_random(const RequestContext& re bookId = mp_nameMapper->getIdForName(bookName); reader = mp_library->getReaderById(bookId); } catch (const std::out_of_range&) { - return build_404(request, bookName); + return Response::build_404(*this, request, bookName); } if (reader == nullptr) { - return build_404(request, bookName); + return Response::build_404(*this, request, bookName); } try { auto entry = reader->getRandomPage(); return build_redirect(bookName, entry.getFinalEntry()); } catch(kiwix::NoEntry& e) { - return build_404(request, bookName); + return Response::build_404(*this, request, bookName); } } @@ -673,13 +630,11 @@ std::unique_ptr InternalServer::handle_captured_external(const Request } catch (const std::out_of_range& e) {} if (source.empty()) - return build_404(request, ""); + return Response::build_404(*this, request, ""); auto data = get_default_data(); data.set("source", source); - auto response = Response::build(*this); - response->set_template(RESOURCE::templates::captured_external_html, data); - response->set_mimeType("text/html; charset=utf-8"); + auto response = ContentResponse::build(*this, RESOURCE::templates::captured_external_html, data, "text/html; charset=utf-8"); response->set_compress(true); return response; } @@ -696,18 +651,16 @@ std::unique_ptr InternalServer::handle_catalog(const RequestContext& r host = request.get_header("Host"); url = request.get_url_part(1); } catch (const std::out_of_range&) { - return build_404(request, ""); + return Response::build_404(*this, request, ""); } if (url != "searchdescription.xml" && url != "root.xml" && url != "search") { - return build_404(request, ""); + return Response::build_404(*this, request, ""); } if (url == "searchdescription.xml") { - auto response = Response::build(*this); + auto response = ContentResponse::build(*this, RESOURCE::opensearchdescription_xml, get_default_data(), "application/opensearchdescription+xml"); response->set_compress(true); - response->set_template(RESOURCE::opensearchdescription_xml, get_default_data()); - response->set_mimeType("application/opensearchdescription+xml"); return response; } @@ -816,7 +769,7 @@ std::unique_ptr InternalServer::handle_content(const RequestContext& r const std::shared_ptr reader = get_reader(bookName); if (reader == nullptr) { - return build_404(request, bookName); + return Response::build_404(*this, request, bookName); } auto urlStr = request.get_url().substr(bookName.size()+1); @@ -837,7 +790,7 @@ std::unique_ptr InternalServer::handle_content(const RequestContext& r if (m_verbose.load()) printf("Failed to find %s\n", urlStr.c_str()); - return build_404(request, bookName); + return Response::build_404(*this, request, bookName); } auto response = EntryResponse::build(*this, request, entry); diff --git a/src/server/internalServer.h b/src/server/internalServer.h index 7e27962fe..21d773548 100644 --- a/src/server/internalServer.h +++ b/src/server/internalServer.h @@ -68,9 +68,6 @@ class InternalServer { private: // functions std::unique_ptr handle_request(const RequestContext& request); - std::unique_ptr build_500(const std::string& msg); - std::unique_ptr build_404(const RequestContext& request, const std::string& zimName); - std::unique_ptr build_304(const RequestContext& request, const ETag& etag) const; std::unique_ptr build_redirect(const std::string& bookName, const kiwix::Entry& entry) const; std::unique_ptr build_homepage(const RequestContext& request); std::unique_ptr handle_skin(const RequestContext& request); @@ -109,6 +106,7 @@ class InternalServer { friend std::unique_ptr RedirectionResponse::build(const InternalServer& server, const std::string& redirectionUrl); friend std::unique_ptr ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype); friend std::unique_ptr EntryResponse::build(const InternalServer& server, const RequestContext& request, const Entry& entry); + friend std::unique_ptr Response::build_500(const InternalServer& server, const std::string& msg); }; diff --git a/src/server/response.cpp b/src/server/response.cpp index f1293f059..7c23d192b 100644 --- a/src/server/response.cpp +++ b/src/server/response.cpp @@ -22,6 +22,17 @@ namespace { // some utilities +std::string render_template(const std::string& template_str, kainjow::mustache::data data) +{ + kainjow::mustache::mustache tmpl(template_str); + kainjow::mustache::data urlencode{kainjow::mustache::lambda2{ + [](const std::string& str,const kainjow::mustache::renderer& r) { return urlEncode(r(str), true); }}}; + data.set("urlencoded", urlencode); + std::stringstream ss; + tmpl.render(data, [&ss](const std::string& str) { ss << str; }); + return ss.str(); +} + std::string get_mime_type(const kiwix::Entry& entry) { try { @@ -45,6 +56,7 @@ bool is_compressible_mime_type(const std::string& mimeType) Response::Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks) : m_verbose(verbose), + m_mode(ResponseMode::RAW_CONTENT), m_root(root), m_content(""), m_mimeType(""), @@ -67,6 +79,39 @@ std::unique_ptr Response::build(const InternalServer& server) server.m_blockExternalLinks)); } +std::unique_ptr Response::build_304(const InternalServer& server, const ETag& etag) +{ + auto response = Response::build(server); + response->set_code(MHD_HTTP_NOT_MODIFIED); + response->m_etag = etag; + return response; +} + +std::unique_ptr Response::build_404(const InternalServer& server, const RequestContext& request, const std::string& bookName) +{ + MustacheData results; + results.set("url", request.get_full_url()); + + auto response = ContentResponse::build(server, RESOURCE::templates::_404_html, results, "text/html"); + response->set_code(MHD_HTTP_NOT_FOUND); + response->set_compress(true); + response->set_taskbar(bookName, ""); + + return response; +} + +std::unique_ptr Response::build_500(const InternalServer& server, const std::string& msg) +{ + MustacheData data; + data.set("error", msg); + auto content = render_template(RESOURCE::templates::_500_html, data); + std::unique_ptr response ( + new ContentResponse(server.m_root, true, false, false, false, content, "text/html")); + response->set_code(MHD_HTTP_INTERNAL_SERVER_ERROR); + return response; +} + + static MHD_Result print_key_value (void *cls, enum MHD_ValueKind kind, const char *key, const char *value) @@ -124,16 +169,6 @@ void print_response_info(int retCode, MHD_Response* response) } -std::string render_template(const std::string& template_str, kainjow::mustache::data data) -{ - kainjow::mustache::mustache tmpl(template_str); - kainjow::mustache::data urlencode{kainjow::mustache::lambda2{ - [](const std::string& str,const kainjow::mustache::renderer& r) { return urlEncode(r(str), true); }}}; - data.set("urlencoded", urlencode); - std::stringstream ss; - tmpl.render(data, [&ss](const std::string& str) { ss << str; }); - return ss.str(); -} void Response::introduce_taskbar() @@ -291,11 +326,6 @@ MHD_Result Response::send(const RequestContext& request, MHD_Connection* connect return ret; } -void Response::set_template(const std::string& template_str, kainjow::mustache::data data) { - m_content = render_template(template_str, data); - m_mode = ResponseMode::RAW_CONTENT; -} - void Response::set_taskbar(const std::string& bookName, const std::string& bookTitle) { m_bookName = bookName; @@ -348,6 +378,12 @@ std::unique_ptr ContentResponse::build(const InternalServer& server, c mimetype)); } +std::unique_ptr ContentResponse::build(const InternalServer& server, const std::string& template_str, kainjow::mustache::data data, const std::string& mimetype) { + auto content = render_template(template_str, data); + return ContentResponse::build(server, content, mimetype); +} + + EntryResponse::EntryResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const Entry& entry, const std::string& mimetype, const ByteRange& byterange) : Response(root, verbose, withTaskbar, withLibraryButton, blockExternalLinks), m_entry(entry) diff --git a/src/server/response.h b/src/server/response.h index 9134e85ba..dd9d82ebc 100644 --- a/src/server/response.h +++ b/src/server/response.h @@ -50,16 +50,15 @@ class Response { virtual ~Response() = default; static std::unique_ptr build(const InternalServer& server); + static std::unique_ptr build_304(const InternalServer& server, const ETag& etag); + static std::unique_ptr build_404(const InternalServer& server, const RequestContext& request, const std::string& bookName); + static std::unique_ptr build_500(const InternalServer& server, const std::string& msg); MHD_Result send(const RequestContext& request, MHD_Connection* connection); - void set_template(const std::string& template_str, kainjow::mustache::data data); - - void set_mimeType(const std::string& mimeType) { m_mimeType = mimeType; } void set_code(int code) { m_returnCode = code; } void set_cacheable() { m_etag.set_option(ETag::CACHEABLE_ENTITY); } void set_server_id(const std::string& id) { m_etag.set_server_id(id); } - void set_etag(const ETag& etag) { m_etag = etag; } void set_compress(bool compress) { m_compress = compress; } void set_taskbar(const std::string& bookName, const std::string& bookTitle); @@ -100,7 +99,6 @@ class Response { class RedirectionResponse : public Response { public: RedirectionResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& redirectionUrl); - static std::unique_ptr build(const InternalServer& server, const std::string& redirectionUrl); @@ -113,14 +111,13 @@ class RedirectionResponse : public Response { class ContentResponse : public Response { public: ContentResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype); - static std::unique_ptr build(const InternalServer& server, const std::string& content, const std::string& mimetype); + static std::unique_ptr build(const InternalServer& server, const std::string& template_str, kainjow::mustache::data data, const std::string& mimetype); }; class EntryResponse : public Response { public: EntryResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const Entry& entry, const std::string& mimetype, const ByteRange& byterange); - static std::unique_ptr build(const InternalServer& server, const RequestContext& request, const Entry& entry); private: From a3939e9a05bb8440549b4470e145c5d4301e29fd Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 29 Jul 2020 16:33:34 +0200 Subject: [PATCH 08/16] Move all the content code in the ContentResponse. --- src/server/internalServer.cpp | 21 ++----- src/server/internalServer.h | 2 +- src/server/response.cpp | 101 +++++++++++----------------------- src/server/response.h | 57 ++++++++++--------- 4 files changed, 70 insertions(+), 111 deletions(-) diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index 140dc1e94..d28ff55e1 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -322,9 +322,7 @@ InternalServer::get_matching_if_none_match_etag(const RequestContext& r) const std::unique_ptr InternalServer::build_homepage(const RequestContext& request) { - auto response = ContentResponse::build(*this, RESOURCE::templates::index_html, homepage_data(), "text/html; charset=utf-8"); - response->set_compress(true); - return response; + return ContentResponse::build(*this, RESOURCE::templates::index_html, homepage_data(), "text/html; charset=utf-8"); } std::unique_ptr InternalServer::handle_meta(const RequestContext& request) @@ -372,7 +370,6 @@ std::unique_ptr InternalServer::handle_meta(const RequestContext& requ } auto response = ContentResponse::build(*this, content, mimeType); - response->set_compress(false); response->set_cacheable(); return response; } @@ -436,7 +433,6 @@ std::unique_ptr InternalServer::handle_suggest(const RequestContext& r data.set("suggestions", results); auto response = ContentResponse::build(*this, RESOURCE::templates::suggestion_json, data, "application/json; charset=utf-8"); - response->set_compress(true); return response; } @@ -452,7 +448,6 @@ std::unique_ptr InternalServer::handle_skin(const RequestContext& requ *this, getResource(resourceName), getMimeTypeForFile(resourceName)); - response->set_compress(true); response->set_cacheable(); return response; } catch (const ResourceNotFound& e) { @@ -527,7 +522,6 @@ std::unique_ptr InternalServer::handle_search(const RequestContext& re data.set("pattern", encodeDiples(patternString)); auto response = ContentResponse::build(*this, RESOURCE::templates::no_search_result_html, data, "text/html; charset=utf-8"); response->set_taskbar(bookName, reader ? reader->getTitle() : ""); - response->set_compress(true); response->set_code(MHD_HTTP_NOT_FOUND); return response; } @@ -579,7 +573,6 @@ std::unique_ptr InternalServer::handle_search(const RequestContext& re renderer.setPageLength(pageLength); auto response = ContentResponse::build(*this, renderer.getHtml(), "text/html; charset=utf-8"); response->set_taskbar(bookName, reader ? reader->getTitle() : ""); - response->set_compress(true); //changing status code if no result obtained if(searcher.getEstimatedResultCount() == 0) { @@ -634,9 +627,7 @@ std::unique_ptr InternalServer::handle_captured_external(const Request auto data = get_default_data(); data.set("source", source); - auto response = ContentResponse::build(*this, RESOURCE::templates::captured_external_html, data, "text/html; charset=utf-8"); - response->set_compress(true); - return response; + return ContentResponse::build(*this, RESOURCE::templates::captured_external_html, data, "text/html; charset=utf-8"); } std::unique_ptr InternalServer::handle_catalog(const RequestContext& request) @@ -660,7 +651,6 @@ std::unique_ptr InternalServer::handle_catalog(const RequestContext& r if (url == "searchdescription.xml") { auto response = ContentResponse::build(*this, RESOURCE::opensearchdescription_xml, get_default_data(), "application/opensearchdescription+xml"); - response->set_compress(true); return response; } @@ -720,7 +710,6 @@ std::unique_ptr InternalServer::handle_catalog(const RequestContext& r *this, opdsDumper.dumpOPDSFeed(bookIdsToDump), "application/atom+xml; profile=opds-catalog; kind=acquisition; charset=utf-8"); - response->set_compress(true); return response; } @@ -794,11 +783,13 @@ std::unique_ptr InternalServer::handle_content(const RequestContext& r } auto response = EntryResponse::build(*this, request, entry); - response->set_taskbar(bookName, reader->getTitle()); + try { + dynamic_cast(*response).set_taskbar(bookName, reader->getTitle()); + } catch (std::bad_cast& e) {} if (m_verbose.load()) { printf("Found %s\n", entry.getPath().c_str()); - printf("mimeType: %s\n", response->get_mimeType().c_str()); + printf("mimeType: %s\n", entry.getMimetype().c_str()); } return response; diff --git a/src/server/internalServer.h b/src/server/internalServer.h index 21d773548..1e2ed994c 100644 --- a/src/server/internalServer.h +++ b/src/server/internalServer.h @@ -104,7 +104,7 @@ class InternalServer { friend std::unique_ptr Response::build(const InternalServer& server); friend std::unique_ptr RedirectionResponse::build(const InternalServer& server, const std::string& redirectionUrl); - friend std::unique_ptr ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype); + friend std::unique_ptr ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype); friend std::unique_ptr EntryResponse::build(const InternalServer& server, const RequestContext& request, const Entry& entry); friend std::unique_ptr Response::build_500(const InternalServer& server, const std::string& msg); diff --git a/src/server/response.cpp b/src/server/response.cpp index 7c23d192b..006f539ad 100644 --- a/src/server/response.cpp +++ b/src/server/response.cpp @@ -54,34 +54,21 @@ bool is_compressible_mime_type(const std::string& mimeType) } // unnamed namespace -Response::Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks) +Response::Response(bool verbose) : m_verbose(verbose), - m_mode(ResponseMode::RAW_CONTENT), - m_root(root), - m_content(""), - m_mimeType(""), - m_returnCode(MHD_HTTP_OK), - m_withTaskbar(withTaskbar), - m_withLibraryButton(withLibraryButton), - m_blockExternalLinks(blockExternalLinks), - m_bookName(""), - m_bookTitle("") + m_mode(ResponseMode::OK_RESPONSE), + m_returnCode(MHD_HTTP_OK) { } std::unique_ptr Response::build(const InternalServer& server) { - return std::unique_ptr(new Response( - server.m_root, - server.m_verbose.load(), - server.m_withTaskbar, - server.m_withLibraryButton, - server.m_blockExternalLinks)); + return std::unique_ptr(new Response(server.m_verbose.load())); } std::unique_ptr Response::build_304(const InternalServer& server, const ETag& etag) { - auto response = Response::build(server); + auto response = ContentResponse::build(server, "", ""); response->set_code(MHD_HTTP_NOT_MODIFIED); response->m_etag = etag; return response; @@ -94,7 +81,6 @@ std::unique_ptr Response::build_404(const InternalServer& server, cons auto response = ContentResponse::build(server, RESOURCE::templates::_404_html, results, "text/html"); response->set_code(MHD_HTTP_NOT_FOUND); - response->set_compress(true); response->set_taskbar(bookName, ""); return response; @@ -171,7 +157,7 @@ void print_response_info(int retCode, MHD_Response* response) -void Response::introduce_taskbar() +void ContentResponse::introduce_taskbar() { kainjow::mustache::data data; data.set("root", m_root); @@ -193,7 +179,7 @@ void Response::introduce_taskbar() } -void Response::inject_externallinks_blocker() +void ContentResponse::inject_externallinks_blocker() { kainjow::mustache::data data; data.set("root", m_root); @@ -205,7 +191,7 @@ void Response::inject_externallinks_blocker() } bool -Response::can_compress(const RequestContext& request) const +ContentResponse::can_compress(const RequestContext& request) const { return request.can_compress() && is_compressible_mime_type(m_mimeType) @@ -213,14 +199,14 @@ Response::can_compress(const RequestContext& request) const } bool -Response::contentDecorationAllowed() const +ContentResponse::contentDecorationAllowed() const { return (startsWith(m_mimeType, "text/html") && m_mimeType.find(";raw=true") == std::string::npos); } MHD_Response* -Response::create_error_response(const RequestContext& request) const +Response::create_mhd_response(const RequestContext& request) { MHD_Response* response = MHD_create_response_from_buffer(0, NULL, MHD_RESPMEM_PERSISTENT); if ( m_returnCode == 416 ) { @@ -234,7 +220,7 @@ Response::create_error_response(const RequestContext& request) const } MHD_Response* -Response::create_raw_content_mhd_response(const RequestContext& request) +ContentResponse::create_mhd_response(const RequestContext& request) { if (contentDecorationAllowed()) { if (m_withTaskbar) { @@ -287,21 +273,6 @@ Response::create_raw_content_mhd_response(const RequestContext& request) return response; } - - -MHD_Response* -Response::create_mhd_response(const RequestContext& request) -{ - switch (m_mode) { - case ResponseMode::ERROR_RESPONSE: - return create_error_response(request); - - case ResponseMode::RAW_CONTENT : - return create_raw_content_mhd_response(request); - } - return nullptr; -} - MHD_Result Response::send(const RequestContext& request, MHD_Connection* connection) { MHD_Response* response = create_mhd_response(request); @@ -326,15 +297,15 @@ MHD_Result Response::send(const RequestContext& request, MHD_Connection* connect return ret; } -void Response::set_taskbar(const std::string& bookName, const std::string& bookTitle) +void ContentResponse::set_taskbar(const std::string& bookName, const std::string& bookTitle) { m_bookName = bookName; m_bookTitle = bookTitle; } -RedirectionResponse::RedirectionResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& redirectionUrl) : - Response(root, verbose, withTaskbar, withLibraryButton, blockExternalLinks), +RedirectionResponse::RedirectionResponse(bool verbose, const std::string& redirectionUrl): + Response(verbose), m_redirectionUrl(redirectionUrl) { m_returnCode = MHD_HTTP_FOUND; @@ -343,11 +314,7 @@ RedirectionResponse::RedirectionResponse(const std::string& root, bool verbose, std::unique_ptr RedirectionResponse::build(const InternalServer& server, const std::string& redirectionUrl) { return std::unique_ptr(new RedirectionResponse( - server.m_root, server.m_verbose.load(), - server.m_withTaskbar, - server.m_withLibraryButton, - server.m_blockExternalLinks, redirectionUrl)); } @@ -359,16 +326,21 @@ MHD_Response* RedirectionResponse::create_mhd_response(const RequestContext& req } ContentResponse::ContentResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype) : - Response(root, verbose, withTaskbar, withLibraryButton, blockExternalLinks) -{ - m_content = content; - m_mimeType = mimetype; - m_mode = ResponseMode::RAW_CONTENT; -} + Response(verbose), + m_root(root), + m_content(content), + m_mimeType(mimetype), + m_withTaskbar(withTaskbar), + m_withLibraryButton(withLibraryButton), + m_blockExternalLinks(blockExternalLinks), + m_compress(is_compressible_mime_type(mimetype)), + m_bookName(""), + m_bookTitle("") +{} -std::unique_ptr ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype) +std::unique_ptr ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype) { - return std::unique_ptr(new ContentResponse( + return std::unique_ptr(new ContentResponse( server.m_root, server.m_verbose.load(), server.m_withTaskbar, @@ -378,19 +350,18 @@ std::unique_ptr ContentResponse::build(const InternalServer& server, c mimetype)); } -std::unique_ptr ContentResponse::build(const InternalServer& server, const std::string& template_str, kainjow::mustache::data data, const std::string& mimetype) { +std::unique_ptr ContentResponse::build(const InternalServer& server, const std::string& template_str, kainjow::mustache::data data, const std::string& mimetype) { auto content = render_template(template_str, data); return ContentResponse::build(server, content, mimetype); } -EntryResponse::EntryResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const Entry& entry, const std::string& mimetype, const ByteRange& byterange) : - Response(root, verbose, withTaskbar, withLibraryButton, blockExternalLinks), - m_entry(entry) +EntryResponse::EntryResponse(bool verbose, const Entry& entry, const std::string& mimetype, const ByteRange& byterange) : + Response(verbose), + m_entry(entry), + m_mimeType(mimetype) { - m_mimeType = mimetype; m_byteRange = byterange; - m_mode = ResponseMode::RAW_CONTENT; // We don't care about raw, but we must init it to something else than error. set_cacheable(); } @@ -405,26 +376,20 @@ std::unique_ptr EntryResponse::build(const InternalServer& server, con const std::string content = string(raw_content.data(), raw_content.size()); auto response = ContentResponse::build(server, content, mimetype); response->set_cacheable(); - response->set_compress(true); response->m_byteRange = byteRange; return response; } if (byteRange.kind() == ByteRange::RESOLVED_UNSATISFIABLE) { - auto response = ContentResponse::build(server, "", mimetype); + auto response = Response::build(server); response->set_cacheable(); response->set_code(416); response->m_byteRange = byteRange; - response->m_mode = ResponseMode::ERROR_RESPONSE; return response; } return std::unique_ptr(new EntryResponse( - server.m_root, server.m_verbose.load(), - server.m_withTaskbar, - server.m_withLibraryButton, - server.m_blockExternalLinks, entry, mimetype, byteRange)); diff --git a/src/server/response.h b/src/server/response.h index dd9d82ebc..2f8995639 100644 --- a/src/server/response.h +++ b/src/server/response.h @@ -35,8 +35,8 @@ extern "C" { namespace kiwix { enum class ResponseMode { + OK_RESPONSE, ERROR_RESPONSE, - RAW_CONTENT, }; class InternalServer; @@ -46,7 +46,7 @@ class EntryResponse; class Response { public: - Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks); + Response(bool verbose); virtual ~Response() = default; static std::unique_ptr build(const InternalServer& server); @@ -59,36 +59,17 @@ class Response { void set_code(int code) { m_returnCode = code; } void set_cacheable() { m_etag.set_option(ETag::CACHEABLE_ENTITY); } void set_server_id(const std::string& id) { m_etag.set_server_id(id); } - void set_compress(bool compress) { m_compress = compress; } - void set_taskbar(const std::string& bookName, const std::string& bookTitle); int getReturnCode() const { return m_returnCode; } - std::string get_mimeType() const { return m_mimeType; } - - void introduce_taskbar(); - void inject_externallinks_blocker(); - - bool can_compress(const RequestContext& request) const; - bool contentDecorationAllowed() const; private: // functions virtual MHD_Response* create_mhd_response(const RequestContext& request); MHD_Response* create_error_response(const RequestContext& request) const; - MHD_Response* create_raw_content_mhd_response(const RequestContext& request); protected: // data bool m_verbose; ResponseMode m_mode; - std::string m_root; - std::string m_content; - std::string m_mimeType; int m_returnCode; - bool m_withTaskbar; - bool m_withLibraryButton; - bool m_blockExternalLinks; - bool m_compress; - std::string m_bookName; - std::string m_bookTitle; ByteRange m_byteRange; ETag m_etag; @@ -98,7 +79,7 @@ class Response { class RedirectionResponse : public Response { public: - RedirectionResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& redirectionUrl); + RedirectionResponse(bool verbose, const std::string& redirectionUrl); static std::unique_ptr build(const InternalServer& server, const std::string& redirectionUrl); @@ -111,20 +92,42 @@ class RedirectionResponse : public Response { class ContentResponse : public Response { public: ContentResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype); - static std::unique_ptr build(const InternalServer& server, const std::string& content, const std::string& mimetype); - static std::unique_ptr build(const InternalServer& server, const std::string& template_str, kainjow::mustache::data data, const std::string& mimetype); -}; + static std::unique_ptr build(const InternalServer& server, const std::string& content, const std::string& mimetype); + static std::unique_ptr build(const InternalServer& server, const std::string& template_str, kainjow::mustache::data data, const std::string& mimetype); + + void set_taskbar(const std::string& bookName, const std::string& bookTitle); + + private: + MHD_Response* create_mhd_response(const RequestContext& request); + + void introduce_taskbar(); + void inject_externallinks_blocker(); + bool can_compress(const RequestContext& request) const; + bool contentDecorationAllowed() const; + + + private: + std::string m_root; + std::string m_content; + std::string m_mimeType; + bool m_withTaskbar; + bool m_withLibraryButton; + bool m_blockExternalLinks; + bool m_compress; + std::string m_bookName; + std::string m_bookTitle; + }; class EntryResponse : public Response { public: - EntryResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const Entry& entry, const std::string& mimetype, const ByteRange& byterange); + EntryResponse(bool verbose, const Entry& entry, const std::string& mimetype, const ByteRange& byterange); static std::unique_ptr build(const InternalServer& server, const RequestContext& request, const Entry& entry); private: MHD_Response* create_mhd_response(const RequestContext& request); Entry m_entry; - + std::string m_mimeType; }; } From 6d5cddca12d90574fda25bceec6c993d737b1c6f Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 29 Jul 2020 17:49:04 +0200 Subject: [PATCH 09/16] Fix android compilation Android clang complains about the fact it cannot move the `std::unique_ptr` into a `std::unique_ptr&&` (for the implicit `std::unique_ptr` constructor). Let's help him a bit. --- src/server/internalServer.cpp | 14 +++++++------- src/server/response.cpp | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index d28ff55e1..a202d5670 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -371,7 +371,7 @@ std::unique_ptr InternalServer::handle_meta(const RequestContext& requ auto response = ContentResponse::build(*this, content, mimeType); response->set_cacheable(); - return response; + return std::move(response); } std::unique_ptr InternalServer::handle_suggest(const RequestContext& request) @@ -433,7 +433,7 @@ std::unique_ptr InternalServer::handle_suggest(const RequestContext& r data.set("suggestions", results); auto response = ContentResponse::build(*this, RESOURCE::templates::suggestion_json, data, "application/json; charset=utf-8"); - return response; + return std::move(response); } std::unique_ptr InternalServer::handle_skin(const RequestContext& request) @@ -449,7 +449,7 @@ std::unique_ptr InternalServer::handle_skin(const RequestContext& requ getResource(resourceName), getMimeTypeForFile(resourceName)); response->set_cacheable(); - return response; + return std::move(response); } catch (const ResourceNotFound& e) { return Response::build_404(*this, request, ""); } @@ -523,7 +523,7 @@ std::unique_ptr InternalServer::handle_search(const RequestContext& re auto response = ContentResponse::build(*this, RESOURCE::templates::no_search_result_html, data, "text/html; charset=utf-8"); response->set_taskbar(bookName, reader ? reader->getTitle() : ""); response->set_code(MHD_HTTP_NOT_FOUND); - return response; + return std::move(response); } Searcher searcher; @@ -579,7 +579,7 @@ std::unique_ptr InternalServer::handle_search(const RequestContext& re response->set_code(MHD_HTTP_NO_CONTENT); } - return response; + return std::move(response); } catch (const std::exception& e) { std::cerr << e.what() << std::endl; return Response::build_500(*this, e.what()); @@ -651,7 +651,7 @@ std::unique_ptr InternalServer::handle_catalog(const RequestContext& r if (url == "searchdescription.xml") { auto response = ContentResponse::build(*this, RESOURCE::opensearchdescription_xml, get_default_data(), "application/opensearchdescription+xml"); - return response; + return std::move(response); } zim::Uuid uuid; @@ -710,7 +710,7 @@ std::unique_ptr InternalServer::handle_catalog(const RequestContext& r *this, opdsDumper.dumpOPDSFeed(bookIdsToDump), "application/atom+xml; profile=opds-catalog; kind=acquisition; charset=utf-8"); - return response; + return std::move(response); } namespace diff --git a/src/server/response.cpp b/src/server/response.cpp index 006f539ad..591fd2582 100644 --- a/src/server/response.cpp +++ b/src/server/response.cpp @@ -71,7 +71,7 @@ std::unique_ptr Response::build_304(const InternalServer& server, cons auto response = ContentResponse::build(server, "", ""); response->set_code(MHD_HTTP_NOT_MODIFIED); response->m_etag = etag; - return response; + return std::move(response); } std::unique_ptr Response::build_404(const InternalServer& server, const RequestContext& request, const std::string& bookName) @@ -83,7 +83,7 @@ std::unique_ptr Response::build_404(const InternalServer& server, cons response->set_code(MHD_HTTP_NOT_FOUND); response->set_taskbar(bookName, ""); - return response; + return std::move(response); } std::unique_ptr Response::build_500(const InternalServer& server, const std::string& msg) @@ -377,7 +377,7 @@ std::unique_ptr EntryResponse::build(const InternalServer& server, con auto response = ContentResponse::build(server, content, mimetype); response->set_cacheable(); response->m_byteRange = byteRange; - return response; + return std::move(response); } if (byteRange.kind() == ByteRange::RESOLVED_UNSATISFIABLE) { From 8d6567d067660ae74cf5aba66a63f98166bbf75d Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 30 Jul 2020 11:21:50 +0200 Subject: [PATCH 10/16] Create a utility builder for 416 response. Also add a map in the response to store specific headers. --- src/server/response.cpp | 25 +++++++++++++++---------- src/server/response.h | 4 ++++ 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/server/response.cpp b/src/server/response.cpp index 591fd2582..e9733552b 100644 --- a/src/server/response.cpp +++ b/src/server/response.cpp @@ -86,6 +86,17 @@ std::unique_ptr Response::build_404(const InternalServer& server, cons return std::move(response); } +std::unique_ptr Response::build_416(const InternalServer& server, size_t resourceLength) +{ + auto response = Response::build(server); + response->set_code(MHD_HTTP_RANGE_NOT_SATISFIABLE); + std::ostringstream oss; + oss << "bytes */" << resourceLength; + response->add_header(MHD_HTTP_HEADER_CONTENT_RANGE, oss.str()); + + return response; +} + std::unique_ptr Response::build_500(const InternalServer& server, const std::string& msg) { MustacheData data; @@ -209,13 +220,6 @@ MHD_Response* Response::create_mhd_response(const RequestContext& request) { MHD_Response* response = MHD_create_response_from_buffer(0, NULL, MHD_RESPMEM_PERSISTENT); - if ( m_returnCode == 416 ) { - std::ostringstream oss; - oss << "bytes */" << m_byteRange.length(); - - MHD_add_response_header(response, - MHD_HTTP_HEADER_CONTENT_RANGE, oss.str().c_str()); - } return response; } @@ -285,6 +289,9 @@ MHD_Result Response::send(const RequestContext& request, MHD_Connection* connect if ( ! etag.empty() ) MHD_add_response_header(response, MHD_HTTP_HEADER_ETAG, etag.c_str()); } + for(auto& p: m_customHeaders) { + MHD_add_response_header(response, p.first.c_str(), p.second.c_str()); + } if (m_returnCode == MHD_HTTP_OK && m_byteRange.kind() == ByteRange::RESOLVED_PARTIAL_CONTENT) m_returnCode = MHD_HTTP_PARTIAL_CONTENT; @@ -381,10 +388,8 @@ std::unique_ptr EntryResponse::build(const InternalServer& server, con } if (byteRange.kind() == ByteRange::RESOLVED_UNSATISFIABLE) { - auto response = Response::build(server); + auto response = Response::build_416(server, entry.getSize()); response->set_cacheable(); - response->set_code(416); - response->m_byteRange = byteRange; return response; } diff --git a/src/server/response.h b/src/server/response.h index 2f8995639..e6f5fbeeb 100644 --- a/src/server/response.h +++ b/src/server/response.h @@ -22,6 +22,7 @@ #define KIWIXLIB_SERVER_RESPONSE_H #include +#include #include #include "byte_range.h" @@ -52,6 +53,7 @@ class Response { static std::unique_ptr build(const InternalServer& server); static std::unique_ptr build_304(const InternalServer& server, const ETag& etag); static std::unique_ptr build_404(const InternalServer& server, const RequestContext& request, const std::string& bookName); + static std::unique_ptr build_416(const InternalServer& server, size_t resourceLength); static std::unique_ptr build_500(const InternalServer& server, const std::string& msg); MHD_Result send(const RequestContext& request, MHD_Connection* connection); @@ -59,6 +61,7 @@ class Response { void set_code(int code) { m_returnCode = code; } void set_cacheable() { m_etag.set_option(ETag::CACHEABLE_ENTITY); } void set_server_id(const std::string& id) { m_etag.set_server_id(id); } + void add_header(const std::string& name, const std::string& value) { m_customHeaders[name] = value; } int getReturnCode() const { return m_returnCode; } @@ -72,6 +75,7 @@ class Response { int m_returnCode; ByteRange m_byteRange; ETag m_etag; + std::map m_customHeaders; friend class EntryResponse; // temporary to allow the builder to change m_mode }; From 9078f0ac6e0be3911ba087cebc7901e689162a4f Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 30 Jul 2020 15:24:46 +0200 Subject: [PATCH 11/16] Remove `ResponseMode`. --- src/server/response.cpp | 15 ++++++--------- src/server/response.h | 6 ------ 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/src/server/response.cpp b/src/server/response.cpp index e9733552b..8ee3799ee 100644 --- a/src/server/response.cpp +++ b/src/server/response.cpp @@ -56,7 +56,6 @@ bool is_compressible_mime_type(const std::string& mimeType) Response::Response(bool verbose) : m_verbose(verbose), - m_mode(ResponseMode::OK_RESPONSE), m_returnCode(MHD_HTTP_OK) { } @@ -281,14 +280,12 @@ MHD_Result Response::send(const RequestContext& request, MHD_Connection* connect { MHD_Response* response = create_mhd_response(request); - if ( m_mode != ResponseMode::ERROR_RESPONSE ) { - MHD_add_response_header(response, "Access-Control-Allow-Origin", "*"); - MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL, - m_etag.get_option(ETag::CACHEABLE_ENTITY) ? "max-age=2723040, public" : "no-cache, no-store, must-revalidate"); - const std::string etag = m_etag.get_etag(); - if ( ! etag.empty() ) - MHD_add_response_header(response, MHD_HTTP_HEADER_ETAG, etag.c_str()); - } + MHD_add_response_header(response, "Access-Control-Allow-Origin", "*"); + MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL, + m_etag.get_option(ETag::CACHEABLE_ENTITY) ? "max-age=2723040, public" : "no-cache, no-store, must-revalidate"); + const std::string etag = m_etag.get_etag(); + if ( ! etag.empty() ) + MHD_add_response_header(response, MHD_HTTP_HEADER_ETAG, etag.c_str()); for(auto& p: m_customHeaders) { MHD_add_response_header(response, p.first.c_str(), p.second.c_str()); } diff --git a/src/server/response.h b/src/server/response.h index e6f5fbeeb..0dbb40813 100644 --- a/src/server/response.h +++ b/src/server/response.h @@ -35,11 +35,6 @@ extern "C" { namespace kiwix { -enum class ResponseMode { - OK_RESPONSE, - ERROR_RESPONSE, -}; - class InternalServer; class RequestContext; @@ -71,7 +66,6 @@ class Response { protected: // data bool m_verbose; - ResponseMode m_mode; int m_returnCode; ByteRange m_byteRange; ETag m_etag; From 77123ac74c6f18c1da78e699ae6372b92b30100a Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 30 Jul 2020 15:37:38 +0200 Subject: [PATCH 12/16] Move the adding of 304 headers in 304 factory. This avoid us to create a ContentResponse just to have some correct headers. --- src/server/response.cpp | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/server/response.cpp b/src/server/response.cpp index 8ee3799ee..8cf21a740 100644 --- a/src/server/response.cpp +++ b/src/server/response.cpp @@ -67,10 +67,13 @@ std::unique_ptr Response::build(const InternalServer& server) std::unique_ptr Response::build_304(const InternalServer& server, const ETag& etag) { - auto response = ContentResponse::build(server, "", ""); + auto response = Response::build(server); response->set_code(MHD_HTTP_NOT_MODIFIED); response->m_etag = etag; - return std::move(response); + if ( etag.get_option(ETag::COMPRESSED_CONTENT) ) { + response->add_header(MHD_HTTP_HEADER_VARY, "Accept-Encoding"); + } + return response; } std::unique_ptr Response::build_404(const InternalServer& server, const RequestContext& request, const std::string& bookName) @@ -258,17 +261,9 @@ ContentResponse::create_mhd_response(const RequestContext& request) MHD_Response* response = MHD_create_response_from_buffer( m_content.size(), const_cast(m_content.data()), MHD_RESPMEM_MUST_COPY); - // At shis point m_etag.get_option(ETag::COMPRESSED_CONTENT) and - // shouldCompress can have different values. This can happen for a 304 (Not - // Modified) response generated while handling a conditional If-None-Match - // request. In that case the m_etag (together with its COMPRESSED_CONTENT - // option) is obtained from the ETag list of the If-None-Match header and the - // response has no body (which shouldn't be compressed). - if ( m_etag.get_option(ETag::COMPRESSED_CONTENT) ) { + if (shouldCompress) { MHD_add_response_header( response, MHD_HTTP_HEADER_VARY, "Accept-Encoding"); - } - if (shouldCompress) { MHD_add_response_header( response, MHD_HTTP_HEADER_CONTENT_ENCODING, "deflate"); } From 3352c95314b231b9b6cfb502e68f0a639d84b494 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 30 Jul 2020 15:59:59 +0200 Subject: [PATCH 13/16] Remove the `RedirectResponse` and use a basic `Response` with header. --- src/server/internalServer.cpp | 4 ++-- src/server/internalServer.h | 1 - src/server/response.cpp | 32 ++++++++++---------------------- src/server/response.h | 13 +------------ 4 files changed, 13 insertions(+), 37 deletions(-) diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index a202d5670..6961cf770 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -511,7 +511,7 @@ std::unique_ptr InternalServer::handle_search(const RequestContext& re /* If article found then redirect directly to it */ if (!patternCorrespondingUrl.empty()) { auto redirectUrl = m_root + "/" + bookName + "/" + patternCorrespondingUrl; - return RedirectionResponse::build(*this, redirectUrl); + return Response::build_redirect(*this, redirectUrl); } } @@ -743,7 +743,7 @@ std::unique_ptr InternalServer::build_redirect(const std::string& bookName, const kiwix::Entry& entry) const { auto redirectUrl = m_root + "/" + bookName + "/" + kiwix::urlEncode(entry.getPath()); - return RedirectionResponse::build(*this, redirectUrl); + return Response::build_redirect(*this, redirectUrl); } std::unique_ptr InternalServer::handle_content(const RequestContext& request) diff --git a/src/server/internalServer.h b/src/server/internalServer.h index 1e2ed994c..24b91ecb5 100644 --- a/src/server/internalServer.h +++ b/src/server/internalServer.h @@ -103,7 +103,6 @@ class InternalServer { std::string m_server_id; friend std::unique_ptr Response::build(const InternalServer& server); - friend std::unique_ptr RedirectionResponse::build(const InternalServer& server, const std::string& redirectionUrl); friend std::unique_ptr ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype); friend std::unique_ptr EntryResponse::build(const InternalServer& server, const RequestContext& request, const Entry& entry); friend std::unique_ptr Response::build_500(const InternalServer& server, const std::string& msg); diff --git a/src/server/response.cpp b/src/server/response.cpp index 8cf21a740..0649fe356 100644 --- a/src/server/response.cpp +++ b/src/server/response.cpp @@ -111,6 +111,15 @@ std::unique_ptr Response::build_500(const InternalServer& server, cons } +std::unique_ptr Response::build_redirect(const InternalServer& server, const std::string& redirectUrl) +{ + auto response = Response::build(server); + response->m_returnCode = MHD_HTTP_FOUND; + response->add_header(MHD_HTTP_HEADER_LOCATION, redirectUrl); + return response; +} + + static MHD_Result print_key_value (void *cls, enum MHD_ValueKind kind, const char *key, const char *value) @@ -221,7 +230,7 @@ ContentResponse::contentDecorationAllowed() const MHD_Response* Response::create_mhd_response(const RequestContext& request) { - MHD_Response* response = MHD_create_response_from_buffer(0, NULL, MHD_RESPMEM_PERSISTENT); + MHD_Response* response = MHD_create_response_from_buffer(0, nullptr, MHD_RESPMEM_PERSISTENT); return response; } @@ -303,27 +312,6 @@ void ContentResponse::set_taskbar(const std::string& bookName, const std::string } -RedirectionResponse::RedirectionResponse(bool verbose, const std::string& redirectionUrl): - Response(verbose), - m_redirectionUrl(redirectionUrl) -{ - m_returnCode = MHD_HTTP_FOUND; -} - -std::unique_ptr RedirectionResponse::build(const InternalServer& server, const std::string& redirectionUrl) -{ - return std::unique_ptr(new RedirectionResponse( - server.m_verbose.load(), - redirectionUrl)); -} - -MHD_Response* RedirectionResponse::create_mhd_response(const RequestContext& request) -{ - MHD_Response* response = MHD_create_response_from_buffer(0, nullptr, MHD_RESPMEM_MUST_COPY); - MHD_add_response_header(response, MHD_HTTP_HEADER_LOCATION, m_redirectionUrl.c_str()); - return response; -} - ContentResponse::ContentResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype) : Response(verbose), m_root(root), diff --git a/src/server/response.h b/src/server/response.h index 0dbb40813..ff21cc8e2 100644 --- a/src/server/response.h +++ b/src/server/response.h @@ -50,6 +50,7 @@ class Response { static std::unique_ptr build_404(const InternalServer& server, const RequestContext& request, const std::string& bookName); static std::unique_ptr build_416(const InternalServer& server, size_t resourceLength); static std::unique_ptr build_500(const InternalServer& server, const std::string& msg); + static std::unique_ptr build_redirect(const InternalServer& server, const std::string& redirectUrl); MHD_Result send(const RequestContext& request, MHD_Connection* connection); @@ -75,18 +76,6 @@ class Response { }; -class RedirectionResponse : public Response { - public: - RedirectionResponse(bool verbose, const std::string& redirectionUrl); - static std::unique_ptr build(const InternalServer& server, const std::string& redirectionUrl); - - - private: - MHD_Response* create_mhd_response(const RequestContext& request); - - std::string m_redirectionUrl; -}; - class ContentResponse : public Response { public: ContentResponse(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks, const std::string& content, const std::string& mimetype); From 47436f7bddf8d8b7120a6374fa69c8f0829bd135 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Thu, 30 Jul 2020 16:05:58 +0200 Subject: [PATCH 14/16] Move some header setting in response's constructors. It make easier to understand what is somehow constant and what depends of the context. --- src/server/response.cpp | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/server/response.cpp b/src/server/response.cpp index 0649fe356..d61abe6df 100644 --- a/src/server/response.cpp +++ b/src/server/response.cpp @@ -58,6 +58,7 @@ Response::Response(bool verbose) : m_verbose(verbose), m_returnCode(MHD_HTTP_OK) { + add_header(MHD_HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN, "*"); } std::unique_ptr Response::build(const InternalServer& server) @@ -119,8 +120,6 @@ std::unique_ptr Response::build_redirect(const InternalServer& server, return response; } - - static MHD_Result print_key_value (void *cls, enum MHD_ValueKind kind, const char *key, const char *value) { @@ -276,7 +275,6 @@ ContentResponse::create_mhd_response(const RequestContext& request) MHD_add_response_header( response, MHD_HTTP_HEADER_CONTENT_ENCODING, "deflate"); } - MHD_add_response_header(response, MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str()); return response; } @@ -284,7 +282,6 @@ MHD_Result Response::send(const RequestContext& request, MHD_Connection* connect { MHD_Response* response = create_mhd_response(request); - MHD_add_response_header(response, "Access-Control-Allow-Origin", "*"); MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL, m_etag.get_option(ETag::CACHEABLE_ENTITY) ? "max-age=2723040, public" : "no-cache, no-store, must-revalidate"); const std::string etag = m_etag.get_etag(); @@ -323,7 +320,9 @@ ContentResponse::ContentResponse(const std::string& root, bool verbose, bool wit m_compress(is_compressible_mime_type(mimetype)), m_bookName(""), m_bookTitle("") -{} +{ + add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType); +} std::unique_ptr ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype) { @@ -342,7 +341,6 @@ std::unique_ptr ContentResponse::build(const InternalServer& se return ContentResponse::build(server, content, mimetype); } - EntryResponse::EntryResponse(bool verbose, const Entry& entry, const std::string& mimetype, const ByteRange& byterange) : Response(verbose), m_entry(entry), @@ -350,6 +348,7 @@ EntryResponse::EntryResponse(bool verbose, const Entry& entry, const std::string { m_byteRange = byterange; set_cacheable(); + add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType); } std::unique_ptr EntryResponse::build(const InternalServer& server, const RequestContext& request, const Entry& entry) @@ -389,8 +388,6 @@ EntryResponse::create_mhd_response(const RequestContext& request) callback_reader_from_entry, new RunningResponse(m_entry, m_byteRange.first()), callback_free_response); - MHD_add_response_header(response, - MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str()); MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes"); if ( m_byteRange.kind() == ByteRange::RESOLVED_PARTIAL_CONTENT ) { std::ostringstream oss; From ee17b0739af7cf3211d141a1fb87e5e6fffd0d66 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 12 Aug 2020 14:47:23 +0200 Subject: [PATCH 15/16] Fix compilation on CI native dyn. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On the CI, the native_dyn docker image is setup with a packaged version on libmicrohttpd for which `MHD_HTTP_RANGE_NOT_SATISFIABLE` is not defined. When the CI will be fixed, we can revert this commit. --- src/server/response.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/server/response.cpp b/src/server/response.cpp index d61abe6df..79509ab6d 100644 --- a/src/server/response.cpp +++ b/src/server/response.cpp @@ -92,7 +92,9 @@ std::unique_ptr Response::build_404(const InternalServer& server, cons std::unique_ptr Response::build_416(const InternalServer& server, size_t resourceLength) { auto response = Response::build(server); - response->set_code(MHD_HTTP_RANGE_NOT_SATISFIABLE); +// [FIXME] (compile with recent enough version of libmicrohttpd) +// response->set_code(MHD_HTTP_RANGE_NOT_SATISFIABLE); + response->set_code(416); std::ostringstream oss; oss << "bytes */" << resourceLength; response->add_header(MHD_HTTP_HEADER_CONTENT_RANGE, oss.str()); From 6f0d3003acf8654ba9984cf2b4348618b6935460 Mon Sep 17 00:00:00 2001 From: Matthieu Gautier Date: Wed, 12 Aug 2020 14:58:58 +0200 Subject: [PATCH 16/16] Remove `m_compress` member. --- src/server/response.cpp | 3 +-- src/server/response.h | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/server/response.cpp b/src/server/response.cpp index 79509ab6d..4bdcb933c 100644 --- a/src/server/response.cpp +++ b/src/server/response.cpp @@ -247,7 +247,7 @@ ContentResponse::create_mhd_response(const RequestContext& request) } } - bool shouldCompress = m_compress && can_compress(request); + bool shouldCompress = can_compress(request); if (shouldCompress) { std::vector compr_buffer(compressBound(m_content.size())); uLongf comprLen = compr_buffer.capacity(); @@ -319,7 +319,6 @@ ContentResponse::ContentResponse(const std::string& root, bool verbose, bool wit m_withTaskbar(withTaskbar), m_withLibraryButton(withLibraryButton), m_blockExternalLinks(blockExternalLinks), - m_compress(is_compressible_mime_type(mimetype)), m_bookName(""), m_bookTitle("") { diff --git a/src/server/response.h b/src/server/response.h index ff21cc8e2..316e745c3 100644 --- a/src/server/response.h +++ b/src/server/response.h @@ -100,7 +100,6 @@ class ContentResponse : public Response { bool m_withTaskbar; bool m_withLibraryButton; bool m_blockExternalLinks; - bool m_compress; std::string m_bookName; std::string m_bookTitle; };