From abb3dec700d019c370ed175f7c833e80610280ae Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Mon, 13 Apr 2020 00:00:37 +0400 Subject: [PATCH 01/18] Refactoring: extracted str2RequestMethod() --- src/server/request_context.cpp | 42 +++++++++++++++------------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/src/server/request_context.cpp b/src/server/request_context.cpp index 0dd3114d2..54d39b808 100644 --- a/src/server/request_context.cpp +++ b/src/server/request_context.cpp @@ -30,42 +30,38 @@ namespace kiwix { static std::atomic_ullong s_requestIndex(0); +namespace { + +RequestMethod str2RequestMethod(const std::string& method) { + if (method == "GET") return RequestMethod::GET; + else if (method == "HEAD") return RequestMethod::HEAD; + else if (method == "POST") return RequestMethod::POST; + else if (method == "PUT") return RequestMethod::PUT; + else if (method == "DELETE") return RequestMethod::DELETE_; + else if (method == "CONNECT") return RequestMethod::CONNECT; + else if (method == "OPTIONS") return RequestMethod::OPTIONS; + else if (method == "TRACE") return RequestMethod::TRACE; + else if (method == "PATCH") return RequestMethod::PATCH; + else return RequestMethod::OTHER; +} + +} // unnamed namespace + RequestContext::RequestContext(struct MHD_Connection* connection, std::string rootLocation, const std::string& _url, - const std::string& method, + const std::string& _method, const std::string& version) : full_url(_url), url(_url), valid_url(true), + method(str2RequestMethod(_method)), version(version), requestIndex(s_requestIndex++), acceptEncodingDeflate(false), accept_range(false), range_pair(0, -1) { - if (method == "GET") { - this->method = RequestMethod::GET; - } else if (method == "HEAD") { - this->method = RequestMethod::HEAD; - } else if (method == "POST") { - this->method = RequestMethod::POST; - } else if (method == "PUT") { - this->method = RequestMethod::PUT; - } else if (method == "DELETE") { - this->method = RequestMethod::DELETE_; - } else if (method == "CONNECT") { - this->method = RequestMethod::CONNECT; - } else if (method == "OPTIONS") { - this->method = RequestMethod::OPTIONS; - } else if (method == "TRACE") { - this->method = RequestMethod::TRACE; - } else if (method == "PATCH") { - this->method = RequestMethod::PATCH; - } else { - this->method = RequestMethod::OTHER; - } - MHD_get_connection_values(connection, MHD_HEADER_KIND, &RequestContext::fill_header, this); MHD_get_connection_values(connection, MHD_GET_ARGUMENT_KIND, &RequestContext::fill_argument, this); From fd80f2a89fe86d1c18fa368cbe70859276ca7f50 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Mon, 13 Apr 2020 00:34:39 +0400 Subject: [PATCH 02/18] Refactoring: extracted fullURL2LocalURL() Also dropped RequestContex::valid_url --- src/server/request_context.cpp | 39 +++++++++++++++++----------------- src/server/request_context.h | 1 - 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/server/request_context.cpp b/src/server/request_context.cpp index 54d39b808..37d490bf9 100644 --- a/src/server/request_context.cpp +++ b/src/server/request_context.cpp @@ -45,6 +45,22 @@ RequestMethod str2RequestMethod(const std::string& method) { else return RequestMethod::OTHER; } +std::string +fullURL2LocalURL(const std::string& full_url, const std::string& rootLocation) +{ + if (rootLocation.empty()) { + // nothing special to handle. + return full_url; + } else if (full_url == rootLocation) { + return "/"; + } else if (full_url.size() > rootLocation.size() && + full_url.substr(0, rootLocation.size()+1) == rootLocation + "/") { + return full_url.substr(rootLocation.size()); + } else { + return ""; + } +} + } // unnamed namespace RequestContext::RequestContext(struct MHD_Connection* connection, @@ -53,8 +69,7 @@ RequestContext::RequestContext(struct MHD_Connection* connection, const std::string& _method, const std::string& version) : full_url(_url), - url(_url), - valid_url(true), + url(fullURL2LocalURL(_url, rootLocation)), method(str2RequestMethod(_method)), version(version), requestIndex(s_requestIndex++), @@ -65,21 +80,6 @@ RequestContext::RequestContext(struct MHD_Connection* connection, MHD_get_connection_values(connection, MHD_HEADER_KIND, &RequestContext::fill_header, this); MHD_get_connection_values(connection, MHD_GET_ARGUMENT_KIND, &RequestContext::fill_argument, this); - valid_url = true; - if (rootLocation.empty()) { - // nothing special to handle. - url = full_url; - } else { - if (full_url == rootLocation) { - url = "/"; - } else if (full_url.size() > rootLocation.size() && - full_url.substr(0, rootLocation.size()+1) == rootLocation + "/") { - url = full_url.substr(rootLocation.size()); - } else { - valid_url = false; - } - } - try { acceptEncodingDeflate = (get_header(MHD_HTTP_HEADER_ACCEPT_ENCODING).find("deflate") != std::string::npos); @@ -143,10 +143,11 @@ void RequestContext::print_debug_info() const { printf(" - %s : '%s'\n", it->first.c_str(), it->second.c_str()); } printf("Parsed : \n"); + printf("full_url: %s\n", full_url.c_str()); printf("url : %s\n", url.c_str()); printf("acceptEncodingDeflate : %d\n", acceptEncodingDeflate); printf("has_range : %d\n", accept_range); - printf("is_valid_url : %d\n", valid_url); + printf("is_valid_url : %d\n", is_valid_url()); printf(".............\n"); } @@ -184,7 +185,7 @@ std::string RequestContext::get_full_url() const { } bool RequestContext::is_valid_url() const { - return valid_url; + return !url.empty(); } bool RequestContext::has_range() const { diff --git a/src/server/request_context.h b/src/server/request_context.h index 5bbed949a..8293498f3 100644 --- a/src/server/request_context.h +++ b/src/server/request_context.h @@ -86,7 +86,6 @@ class RequestContext { private: std::string full_url; std::string url; - bool valid_url; RequestMethod method; std::string version; unsigned long long requestIndex; From 2d3bf9b981d65fd3a328979760c9c4a2d3126cd5 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Tue, 14 Apr 2020 22:45:53 +0400 Subject: [PATCH 03/18] Refactoring: extracted InternalServer::homepage_data() Also typedef'ed kainjow::mustache::data as MustacheData --- src/server.cpp | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/src/server.cpp b/src/server.cpp index 511a8936d..49700df3f 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -75,6 +75,8 @@ namespace kiwix { static IdNameMapper defaultNameMapper; +typedef kainjow::mustache::data MustacheData; + static int staticHandlerCallback(void* cls, struct MHD_Connection* connection, const char* url, @@ -123,7 +125,8 @@ class InternalServer { Response handle_captured_external(const RequestContext& request); Response handle_content(const RequestContext& request); - kainjow::mustache::data get_default_data(); + MustacheData get_default_data() const; + MustacheData homepage_data() const; Response get_default_response(); std::string m_addr; @@ -359,9 +362,9 @@ Response InternalServer::handle_request(const RequestContext& request) } } -kainjow::mustache::data InternalServer::get_default_data() +MustacheData InternalServer::get_default_data() const { - kainjow::mustache::data data; + MustacheData data; data.set("root", m_root); return data; } @@ -375,7 +378,7 @@ Response InternalServer::get_default_response() Response InternalServer::build_404(const RequestContext& request, const std::string& bookName) { - kainjow::mustache::data results; + MustacheData results; results.set("url", request.get_full_url()); auto response = get_default_response(); @@ -390,7 +393,7 @@ Response InternalServer::build_404(const RequestContext& request, Response InternalServer::build_500(const std::string& msg) { - kainjow::mustache::data data; + MustacheData data; data.set("error", msg); Response response(m_root, true, false, false, false); response.set_template(RESOURCE::templates::_500_html, data); @@ -399,15 +402,15 @@ Response InternalServer::build_500(const std::string& msg) return response; } -Response InternalServer::build_homepage(const RequestContext& request) +MustacheData InternalServer::homepage_data() const { auto data = get_default_data(); - kainjow::mustache::data books{kainjow::mustache::data::type::list}; + MustacheData books{MustacheData::type::list}; for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) { auto& currentBook = mp_library->getBookById(bookId); - kainjow::mustache::data book; + MustacheData book; book.set("name", mp_nameMapper->getNameForId(bookId)); book.set("title", currentBook.getTitle()); book.set("description", currentBook.getDescription()); @@ -417,9 +420,13 @@ Response InternalServer::build_homepage(const RequestContext& request) } data.set("books", books); + return data; +} +Response InternalServer::build_homepage(const RequestContext& request) +{ auto response = get_default_response(); - response.set_template(RESOURCE::templates::index_html, data); + response.set_template(RESOURCE::templates::index_html, homepage_data()); response.set_mimeType("text/html; charset=utf-8"); response.set_compress(true); response.set_taskbar("", ""); @@ -507,14 +514,14 @@ Response InternalServer::handle_suggest(const RequestContext& request) printf("Searching suggestions for: \"%s\"\n", term.c_str()); } - kainjow::mustache::data results{kainjow::mustache::data::type::list}; + MustacheData results{MustacheData::type::list}; bool first = true; if (reader != nullptr) { /* Get the suggestions */ reader->searchSuggestionsSmart(term, maxSuggestionCount); while (reader->getNextSuggestion(suggestion)) { - kainjow::mustache::data result; + MustacheData result; result.set("label", suggestion); result.set("value", suggestion); result.set("first", first); @@ -526,7 +533,7 @@ Response InternalServer::handle_suggest(const RequestContext& request) /* Propose the fulltext search if possible */ if (reader->hasFulltextIndex()) { - kainjow::mustache::data result; + MustacheData result; result.set("label", "containing '" + term + "'..."); result.set("value", term + " "); result.set("first", first); From bbc06931ada25ec2fbbde6c722b68bfb39e81a76 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Thu, 16 Apr 2020 13:54:25 +0400 Subject: [PATCH 04/18] Refactoring: extracted get_book_name() --- src/server.cpp | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/server.cpp b/src/server.cpp index 49700df3f..e44a08adf 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -831,6 +831,20 @@ Response InternalServer::handle_catalog(const RequestContext& request) return response; } +namespace +{ + +std::string get_book_name(const RequestContext& request) +{ + try { + return request.get_url_part(0); + } catch (const std::out_of_range& e) { + return std::string(); + } +} + +} // unnamed namespace + Response InternalServer::handle_content(const RequestContext& request) { if (m_verbose.load()) { @@ -843,12 +857,7 @@ Response InternalServer::handle_content(const RequestContext& request) kiwix::Entry entry; - std::string bookName; - try { - bookName = request.get_url_part(0); - } catch (const std::out_of_range& e) { - return build_homepage(request); - } + const std::string bookName = get_book_name(request); if (bookName.empty()) return build_homepage(request); From 1ef5ebfb5261996237bd5358ab3090e4911c4404 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Thu, 16 Apr 2020 13:55:54 +0400 Subject: [PATCH 05/18] Refactoring: extracted InternalServer::get_reader() --- src/server.cpp | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/server.cpp b/src/server.cpp index e44a08adf..8b1dff706 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -129,6 +129,9 @@ class InternalServer { MustacheData homepage_data() const; Response get_default_response(); + std::shared_ptr get_reader(const std::string& bookName) const; + + private: // data std::string m_addr; int m_port; std::string m_root; @@ -845,6 +848,18 @@ std::string get_book_name(const RequestContext& request) } // unnamed namespace +std::shared_ptr +InternalServer::get_reader(const std::string& bookName) const +{ + std::shared_ptr reader; + try { + const std::string bookId = mp_nameMapper->getIdForName(bookName); + reader = mp_library->getReaderById(bookId); + } catch (const std::out_of_range& e) { + } + return reader; +} + Response InternalServer::handle_content(const RequestContext& request) { if (m_verbose.load()) { @@ -861,15 +876,7 @@ Response InternalServer::handle_content(const RequestContext& request) if (bookName.empty()) return build_homepage(request); - std::string bookId; - std::shared_ptr reader; - try { - bookId = mp_nameMapper->getIdForName(bookName); - reader = mp_library->getReaderById(bookId); - } catch (const std::out_of_range& e) { - return build_404(request, bookName); - } - + const std::shared_ptr reader = get_reader(bookName); if (reader == nullptr) { return build_404(request, bookName); } From a058520628395fdfa96fdb03d5daf3731920392d Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Thu, 16 Apr 2020 13:57:01 +0400 Subject: [PATCH 06/18] Refactoring: extracted get_mime_type() --- src/server.cpp | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/server.cpp b/src/server.cpp index 8b1dff706..39e641a16 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -846,6 +846,15 @@ std::string get_book_name(const RequestContext& request) } } +std::string get_mime_type(const kiwix::Entry& entry) +{ + try { + return entry.getMimetype(); + } catch (exception& e) { + return "application/octet-stream"; + } +} + } // unnamed namespace std::shared_ptr @@ -866,9 +875,7 @@ Response InternalServer::handle_content(const RequestContext& request) printf("** running handle_content\n"); } - std::string baseUrl; std::string content; - std::string mimeType; kiwix::Entry entry; @@ -904,11 +911,7 @@ Response InternalServer::handle_content(const RequestContext& request) return build_404(request, bookName); } - try { - mimeType = entry.getMimetype(); - } catch (exception& e) { - mimeType = "application/octet-stream"; - } + const std::string mimeType = get_mime_type(entry); if (m_verbose.load()) { printf("Found %s\n", urlStr.c_str()); From 87cbbed9e35e73f2be8caeb96a7b342305427fa9 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Thu, 16 Apr 2020 13:58:15 +0400 Subject: [PATCH 07/18] Refactoring: extracted is_compressible_mime_type() --- src/server.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/server.cpp b/src/server.cpp index 39e641a16..fc5e8e00b 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -855,6 +855,13 @@ std::string get_mime_type(const kiwix::Entry& entry) } } +bool is_compressible_mime_type(const std::string& mimeType) +{ + return mimeType.find("text/") != string::npos + || mimeType.find("application/javascript") != string::npos + || mimeType.find("application/json") != string::npos; +} + } // unnamed namespace std::shared_ptr @@ -918,9 +925,7 @@ Response InternalServer::handle_content(const RequestContext& request) printf("mimeType: %s\n", mimeType.c_str()); } - if (mimeType.find("text/") != string::npos - || mimeType.find("application/javascript") != string::npos - || mimeType.find("application/json") != string::npos) { + if ( is_compressible_mime_type(mimeType) ) { zim::Blob raw_content = entry.getBlob(); content = string(raw_content.data(), raw_content.size()); auto response = get_default_response(); From 83ee8dec1575cf7e056c2b79d0f852f8e8bb5772 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Thu, 16 Apr 2020 14:00:30 +0400 Subject: [PATCH 08/18] Made InternalServer::get_default_response() const --- src/server.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server.cpp b/src/server.cpp index fc5e8e00b..e9c40bf14 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -127,7 +127,7 @@ class InternalServer { MustacheData get_default_data() const; MustacheData homepage_data() const; - Response get_default_response(); + Response get_default_response() const; std::shared_ptr get_reader(const std::string& bookName) const; @@ -372,7 +372,7 @@ MustacheData InternalServer::get_default_data() const return data; } -Response InternalServer::get_default_response() +Response InternalServer::get_default_response() const { return Response(m_root, m_verbose.load(), m_withTaskbar, m_withLibraryButton, m_blockExternalLinks); } From 659ee6ba7143516106a04d5a8efe69d74489ab80 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Thu, 16 Apr 2020 14:01:27 +0400 Subject: [PATCH 09/18] Refactoring: extracted InternalServer::build_redirect() --- src/server.cpp | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/server.cpp b/src/server.cpp index e9c40bf14..15fb7bbc6 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -115,6 +115,7 @@ class InternalServer { Response handle_request(const RequestContext& request); Response build_500(const std::string& msg); Response build_404(const RequestContext& request, const std::string& zimName); + Response build_redirect(const std::string& bookName, const kiwix::Entry& entry) const; Response build_homepage(const RequestContext& request); Response handle_skin(const RequestContext& request); Response handle_catalog(const RequestContext& request); @@ -721,10 +722,7 @@ Response InternalServer::handle_random(const RequestContext& request) try { auto entry = reader->getRandomPage(); - entry = entry.getFinalEntry(); - auto response = get_default_response(); - response.set_redirection(m_root + "/" + bookName + "/" + kiwix::urlEncode(entry.getPath())); - return response; + return build_redirect(bookName, entry.getFinalEntry()); } catch(kiwix::NoEntry& e) { return build_404(request, bookName); } @@ -876,6 +874,15 @@ InternalServer::get_reader(const std::string& bookName) const return reader; } +Response +InternalServer::build_redirect(const std::string& bookName, const kiwix::Entry& entry) const +{ + auto response = get_default_response(); + response.set_redirection(m_root + "/" + bookName + "/" + + kiwix::urlEncode(entry.getPath())); + return response; +} + Response InternalServer::handle_content(const RequestContext& request) { if (m_verbose.load()) { @@ -905,11 +912,7 @@ Response InternalServer::handle_content(const RequestContext& request) if (entry.isRedirect() || urlStr.empty()) { // If urlStr is empty, we want to mainPage. // We must do a redirection to the real page. - entry = entry.getFinalEntry(); - auto response = get_default_response(); - response.set_redirection(m_root + "/" + bookName + "/" + - kiwix::urlEncode(entry.getPath())); - return response; + return build_redirect(bookName, entry.getFinalEntry()); } } catch(kiwix::NoEntry& e) { if (m_verbose.load()) From 6c7ab6ff54e67c9c23c2b271caaf4b7e99934bf5 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Thu, 16 Apr 2020 14:28:44 +0400 Subject: [PATCH 10/18] Refactoring: moved local variable declarations --- src/server.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/server.cpp b/src/server.cpp index 15fb7bbc6..5b680328d 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -889,10 +889,6 @@ Response InternalServer::handle_content(const RequestContext& request) printf("** running handle_content\n"); } - std::string content; - - kiwix::Entry entry; - const std::string bookName = get_book_name(request); if (bookName.empty()) return build_homepage(request); @@ -907,6 +903,8 @@ Response InternalServer::handle_content(const RequestContext& request) urlStr = urlStr.substr(1); } + kiwix::Entry entry; + try { entry = reader->getEntryFromPath(urlStr); if (entry.isRedirect() || urlStr.empty()) { @@ -928,6 +926,8 @@ Response InternalServer::handle_content(const RequestContext& request) printf("mimeType: %s\n", mimeType.c_str()); } + std::string content; + if ( is_compressible_mime_type(mimeType) ) { zim::Blob raw_content = entry.getBlob(); content = string(raw_content.data(), raw_content.size()); From a8e78f27e13d62aa59707c89113d297560bb5b80 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Thu, 16 Apr 2020 17:32:07 +0400 Subject: [PATCH 11/18] Refactoring: extracted Response::create_mhd_response() --- src/server/response.cpp | 10 ++++++++-- src/server/response.h | 5 ++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/server/response.cpp b/src/server/response.cpp index bba6d3028..00e7abb9a 100644 --- a/src/server/response.cpp +++ b/src/server/response.cpp @@ -139,8 +139,8 @@ void Response::inject_externallinks_blocker() script_tag); } - -int Response::send(const RequestContext& request, MHD_Connection* connection) +MHD_Response* +Response::create_mhd_response(const RequestContext& request) { MHD_Response* response = nullptr; switch (m_mode) { @@ -218,6 +218,12 @@ int Response::send(const RequestContext& request, MHD_Connection* connection) break; } } + return response; +} + +int Response::send(const RequestContext& request, MHD_Connection* connection) +{ + MHD_Response* response = create_mhd_response(request); MHD_add_response_header(response, "Access-Control-Allow-Origin", "*"); MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL, diff --git a/src/server/response.h b/src/server/response.h index 89f235072..2e0881346 100644 --- a/src/server/response.h +++ b/src/server/response.h @@ -66,7 +66,10 @@ class Response { void introduce_taskbar(); void inject_externallinks_blocker(); - private: + private: // functions + MHD_Response* create_mhd_response(const RequestContext& request); + + private: // data bool m_verbose; ResponseMode m_mode; std::string m_root; From 21c6de2f805991412766b7448e62485719df08d1 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Thu, 16 Apr 2020 17:41:18 +0400 Subject: [PATCH 12/18] Refactoring: split Response::create_mhd_response() The changes are easier to understand in ignore-white-space mode (git diff -w, git show -w). --- src/server/response.cpp | 158 ++++++++++++++++++++++------------------ src/server/response.h | 3 + 2 files changed, 89 insertions(+), 72 deletions(-) diff --git a/src/server/response.cpp b/src/server/response.cpp index 00e7abb9a..5c7482152 100644 --- a/src/server/response.cpp +++ b/src/server/response.cpp @@ -140,87 +140,101 @@ void Response::inject_externallinks_blocker() } MHD_Response* -Response::create_mhd_response(const RequestContext& request) +Response::create_raw_content_mhd_response(const RequestContext& request) { - MHD_Response* response = nullptr; - switch (m_mode) { - case ResponseMode::RAW_CONTENT : { - if (m_addTaskbar) { - introduce_taskbar(); - } - if ( m_blockExternalLinks ) { - inject_externallinks_blocker(); - } + if (m_addTaskbar) { + introduce_taskbar(); + } + if ( m_blockExternalLinks ) { + inject_externallinks_blocker(); + } - bool shouldCompress = m_compress && request.can_compress(); - shouldCompress &= m_mimeType.find("text/") != string::npos - || m_mimeType.find("application/javascript") != string::npos - || m_mimeType.find("application/json") != string::npos; + bool shouldCompress = m_compress && request.can_compress(); + shouldCompress &= m_mimeType.find("text/") != string::npos + || m_mimeType.find("application/javascript") != string::npos + || m_mimeType.find("application/json") != string::npos; - shouldCompress &= (m_content.size() > KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE); + shouldCompress &= (m_content.size() > KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE); - if (shouldCompress) { - std::vector compr_buffer(compressBound(m_content.size())); - uLongf comprLen = compr_buffer.capacity(); - int err = compress(&compr_buffer[0], - &comprLen, - (const Bytef*)(m_content.data()), - m_content.size()); - if (err == Z_OK && comprLen > 2 && comprLen < (m_content.size() + 2)) { - /* /!\ Internet Explorer has a bug with deflate compression. - It can not handle the first two bytes (compression headers) - We need to chunk them off (move the content 2bytes) - It has no incidence on other browsers - See http://www.subbu.org/blog/2008/03/ie7-deflate-or-not and comments */ - m_content = string((char*)&compr_buffer[2], comprLen - 2); - } else { - shouldCompress = false; - } - } - - response = MHD_create_response_from_buffer( - m_content.size(), const_cast(m_content.data()), MHD_RESPMEM_MUST_COPY); - - if (shouldCompress) { - MHD_add_response_header( - response, MHD_HTTP_HEADER_VARY, "Accept-Encoding"); - MHD_add_response_header( - response, MHD_HTTP_HEADER_CONTENT_ENCODING, "deflate"); - } - MHD_add_response_header(response, MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str()); - break; - } - - case ResponseMode::REDIRECTION : { - response = MHD_create_response_from_buffer(0, nullptr, MHD_RESPMEM_MUST_COPY); - MHD_add_response_header(response, MHD_HTTP_HEADER_LOCATION, m_content.c_str()); - break; - } - - case ResponseMode::ENTRY : { - response = MHD_create_response_from_callback(m_entry.getSize(), - 16384, - callback_reader_from_entry, - new RunningResponse(m_entry, m_startRange), - callback_free_response); - MHD_add_response_header(response, - MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str()); - MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes"); - std::ostringstream oss; - oss << "bytes " << m_startRange << "-" << m_startRange + m_lenRange - 1 - << "/" << m_entry.getSize(); - - MHD_add_response_header(response, - MHD_HTTP_HEADER_CONTENT_RANGE, oss.str().c_str()); - - MHD_add_response_header(response, - MHD_HTTP_HEADER_CONTENT_LENGTH, kiwix::to_string(m_lenRange).c_str()); - break; + if (shouldCompress) { + std::vector compr_buffer(compressBound(m_content.size())); + uLongf comprLen = compr_buffer.capacity(); + int err = compress(&compr_buffer[0], + &comprLen, + (const Bytef*)(m_content.data()), + m_content.size()); + if (err == Z_OK && comprLen > 2 && comprLen < (m_content.size() + 2)) { + /* /!\ Internet Explorer has a bug with deflate compression. + It can not handle the first two bytes (compression headers) + We need to chunk them off (move the content 2bytes) + It has no incidence on other browsers + See http://www.subbu.org/blog/2008/03/ie7-deflate-or-not and comments */ + m_content = string((char*)&compr_buffer[2], comprLen - 2); + } else { + shouldCompress = false; } } + + MHD_Response* response = MHD_create_response_from_buffer( + m_content.size(), const_cast(m_content.data()), MHD_RESPMEM_MUST_COPY); + + if (shouldCompress) { + MHD_add_response_header( + response, MHD_HTTP_HEADER_VARY, "Accept-Encoding"); + MHD_add_response_header( + response, MHD_HTTP_HEADER_CONTENT_ENCODING, "deflate"); + } + MHD_add_response_header(response, MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str()); return response; } +MHD_Response* +Response::create_redirection_mhd_response() const +{ + MHD_Response* response = MHD_create_response_from_buffer(0, nullptr, MHD_RESPMEM_MUST_COPY); + MHD_add_response_header(response, MHD_HTTP_HEADER_LOCATION, m_content.c_str()); + return response; +} + +MHD_Response* +Response::create_entry_mhd_response() const +{ + MHD_Response* response = MHD_create_response_from_callback(m_entry.getSize(), + 16384, + callback_reader_from_entry, + new RunningResponse(m_entry, m_startRange), + callback_free_response); + MHD_add_response_header(response, + MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str()); + MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes"); + std::ostringstream oss; + oss << "bytes " << m_startRange << "-" << m_startRange + m_lenRange - 1 + << "/" << m_entry.getSize(); + + MHD_add_response_header(response, + MHD_HTTP_HEADER_CONTENT_RANGE, oss.str().c_str()); + + MHD_add_response_header(response, + MHD_HTTP_HEADER_CONTENT_LENGTH, kiwix::to_string(m_lenRange).c_str()); + return response; +} + +MHD_Response* +Response::create_mhd_response(const RequestContext& request) +{ + switch (m_mode) { + case ResponseMode::RAW_CONTENT : + return create_raw_content_mhd_response(request); + + case ResponseMode::REDIRECTION : + return create_redirection_mhd_response(); + + case ResponseMode::ENTRY : + return create_entry_mhd_response(); + } + return nullptr; +} + int Response::send(const RequestContext& request, MHD_Connection* connection) { MHD_Response* response = create_mhd_response(request); diff --git a/src/server/response.h b/src/server/response.h index 2e0881346..d3223673d 100644 --- a/src/server/response.h +++ b/src/server/response.h @@ -68,6 +68,9 @@ class Response { private: // functions MHD_Response* create_mhd_response(const RequestContext& request); + MHD_Response* create_raw_content_mhd_response(const RequestContext& request); + MHD_Response* create_redirection_mhd_response() const; + MHD_Response* create_entry_mhd_response() const; private: // data bool m_verbose; From a004d96cd715b6dd1ca5900d100e150af1e46c41 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Thu, 16 Apr 2020 18:05:05 +0400 Subject: [PATCH 13/18] Refactoring: extracted get_range_len() --- src/server.cpp | 18 +++++++++--------- src/server/request_context.h | 12 ++++++++---- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/server.cpp b/src/server.cpp index 5b680328d..dd50afcd3 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -860,6 +860,13 @@ bool is_compressible_mime_type(const std::string& mimeType) || mimeType.find("application/json") != string::npos; } +int get_range_len(const kiwix::Entry& entry, RequestContext::ByteRange range) +{ + return range.second == -1 + ? entry.getSize() - range.first + : range.second - range.first; +} + } // unnamed namespace std::shared_ptr @@ -926,11 +933,9 @@ Response InternalServer::handle_content(const RequestContext& request) printf("mimeType: %s\n", mimeType.c_str()); } - std::string content; - if ( is_compressible_mime_type(mimeType) ) { zim::Blob raw_content = entry.getBlob(); - content = string(raw_content.data(), raw_content.size()); + const std::string content = string(raw_content.data(), raw_content.size()); auto response = get_default_response(); if (mimeType.find("text/html") != string::npos) @@ -942,12 +947,7 @@ Response InternalServer::handle_content(const RequestContext& request) response.set_cache(true); return response; } else { - int range_len; - if (request.get_range().second == -1) { - range_len = entry.getSize() - request.get_range().first; - } else { - range_len = request.get_range().second - request.get_range().first; - } + const int range_len = get_range_len(entry, request.get_range()); auto response = get_default_response(); response.set_entry(entry); response.set_mimeType(mimeType); diff --git a/src/server/request_context.h b/src/server/request_context.h index 8293498f3..d58040f07 100644 --- a/src/server/request_context.h +++ b/src/server/request_context.h @@ -51,7 +51,10 @@ class IndexError: public std::runtime_error {}; class RequestContext { - public: + public: // types + typedef std::pair ByteRange; + + public: // functions RequestContext(struct MHD_Connection* connection, std::string rootLocation, const std::string& url, @@ -79,11 +82,11 @@ class RequestContext { std::string get_full_url() const; bool has_range() const; - std::pair get_range() const; + ByteRange get_range() const; bool can_compress() const { return acceptEncodingDeflate; } - private: + private: // data std::string full_url; std::string url; RequestMethod method; @@ -93,10 +96,11 @@ class RequestContext { bool acceptEncodingDeflate; bool accept_range; - std::pair range_pair; + ByteRange range_pair; std::map headers; std::map arguments; + private: // functions static int fill_header(void *, enum MHD_ValueKind, const char*, const char*); static int fill_argument(void *, enum MHD_ValueKind, const char*, const char*); }; From 14d8583c8315eca6e27e7d21852866d4d3573afc Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Thu, 16 Apr 2020 18:07:51 +0400 Subject: [PATCH 14/18] Refactoring in InternalServer::handle_content() Deduplicated common code found in the two branches of the last if(){}else{} statement in InternalServer::handle_content(). --- src/server.cpp | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/server.cpp b/src/server.cpp index dd50afcd3..4a383e468 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -933,29 +933,25 @@ Response InternalServer::handle_content(const RequestContext& request) printf("mimeType: %s\n", mimeType.c_str()); } + auto response = get_default_response(); + response.set_mimeType(mimeType); + response.set_cache(true); if ( is_compressible_mime_type(mimeType) ) { zim::Blob raw_content = entry.getBlob(); const std::string content = string(raw_content.data(), raw_content.size()); - auto response = get_default_response(); if (mimeType.find("text/html") != string::npos) response.set_taskbar(bookName, reader->getTitle()); - response.set_mimeType(mimeType); response.set_content(content); response.set_compress(true); - response.set_cache(true); - return response; } else { const int range_len = get_range_len(entry, request.get_range()); - auto response = get_default_response(); response.set_entry(entry); - response.set_mimeType(mimeType); response.set_range_first(request.get_range().first); response.set_range_len(range_len); - response.set_cache(true); - return response; } + return response; } } From 7bd7ec493759c42d9623ca90a435c0e81b6c728d Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Thu, 16 Apr 2020 18:29:44 +0400 Subject: [PATCH 15/18] Refactoring: preparing to move some code --- src/server.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/server.cpp b/src/server.cpp index 4a383e468..72541d408 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -934,15 +934,16 @@ Response InternalServer::handle_content(const RequestContext& request) } auto response = get_default_response(); + + /////////////////////////////////////////////////////////// + // This chunk of code should go into Response::set_entry() response.set_mimeType(mimeType); response.set_cache(true); + if ( is_compressible_mime_type(mimeType) ) { zim::Blob raw_content = entry.getBlob(); const std::string content = string(raw_content.data(), raw_content.size()); - if (mimeType.find("text/html") != string::npos) - response.set_taskbar(bookName, reader->getTitle()); - response.set_content(content); response.set_compress(true); } else { @@ -951,6 +952,12 @@ Response InternalServer::handle_content(const RequestContext& request) response.set_range_first(request.get_range().first); response.set_range_len(range_len); } + // This chunk of code should go into Response::set_entry() + /////////////////////////////////////////////////////////// + + if (mimeType.find("text/html") != string::npos) + response.set_taskbar(bookName, reader->getTitle()); + return response; } From 9ec7757efe0526cb38cc6b3043767311a95d53d7 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Thu, 16 Apr 2020 18:48:32 +0400 Subject: [PATCH 16/18] Refactoring: smart Response::set_entry() Response::set_entry() was upgraded from a simple setter to a method performing certain business logic that was previously taken care of by InternalServer::handle_content(). --- src/server.cpp | 34 +---------------------------- src/server/response.cpp | 47 ++++++++++++++++++++++++++++++++++++++++- src/server/response.h | 2 +- 3 files changed, 48 insertions(+), 35 deletions(-) diff --git a/src/server.cpp b/src/server.cpp index 72541d408..d3845f2e4 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -853,20 +853,6 @@ std::string get_mime_type(const kiwix::Entry& entry) } } -bool is_compressible_mime_type(const std::string& mimeType) -{ - return mimeType.find("text/") != string::npos - || mimeType.find("application/javascript") != string::npos - || mimeType.find("application/json") != string::npos; -} - -int get_range_len(const kiwix::Entry& entry, RequestContext::ByteRange range) -{ - return range.second == -1 - ? entry.getSize() - range.first - : range.second - range.first; -} - } // unnamed namespace std::shared_ptr @@ -935,25 +921,7 @@ Response InternalServer::handle_content(const RequestContext& request) auto response = get_default_response(); - /////////////////////////////////////////////////////////// - // This chunk of code should go into Response::set_entry() - response.set_mimeType(mimeType); - response.set_cache(true); - - if ( is_compressible_mime_type(mimeType) ) { - zim::Blob raw_content = entry.getBlob(); - const std::string content = string(raw_content.data(), raw_content.size()); - - response.set_content(content); - response.set_compress(true); - } else { - const int range_len = get_range_len(entry, request.get_range()); - response.set_entry(entry); - response.set_range_first(request.get_range().first); - response.set_range_len(range_len); - } - // This chunk of code should go into Response::set_entry() - /////////////////////////////////////////////////////////// + response.set_entry(entry, request); if (mimeType.find("text/html") != string::npos) response.set_taskbar(bookName, reader->getTitle()); diff --git a/src/server/response.cpp b/src/server/response.cpp index 5c7482152..7540fd687 100644 --- a/src/server/response.cpp +++ b/src/server/response.cpp @@ -17,6 +17,35 @@ namespace kiwix { +namespace +{ +// some utilities + +std::string get_mime_type(const kiwix::Entry& entry) +{ + try { + return entry.getMimetype(); + } catch (exception& e) { + return "application/octet-stream"; + } +} + +bool is_compressible_mime_type(const std::string& mimeType) +{ + return mimeType.find("text/") != string::npos + || mimeType.find("application/javascript") != string::npos + || mimeType.find("application/json") != string::npos; +} + +int get_range_len(const kiwix::Entry& entry, RequestContext::ByteRange range) +{ + return range.second == -1 + ? entry.getSize() - range.first + : range.second - range.first; +} + +} // unnamed namespace + Response::Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks) : m_verbose(verbose), m_root(root), @@ -269,9 +298,25 @@ void Response::set_redirection(const std::string& url) { m_returnCode = MHD_HTTP_FOUND; } -void Response::set_entry(const Entry& entry) { +void Response::set_entry(const Entry& entry, const RequestContext& request) { m_entry = entry; m_mode = ResponseMode::ENTRY; + + const std::string mimeType = get_mime_type(entry); + set_mimeType(mimeType); + set_cache(true); + + if ( is_compressible_mime_type(mimeType) ) { + zim::Blob raw_content = entry.getBlob(); + const std::string content = string(raw_content.data(), raw_content.size()); + + set_content(content); + set_compress(true); + } else { + const int range_len = get_range_len(entry, request.get_range()); + set_range_first(request.get_range().first); + set_range_len(range_len); + } } void Response::set_taskbar(const std::string& bookName, const std::string& bookTitle) diff --git a/src/server/response.h b/src/server/response.h index d3223673d..9ae0b8854 100644 --- a/src/server/response.h +++ b/src/server/response.h @@ -50,7 +50,7 @@ class Response { void set_template(const std::string& template_str, kainjow::mustache::data data); void set_content(const std::string& content); void set_redirection(const std::string& url); - void set_entry(const Entry& entry); + void set_entry(const Entry& entry, const RequestContext& request); void set_mimeType(const std::string& mimeType) { m_mimeType = mimeType; } From 81e781133dcc0e3736c7abb126a275b07c1264f5 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Wed, 29 Apr 2020 18:33:01 +0400 Subject: [PATCH 17/18] Refactoring: utilized is_compressible_mime_type() --- src/server/response.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/server/response.cpp b/src/server/response.cpp index 7540fd687..7814f5d75 100644 --- a/src/server/response.cpp +++ b/src/server/response.cpp @@ -179,10 +179,7 @@ Response::create_raw_content_mhd_response(const RequestContext& request) } bool shouldCompress = m_compress && request.can_compress(); - shouldCompress &= m_mimeType.find("text/") != string::npos - || m_mimeType.find("application/javascript") != string::npos - || m_mimeType.find("application/json") != string::npos; - + shouldCompress &= is_compressible_mime_type(m_mimeType); shouldCompress &= (m_content.size() > KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE); if (shouldCompress) { From bfa51c2d87362006276c54ec3e7fca36872b851a Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Wed, 29 Apr 2020 18:33:25 +0400 Subject: [PATCH 18/18] Refactoring: got rid of duplicate get_mime_type() --- src/server.cpp | 23 ++++++----------------- src/server/response.h | 1 + 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/src/server.cpp b/src/server.cpp index d3845f2e4..4a99a9609 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -844,15 +844,6 @@ std::string get_book_name(const RequestContext& request) } } -std::string get_mime_type(const kiwix::Entry& entry) -{ - try { - return entry.getMimetype(); - } catch (exception& e) { - return "application/octet-stream"; - } -} - } // unnamed namespace std::shared_ptr @@ -912,18 +903,16 @@ Response InternalServer::handle_content(const RequestContext& request) return build_404(request, bookName); } - const std::string mimeType = get_mime_type(entry); - - if (m_verbose.load()) { - printf("Found %s\n", urlStr.c_str()); - printf("mimeType: %s\n", mimeType.c_str()); - } - auto response = get_default_response(); response.set_entry(entry, request); - if (mimeType.find("text/html") != string::npos) + if (m_verbose.load()) { + printf("Found %s\n", entry.getPath().c_str()); + printf("mimeType: %s\n", response.get_mimeType().c_str()); + } + + if (response.get_mimeType().find("text/html") != string::npos) response.set_taskbar(bookName, reader->getTitle()); return response; diff --git a/src/server/response.h b/src/server/response.h index 9ae0b8854..71364c638 100644 --- a/src/server/response.h +++ b/src/server/response.h @@ -62,6 +62,7 @@ class Response { void set_range_len(uint64_t len) { m_lenRange = len; } int getReturnCode() { return m_returnCode; } + std::string get_mimeType() const { return m_mimeType; } void introduce_taskbar(); void inject_externallinks_blocker();