diff --git a/src/server.cpp b/src/server.cpp index 511a8936d..4a99a9609 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -75,6 +75,8 @@ namespace kiwix { static IdNameMapper defaultNameMapper; +typedef kainjow::mustache::data MustacheData; + static int staticHandlerCallback(void* cls, struct MHD_Connection* connection, const char* url, @@ -113,6 +115,7 @@ class InternalServer { Response handle_request(const RequestContext& request); Response build_500(const std::string& msg); Response build_404(const RequestContext& request, const std::string& zimName); + Response build_redirect(const std::string& bookName, const kiwix::Entry& entry) const; Response build_homepage(const RequestContext& request); Response handle_skin(const RequestContext& request); Response handle_catalog(const RequestContext& request); @@ -123,9 +126,13 @@ class InternalServer { Response handle_captured_external(const RequestContext& request); Response handle_content(const RequestContext& request); - kainjow::mustache::data get_default_data(); - Response get_default_response(); + MustacheData get_default_data() const; + MustacheData homepage_data() const; + Response get_default_response() const; + std::shared_ptr get_reader(const std::string& bookName) const; + + private: // data std::string m_addr; int m_port; std::string m_root; @@ -359,14 +366,14 @@ Response InternalServer::handle_request(const RequestContext& request) } } -kainjow::mustache::data InternalServer::get_default_data() +MustacheData InternalServer::get_default_data() const { - kainjow::mustache::data data; + MustacheData data; data.set("root", m_root); return data; } -Response InternalServer::get_default_response() +Response InternalServer::get_default_response() const { return Response(m_root, m_verbose.load(), m_withTaskbar, m_withLibraryButton, m_blockExternalLinks); } @@ -375,7 +382,7 @@ Response InternalServer::get_default_response() Response InternalServer::build_404(const RequestContext& request, const std::string& bookName) { - kainjow::mustache::data results; + MustacheData results; results.set("url", request.get_full_url()); auto response = get_default_response(); @@ -390,7 +397,7 @@ Response InternalServer::build_404(const RequestContext& request, Response InternalServer::build_500(const std::string& msg) { - kainjow::mustache::data data; + MustacheData data; data.set("error", msg); Response response(m_root, true, false, false, false); response.set_template(RESOURCE::templates::_500_html, data); @@ -399,15 +406,15 @@ Response InternalServer::build_500(const std::string& msg) return response; } -Response InternalServer::build_homepage(const RequestContext& request) +MustacheData InternalServer::homepage_data() const { auto data = get_default_data(); - kainjow::mustache::data books{kainjow::mustache::data::type::list}; + MustacheData books{MustacheData::type::list}; for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) { auto& currentBook = mp_library->getBookById(bookId); - kainjow::mustache::data book; + MustacheData book; book.set("name", mp_nameMapper->getNameForId(bookId)); book.set("title", currentBook.getTitle()); book.set("description", currentBook.getDescription()); @@ -417,9 +424,13 @@ Response InternalServer::build_homepage(const RequestContext& request) } data.set("books", books); + return data; +} +Response InternalServer::build_homepage(const RequestContext& request) +{ auto response = get_default_response(); - response.set_template(RESOURCE::templates::index_html, data); + response.set_template(RESOURCE::templates::index_html, homepage_data()); response.set_mimeType("text/html; charset=utf-8"); response.set_compress(true); response.set_taskbar("", ""); @@ -507,14 +518,14 @@ Response InternalServer::handle_suggest(const RequestContext& request) printf("Searching suggestions for: \"%s\"\n", term.c_str()); } - kainjow::mustache::data results{kainjow::mustache::data::type::list}; + MustacheData results{MustacheData::type::list}; bool first = true; if (reader != nullptr) { /* Get the suggestions */ reader->searchSuggestionsSmart(term, maxSuggestionCount); while (reader->getNextSuggestion(suggestion)) { - kainjow::mustache::data result; + MustacheData result; result.set("label", suggestion); result.set("value", suggestion); result.set("first", first); @@ -526,7 +537,7 @@ Response InternalServer::handle_suggest(const RequestContext& request) /* Propose the fulltext search if possible */ if (reader->hasFulltextIndex()) { - kainjow::mustache::data result; + MustacheData result; result.set("label", "containing '" + term + "'..."); result.set("value", term + " "); result.set("first", first); @@ -711,10 +722,7 @@ Response InternalServer::handle_random(const RequestContext& request) try { auto entry = reader->getRandomPage(); - entry = entry.getFinalEntry(); - auto response = get_default_response(); - response.set_redirection(m_root + "/" + bookName + "/" + kiwix::urlEncode(entry.getPath())); - return response; + return build_redirect(bookName, entry.getFinalEntry()); } catch(kiwix::NoEntry& e) { return build_404(request, bookName); } @@ -824,36 +832,52 @@ Response InternalServer::handle_catalog(const RequestContext& request) return response; } +namespace +{ + +std::string get_book_name(const RequestContext& request) +{ + try { + return request.get_url_part(0); + } catch (const std::out_of_range& e) { + return std::string(); + } +} + +} // unnamed namespace + +std::shared_ptr +InternalServer::get_reader(const std::string& bookName) const +{ + std::shared_ptr reader; + try { + const std::string bookId = mp_nameMapper->getIdForName(bookName); + reader = mp_library->getReaderById(bookId); + } catch (const std::out_of_range& e) { + } + return reader; +} + +Response +InternalServer::build_redirect(const std::string& bookName, const kiwix::Entry& entry) const +{ + auto response = get_default_response(); + response.set_redirection(m_root + "/" + bookName + "/" + + kiwix::urlEncode(entry.getPath())); + return response; +} + Response InternalServer::handle_content(const RequestContext& request) { if (m_verbose.load()) { printf("** running handle_content\n"); } - std::string baseUrl; - std::string content; - std::string mimeType; - - kiwix::Entry entry; - - std::string bookName; - try { - bookName = request.get_url_part(0); - } catch (const std::out_of_range& e) { - return build_homepage(request); - } + const std::string bookName = get_book_name(request); if (bookName.empty()) return build_homepage(request); - std::string bookId; - std::shared_ptr reader; - try { - bookId = mp_nameMapper->getIdForName(bookName); - reader = mp_library->getReaderById(bookId); - } catch (const std::out_of_range& e) { - return build_404(request, bookName); - } - + const std::shared_ptr reader = get_reader(bookName); if (reader == nullptr) { return build_404(request, bookName); } @@ -863,16 +887,14 @@ Response InternalServer::handle_content(const RequestContext& request) urlStr = urlStr.substr(1); } + kiwix::Entry entry; + try { entry = reader->getEntryFromPath(urlStr); if (entry.isRedirect() || urlStr.empty()) { // If urlStr is empty, we want to mainPage. // We must do a redirection to the real page. - entry = entry.getFinalEntry(); - auto response = get_default_response(); - response.set_redirection(m_root + "/" + bookName + "/" + - kiwix::urlEncode(entry.getPath())); - return response; + return build_redirect(bookName, entry.getFinalEntry()); } } catch(kiwix::NoEntry& e) { if (m_verbose.load()) @@ -881,47 +903,19 @@ Response InternalServer::handle_content(const RequestContext& request) return build_404(request, bookName); } - try { - mimeType = entry.getMimetype(); - } catch (exception& e) { - mimeType = "application/octet-stream"; - } + auto response = get_default_response(); + + response.set_entry(entry, request); if (m_verbose.load()) { - printf("Found %s\n", urlStr.c_str()); - printf("mimeType: %s\n", mimeType.c_str()); + printf("Found %s\n", entry.getPath().c_str()); + printf("mimeType: %s\n", response.get_mimeType().c_str()); } - if (mimeType.find("text/") != string::npos - || mimeType.find("application/javascript") != string::npos - || mimeType.find("application/json") != string::npos) { - zim::Blob raw_content = entry.getBlob(); - content = string(raw_content.data(), raw_content.size()); - auto response = get_default_response(); + if (response.get_mimeType().find("text/html") != string::npos) + response.set_taskbar(bookName, reader->getTitle()); - if (mimeType.find("text/html") != string::npos) - response.set_taskbar(bookName, reader->getTitle()); - - response.set_mimeType(mimeType); - response.set_content(content); - response.set_compress(true); - response.set_cache(true); - return response; - } else { - int range_len; - if (request.get_range().second == -1) { - range_len = entry.getSize() - request.get_range().first; - } else { - range_len = request.get_range().second - request.get_range().first; - } - auto response = get_default_response(); - response.set_entry(entry); - response.set_mimeType(mimeType); - response.set_range_first(request.get_range().first); - response.set_range_len(range_len); - response.set_cache(true); - return response; - } + return response; } } diff --git a/src/server/request_context.cpp b/src/server/request_context.cpp index 0dd3114d2..37d490bf9 100644 --- a/src/server/request_context.cpp +++ b/src/server/request_context.cpp @@ -30,60 +30,56 @@ namespace kiwix { static std::atomic_ullong s_requestIndex(0); +namespace { + +RequestMethod str2RequestMethod(const std::string& method) { + if (method == "GET") return RequestMethod::GET; + else if (method == "HEAD") return RequestMethod::HEAD; + else if (method == "POST") return RequestMethod::POST; + else if (method == "PUT") return RequestMethod::PUT; + else if (method == "DELETE") return RequestMethod::DELETE_; + else if (method == "CONNECT") return RequestMethod::CONNECT; + else if (method == "OPTIONS") return RequestMethod::OPTIONS; + else if (method == "TRACE") return RequestMethod::TRACE; + else if (method == "PATCH") return RequestMethod::PATCH; + else return RequestMethod::OTHER; +} + +std::string +fullURL2LocalURL(const std::string& full_url, const std::string& rootLocation) +{ + if (rootLocation.empty()) { + // nothing special to handle. + return full_url; + } else if (full_url == rootLocation) { + return "/"; + } else if (full_url.size() > rootLocation.size() && + full_url.substr(0, rootLocation.size()+1) == rootLocation + "/") { + return full_url.substr(rootLocation.size()); + } else { + return ""; + } +} + +} // unnamed namespace + RequestContext::RequestContext(struct MHD_Connection* connection, std::string rootLocation, const std::string& _url, - const std::string& method, + const std::string& _method, const std::string& version) : full_url(_url), - url(_url), - valid_url(true), + url(fullURL2LocalURL(_url, rootLocation)), + method(str2RequestMethod(_method)), version(version), requestIndex(s_requestIndex++), acceptEncodingDeflate(false), accept_range(false), range_pair(0, -1) { - if (method == "GET") { - this->method = RequestMethod::GET; - } else if (method == "HEAD") { - this->method = RequestMethod::HEAD; - } else if (method == "POST") { - this->method = RequestMethod::POST; - } else if (method == "PUT") { - this->method = RequestMethod::PUT; - } else if (method == "DELETE") { - this->method = RequestMethod::DELETE_; - } else if (method == "CONNECT") { - this->method = RequestMethod::CONNECT; - } else if (method == "OPTIONS") { - this->method = RequestMethod::OPTIONS; - } else if (method == "TRACE") { - this->method = RequestMethod::TRACE; - } else if (method == "PATCH") { - this->method = RequestMethod::PATCH; - } else { - this->method = RequestMethod::OTHER; - } - MHD_get_connection_values(connection, MHD_HEADER_KIND, &RequestContext::fill_header, this); MHD_get_connection_values(connection, MHD_GET_ARGUMENT_KIND, &RequestContext::fill_argument, this); - valid_url = true; - if (rootLocation.empty()) { - // nothing special to handle. - url = full_url; - } else { - if (full_url == rootLocation) { - url = "/"; - } else if (full_url.size() > rootLocation.size() && - full_url.substr(0, rootLocation.size()+1) == rootLocation + "/") { - url = full_url.substr(rootLocation.size()); - } else { - valid_url = false; - } - } - try { acceptEncodingDeflate = (get_header(MHD_HTTP_HEADER_ACCEPT_ENCODING).find("deflate") != std::string::npos); @@ -147,10 +143,11 @@ void RequestContext::print_debug_info() const { printf(" - %s : '%s'\n", it->first.c_str(), it->second.c_str()); } printf("Parsed : \n"); + printf("full_url: %s\n", full_url.c_str()); printf("url : %s\n", url.c_str()); printf("acceptEncodingDeflate : %d\n", acceptEncodingDeflate); printf("has_range : %d\n", accept_range); - printf("is_valid_url : %d\n", valid_url); + printf("is_valid_url : %d\n", is_valid_url()); printf(".............\n"); } @@ -188,7 +185,7 @@ std::string RequestContext::get_full_url() const { } bool RequestContext::is_valid_url() const { - return valid_url; + return !url.empty(); } bool RequestContext::has_range() const { diff --git a/src/server/request_context.h b/src/server/request_context.h index 5bbed949a..d58040f07 100644 --- a/src/server/request_context.h +++ b/src/server/request_context.h @@ -51,7 +51,10 @@ class IndexError: public std::runtime_error {}; class RequestContext { - public: + public: // types + typedef std::pair ByteRange; + + public: // functions RequestContext(struct MHD_Connection* connection, std::string rootLocation, const std::string& url, @@ -79,14 +82,13 @@ class RequestContext { std::string get_full_url() const; bool has_range() const; - std::pair get_range() const; + ByteRange get_range() const; bool can_compress() const { return acceptEncodingDeflate; } - private: + private: // data std::string full_url; std::string url; - bool valid_url; RequestMethod method; std::string version; unsigned long long requestIndex; @@ -94,10 +96,11 @@ class RequestContext { bool acceptEncodingDeflate; bool accept_range; - std::pair range_pair; + ByteRange range_pair; std::map headers; std::map arguments; + private: // functions static int fill_header(void *, enum MHD_ValueKind, const char*, const char*); static int fill_argument(void *, enum MHD_ValueKind, const char*, const char*); }; diff --git a/src/server/response.cpp b/src/server/response.cpp index bba6d3028..7814f5d75 100644 --- a/src/server/response.cpp +++ b/src/server/response.cpp @@ -17,6 +17,35 @@ namespace kiwix { +namespace +{ +// some utilities + +std::string get_mime_type(const kiwix::Entry& entry) +{ + try { + return entry.getMimetype(); + } catch (exception& e) { + return "application/octet-stream"; + } +} + +bool is_compressible_mime_type(const std::string& mimeType) +{ + return mimeType.find("text/") != string::npos + || mimeType.find("application/javascript") != string::npos + || mimeType.find("application/json") != string::npos; +} + +int get_range_len(const kiwix::Entry& entry, RequestContext::ByteRange range) +{ + return range.second == -1 + ? entry.getSize() - range.first + : range.second - range.first; +} + +} // unnamed namespace + Response::Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks) : m_verbose(verbose), m_root(root), @@ -139,85 +168,102 @@ void Response::inject_externallinks_blocker() script_tag); } +MHD_Response* +Response::create_raw_content_mhd_response(const RequestContext& request) +{ + if (m_addTaskbar) { + introduce_taskbar(); + } + if ( m_blockExternalLinks ) { + inject_externallinks_blocker(); + } + + bool shouldCompress = m_compress && request.can_compress(); + shouldCompress &= is_compressible_mime_type(m_mimeType); + shouldCompress &= (m_content.size() > KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE); + + if (shouldCompress) { + std::vector compr_buffer(compressBound(m_content.size())); + uLongf comprLen = compr_buffer.capacity(); + int err = compress(&compr_buffer[0], + &comprLen, + (const Bytef*)(m_content.data()), + m_content.size()); + if (err == Z_OK && comprLen > 2 && comprLen < (m_content.size() + 2)) { + /* /!\ Internet Explorer has a bug with deflate compression. + It can not handle the first two bytes (compression headers) + We need to chunk them off (move the content 2bytes) + It has no incidence on other browsers + See http://www.subbu.org/blog/2008/03/ie7-deflate-or-not and comments */ + m_content = string((char*)&compr_buffer[2], comprLen - 2); + } else { + shouldCompress = false; + } + } + + MHD_Response* response = MHD_create_response_from_buffer( + m_content.size(), const_cast(m_content.data()), MHD_RESPMEM_MUST_COPY); + + if (shouldCompress) { + MHD_add_response_header( + response, MHD_HTTP_HEADER_VARY, "Accept-Encoding"); + MHD_add_response_header( + response, MHD_HTTP_HEADER_CONTENT_ENCODING, "deflate"); + } + MHD_add_response_header(response, MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str()); + return response; +} + +MHD_Response* +Response::create_redirection_mhd_response() const +{ + MHD_Response* response = MHD_create_response_from_buffer(0, nullptr, MHD_RESPMEM_MUST_COPY); + MHD_add_response_header(response, MHD_HTTP_HEADER_LOCATION, m_content.c_str()); + return response; +} + +MHD_Response* +Response::create_entry_mhd_response() const +{ + MHD_Response* response = MHD_create_response_from_callback(m_entry.getSize(), + 16384, + callback_reader_from_entry, + new RunningResponse(m_entry, m_startRange), + callback_free_response); + MHD_add_response_header(response, + MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str()); + MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes"); + std::ostringstream oss; + oss << "bytes " << m_startRange << "-" << m_startRange + m_lenRange - 1 + << "/" << m_entry.getSize(); + + MHD_add_response_header(response, + MHD_HTTP_HEADER_CONTENT_RANGE, oss.str().c_str()); + + MHD_add_response_header(response, + MHD_HTTP_HEADER_CONTENT_LENGTH, kiwix::to_string(m_lenRange).c_str()); + return response; +} + +MHD_Response* +Response::create_mhd_response(const RequestContext& request) +{ + switch (m_mode) { + case ResponseMode::RAW_CONTENT : + return create_raw_content_mhd_response(request); + + case ResponseMode::REDIRECTION : + return create_redirection_mhd_response(); + + case ResponseMode::ENTRY : + return create_entry_mhd_response(); + } + return nullptr; +} int Response::send(const RequestContext& request, MHD_Connection* connection) { - MHD_Response* response = nullptr; - switch (m_mode) { - case ResponseMode::RAW_CONTENT : { - if (m_addTaskbar) { - introduce_taskbar(); - } - if ( m_blockExternalLinks ) { - inject_externallinks_blocker(); - } - - bool shouldCompress = m_compress && request.can_compress(); - shouldCompress &= m_mimeType.find("text/") != string::npos - || m_mimeType.find("application/javascript") != string::npos - || m_mimeType.find("application/json") != string::npos; - - shouldCompress &= (m_content.size() > KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE); - - if (shouldCompress) { - std::vector compr_buffer(compressBound(m_content.size())); - uLongf comprLen = compr_buffer.capacity(); - int err = compress(&compr_buffer[0], - &comprLen, - (const Bytef*)(m_content.data()), - m_content.size()); - if (err == Z_OK && comprLen > 2 && comprLen < (m_content.size() + 2)) { - /* /!\ Internet Explorer has a bug with deflate compression. - It can not handle the first two bytes (compression headers) - We need to chunk them off (move the content 2bytes) - It has no incidence on other browsers - See http://www.subbu.org/blog/2008/03/ie7-deflate-or-not and comments */ - m_content = string((char*)&compr_buffer[2], comprLen - 2); - } else { - shouldCompress = false; - } - } - - response = MHD_create_response_from_buffer( - m_content.size(), const_cast(m_content.data()), MHD_RESPMEM_MUST_COPY); - - if (shouldCompress) { - MHD_add_response_header( - response, MHD_HTTP_HEADER_VARY, "Accept-Encoding"); - MHD_add_response_header( - response, MHD_HTTP_HEADER_CONTENT_ENCODING, "deflate"); - } - MHD_add_response_header(response, MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str()); - break; - } - - case ResponseMode::REDIRECTION : { - response = MHD_create_response_from_buffer(0, nullptr, MHD_RESPMEM_MUST_COPY); - MHD_add_response_header(response, MHD_HTTP_HEADER_LOCATION, m_content.c_str()); - break; - } - - case ResponseMode::ENTRY : { - response = MHD_create_response_from_callback(m_entry.getSize(), - 16384, - callback_reader_from_entry, - new RunningResponse(m_entry, m_startRange), - callback_free_response); - MHD_add_response_header(response, - MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str()); - MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes"); - std::ostringstream oss; - oss << "bytes " << m_startRange << "-" << m_startRange + m_lenRange - 1 - << "/" << m_entry.getSize(); - - MHD_add_response_header(response, - MHD_HTTP_HEADER_CONTENT_RANGE, oss.str().c_str()); - - MHD_add_response_header(response, - MHD_HTTP_HEADER_CONTENT_LENGTH, kiwix::to_string(m_lenRange).c_str()); - break; - } - } + MHD_Response* response = create_mhd_response(request); MHD_add_response_header(response, "Access-Control-Allow-Origin", "*"); MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL, @@ -249,9 +295,25 @@ void Response::set_redirection(const std::string& url) { m_returnCode = MHD_HTTP_FOUND; } -void Response::set_entry(const Entry& entry) { +void Response::set_entry(const Entry& entry, const RequestContext& request) { m_entry = entry; m_mode = ResponseMode::ENTRY; + + const std::string mimeType = get_mime_type(entry); + set_mimeType(mimeType); + set_cache(true); + + if ( is_compressible_mime_type(mimeType) ) { + zim::Blob raw_content = entry.getBlob(); + const std::string content = string(raw_content.data(), raw_content.size()); + + set_content(content); + set_compress(true); + } else { + const int range_len = get_range_len(entry, request.get_range()); + set_range_first(request.get_range().first); + set_range_len(range_len); + } } void Response::set_taskbar(const std::string& bookName, const std::string& bookTitle) diff --git a/src/server/response.h b/src/server/response.h index 89f235072..71364c638 100644 --- a/src/server/response.h +++ b/src/server/response.h @@ -50,7 +50,7 @@ class Response { void set_template(const std::string& template_str, kainjow::mustache::data data); void set_content(const std::string& content); void set_redirection(const std::string& url); - void set_entry(const Entry& entry); + void set_entry(const Entry& entry, const RequestContext& request); void set_mimeType(const std::string& mimeType) { m_mimeType = mimeType; } @@ -62,11 +62,18 @@ class Response { void set_range_len(uint64_t len) { m_lenRange = len; } int getReturnCode() { return m_returnCode; } + std::string get_mimeType() const { return m_mimeType; } void introduce_taskbar(); void inject_externallinks_blocker(); - private: + private: // functions + MHD_Response* create_mhd_response(const RequestContext& request); + MHD_Response* create_raw_content_mhd_response(const RequestContext& request); + MHD_Response* create_redirection_mhd_response() const; + MHD_Response* create_entry_mhd_response() const; + + private: // data bool m_verbose; ResponseMode m_mode; std::string m_root;