smallsolar
/
libkiwix
mirror of https://github.com/kiwix/libkiwix.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #345 from kiwix/server_refactoring

This commit is contained in:
Matthieu Gautier 2020-04-29 17:38:41 +02:00 committed by GitHub
parent 80fcbceeb3 bfa51c2d87
commit 1842e8f51c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 269 additions and 206 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

154
src/server.cpp
View File

@ -75,6 +75,8 @@ namespace kiwix {
static IdNameMapper defaultNameMapper;
typedef kainjow::mustache::data MustacheData;
static int staticHandlerCallback(void* cls,
struct MHD_Connection* connection,
const char* url,
@ -113,6 +115,7 @@ class InternalServer {
Response handle_request(const RequestContext& request);
Response build_500(const std::string& msg);
Response build_404(const RequestContext& request, const std::string& zimName);
Response build_redirect(const std::string& bookName, const kiwix::Entry& entry) const;
Response build_homepage(const RequestContext& request);
Response handle_skin(const RequestContext& request);
Response handle_catalog(const RequestContext& request);
@ -123,9 +126,13 @@ class InternalServer {
Response handle_captured_external(const RequestContext& request);
Response handle_content(const RequestContext& request);
kainjow::mustache::data get_default_data();
Response get_default_response();
MustacheData get_default_data() const;
MustacheData homepage_data() const;
Response get_default_response() const;
std::shared_ptr<Reader> get_reader(const std::string& bookName) const;
private: // data
std::string m_addr;
int m_port;
std::string m_root;
@ -359,14 +366,14 @@ Response InternalServer::handle_request(const RequestContext& request)
}
}
kainjow::mustache::data InternalServer::get_default_data()
MustacheData InternalServer::get_default_data() const
{
kainjow::mustache::data data;
MustacheData data;
data.set("root", m_root);
return data;
}
Response InternalServer::get_default_response()
Response InternalServer::get_default_response() const
{
return Response(m_root, m_verbose.load(), m_withTaskbar, m_withLibraryButton, m_blockExternalLinks);
}
@ -375,7 +382,7 @@ Response InternalServer::get_default_response()
Response InternalServer::build_404(const RequestContext& request,
const std::string& bookName)
{
kainjow::mustache::data results;
MustacheData results;
results.set("url", request.get_full_url());
auto response = get_default_response();
@ -390,7 +397,7 @@ Response InternalServer::build_404(const RequestContext& request,
Response InternalServer::build_500(const std::string& msg)
{
kainjow::mustache::data data;
MustacheData data;
data.set("error", msg);
Response response(m_root, true, false, false, false);
response.set_template(RESOURCE::templates::_500_html, data);
@ -399,15 +406,15 @@ Response InternalServer::build_500(const std::string& msg)
return response;
}
Response InternalServer::build_homepage(const RequestContext& request)
MustacheData InternalServer::homepage_data() const
{
auto data = get_default_data();
kainjow::mustache::data books{kainjow::mustache::data::type::list};
MustacheData books{MustacheData::type::list};
for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
auto& currentBook = mp_library->getBookById(bookId);
kainjow::mustache::data book;
MustacheData book;
book.set("name", mp_nameMapper->getNameForId(bookId));
book.set("title", currentBook.getTitle());
book.set("description", currentBook.getDescription());
@ -417,9 +424,13 @@ Response InternalServer::build_homepage(const RequestContext& request)
}
data.set("books", books);
return data;
}
Response InternalServer::build_homepage(const RequestContext& request)
{
auto response = get_default_response();
response.set_template(RESOURCE::templates::index_html, data);
response.set_template(RESOURCE::templates::index_html, homepage_data());
response.set_mimeType("text/html; charset=utf-8");
response.set_compress(true);
response.set_taskbar("", "");
@ -507,14 +518,14 @@ Response InternalServer::handle_suggest(const RequestContext& request)
printf("Searching suggestions for: \"%s\"\n", term.c_str());
}
kainjow::mustache::data results{kainjow::mustache::data::type::list};
MustacheData results{MustacheData::type::list};
bool first = true;
if (reader != nullptr) {
/* Get the suggestions */
reader->searchSuggestionsSmart(term, maxSuggestionCount);
while (reader->getNextSuggestion(suggestion)) {
kainjow::mustache::data result;
MustacheData result;
result.set("label", suggestion);
result.set("value", suggestion);
result.set("first", first);
@ -526,7 +537,7 @@ Response InternalServer::handle_suggest(const RequestContext& request)
/* Propose the fulltext search if possible */
if (reader->hasFulltextIndex()) {
kainjow::mustache::data result;
MustacheData result;
result.set("label", "containing '" + term + "'...");
result.set("value", term + " ");
result.set("first", first);
@ -711,10 +722,7 @@ Response InternalServer::handle_random(const RequestContext& request)
try {
auto entry = reader->getRandomPage();
entry = entry.getFinalEntry();
auto response = get_default_response();
response.set_redirection(m_root + "/" + bookName + "/" + kiwix::urlEncode(entry.getPath()));
return response;
return build_redirect(bookName, entry.getFinalEntry());
} catch(kiwix::NoEntry& e) {
return build_404(request, bookName);
}
@ -824,36 +832,52 @@ Response InternalServer::handle_catalog(const RequestContext& request)
return response;
}
namespace
{
std::string get_book_name(const RequestContext& request)
{
try {
return request.get_url_part(0);
} catch (const std::out_of_range& e) {
return std::string();
}
}
} // unnamed namespace
std::shared_ptr<Reader>
InternalServer::get_reader(const std::string& bookName) const
{
std::shared_ptr<Reader> reader;
try {
const std::string bookId = mp_nameMapper->getIdForName(bookName);
reader = mp_library->getReaderById(bookId);
} catch (const std::out_of_range& e) {
}
return reader;
}
Response
InternalServer::build_redirect(const std::string& bookName, const kiwix::Entry& entry) const
{
auto response = get_default_response();
response.set_redirection(m_root + "/" + bookName + "/" +
kiwix::urlEncode(entry.getPath()));
return response;
}
Response InternalServer::handle_content(const RequestContext& request)
{
if (m_verbose.load()) {
printf("** running handle_content\n");
}
std::string baseUrl;
std::string content;
std::string mimeType;
kiwix::Entry entry;
std::string bookName;
try {
bookName = request.get_url_part(0);
} catch (const std::out_of_range& e) {
return build_homepage(request);
}
const std::string bookName = get_book_name(request);
if (bookName.empty())
return build_homepage(request);
std::string bookId;
std::shared_ptr<Reader> reader;
try {
bookId = mp_nameMapper->getIdForName(bookName);
reader = mp_library->getReaderById(bookId);
} catch (const std::out_of_range& e) {
return build_404(request, bookName);
}
const std::shared_ptr<Reader> reader = get_reader(bookName);
if (reader == nullptr) {
return build_404(request, bookName);
}
@ -863,16 +887,14 @@ Response InternalServer::handle_content(const RequestContext& request)
urlStr = urlStr.substr(1);
}
kiwix::Entry entry;
try {
entry = reader->getEntryFromPath(urlStr);
if (entry.isRedirect() || urlStr.empty()) {
// If urlStr is empty, we want to mainPage.
// We must do a redirection to the real page.
entry = entry.getFinalEntry();
auto response = get_default_response();
response.set_redirection(m_root + "/" + bookName + "/" +
kiwix::urlEncode(entry.getPath()));
return response;
return build_redirect(bookName, entry.getFinalEntry());
}
} catch(kiwix::NoEntry& e) {
if (m_verbose.load())
@ -881,47 +903,19 @@ Response InternalServer::handle_content(const RequestContext& request)
return build_404(request, bookName);
}
try {
mimeType = entry.getMimetype();
} catch (exception& e) {
mimeType = "application/octet-stream";
}
auto response = get_default_response();
response.set_entry(entry, request);
if (m_verbose.load()) {
printf("Found %s\n", urlStr.c_str());
printf("mimeType: %s\n", mimeType.c_str());
printf("Found %s\n", entry.getPath().c_str());
printf("mimeType: %s\n", response.get_mimeType().c_str());
}
if (mimeType.find("text/") != string::npos
|| mimeType.find("application/javascript") != string::npos
|| mimeType.find("application/json") != string::npos) {
zim::Blob raw_content = entry.getBlob();
content = string(raw_content.data(), raw_content.size());
auto response = get_default_response();
if (response.get_mimeType().find("text/html") != string::npos)
response.set_taskbar(bookName, reader->getTitle());
if (mimeType.find("text/html") != string::npos)
response.set_taskbar(bookName, reader->getTitle());
response.set_mimeType(mimeType);
response.set_content(content);
response.set_compress(true);
response.set_cache(true);
return response;
} else {
int range_len;
if (request.get_range().second == -1) {
range_len = entry.getSize() - request.get_range().first;
} else {
range_len = request.get_range().second - request.get_range().first;
}
auto response = get_default_response();
response.set_entry(entry);
response.set_mimeType(mimeType);
response.set_range_first(request.get_range().first);
response.set_range_len(range_len);
response.set_cache(true);
return response;
}
return response;
}
}

81
src/server/request_context.cpp
View File

@ -30,60 +30,56 @@ namespace kiwix {
static std::atomic_ullong s_requestIndex(0);
namespace {
RequestMethod str2RequestMethod(const std::string& method) {
if (method == "GET") return RequestMethod::GET;
else if (method == "HEAD") return RequestMethod::HEAD;
else if (method == "POST") return RequestMethod::POST;
else if (method == "PUT") return RequestMethod::PUT;
else if (method == "DELETE") return RequestMethod::DELETE_;
else if (method == "CONNECT") return RequestMethod::CONNECT;
else if (method == "OPTIONS") return RequestMethod::OPTIONS;
else if (method == "TRACE") return RequestMethod::TRACE;
else if (method == "PATCH") return RequestMethod::PATCH;
else return RequestMethod::OTHER;
}
std::string
fullURL2LocalURL(const std::string& full_url, const std::string& rootLocation)
{
if (rootLocation.empty()) {
// nothing special to handle.
return full_url;
} else if (full_url == rootLocation) {
return "/";
} else if (full_url.size() > rootLocation.size() &&
full_url.substr(0, rootLocation.size()+1) == rootLocation + "/") {
return full_url.substr(rootLocation.size());
} else {
return "";
}
}
} // unnamed namespace
RequestContext::RequestContext(struct MHD_Connection* connection,
std::string rootLocation,
const std::string& _url,
const std::string& method,
const std::string& _method,
const std::string& version) :
full_url(_url),
url(_url),
valid_url(true),
url(fullURL2LocalURL(_url, rootLocation)),
method(str2RequestMethod(_method)),
version(version),
requestIndex(s_requestIndex++),
acceptEncodingDeflate(false),
accept_range(false),
range_pair(0, -1)
{
if (method == "GET") {
this->method = RequestMethod::GET;
} else if (method == "HEAD") {
this->method = RequestMethod::HEAD;
} else if (method == "POST") {
this->method = RequestMethod::POST;
} else if (method == "PUT") {
this->method = RequestMethod::PUT;
} else if (method == "DELETE") {
this->method = RequestMethod::DELETE_;
} else if (method == "CONNECT") {
this->method = RequestMethod::CONNECT;
} else if (method == "OPTIONS") {
this->method = RequestMethod::OPTIONS;
} else if (method == "TRACE") {
this->method = RequestMethod::TRACE;
} else if (method == "PATCH") {
this->method = RequestMethod::PATCH;
} else {
this->method = RequestMethod::OTHER;
}
MHD_get_connection_values(connection, MHD_HEADER_KIND, &RequestContext::fill_header, this);
MHD_get_connection_values(connection, MHD_GET_ARGUMENT_KIND, &RequestContext::fill_argument, this);
valid_url = true;
if (rootLocation.empty()) {
// nothing special to handle.
url = full_url;
} else {
if (full_url == rootLocation) {
url = "/";
} else if (full_url.size() > rootLocation.size() &&
full_url.substr(0, rootLocation.size()+1) == rootLocation + "/") {
url = full_url.substr(rootLocation.size());
} else {
valid_url = false;
}
}
try {
acceptEncodingDeflate =
(get_header(MHD_HTTP_HEADER_ACCEPT_ENCODING).find("deflate") != std::string::npos);
@ -147,10 +143,11 @@ void RequestContext::print_debug_info() const {
printf(" - %s : '%s'\n", it->first.c_str(), it->second.c_str());
}
printf("Parsed : \n");
printf("full_url: %s\n", full_url.c_str());
printf("url : %s\n", url.c_str());
printf("acceptEncodingDeflate : %d\n", acceptEncodingDeflate);
printf("has_range : %d\n", accept_range);
printf("is_valid_url : %d\n", valid_url);
printf("is_valid_url : %d\n", is_valid_url());
printf(".............\n");
}
@ -188,7 +185,7 @@ std::string RequestContext::get_full_url() const {
}
bool RequestContext::is_valid_url() const {
return valid_url;
return !url.empty();
}
bool RequestContext::has_range() const {

13
src/server/request_context.h
View File

@ -51,7 +51,10 @@ class IndexError: public std::runtime_error {};
class RequestContext {
public:
public: // types
typedef std::pair<int, int> ByteRange;
public: // functions
RequestContext(struct MHD_Connection* connection,
std::string rootLocation,
const std::string& url,
@ -79,14 +82,13 @@ class RequestContext {
std::string get_full_url() const;
bool has_range() const;
std::pair<int, int> get_range() const;
ByteRange get_range() const;
bool can_compress() const { return acceptEncodingDeflate; }
private:
private: // data
std::string full_url;
std::string url;
bool valid_url;
RequestMethod method;
std::string version;
unsigned long long requestIndex;
@ -94,10 +96,11 @@ class RequestContext {
bool acceptEncodingDeflate;
bool accept_range;
std::pair<int, int> range_pair;
ByteRange range_pair;
std::map<std::string, std::string> headers;
std::map<std::string, std::string> arguments;
private: // functions
static int fill_header(void *, enum MHD_ValueKind, const char*, const char*);
static int fill_argument(void *, enum MHD_ValueKind, const char*, const char*);
};

216
src/server/response.cpp
View File

@ -17,6 +17,35 @@
namespace kiwix {
namespace
{
// some utilities
std::string get_mime_type(const kiwix::Entry& entry)
{
try {
return entry.getMimetype();
} catch (exception& e) {
return "application/octet-stream";
}
}
bool is_compressible_mime_type(const std::string& mimeType)
{
return mimeType.find("text/") != string::npos
|| mimeType.find("application/javascript") != string::npos
|| mimeType.find("application/json") != string::npos;
}
int get_range_len(const kiwix::Entry& entry, RequestContext::ByteRange range)
{
return range.second == -1
? entry.getSize() - range.first
: range.second - range.first;
}
} // unnamed namespace
Response::Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks)
: m_verbose(verbose),
m_root(root),
@ -139,85 +168,102 @@ void Response::inject_externallinks_blocker()
script_tag);
}
MHD_Response*
Response::create_raw_content_mhd_response(const RequestContext& request)
{
if (m_addTaskbar) {
introduce_taskbar();
}
if ( m_blockExternalLinks ) {
inject_externallinks_blocker();
}
bool shouldCompress = m_compress && request.can_compress();
shouldCompress &= is_compressible_mime_type(m_mimeType);
shouldCompress &= (m_content.size() > KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE);
if (shouldCompress) {
std::vector<Bytef> compr_buffer(compressBound(m_content.size()));
uLongf comprLen = compr_buffer.capacity();
int err = compress(&compr_buffer[0],
&comprLen,
(const Bytef*)(m_content.data()),
m_content.size());
if (err == Z_OK && comprLen > 2 && comprLen < (m_content.size() + 2)) {
/* /!\ Internet Explorer has a bug with deflate compression.
It can not handle the first two bytes (compression headers)
We need to chunk them off (move the content 2bytes)
It has no incidence on other browsers
See http://www.subbu.org/blog/2008/03/ie7-deflate-or-not and comments */
m_content = string((char*)&compr_buffer[2], comprLen - 2);
} else {
shouldCompress = false;
}
}
MHD_Response* response = MHD_create_response_from_buffer(
m_content.size(), const_cast<char*>(m_content.data()), MHD_RESPMEM_MUST_COPY);
if (shouldCompress) {
MHD_add_response_header(
response, MHD_HTTP_HEADER_VARY, "Accept-Encoding");
MHD_add_response_header(
response, MHD_HTTP_HEADER_CONTENT_ENCODING, "deflate");
}
MHD_add_response_header(response, MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str());
return response;
}
MHD_Response*
Response::create_redirection_mhd_response() const
{
MHD_Response* response = MHD_create_response_from_buffer(0, nullptr, MHD_RESPMEM_MUST_COPY);
MHD_add_response_header(response, MHD_HTTP_HEADER_LOCATION, m_content.c_str());
return response;
}
MHD_Response*
Response::create_entry_mhd_response() const
{
MHD_Response* response = MHD_create_response_from_callback(m_entry.getSize(),
16384,
callback_reader_from_entry,
new RunningResponse(m_entry, m_startRange),
callback_free_response);
MHD_add_response_header(response,
MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str());
MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes");
std::ostringstream oss;
oss << "bytes " << m_startRange << "-" << m_startRange + m_lenRange - 1
<< "/" << m_entry.getSize();
MHD_add_response_header(response,
MHD_HTTP_HEADER_CONTENT_RANGE, oss.str().c_str());
MHD_add_response_header(response,
MHD_HTTP_HEADER_CONTENT_LENGTH, kiwix::to_string(m_lenRange).c_str());
return response;
}
MHD_Response*
Response::create_mhd_response(const RequestContext& request)
{
switch (m_mode) {
case ResponseMode::RAW_CONTENT :
return create_raw_content_mhd_response(request);
case ResponseMode::REDIRECTION :
return create_redirection_mhd_response();
case ResponseMode::ENTRY :
return create_entry_mhd_response();
}
return nullptr;
}
int Response::send(const RequestContext& request, MHD_Connection* connection)
{
MHD_Response* response = nullptr;
switch (m_mode) {
case ResponseMode::RAW_CONTENT : {
if (m_addTaskbar) {
introduce_taskbar();
}
if ( m_blockExternalLinks ) {
inject_externallinks_blocker();
}
bool shouldCompress = m_compress && request.can_compress();
shouldCompress &= m_mimeType.find("text/") != string::npos
|| m_mimeType.find("application/javascript") != string::npos
|| m_mimeType.find("application/json") != string::npos;
shouldCompress &= (m_content.size() > KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE);
if (shouldCompress) {
std::vector<Bytef> compr_buffer(compressBound(m_content.size()));
uLongf comprLen = compr_buffer.capacity();
int err = compress(&compr_buffer[0],
&comprLen,
(const Bytef*)(m_content.data()),
m_content.size());
if (err == Z_OK && comprLen > 2 && comprLen < (m_content.size() + 2)) {
/* /!\ Internet Explorer has a bug with deflate compression.
It can not handle the first two bytes (compression headers)
We need to chunk them off (move the content 2bytes)
It has no incidence on other browsers
See http://www.subbu.org/blog/2008/03/ie7-deflate-or-not and comments */
m_content = string((char*)&compr_buffer[2], comprLen - 2);
} else {
shouldCompress = false;
}
}
response = MHD_create_response_from_buffer(
m_content.size(), const_cast<char*>(m_content.data()), MHD_RESPMEM_MUST_COPY);
if (shouldCompress) {
MHD_add_response_header(
response, MHD_HTTP_HEADER_VARY, "Accept-Encoding");
MHD_add_response_header(
response, MHD_HTTP_HEADER_CONTENT_ENCODING, "deflate");
}
MHD_add_response_header(response, MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str());
break;
}
case ResponseMode::REDIRECTION : {
response = MHD_create_response_from_buffer(0, nullptr, MHD_RESPMEM_MUST_COPY);
MHD_add_response_header(response, MHD_HTTP_HEADER_LOCATION, m_content.c_str());
break;
}
case ResponseMode::ENTRY : {
response = MHD_create_response_from_callback(m_entry.getSize(),
16384,
callback_reader_from_entry,
new RunningResponse(m_entry, m_startRange),
callback_free_response);
MHD_add_response_header(response,
MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str());
MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes");
std::ostringstream oss;
oss << "bytes " << m_startRange << "-" << m_startRange + m_lenRange - 1
<< "/" << m_entry.getSize();
MHD_add_response_header(response,
MHD_HTTP_HEADER_CONTENT_RANGE, oss.str().c_str());
MHD_add_response_header(response,
MHD_HTTP_HEADER_CONTENT_LENGTH, kiwix::to_string(m_lenRange).c_str());
break;
}
}
MHD_Response* response = create_mhd_response(request);
MHD_add_response_header(response, "Access-Control-Allow-Origin", "*");
MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL,
@ -249,9 +295,25 @@ void Response::set_redirection(const std::string& url) {
m_returnCode = MHD_HTTP_FOUND;
}
void Response::set_entry(const Entry& entry) {
void Response::set_entry(const Entry& entry, const RequestContext& request) {
m_entry = entry;
m_mode = ResponseMode::ENTRY;
const std::string mimeType = get_mime_type(entry);
set_mimeType(mimeType);
set_cache(true);
if ( is_compressible_mime_type(mimeType) ) {
zim::Blob raw_content = entry.getBlob();
const std::string content = string(raw_content.data(), raw_content.size());
set_content(content);
set_compress(true);
} else {
const int range_len = get_range_len(entry, request.get_range());
set_range_first(request.get_range().first);
set_range_len(range_len);
}
}
void Response::set_taskbar(const std::string& bookName, const std::string& bookTitle)

11
src/server/response.h
View File

@ -50,7 +50,7 @@ class Response {
void set_template(const std::string& template_str, kainjow::mustache::data data);
void set_content(const std::string& content);
void set_redirection(const std::string& url);
void set_entry(const Entry& entry);
void set_entry(const Entry& entry, const RequestContext& request);
void set_mimeType(const std::string& mimeType) { m_mimeType = mimeType; }
@ -62,11 +62,18 @@ class Response {
void set_range_len(uint64_t len) { m_lenRange = len; }
int getReturnCode() { return m_returnCode; }
std::string get_mimeType() const { return m_mimeType; }
void introduce_taskbar();
void inject_externallinks_blocker();
private:
private: // functions
MHD_Response* create_mhd_response(const RequestContext& request);
MHD_Response* create_raw_content_mhd_response(const RequestContext& request);
MHD_Response* create_redirection_mhd_response() const;
MHD_Response* create_entry_mhd_response() const;
private: // data
bool m_verbose;
ResponseMode m_mode;
std::string m_root;