smallsolar
/
libkiwix
mirror of https://github.com/kiwix/libkiwix.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #345 from kiwix/server_refactoring

This commit is contained in:
Matthieu Gautier 2020-04-29 17:38:41 +02:00 committed by GitHub
parent 80fcbceeb3 bfa51c2d87
commit 1842e8f51c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 269 additions and 206 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

150
src/server.cpp
View File

@ -75,6 +75,8 @@ namespace kiwix {
static IdNameMapper defaultNameMapper;
typedef kainjow::mustache::data MustacheData;
static int staticHandlerCallback(void* cls,
struct MHD_Connection* connection,
const char* url,
@ -113,6 +115,7 @@ class InternalServer {
Response handle_request(const RequestContext& request);
Response build_500(const std::string& msg);
Response build_404(const RequestContext& request, const std::string& zimName);
Response build_redirect(const std::string& bookName, const kiwix::Entry& entry) const;
Response build_homepage(const RequestContext& request);
Response handle_skin(const RequestContext& request);
Response handle_catalog(const RequestContext& request);
@ -123,9 +126,13 @@ class InternalServer {
Response handle_captured_external(const RequestContext& request);
Response handle_content(const RequestContext& request);
kainjow::mustache::data get_default_data();
Response get_default_response();
MustacheData get_default_data() const;
MustacheData homepage_data() const;
Response get_default_response() const;
std::shared_ptr<Reader> get_reader(const std::string& bookName) const;
private: // data
std::string m_addr;
int m_port;
std::string m_root;
@ -359,14 +366,14 @@ Response InternalServer::handle_request(const RequestContext& request)
}
}
kainjow::mustache::data InternalServer::get_default_data()
MustacheData InternalServer::get_default_data() const
{
kainjow::mustache::data data;
MustacheData data;
data.set("root", m_root);
return data;
}
Response InternalServer::get_default_response()
Response InternalServer::get_default_response() const
{
return Response(m_root, m_verbose.load(), m_withTaskbar, m_withLibraryButton, m_blockExternalLinks);
}
@ -375,7 +382,7 @@ Response InternalServer::get_default_response()
Response InternalServer::build_404(const RequestContext& request,
const std::string& bookName)
{
kainjow::mustache::data results;
MustacheData results;
results.set("url", request.get_full_url());
auto response = get_default_response();
@ -390,7 +397,7 @@ Response InternalServer::build_404(const RequestContext& request,
Response InternalServer::build_500(const std::string& msg)
{
kainjow::mustache::data data;
MustacheData data;
data.set("error", msg);
Response response(m_root, true, false, false, false);
response.set_template(RESOURCE::templates::_500_html, data);
@ -399,15 +406,15 @@ Response InternalServer::build_500(const std::string& msg)
return response;
}
Response InternalServer::build_homepage(const RequestContext& request)
MustacheData InternalServer::homepage_data() const
{
auto data = get_default_data();
kainjow::mustache::data books{kainjow::mustache::data::type::list};
MustacheData books{MustacheData::type::list};
for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
auto& currentBook = mp_library->getBookById(bookId);
kainjow::mustache::data book;
MustacheData book;
book.set("name", mp_nameMapper->getNameForId(bookId));
book.set("title", currentBook.getTitle());
book.set("description", currentBook.getDescription());
@ -417,9 +424,13 @@ Response InternalServer::build_homepage(const RequestContext& request)
}
data.set("books", books);
return data;
}
Response InternalServer::build_homepage(const RequestContext& request)
{
auto response = get_default_response();
response.set_template(RESOURCE::templates::index_html, data);
response.set_template(RESOURCE::templates::index_html, homepage_data());
response.set_mimeType("text/html; charset=utf-8");
response.set_compress(true);
response.set_taskbar("", "");
@ -507,14 +518,14 @@ Response InternalServer::handle_suggest(const RequestContext& request)
printf("Searching suggestions for: \"%s\"\n", term.c_str());
}
kainjow::mustache::data results{kainjow::mustache::data::type::list};
MustacheData results{MustacheData::type::list};
bool first = true;
if (reader != nullptr) {
/* Get the suggestions */
reader->searchSuggestionsSmart(term, maxSuggestionCount);
while (reader->getNextSuggestion(suggestion)) {
kainjow::mustache::data result;
MustacheData result;
result.set("label", suggestion);
result.set("value", suggestion);
result.set("first", first);
@ -526,7 +537,7 @@ Response InternalServer::handle_suggest(const RequestContext& request)
/* Propose the fulltext search if possible */
if (reader->hasFulltextIndex()) {
kainjow::mustache::data result;
MustacheData result;
result.set("label", "containing '" + term + "'...");
result.set("value", term + " ");
result.set("first", first);
@ -711,10 +722,7 @@ Response InternalServer::handle_random(const RequestContext& request)
try {
auto entry = reader->getRandomPage();
entry = entry.getFinalEntry();
auto response = get_default_response();
response.set_redirection(m_root + "/" + bookName + "/" + kiwix::urlEncode(entry.getPath()));
return response;
return build_redirect(bookName, entry.getFinalEntry());
} catch(kiwix::NoEntry& e) {
return build_404(request, bookName);
}
@ -824,36 +832,52 @@ Response InternalServer::handle_catalog(const RequestContext& request)
return response;
}
namespace
{
std::string get_book_name(const RequestContext& request)
{
try {
return request.get_url_part(0);
} catch (const std::out_of_range& e) {
return std::string();
}
}
} // unnamed namespace
std::shared_ptr<Reader>
InternalServer::get_reader(const std::string& bookName) const
{
std::shared_ptr<Reader> reader;
try {
const std::string bookId = mp_nameMapper->getIdForName(bookName);
reader = mp_library->getReaderById(bookId);
} catch (const std::out_of_range& e) {
}
return reader;
}
Response
InternalServer::build_redirect(const std::string& bookName, const kiwix::Entry& entry) const
{
auto response = get_default_response();
response.set_redirection(m_root + "/" + bookName + "/" +
kiwix::urlEncode(entry.getPath()));
return response;
}
Response InternalServer::handle_content(const RequestContext& request)
{
if (m_verbose.load()) {
printf("** running handle_content\n");
}
std::string baseUrl;
std::string content;
std::string mimeType;
kiwix::Entry entry;
std::string bookName;
try {
bookName = request.get_url_part(0);
} catch (const std::out_of_range& e) {
return build_homepage(request);
}
const std::string bookName = get_book_name(request);
if (bookName.empty())
return build_homepage(request);
std::string bookId;
std::shared_ptr<Reader> reader;
try {
bookId = mp_nameMapper->getIdForName(bookName);
reader = mp_library->getReaderById(bookId);
} catch (const std::out_of_range& e) {
return build_404(request, bookName);
}
const std::shared_ptr<Reader> reader = get_reader(bookName);
if (reader == nullptr) {
return build_404(request, bookName);
}
@ -863,16 +887,14 @@ Response InternalServer::handle_content(const RequestContext& request)
urlStr = urlStr.substr(1);
}
kiwix::Entry entry;
try {
entry = reader->getEntryFromPath(urlStr);
if (entry.isRedirect() || urlStr.empty()) {
// If urlStr is empty, we want to mainPage.
// We must do a redirection to the real page.
entry = entry.getFinalEntry();
auto response = get_default_response();
response.set_redirection(m_root + "/" + bookName + "/" +
kiwix::urlEncode(entry.getPath()));
return response;
return build_redirect(bookName, entry.getFinalEntry());
}
} catch(kiwix::NoEntry& e) {
if (m_verbose.load())
@ -881,47 +903,19 @@ Response InternalServer::handle_content(const RequestContext& request)
return build_404(request, bookName);
}
try {
mimeType = entry.getMimetype();
} catch (exception& e) {
mimeType = "application/octet-stream";
}
auto response = get_default_response();
response.set_entry(entry, request);
if (m_verbose.load()) {
printf("Found %s\n", urlStr.c_str());
printf("mimeType: %s\n", mimeType.c_str());
printf("Found %s\n", entry.getPath().c_str());
printf("mimeType: %s\n", response.get_mimeType().c_str());
}
if (mimeType.find("text/") != string::npos
|| mimeType.find("application/javascript") != string::npos
|| mimeType.find("application/json") != string::npos) {
zim::Blob raw_content = entry.getBlob();
content = string(raw_content.data(), raw_content.size());
auto response = get_default_response();
if (mimeType.find("text/html") != string::npos)
if (response.get_mimeType().find("text/html") != string::npos)
response.set_taskbar(bookName, reader->getTitle());
response.set_mimeType(mimeType);
response.set_content(content);
response.set_compress(true);
response.set_cache(true);
return response;
} else {
int range_len;
if (request.get_range().second == -1) {
range_len = entry.getSize() - request.get_range().first;
} else {
range_len = request.get_range().second - request.get_range().first;
}
auto response = get_default_response();
response.set_entry(entry);
response.set_mimeType(mimeType);
response.set_range_first(request.get_range().first);
response.set_range_len(range_len);
response.set_cache(true);
return response;
}
}
}

81
src/server/request_context.cpp
View File

@ -30,60 +30,56 @@ namespace kiwix {
static std::atomic_ullong s_requestIndex(0);
namespace {
RequestMethod str2RequestMethod(const std::string& method) {
if (method == "GET") return RequestMethod::GET;
else if (method == "HEAD") return RequestMethod::HEAD;
else if (method == "POST") return RequestMethod::POST;
else if (method == "PUT") return RequestMethod::PUT;
else if (method == "DELETE") return RequestMethod::DELETE_;
else if (method == "CONNECT") return RequestMethod::CONNECT;
else if (method == "OPTIONS") return RequestMethod::OPTIONS;
else if (method == "TRACE") return RequestMethod::TRACE;
else if (method == "PATCH") return RequestMethod::PATCH;
else return RequestMethod::OTHER;
}
std::string
fullURL2LocalURL(const std::string& full_url, const std::string& rootLocation)
{
if (rootLocation.empty()) {
// nothing special to handle.
return full_url;
} else if (full_url == rootLocation) {
return "/";
} else if (full_url.size() > rootLocation.size() &&
full_url.substr(0, rootLocation.size()+1) == rootLocation + "/") {
return full_url.substr(rootLocation.size());
} else {
return "";
}
}
} // unnamed namespace
RequestContext::RequestContext(struct MHD_Connection* connection,
std::string rootLocation,
const std::string& _url,
const std::string& method,
const std::string& _method,
const std::string& version) :
full_url(_url),
url(_url),
valid_url(true),
url(fullURL2LocalURL(_url, rootLocation)),
method(str2RequestMethod(_method)),
version(version),
requestIndex(s_requestIndex++),
acceptEncodingDeflate(false),
accept_range(false),
range_pair(0, -1)
{
if (method == "GET") {
this->method = RequestMethod::GET;
} else if (method == "HEAD") {
this->method = RequestMethod::HEAD;
} else if (method == "POST") {
this->method = RequestMethod::POST;
} else if (method == "PUT") {
this->method = RequestMethod::PUT;
} else if (method == "DELETE") {
this->method = RequestMethod::DELETE_;
} else if (method == "CONNECT") {
this->method = RequestMethod::CONNECT;
} else if (method == "OPTIONS") {
this->method = RequestMethod::OPTIONS;
} else if (method == "TRACE") {
this->method = RequestMethod::TRACE;
} else if (method == "PATCH") {
this->method = RequestMethod::PATCH;
} else {
this->method = RequestMethod::OTHER;
}
MHD_get_connection_values(connection, MHD_HEADER_KIND, &RequestContext::fill_header, this);
MHD_get_connection_values(connection, MHD_GET_ARGUMENT_KIND, &RequestContext::fill_argument, this);
valid_url = true;
if (rootLocation.empty()) {
// nothing special to handle.
url = full_url;
} else {
if (full_url == rootLocation) {
url = "/";
} else if (full_url.size() > rootLocation.size() &&
full_url.substr(0, rootLocation.size()+1) == rootLocation + "/") {
url = full_url.substr(rootLocation.size());
} else {
valid_url = false;
}
}
try {
acceptEncodingDeflate =
(get_header(MHD_HTTP_HEADER_ACCEPT_ENCODING).find("deflate") != std::string::npos);
@ -147,10 +143,11 @@ void RequestContext::print_debug_info() const {
printf(" - %s : '%s'\n", it->first.c_str(), it->second.c_str());
}
printf("Parsed : \n");
printf("full_url: %s\n", full_url.c_str());
printf("url : %s\n", url.c_str());
printf("acceptEncodingDeflate : %d\n", acceptEncodingDeflate);
printf("has_range : %d\n", accept_range);
printf("is_valid_url : %d\n", valid_url);
printf("is_valid_url : %d\n", is_valid_url());
printf(".............\n");
}
@ -188,7 +185,7 @@ std::string RequestContext::get_full_url() const {
}
bool RequestContext::is_valid_url() const {
return valid_url;
return !url.empty();
}
bool RequestContext::has_range() const {

13
src/server/request_context.h
View File

@ -51,7 +51,10 @@ class IndexError: public std::runtime_error {};
class RequestContext {
public:
public: // types
typedef std::pair<int, int> ByteRange;
public: // functions
RequestContext(struct MHD_Connection* connection,
std::string rootLocation,
const std::string& url,
@ -79,14 +82,13 @@ class RequestContext {
std::string get_full_url() const;
bool has_range() const;
std::pair<int, int> get_range() const;
ByteRange get_range() const;
bool can_compress() const { return acceptEncodingDeflate; }
private:
private: // data
std::string full_url;
std::string url;
bool valid_url;
RequestMethod method;
std::string version;
unsigned long long requestIndex;
@ -94,10 +96,11 @@ class RequestContext {
bool acceptEncodingDeflate;
bool accept_range;
std::pair<int, int> range_pair;
ByteRange range_pair;
std::map<std::string, std::string> headers;
std::map<std::string, std::string> arguments;
private: // functions
static int fill_header(void *, enum MHD_ValueKind, const char*, const char*);
static int fill_argument(void *, enum MHD_ValueKind, const char*, const char*);
};

98
src/server/response.cpp
View File

@ -17,6 +17,35 @@
namespace kiwix {
namespace
{
// some utilities
std::string get_mime_type(const kiwix::Entry& entry)
{
try {
return entry.getMimetype();
} catch (exception& e) {
return "application/octet-stream";
}
}
bool is_compressible_mime_type(const std::string& mimeType)
{
return mimeType.find("text/") != string::npos
|| mimeType.find("application/javascript") != string::npos
|| mimeType.find("application/json") != string::npos;
}
int get_range_len(const kiwix::Entry& entry, RequestContext::ByteRange range)
{
return range.second == -1
? entry.getSize() - range.first
: range.second - range.first;
}
} // unnamed namespace
Response::Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton, bool blockExternalLinks)
: m_verbose(verbose),
m_root(root),
@ -139,12 +168,9 @@ void Response::inject_externallinks_blocker()
script_tag);
}
int Response::send(const RequestContext& request, MHD_Connection* connection)
MHD_Response*
Response::create_raw_content_mhd_response(const RequestContext& request)
{
MHD_Response* response = nullptr;
switch (m_mode) {
case ResponseMode::RAW_CONTENT : {
if (m_addTaskbar) {
introduce_taskbar();
}
@ -153,10 +179,7 @@ int Response::send(const RequestContext& request, MHD_Connection* connection)
}
bool shouldCompress = m_compress && request.can_compress();
shouldCompress &= m_mimeType.find("text/") != string::npos
|| m_mimeType.find("application/javascript") != string::npos
|| m_mimeType.find("application/json") != string::npos;
shouldCompress &= is_compressible_mime_type(m_mimeType);
shouldCompress &= (m_content.size() > KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE);
if (shouldCompress) {
@ -178,7 +201,7 @@ int Response::send(const RequestContext& request, MHD_Connection* connection)
}
}
response = MHD_create_response_from_buffer(
MHD_Response* response = MHD_create_response_from_buffer(
m_content.size(), const_cast<char*>(m_content.data()), MHD_RESPMEM_MUST_COPY);
if (shouldCompress) {
@ -188,17 +211,21 @@ int Response::send(const RequestContext& request, MHD_Connection* connection)
response, MHD_HTTP_HEADER_CONTENT_ENCODING, "deflate");
}
MHD_add_response_header(response, MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str());
break;
return response;
}
case ResponseMode::REDIRECTION : {
response = MHD_create_response_from_buffer(0, nullptr, MHD_RESPMEM_MUST_COPY);
MHD_Response*
Response::create_redirection_mhd_response() const
{
MHD_Response* response = MHD_create_response_from_buffer(0, nullptr, MHD_RESPMEM_MUST_COPY);
MHD_add_response_header(response, MHD_HTTP_HEADER_LOCATION, m_content.c_str());
break;
return response;
}
case ResponseMode::ENTRY : {
response = MHD_create_response_from_callback(m_entry.getSize(),
MHD_Response*
Response::create_entry_mhd_response() const
{
MHD_Response* response = MHD_create_response_from_callback(m_entry.getSize(),
16384,
callback_reader_from_entry,
new RunningResponse(m_entry, m_startRange),
@ -215,9 +242,28 @@ int Response::send(const RequestContext& request, MHD_Connection* connection)
MHD_add_response_header(response,
MHD_HTTP_HEADER_CONTENT_LENGTH, kiwix::to_string(m_lenRange).c_str());
break;
return response;
}
MHD_Response*
Response::create_mhd_response(const RequestContext& request)
{
switch (m_mode) {
case ResponseMode::RAW_CONTENT :
return create_raw_content_mhd_response(request);
case ResponseMode::REDIRECTION :
return create_redirection_mhd_response();
case ResponseMode::ENTRY :
return create_entry_mhd_response();
}
return nullptr;
}
int Response::send(const RequestContext& request, MHD_Connection* connection)
{
MHD_Response* response = create_mhd_response(request);
MHD_add_response_header(response, "Access-Control-Allow-Origin", "*");
MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL,
@ -249,9 +295,25 @@ void Response::set_redirection(const std::string& url) {
m_returnCode = MHD_HTTP_FOUND;
}
void Response::set_entry(const Entry& entry) {
void Response::set_entry(const Entry& entry, const RequestContext& request) {
m_entry = entry;
m_mode = ResponseMode::ENTRY;
const std::string mimeType = get_mime_type(entry);
set_mimeType(mimeType);
set_cache(true);
if ( is_compressible_mime_type(mimeType) ) {
zim::Blob raw_content = entry.getBlob();
const std::string content = string(raw_content.data(), raw_content.size());
set_content(content);
set_compress(true);
} else {
const int range_len = get_range_len(entry, request.get_range());
set_range_first(request.get_range().first);
set_range_len(range_len);
}
}
void Response::set_taskbar(const std::string& bookName, const std::string& bookTitle)

11
src/server/response.h
View File

@ -50,7 +50,7 @@ class Response {
void set_template(const std::string& template_str, kainjow::mustache::data data);
void set_content(const std::string& content);
void set_redirection(const std::string& url);
void set_entry(const Entry& entry);
void set_entry(const Entry& entry, const RequestContext& request);
void set_mimeType(const std::string& mimeType) { m_mimeType = mimeType; }
@ -62,11 +62,18 @@ class Response {
void set_range_len(uint64_t len) { m_lenRange = len; }
int getReturnCode() { return m_returnCode; }
std::string get_mimeType() const { return m_mimeType; }
void introduce_taskbar();
void inject_externallinks_blocker();
private:
private: // functions
MHD_Response* create_mhd_response(const RequestContext& request);
MHD_Response* create_raw_content_mhd_response(const RequestContext& request);
MHD_Response* create_redirection_mhd_response() const;
MHD_Response* create_entry_mhd_response() const;
private: // data
bool m_verbose;
ResponseMode m_mode;
std::string m_root;