diff --git a/include/library.h b/include/library.h index f358e6746..95d22e4bd 100644 --- a/include/library.h +++ b/include/library.h @@ -233,12 +233,19 @@ class Library unsigned int getBookCount(const bool localBooks, const bool remoteBooks) const; /** - * Get all langagues of the books in the library. + * Get all languagues of the books in the library. * * @return A list of languages. */ std::vector getBooksLanguages() const; + /** + * Get all categories of the books in the library. + * + * @return A list of categories. + */ + std::vector getBooksCategories() const; + /** * Get all book creators of the books in the library. * diff --git a/include/opds_dumper.h b/include/opds_dumper.h index c6cc60d3d..230d094a1 100644 --- a/include/opds_dumper.h +++ b/include/opds_dumper.h @@ -51,24 +51,35 @@ class OPDSDumper /** * Dump the OPDS feed. * - * @param id The id of the library. + * @param bookIds the ids of the books to include in the feed + * @param query the query used to obtain the list of book ids * @return The OPDS feed. */ - std::string dumpOPDSFeed(const std::vector& bookIds); + std::string dumpOPDSFeed(const std::vector& bookIds, const std::string& query) const; /** - * Set the id of the opds stream. + * Dump the OPDS feed. + * + * @param bookIds the ids of the books to include in the feed + * @param query the query used to obtain the list of book ids + * @return The OPDS feed. + */ + std::string dumpOPDSFeedV2(const std::vector& bookIds, const std::string& query) const; + + /** + * Dump the categories OPDS feed. + * + * @param categories list of category names + * @return The OPDS feed. + */ + std::string categoriesOPDSFeed(const std::vector& categories) const; + + /** + * Set the id of the library. * * @param id the id to use. */ - void setId(const std::string& id) { this->id = id;} - - /** - * Set the title oft the opds stream. - * - * @param title the title to use. - */ - void setTitle(const std::string& title) { this->title = title; } + void setLibraryId(const std::string& id) { this->libraryId = id;} /** * Set the root location used when generating url. @@ -77,13 +88,6 @@ class OPDSDumper */ void setRootLocation(const std::string& rootLocation) { this->rootLocation = rootLocation; } - /** - * Set the search url. - * - * @param searchUrl the search url to use. - */ - void setSearchDescriptionUrl(const std::string& searchDescriptionUrl) { this->searchDescriptionUrl = searchDescriptionUrl; } - /** * Set some informations about the search results. * @@ -93,27 +97,13 @@ class OPDSDumper */ void setOpenSearchInfo(int totalResult, int startIndex, int count); - /** - * Set the library to dump. - * - * @param library The library to dump. - */ - void setLibrary(Library* library) { this->library = library; } - protected: kiwix::Library* library; - std::string id; - std::string title; - std::string date; + std::string libraryId; std::string rootLocation; - std::string searchDescriptionUrl; int m_totalResults; int m_startIndex; int m_count; - bool m_isSearchResult = false; - - private: - pugi::xml_node handleBook(Book book, pugi::xml_node root_node); }; } diff --git a/include/tools/otherTools.h b/include/tools/otherTools.h index 726c332b3..c105d52f5 100644 --- a/include/tools/otherTools.h +++ b/include/tools/otherTools.h @@ -24,6 +24,7 @@ #include #include #include +#include namespace pugi { class xml_node; @@ -45,6 +46,11 @@ namespace kiwix using MimeCounterType = std::map; MimeCounterType parseMimetypeCounter(const std::string& counterData); + + std::string gen_date_str(); + std::string gen_uuid(const std::string& s); + + std::string render_template(const std::string& template_str, kainjow::mustache::data data); } #endif diff --git a/src/library.cpp b/src/library.cpp index d8307c9ae..1147282df 100644 --- a/src/library.cpp +++ b/src/library.cpp @@ -201,6 +201,21 @@ std::vector Library::getBooksLanguages() const return booksLanguages; } +std::vector Library::getBooksCategories() const +{ + std::set categories; + + for (const auto& pair: m_books) { + const auto& book = pair.second; + const auto& c = book.getCategory(); + if ( !c.empty() ) { + categories.insert(c); + } + } + + return std::vector(categories.begin(), categories.end()); +} + std::vector Library::getBooksCreators() const { std::vector booksCreators; diff --git a/src/meson.build b/src/meson.build index 333c3a22f..d4c4f9697 100644 --- a/src/meson.build +++ b/src/meson.build @@ -25,7 +25,8 @@ kiwix_sources = [ 'server/etag.cpp', 'server/request_context.cpp', 'server/response.cpp', - 'server/internalServer.cpp' + 'server/internalServer.cpp', + 'server/internalServer_catalog_v2.cpp' ] kiwix_sources += lib_resources diff --git a/src/opds_dumper.cpp b/src/opds_dumper.cpp index 8f14e0949..9d6fdfd5e 100644 --- a/src/opds_dumper.cpp +++ b/src/opds_dumper.cpp @@ -21,10 +21,13 @@ #include "book.h" #include "tools/otherTools.h" -#include + +#include "kiwixlib-resources.h" +#include namespace kiwix { + /* Constructor */ OPDSDumper::OPDSDumper(Library* library) : library(library) @@ -35,121 +38,111 @@ OPDSDumper::~OPDSDumper() { } -std::string gen_date_str() -{ - auto now = time(0); - auto tm = localtime(&now); - - std::stringstream is; - is << std::setw(2) << std::setfill('0') - << 1900+tm->tm_year << "-" - << std::setw(2) << std::setfill('0') << tm->tm_mon+1 << "-" - << std::setw(2) << std::setfill('0') << tm->tm_mday << "T" - << std::setw(2) << std::setfill('0') << tm->tm_hour << ":" - << std::setw(2) << std::setfill('0') << tm->tm_min << ":" - << std::setw(2) << std::setfill('0') << tm->tm_sec << "Z"; - return is.str(); -} - -static std::string gen_date_from_yyyy_mm_dd(const std::string& date) -{ - std::stringstream is; - is << date << "T00:00::00Z"; - return is.str(); -} - void OPDSDumper::setOpenSearchInfo(int totalResults, int startIndex, int count) { m_totalResults = totalResults; m_startIndex = startIndex, m_count = count; - m_isSearchResult = true; } -#define ADD_TEXT_ENTRY(node, child, value) (node).append_child((child)).append_child(pugi::node_pcdata).set_value((value).c_str()) - -pugi::xml_node OPDSDumper::handleBook(Book book, pugi::xml_node root_node) { - auto entry_node = root_node.append_child("entry"); - ADD_TEXT_ENTRY(entry_node, "id", "urn:uuid:"+book.getId()); - ADD_TEXT_ENTRY(entry_node, "title", book.getTitle()); - ADD_TEXT_ENTRY(entry_node, "summary", book.getDescription()); - ADD_TEXT_ENTRY(entry_node, "language", book.getLanguage()); - ADD_TEXT_ENTRY(entry_node, "updated", gen_date_from_yyyy_mm_dd(book.getDate())); - ADD_TEXT_ENTRY(entry_node, "name", book.getName()); - ADD_TEXT_ENTRY(entry_node, "flavour", book.getFlavour()); - ADD_TEXT_ENTRY(entry_node, "category", book.getCategory()); - ADD_TEXT_ENTRY(entry_node, "tags", book.getTags()); - ADD_TEXT_ENTRY(entry_node, "articleCount", to_string(book.getArticleCount())); - ADD_TEXT_ENTRY(entry_node, "mediaCount", to_string(book.getMediaCount())); - ADD_TEXT_ENTRY(entry_node, "icon", rootLocation + "/meta?name=favicon&content=" + book.getHumanReadableIdFromPath()); - - auto content_node = entry_node.append_child("link"); - content_node.append_attribute("type") = "text/html"; - content_node.append_attribute("href") = (rootLocation + "/" + book.getHumanReadableIdFromPath()).c_str(); - - auto author_node = entry_node.append_child("author"); - ADD_TEXT_ENTRY(author_node, "name", book.getCreator()); - - auto publisher_node = entry_node.append_child("publisher"); - ADD_TEXT_ENTRY(publisher_node, "name", book.getPublisher()); - - if (! book.getUrl().empty()) { - auto acquisition_link = entry_node.append_child("link"); - acquisition_link.append_attribute("rel") = "http://opds-spec.org/acquisition/open-access"; - acquisition_link.append_attribute("type") = "application/x-zim"; - acquisition_link.append_attribute("href") = book.getUrl().c_str(); - acquisition_link.append_attribute("length") = to_string(book.getSize()).c_str(); - } - - if (! book.getFaviconMimeType().empty() ) { - auto image_link = entry_node.append_child("link"); - image_link.append_attribute("rel") = "http://opds-spec.org/image/thumbnail"; - image_link.append_attribute("type") = book.getFaviconMimeType().c_str(); - image_link.append_attribute("href") = (rootLocation + "/meta?name=favicon&content=" + book.getHumanReadableIdFromPath()).c_str(); - } - return entry_node; -} - -string OPDSDumper::dumpOPDSFeed(const std::vector& bookIds) +namespace { - date = gen_date_str(); - pugi::xml_document doc; - auto root_node = doc.append_child("feed"); - root_node.append_attribute("xmlns") = "http://www.w3.org/2005/Atom"; - root_node.append_attribute("xmlns:opds") = "http://opds-spec.org/2010/catalog"; +typedef kainjow::mustache::data MustacheData; +typedef kainjow::mustache::list BookData; - ADD_TEXT_ENTRY(root_node, "id", id); - - ADD_TEXT_ENTRY(root_node, "title", title); - ADD_TEXT_ENTRY(root_node, "updated", date); - - if (m_isSearchResult) { - ADD_TEXT_ENTRY(root_node, "totalResults", to_string(m_totalResults)); - ADD_TEXT_ENTRY(root_node, "startIndex", to_string(m_startIndex)); - ADD_TEXT_ENTRY(root_node, "itemsPerPage", to_string(m_count)); +BookData getBookData(const Library* library, const std::vector& bookIds) +{ + BookData bookData; + for ( const auto& bookId : bookIds ) { + const Book& book = library->getBookById(bookId); + const MustacheData bookUrl = book.getUrl().empty() + ? MustacheData(false) + : MustacheData(book.getUrl()); + bookData.push_back(kainjow::mustache::object{ + {"id", "urn:uuid:"+book.getId()}, + {"name", book.getName()}, + {"title", book.getTitle()}, + {"description", book.getDescription()}, + {"language", book.getLanguage()}, + {"content_id", book.getHumanReadableIdFromPath()}, + {"updated", book.getDate() + "T00:00:00Z"}, + {"category", book.getCategory()}, + {"flavour", book.getFlavour()}, + {"tags", book.getTags()}, + {"article_count", to_string(book.getArticleCount())}, + {"media_count", to_string(book.getMediaCount())}, + {"author_name", book.getCreator()}, + {"publisher_name", book.getPublisher()}, + {"url", bookUrl}, + {"size", to_string(book.getSize())}, + }); } - auto self_link_node = root_node.append_child("link"); - self_link_node.append_attribute("rel") = "self"; - self_link_node.append_attribute("href") = ""; - self_link_node.append_attribute("type") = "application/atom+xml"; + return bookData; +} +} // unnamed namespace - if (!searchDescriptionUrl.empty() ) { - auto search_link = root_node.append_child("link"); - search_link.append_attribute("rel") = "search"; - search_link.append_attribute("type") = "application/opensearchdescription+xml"; - search_link.append_attribute("href") = searchDescriptionUrl.c_str(); +string OPDSDumper::dumpOPDSFeed(const std::vector& bookIds, const std::string& query) const +{ + const auto bookData = getBookData(library, bookIds); + const kainjow::mustache::object template_data{ + {"date", gen_date_str()}, + {"root", rootLocation}, + {"feed_id", gen_uuid(libraryId + "/catalog/search?"+query)}, + {"filter", query.empty() ? MustacheData(false) : MustacheData(query)}, + {"totalResults", to_string(m_totalResults)}, + {"startIndex", to_string(m_startIndex)}, + {"itemsPerPage", to_string(m_count)}, + {"books", bookData } + }; + + return render_template(RESOURCE::templates::catalog_entries_xml, template_data); +} + +string OPDSDumper::dumpOPDSFeedV2(const std::vector& bookIds, const std::string& query) const +{ + const auto bookData = getBookData(library, bookIds); + + const kainjow::mustache::object template_data{ + {"date", gen_date_str()}, + {"endpoint_root", rootLocation + "/catalog/v2"}, + {"feed_id", gen_uuid(libraryId + "/entries?"+query)}, + {"filter", query.empty() ? MustacheData(false) : MustacheData(query)}, + {"query", query.empty() ? "" : "?" + urlEncode(query)}, + {"totalResults", to_string(m_totalResults)}, + {"startIndex", to_string(m_startIndex)}, + {"itemsPerPage", to_string(m_count)}, + {"books", bookData } + }; + + return render_template(RESOURCE::templates::catalog_v2_entries_xml, template_data); +} + +std::string OPDSDumper::categoriesOPDSFeed(const std::vector& categories) const +{ + const auto now = gen_date_str(); + kainjow::mustache::list categoryData; + for ( const auto& category : categories ) { + const auto urlencodedCategoryName = urlEncode(category); + categoryData.push_back(kainjow::mustache::object{ + {"name", category}, + {"urlencoded_name", urlencodedCategoryName}, + {"updated", now}, + {"id", gen_uuid(libraryId + "/categories/" + urlencodedCategoryName)} + }); } - if (library) { - for (auto& bookId: bookIds) { - handleBook(library->getBookById(bookId), root_node); - } - } - - return nodeToString(root_node); + return render_template( + RESOURCE::templates::catalog_v2_categories_xml, + kainjow::mustache::object{ + {"date", now}, + {"endpoint_root", rootLocation + "/catalog/v2"}, + {"feed_id", gen_uuid(libraryId + "/categories")}, + {"categories", categoryData } + } + ); } } diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index 4d344edc0..e395e0dc4 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -76,6 +76,21 @@ extern "C" { namespace kiwix { +namespace +{ + +inline std::string normalizeRootUrl(std::string rootUrl) +{ + while ( !rootUrl.empty() && rootUrl.back() == '/' ) + rootUrl.pop_back(); + + while ( !rootUrl.empty() && rootUrl.front() == '/' ) + rootUrl = rootUrl.substr(1); + return rootUrl.empty() ? rootUrl : "/" + rootUrl; +} + +} // unnamed namespace + static IdNameMapper defaultNameMapper; static MHD_Result staticHandlerCallback(void* cls, @@ -100,7 +115,7 @@ InternalServer::InternalServer(Library* library, bool blockExternalLinks) : m_addr(addr), m_port(port), - m_root(root), + m_root(normalizeRootUrl(root)), m_nbThreads(nbThreads), m_verbose(verbose), m_withTaskbar(withTaskbar), @@ -153,6 +168,7 @@ bool InternalServer::start() { } auto server_start_time = std::chrono::system_clock::now().time_since_epoch(); m_server_id = kiwix::to_string(server_start_time.count()); + m_library_id = m_server_id; return true; } @@ -243,10 +259,10 @@ std::unique_ptr InternalServer::handle_request(const RequestContext& r if ( etag ) return Response::build_304(*this, etag); - if (kiwix::startsWith(request.get_url(), "/skin/")) + if (startsWith(request.get_url(), "/skin/")) return handle_skin(request); - if (startsWith(request.get_url(), "/catalog")) + if (startsWith(request.get_url(), "/catalog/")) return handle_catalog(request); if (request.get_url() == "/meta") @@ -606,6 +622,10 @@ std::unique_ptr InternalServer::handle_catalog(const RequestContext& r return Response::build_404(*this, request, ""); } + if (url == "v2") { + return handle_catalog_v2(request); + } + if (url != "searchdescription.xml" && url != "root.xml" && url != "search") { return Response::build_404(*this, request, ""); } @@ -616,13 +636,11 @@ std::unique_ptr InternalServer::handle_catalog(const RequestContext& r } zim::Uuid uuid; - kiwix::OPDSDumper opdsDumper; + kiwix::OPDSDumper opdsDumper(mp_library); opdsDumper.setRootLocation(m_root); - opdsDumper.setSearchDescriptionUrl("catalog/searchdescription.xml"); - opdsDumper.setLibrary(mp_library); + opdsDumper.setLibraryId(m_library_id); std::vector bookIdsToDump; if (url == "root.xml") { - opdsDumper.setTitle("All zims"); uuid = zim::Uuid::generate(host); bookIdsToDump = mp_library->filter(kiwix::Filter().valid(true).local(true).remote(true)); } else if (url == "search") { @@ -630,28 +648,24 @@ std::unique_ptr InternalServer::handle_catalog(const RequestContext& r uuid = zim::Uuid::generate(); } - opdsDumper.setId(kiwix::to_string(uuid)); auto response = ContentResponse::build( *this, - opdsDumper.dumpOPDSFeed(bookIdsToDump), + opdsDumper.dumpOPDSFeed(bookIdsToDump, request.get_query()), "application/atom+xml; profile=opds-catalog; kind=acquisition; charset=utf-8"); return std::move(response); } -std::vector -InternalServer::search_catalog(const RequestContext& request, - kiwix::OPDSDumper& opdsDumper) +namespace +{ + +Filter get_search_filter(const RequestContext& request) { auto filter = kiwix::Filter().valid(true).local(true); - string query(""); - size_t count(10); - size_t startIndex(0); try { - query = request.get_argument("q"); - filter.query(query); + filter.query(request.get_argument("q")); } catch (const std::out_of_range&) {} try { - filter.maxSize(extractFromString(request.get_argument("maxsize"))); + filter.maxSize(request.get_argument("maxsize")); } catch (...) {} try { filter.name(request.get_argument("name")); @@ -662,26 +676,37 @@ InternalServer::search_catalog(const RequestContext& request, try { filter.lang(request.get_argument("lang")); } catch (const std::out_of_range&) {} - try { - count = extractFromString(request.get_argument("count")); - } catch (...) {} - try { - startIndex = extractFromString(request.get_argument("start")); - } catch (...) {} try { filter.acceptTags(kiwix::split(request.get_argument("tag"), ";")); } catch (...) {} try { filter.rejectTags(kiwix::split(request.get_argument("notag"), ";")); } catch (...) {} - opdsDumper.setTitle("Search result for " + query); + return filter; +} + +template +std::vector subrange(const std::vector& v, size_t s, size_t n) +{ + const size_t e = std::min(v.size(), s+n); + return std::vector(v.begin()+std::min(v.size(), s), v.begin()+e); +} + +} // unnamed namespace + +std::vector +InternalServer::search_catalog(const RequestContext& request, + kiwix::OPDSDumper& opdsDumper) +{ + const auto filter = get_search_filter(request); + const std::string q = filter.hasQuery() + ? filter.getQuery() + : ""; std::vector bookIdsToDump = mp_library->filter(filter); const auto totalResults = bookIdsToDump.size(); - const auto s = std::min(startIndex, totalResults); - bookIdsToDump.erase(bookIdsToDump.begin(), bookIdsToDump.begin()+s); - if (count>0 && bookIdsToDump.size() > count) { - bookIdsToDump.resize(count); - } + const size_t count = request.get_optional_param("count", 10UL); + const size_t startIndex = request.get_optional_param("start", 0UL); + bookIdsToDump = subrange(bookIdsToDump, startIndex, count); opdsDumper.setOpenSearchInfo(totalResults, startIndex, bookIdsToDump.size()); return bookIdsToDump; } diff --git a/src/server/internalServer.h b/src/server/internalServer.h index c37c736f2..142d3fc37 100644 --- a/src/server/internalServer.h +++ b/src/server/internalServer.h @@ -73,6 +73,10 @@ class InternalServer { std::unique_ptr build_homepage(const RequestContext& request); std::unique_ptr handle_skin(const RequestContext& request); std::unique_ptr handle_catalog(const RequestContext& request); + std::unique_ptr handle_catalog_v2(const RequestContext& request); + std::unique_ptr handle_catalog_v2_root(const RequestContext& request); + std::unique_ptr handle_catalog_v2_entries(const RequestContext& request); + std::unique_ptr handle_catalog_v2_categories(const RequestContext& request); std::unique_ptr handle_meta(const RequestContext& request); std::unique_ptr handle_search(const RequestContext& request); std::unique_ptr handle_suggest(const RequestContext& request); @@ -104,6 +108,7 @@ class InternalServer { NameMapper* mp_nameMapper; std::string m_server_id; + std::string m_library_id; friend std::unique_ptr Response::build(const InternalServer& server); friend std::unique_ptr ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype, bool isHomePage); diff --git a/src/server/internalServer_catalog_v2.cpp b/src/server/internalServer_catalog_v2.cpp new file mode 100644 index 000000000..e7c05bac6 --- /dev/null +++ b/src/server/internalServer_catalog_v2.cpp @@ -0,0 +1,109 @@ +/* + * Copyright 2021 Veloman Yunkan + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301, USA. + */ + +#include "internalServer.h" + +#include "library.h" +#include "opds_dumper.h" +#include "request_context.h" +#include "response.h" +#include "tools/otherTools.h" +#include "kiwixlib-resources.h" + +#include + +#include +#include + +namespace kiwix { + +std::unique_ptr InternalServer::handle_catalog_v2(const RequestContext& request) +{ + if (m_verbose.load()) { + printf("** running handle_catalog_v2"); + } + + std::string url; + try { + url = request.get_url_part(2); + } catch (const std::out_of_range&) { + return Response::build_404(*this, request, ""); + } + + if (url == "root.xml") { + return handle_catalog_v2_root(request); + } else if (url == "searchdescription.xml") { + const std::string endpoint_root = m_root + "/catalog/v2"; + return ContentResponse::build(*this, + RESOURCE::catalog_v2_searchdescription_xml, + kainjow::mustache::object({{"endpoint_root", endpoint_root}}), + "application/opensearchdescription+xml" + ); + } else if (url == "entries") { + return handle_catalog_v2_entries(request); + } else if (url == "categories") { + return handle_catalog_v2_categories(request); + } else { + return Response::build_404(*this, request, ""); + } +} + +std::unique_ptr InternalServer::handle_catalog_v2_root(const RequestContext& request) +{ + return ContentResponse::build( + *this, + RESOURCE::templates::catalog_v2_root_xml, + kainjow::mustache::object{ + {"date", gen_date_str()}, + {"endpoint_root", m_root + "/catalog/v2"}, + {"feed_id", gen_uuid(m_library_id)}, + {"all_entries_feed_id", gen_uuid(m_library_id + "/entries")}, + {"category_list_feed_id", gen_uuid(m_library_id + "/categories")} + }, + "application/atom+xml;profile=opds-catalog;kind=navigation" + ); +} + +std::unique_ptr InternalServer::handle_catalog_v2_entries(const RequestContext& request) +{ + OPDSDumper opdsDumper(mp_library); + opdsDumper.setRootLocation(m_root); + opdsDumper.setLibraryId(m_library_id); + const auto bookIds = search_catalog(request, opdsDumper); + const auto opdsFeed = opdsDumper.dumpOPDSFeedV2(bookIds, request.get_query()); + return ContentResponse::build( + *this, + opdsFeed, + "application/atom+xml;profile=opds-catalog;kind=acquisition" + ); +} + +std::unique_ptr InternalServer::handle_catalog_v2_categories(const RequestContext& request) +{ + OPDSDumper opdsDumper(mp_library); + opdsDumper.setRootLocation(m_root); + opdsDumper.setLibraryId(m_library_id); + return ContentResponse::build( + *this, + opdsDumper.categoriesOPDSFeed(mp_library->getBooksCategories()), + "application/atom+xml;profile=opds-catalog;kind=navigation" + ); +} + +} // namespace kiwix diff --git a/src/server/request_context.cpp b/src/server/request_context.cpp index cad39eefc..765d01adf 100644 --- a/src/server/request_context.cpp +++ b/src/server/request_context.cpp @@ -183,4 +183,14 @@ std::string RequestContext::get_header(const std::string& name) const { return headers.at(lcAll(name)); } +std::string RequestContext::get_query() const { + std::string q; + const char* sep = ""; + for ( const auto& a : arguments ) { + q += sep + a.first + '=' + a.second; + sep = "&"; + } + return q; +} + } diff --git a/src/server/request_context.h b/src/server/request_context.h index 7cb8f3da5..5457ae4bf 100644 --- a/src/server/request_context.h +++ b/src/server/request_context.h @@ -74,11 +74,21 @@ class RequestContext { return v; } + template + T get_optional_param(const std::string& name, T default_value) const + { + try { + return get_argument(name); + } catch (...) {} + return default_value; + } + RequestMethod get_method() const; std::string get_url() const; std::string get_url_part(int part) const; std::string get_full_url() const; + std::string get_query() const; ByteRange get_range() const; diff --git a/src/server/response.cpp b/src/server/response.cpp index 3ebec43d2..1f0a8f401 100644 --- a/src/server/response.cpp +++ b/src/server/response.cpp @@ -24,6 +24,7 @@ #include "tools/regexTools.h" #include "tools/stringTools.h" +#include "tools/otherTools.h" #include "string.h" #include @@ -38,17 +39,6 @@ namespace { // some utilities -std::string render_template(const std::string& template_str, kainjow::mustache::data data) -{ - kainjow::mustache::mustache tmpl(template_str); - kainjow::mustache::data urlencode{kainjow::mustache::lambda2{ - [](const std::string& str,const kainjow::mustache::renderer& r) { return urlEncode(r(str), true); }}}; - data.set("urlencoded", urlencode); - std::stringstream ss; - tmpl.render(data, [&ss](const std::string& str) { ss << str; }); - return ss.str(); -} - std::string get_mime_type(const zim::Item& item) { try { diff --git a/src/tools/otherTools.cpp b/src/tools/otherTools.cpp index b5db9d492..eb081efb7 100644 --- a/src/tools/otherTools.cpp +++ b/src/tools/otherTools.cpp @@ -19,6 +19,7 @@ #include "tools/otherTools.h" #include +#include #ifdef _WIN32 #include @@ -32,6 +33,8 @@ #include #include +#include + static std::map codeisomapping { { "aa", "aar" }, @@ -341,3 +344,35 @@ kiwix::MimeCounterType kiwix::parseMimetypeCounter(const std::string& counterDat return counters; } + +std::string kiwix::gen_date_str() +{ + auto now = std::time(0); + auto tm = std::localtime(&now); + + std::stringstream is; + is << std::setw(2) << std::setfill('0') + << 1900+tm->tm_year << "-" + << std::setw(2) << std::setfill('0') << tm->tm_mon+1 << "-" + << std::setw(2) << std::setfill('0') << tm->tm_mday << "T" + << std::setw(2) << std::setfill('0') << tm->tm_hour << ":" + << std::setw(2) << std::setfill('0') << tm->tm_min << ":" + << std::setw(2) << std::setfill('0') << tm->tm_sec << "Z"; + return is.str(); +} + +std::string kiwix::gen_uuid(const std::string& s) +{ + return kiwix::to_string(zim::Uuid::generate(s)); +} + +std::string kiwix::render_template(const std::string& template_str, kainjow::mustache::data data) +{ + kainjow::mustache::mustache tmpl(template_str); + kainjow::mustache::data urlencode{kainjow::mustache::lambda2{ + [](const std::string& str,const kainjow::mustache::renderer& r) { return urlEncode(r(str), true); }}}; + data.set("urlencoded", urlencode); + std::stringstream ss; + tmpl.render(data, [&ss](const std::string& str) { ss << str; }); + return ss.str(); +} diff --git a/static/catalog_v2_searchdescription.xml b/static/catalog_v2_searchdescription.xml new file mode 100644 index 000000000..d9c94967a --- /dev/null +++ b/static/catalog_v2_searchdescription.xml @@ -0,0 +1,10 @@ + + + Zim catalog search + Search zim files in the catalog. + + diff --git a/static/opensearchdescription.xml b/static/opensearchdescription.xml index 47fd790b5..702152f8b 100644 --- a/static/opensearchdescription.xml +++ b/static/opensearchdescription.xml @@ -6,5 +6,5 @@ xmlns:atom="http://www.w3.org/2005/Atom" xmlns:k="http://kiwix.org/opensearchextension/1.0" indexOffset="0" - template="/{{root}}/catalog/search?q={searchTerms?}&lang={language?}&name={k:name?}&tag={k:tag?}¬ag={k:notag?}&maxsize={k:maxsize?}&count={count?}&start={startIndex?}"/> + template="{{root}}/catalog/search?q={searchTerms?}&lang={language?}&name={k:name?}&tag={k:tag?}¬ag={k:notag?}&maxsize={k:maxsize?}&count={count?}&start={startIndex?}"/> diff --git a/static/resources_list.txt b/static/resources_list.txt index 383c8228e..025742834 100644 --- a/static/resources_list.txt +++ b/static/resources_list.txt @@ -36,4 +36,9 @@ templates/head_taskbar.html templates/taskbar_part.html templates/external_blocker_part.html templates/captured_external.html +templates/catalog_entries.xml +templates/catalog_v2_root.xml +templates/catalog_v2_entries.xml +templates/catalog_v2_categories.xml opensearchdescription.xml +catalog_v2_searchdescription.xml diff --git a/static/templates/catalog_entries.xml b/static/templates/catalog_entries.xml new file mode 100644 index 000000000..32754d5b0 --- /dev/null +++ b/static/templates/catalog_entries.xml @@ -0,0 +1,38 @@ + + {{feed_id}} + {{^filter}}All zims{{/filter}}{{#filter}}Filtered zims ({{filter}}){{/filter}} + {{date}} +{{#filter}} + {{totalResults}} + {{startIndex}} + {{itemsPerPage}} +{{/filter}} + + + {{#books}} + + {{id}} + {{title}} + {{description}} + {{language}} + {{updated}} + {{name}} + {{flavour}} + {{category}} + {{tags}} + {{article_count}} + {{media_count}} + /meta?name=favicon&content={{{content_id}}} + + + {{author_name}} + + + {{publisher_name}} + + {{#url}} + + {{/url}} + + {{/books}} + diff --git a/static/templates/catalog_v2_categories.xml b/static/templates/catalog_v2_categories.xml new file mode 100644 index 000000000..4955b52aa --- /dev/null +++ b/static/templates/catalog_v2_categories.xml @@ -0,0 +1,25 @@ + + + {{feed_id}} + + + List of categories + {{date}} + + {{#categories}} + + {{name}} + + {{updated}} + {{id}} + All entries with category of '{{name}}'. + + {{/categories}} + diff --git a/static/templates/catalog_v2_entries.xml b/static/templates/catalog_v2_entries.xml new file mode 100644 index 000000000..05cbb5054 --- /dev/null +++ b/static/templates/catalog_v2_entries.xml @@ -0,0 +1,50 @@ + + + {{feed_id}} + + + + + + {{^filter}}All Entries{{/filter}}{{#filter}}Filtered Entries ({{filter}}){{/filter}} + {{date}} +{{#filter}} + {{totalResults}} + {{startIndex}} + {{itemsPerPage}} +{{/filter}} + {{#books}} + + {{id}} + {{title}} + {{description}} + {{language}} + {{updated}} + {{name}} + {{flavour}} + {{category}} + {{tags}} + {{article_count}} + {{media_count}} + /meta?name=favicon&content={{{content_id}}} + + + {{author_name}} + + + {{publisher_name}} + + {{#url}} + + {{/url}} + + {{/books}} + diff --git a/static/templates/catalog_v2_root.xml b/static/templates/catalog_v2_root.xml new file mode 100644 index 000000000..44db61c13 --- /dev/null +++ b/static/templates/catalog_v2_root.xml @@ -0,0 +1,35 @@ + + + {{feed_id}} + + + + OPDS Catalog Root + {{date}} + + + All entries + + {{date}} + {{all_entries_feed_id}} + All entries from this catalog. + + + List of categories + + {{date}} + {{category_list_feed_id}} + List of all categories in this catalog. + + diff --git a/test/server.cpp b/test/server.cpp index 612cfd658..87f1d47b3 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -240,6 +240,7 @@ const char* urls404[] = { "/skin/non-existent-skin-resource", "/catalog", "/catalog/non-existent-item", + "/catalogBLABLABLA/root.xml", "/meta", "/meta?content=zimfile", "/meta?content=zimfile&name=non-existent-item", @@ -570,18 +571,21 @@ protected: } }; -// Returns a copy of 'text' with every line that fully matches 'pattern' -// replaced with the fixed string 'replacement' +// Returns a copy of 'text' where every line that fully matches 'pattern' +// preceded by optional whitespace is replaced with the fixed string +// 'replacement' preserving the leading whitespace std::string replaceLines(const std::string& text, const std::string& pattern, const std::string& replacement) { - std::regex regex("^" + pattern + "$"); + std::regex regex("^ *" + pattern + "$"); std::ostringstream oss; std::istringstream iss(text); std::string line; while ( std::getline(iss, line) ) { if ( std::regex_match(line, regex) ) { + for ( size_t i = 0; i < line.size() && line[i] == ' '; ++i ) + oss << ' '; oss << replacement << "\n"; } else { oss << line << "\n"; @@ -592,10 +596,10 @@ std::string replaceLines(const std::string& text, std::string maskVariableOPDSFeedData(std::string s) { - s = replaceLines(s, " .+", - " YYYY-MM-DDThh:mm:ssZ"); - s = replaceLines(s, " .+", - " 12345678-90ab-cdef-1234-567890abcdef"); + s = replaceLines(s, R"(\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ)", + "YYYY-MM-DDThh:mm:ssZ"); + s = replaceLines(s, "[[:xdigit:]]{8}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{12}", + "12345678-90ab-cdef-1234-567890abcdef"); return s; } @@ -607,7 +611,7 @@ std::string maskVariableOPDSFeedData(std::string s) " \n" \ " \n" + " href=\"/catalog/searchdescription.xml\" />\n" #define CHARLES_RAY_CATALOG_ENTRY \ " \n" \ @@ -615,7 +619,7 @@ std::string maskVariableOPDSFeedData(std::string s) " Charles, Ray\n" \ " Wikipedia articles about Ray Charles\n" \ " eng\n" \ - " 2020-03-31T00:00::00Z\n" \ + " YYYY-MM-DDThh:mm:ssZ\n" \ " wikipedia_en_ray_charles\n" \ " \n" \ " jazz\n" \ @@ -639,7 +643,7 @@ std::string maskVariableOPDSFeedData(std::string s) " Ray Charles\n" \ " Wikipedia articles about Ray Charles\n" \ " eng\n" \ - " 2020-03-31T00:00::00Z\n" \ + " YYYY-MM-DDThh:mm:ssZ\n" \ " wikipedia_en_ray_charles\n" \ " \n" \ " wikipedia\n" \ @@ -663,7 +667,7 @@ std::string maskVariableOPDSFeedData(std::string s) " Ray (uncategorized) Charles\n" \ " No category is assigned to this library entry.\n" \ " eng\n" \ - " 2020-03-31T00:00::00Z\n" \ + " YYYY-MM-DDThh:mm:ssZ\n" \ " wikipedia_en_ray_charles\n" \ " \n" \ " \n" \ @@ -690,6 +694,7 @@ TEST_F(LibraryServerTest, catalog_root_xml) " 12345678-90ab-cdef-1234-567890abcdef\n" " All zims\n" " YYYY-MM-DDThh:mm:ssZ\n" + "\n" CATALOG_LINK_TAGS CHARLES_RAY_CATALOG_ENTRY RAY_CHARLES_CATALOG_ENTRY @@ -711,7 +716,7 @@ TEST_F(LibraryServerTest, catalog_searchdescription_xml) " xmlns:atom=\"http://www.w3.org/2005/Atom\"\n" " xmlns:k=\"http://kiwix.org/opensearchextension/1.0\"\n" " indexOffset=\"0\"\n" - " template=\"//catalog/search?q={searchTerms?}&lang={language?}&name={k:name?}&tag={k:tag?}¬ag={k:notag?}&maxsize={k:maxsize?}&count={count?}&start={startIndex?}\"/>\n" + " template=\"/catalog/search?q={searchTerms?}&lang={language?}&name={k:name?}&tag={k:tag?}¬ag={k:notag?}&maxsize={k:maxsize?}&count={count?}&start={startIndex?}\"/>\n" "\n" ); } @@ -723,7 +728,7 @@ TEST_F(LibraryServerTest, catalog_search_by_phrase) EXPECT_EQ(maskVariableOPDSFeedData(r->body), OPDS_FEED_TAG " 12345678-90ab-cdef-1234-567890abcdef\n" - " Search result for \"ray charles\"\n" + " Filtered zims (q="ray charles")\n" " YYYY-MM-DDThh:mm:ssZ\n" " 2\n" " 0\n" @@ -742,7 +747,7 @@ TEST_F(LibraryServerTest, catalog_search_by_words) EXPECT_EQ(maskVariableOPDSFeedData(r->body), OPDS_FEED_TAG " 12345678-90ab-cdef-1234-567890abcdef\n" - " Search result for ray charles\n" + " Filtered zims (q=ray charles)\n" " YYYY-MM-DDThh:mm:ssZ\n" " 3\n" " 0\n" @@ -763,7 +768,7 @@ TEST_F(LibraryServerTest, catalog_prefix_search) EXPECT_EQ(maskVariableOPDSFeedData(r->body), OPDS_FEED_TAG " 12345678-90ab-cdef-1234-567890abcdef\n" - " Search result for description:ray description:charles\n" + " Filtered zims (q=description:ray description:charles)\n" " YYYY-MM-DDThh:mm:ssZ\n" " 2\n" " 0\n" @@ -780,7 +785,7 @@ TEST_F(LibraryServerTest, catalog_prefix_search) EXPECT_EQ(maskVariableOPDSFeedData(r->body), OPDS_FEED_TAG " 12345678-90ab-cdef-1234-567890abcdef\n" - " Search result for title:\"ray charles\"\n" + " Filtered zims (q=title:"ray charles")\n" " YYYY-MM-DDThh:mm:ssZ\n" " 1\n" " 0\n" @@ -799,7 +804,7 @@ TEST_F(LibraryServerTest, catalog_search_with_word_exclusion) EXPECT_EQ(maskVariableOPDSFeedData(r->body), OPDS_FEED_TAG " 12345678-90ab-cdef-1234-567890abcdef\n" - " Search result for ray -uncategorized\n" + " Filtered zims (q=ray -uncategorized)\n" " YYYY-MM-DDThh:mm:ssZ\n" " 2\n" " 0\n" @@ -818,7 +823,7 @@ TEST_F(LibraryServerTest, catalog_search_by_tag) EXPECT_EQ(maskVariableOPDSFeedData(r->body), OPDS_FEED_TAG " 12345678-90ab-cdef-1234-567890abcdef\n" - " Search result for <Empty query>\n" + " Filtered zims (tag=_category:jazz)\n" " YYYY-MM-DDThh:mm:ssZ\n" " 1\n" " 0\n" @@ -836,7 +841,7 @@ TEST_F(LibraryServerTest, catalog_search_by_category) EXPECT_EQ(maskVariableOPDSFeedData(r->body), OPDS_FEED_TAG " 12345678-90ab-cdef-1234-567890abcdef\n" - " Search result for <Empty query>\n" + " Filtered zims (category=jazz)\n" " YYYY-MM-DDThh:mm:ssZ\n" " 1\n" " 0\n" @@ -855,7 +860,7 @@ TEST_F(LibraryServerTest, catalog_search_results_pagination) EXPECT_EQ(maskVariableOPDSFeedData(r->body), OPDS_FEED_TAG " 12345678-90ab-cdef-1234-567890abcdef\n" - " Search result for <Empty query>\n" + " Filtered zims (count=1)\n" " YYYY-MM-DDThh:mm:ssZ\n" " 3\n" " 0\n" @@ -871,7 +876,7 @@ TEST_F(LibraryServerTest, catalog_search_results_pagination) EXPECT_EQ(maskVariableOPDSFeedData(r->body), OPDS_FEED_TAG " 12345678-90ab-cdef-1234-567890abcdef\n" - " Search result for <Empty query>\n" + " Filtered zims (count=1&start=1)\n" " YYYY-MM-DDThh:mm:ssZ\n" " 3\n" " 1\n" @@ -887,13 +892,217 @@ TEST_F(LibraryServerTest, catalog_search_results_pagination) EXPECT_EQ(maskVariableOPDSFeedData(r->body), OPDS_FEED_TAG " 12345678-90ab-cdef-1234-567890abcdef\n" - " Search result for <Empty query>\n" + " Filtered zims (count=10&start=100)\n" " YYYY-MM-DDThh:mm:ssZ\n" " 3\n" " 100\n" " 0\n" CATALOG_LINK_TAGS + " \n" "\n" ); } } + +TEST_F(LibraryServerTest, catalog_v2_root) +{ + const auto r = zfs1_->GET("/catalog/v2/root.xml"); + EXPECT_EQ(r->status, 200); + const char expected_output[] = R"( + + 12345678-90ab-cdef-1234-567890abcdef + + + + OPDS Catalog Root + YYYY-MM-DDThh:mm:ssZ + + + All entries + + YYYY-MM-DDThh:mm:ssZ + 12345678-90ab-cdef-1234-567890abcdef + All entries from this catalog. + + + List of categories + + YYYY-MM-DDThh:mm:ssZ + 12345678-90ab-cdef-1234-567890abcdef + List of all categories in this catalog. + + +)"; + EXPECT_EQ(maskVariableOPDSFeedData(r->body), expected_output); +} + +TEST_F(LibraryServerTest, catalog_v2_searchdescription_xml) +{ + const auto r = zfs1_->GET("/catalog/v2/searchdescription.xml"); + EXPECT_EQ(r->status, 200); + EXPECT_EQ(r->body, + "\n" + "\n" + " Zim catalog search\n" + " Search zim files in the catalog.\n" + " \n" + "\n" + ); +} + +TEST_F(LibraryServerTest, catalog_v2_categories) +{ + const auto r = zfs1_->GET("/catalog/v2/categories"); + EXPECT_EQ(r->status, 200); + const char expected_output[] = R"( + + 12345678-90ab-cdef-1234-567890abcdef + + + List of categories + YYYY-MM-DDThh:mm:ssZ + + + jazz + + YYYY-MM-DDThh:mm:ssZ + 12345678-90ab-cdef-1234-567890abcdef + All entries with category of 'jazz'. + + + wikipedia + + YYYY-MM-DDThh:mm:ssZ + 12345678-90ab-cdef-1234-567890abcdef + All entries with category of 'wikipedia'. + + +)"; + EXPECT_EQ(maskVariableOPDSFeedData(r->body), expected_output); +} + +#define CATALOG_V2_ENTRIES_PREAMBLE(q) \ + "\n" \ + "\n" \ + " 12345678-90ab-cdef-1234-567890abcdef\n" \ + "\n" \ + " \n" \ + " \n" \ + " \n" \ + "\n" \ + + +TEST_F(LibraryServerTest, catalog_v2_entries) +{ + const auto r = zfs1_->GET("/catalog/v2/entries"); + EXPECT_EQ(r->status, 200); + EXPECT_EQ(maskVariableOPDSFeedData(r->body), + CATALOG_V2_ENTRIES_PREAMBLE("") + " All Entries\n" + " YYYY-MM-DDThh:mm:ssZ\n" + "\n" + CHARLES_RAY_CATALOG_ENTRY + RAY_CHARLES_CATALOG_ENTRY + UNCATEGORIZED_RAY_CHARLES_CATALOG_ENTRY + "\n" + ); +} + +TEST_F(LibraryServerTest, catalog_v2_entries_filtered_by_range) +{ + { + const auto r = zfs1_->GET("/catalog/v2/entries?start=1"); + EXPECT_EQ(r->status, 200); + EXPECT_EQ(maskVariableOPDSFeedData(r->body), + CATALOG_V2_ENTRIES_PREAMBLE("?start=1") + " Filtered Entries (start=1)\n" + " YYYY-MM-DDThh:mm:ssZ\n" + " 3\n" + " 1\n" + " 2\n" + RAY_CHARLES_CATALOG_ENTRY + UNCATEGORIZED_RAY_CHARLES_CATALOG_ENTRY + "\n" + ); + } + + { + const auto r = zfs1_->GET("/catalog/v2/entries?count=2"); + EXPECT_EQ(r->status, 200); + EXPECT_EQ(maskVariableOPDSFeedData(r->body), + CATALOG_V2_ENTRIES_PREAMBLE("?count=2") + " Filtered Entries (count=2)\n" + " YYYY-MM-DDThh:mm:ssZ\n" + " 3\n" + " 0\n" + " 2\n" + CHARLES_RAY_CATALOG_ENTRY + RAY_CHARLES_CATALOG_ENTRY + "\n" + ); + } + + { + const auto r = zfs1_->GET("/catalog/v2/entries?start=1&count=1"); + EXPECT_EQ(r->status, 200); + EXPECT_EQ(maskVariableOPDSFeedData(r->body), + CATALOG_V2_ENTRIES_PREAMBLE("?count=1&start=1") + " Filtered Entries (count=1&start=1)\n" + " YYYY-MM-DDThh:mm:ssZ\n" + " 3\n" + " 1\n" + " 1\n" + RAY_CHARLES_CATALOG_ENTRY + "\n" + ); + } +} + +TEST_F(LibraryServerTest, catalog_v2_entries_filtered_by_search_terms) +{ + const auto r = zfs1_->GET("/catalog/v2/entries?q=\"ray%20charles\""); + EXPECT_EQ(r->status, 200); + EXPECT_EQ(maskVariableOPDSFeedData(r->body), + CATALOG_V2_ENTRIES_PREAMBLE("?q=%22ray%20charles%22") + " Filtered Entries (q="ray charles")\n" + " YYYY-MM-DDThh:mm:ssZ\n" + " 2\n" + " 0\n" + " 2\n" + RAY_CHARLES_CATALOG_ENTRY + CHARLES_RAY_CATALOG_ENTRY + "\n" + ); +}