From 0a3d293ae0724efa606af7fd6412255be3bfe1b9 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Thu, 15 Apr 2021 18:38:38 +0400 Subject: [PATCH 01/26] Broke Server.404 with /catalogBLABLABLA/root.xml The new negative test-point demonstrates that Kiwix server doesn't distinguish /catalogBLABLABLA from /catalog. --- test/server.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/test/server.cpp b/test/server.cpp index 612cfd658..7333c2f2c 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -240,6 +240,7 @@ const char* urls404[] = { "/skin/non-existent-skin-resource", "/catalog", "/catalog/non-existent-item", + "/catalogBLABLABLA/root.xml", "/meta", "/meta?content=zimfile", "/meta?content=zimfile&name=non-existent-item", From 5b272ac49c83d7779f1d9978c653f7f46bc7d234 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Thu, 15 Apr 2021 18:53:53 +0400 Subject: [PATCH 02/26] Fixed handling of /catalogBLABLA/root.xml & alike Also removed an unneeded namespace qualifier. --- src/server/internalServer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index 4d344edc0..c626da4e7 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -243,10 +243,10 @@ std::unique_ptr InternalServer::handle_request(const RequestContext& r if ( etag ) return Response::build_304(*this, etag); - if (kiwix::startsWith(request.get_url(), "/skin/")) + if (startsWith(request.get_url(), "/skin/")) return handle_skin(request); - if (startsWith(request.get_url(), "/catalog")) + if (startsWith(request.get_url(), "/catalog/")) return handle_catalog(request); if (request.get_url() == "/meta") From 1e0ff1fbb023aed3e1c865332b5cc9b9fd6d357e Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Fri, 16 Apr 2021 23:49:45 +0400 Subject: [PATCH 03/26] Fixed the double colon in OPDS date string --- src/opds_dumper.cpp | 2 +- test/server.cpp | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/opds_dumper.cpp b/src/opds_dumper.cpp index 8f14e0949..aff73898e 100644 --- a/src/opds_dumper.cpp +++ b/src/opds_dumper.cpp @@ -54,7 +54,7 @@ std::string gen_date_str() static std::string gen_date_from_yyyy_mm_dd(const std::string& date) { std::stringstream is; - is << date << "T00:00::00Z"; + is << date << "T00:00:00Z"; return is.str(); } diff --git a/test/server.cpp b/test/server.cpp index 7333c2f2c..68b18fef5 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -616,7 +616,7 @@ std::string maskVariableOPDSFeedData(std::string s) " Charles, Ray\n" \ " Wikipedia articles about Ray Charles\n" \ " eng\n" \ - " 2020-03-31T00:00::00Z\n" \ + " 2020-03-31T00:00:00Z\n" \ " wikipedia_en_ray_charles\n" \ " \n" \ " jazz\n" \ @@ -640,7 +640,7 @@ std::string maskVariableOPDSFeedData(std::string s) " Ray Charles\n" \ " Wikipedia articles about Ray Charles\n" \ " eng\n" \ - " 2020-03-31T00:00::00Z\n" \ + " 2020-03-31T00:00:00Z\n" \ " wikipedia_en_ray_charles\n" \ " \n" \ " wikipedia\n" \ @@ -664,7 +664,7 @@ std::string maskVariableOPDSFeedData(std::string s) " Ray (uncategorized) Charles\n" \ " No category is assigned to this library entry.\n" \ " eng\n" \ - " 2020-03-31T00:00::00Z\n" \ + " 2020-03-31T00:00:00Z\n" \ " wikipedia_en_ray_charles\n" \ " \n" \ " \n" \ From 54b78eaf568e9c5fa3fcfba91e4f54ddb16344aa Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Thu, 15 Apr 2021 21:43:49 +0400 Subject: [PATCH 04/26] Moved gen_date_str() to tools/otherTools.cpp --- include/tools/otherTools.h | 2 ++ src/opds_dumper.cpp | 17 ----------------- src/tools/otherTools.cpp | 17 +++++++++++++++++ 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/include/tools/otherTools.h b/include/tools/otherTools.h index 726c332b3..4b7d2e184 100644 --- a/include/tools/otherTools.h +++ b/include/tools/otherTools.h @@ -45,6 +45,8 @@ namespace kiwix using MimeCounterType = std::map; MimeCounterType parseMimetypeCounter(const std::string& counterData); + + std::string gen_date_str(); } #endif diff --git a/src/opds_dumper.cpp b/src/opds_dumper.cpp index aff73898e..2147e0866 100644 --- a/src/opds_dumper.cpp +++ b/src/opds_dumper.cpp @@ -21,7 +21,6 @@ #include "book.h" #include "tools/otherTools.h" -#include namespace kiwix { @@ -35,22 +34,6 @@ OPDSDumper::~OPDSDumper() { } -std::string gen_date_str() -{ - auto now = time(0); - auto tm = localtime(&now); - - std::stringstream is; - is << std::setw(2) << std::setfill('0') - << 1900+tm->tm_year << "-" - << std::setw(2) << std::setfill('0') << tm->tm_mon+1 << "-" - << std::setw(2) << std::setfill('0') << tm->tm_mday << "T" - << std::setw(2) << std::setfill('0') << tm->tm_hour << ":" - << std::setw(2) << std::setfill('0') << tm->tm_min << ":" - << std::setw(2) << std::setfill('0') << tm->tm_sec << "Z"; - return is.str(); -} - static std::string gen_date_from_yyyy_mm_dd(const std::string& date) { std::stringstream is; diff --git a/src/tools/otherTools.cpp b/src/tools/otherTools.cpp index b5db9d492..a51ed5ff3 100644 --- a/src/tools/otherTools.cpp +++ b/src/tools/otherTools.cpp @@ -19,6 +19,7 @@ #include "tools/otherTools.h" #include +#include #ifdef _WIN32 #include @@ -341,3 +342,19 @@ kiwix::MimeCounterType kiwix::parseMimetypeCounter(const std::string& counterDat return counters; } + +std::string kiwix::gen_date_str() +{ + auto now = std::time(0); + auto tm = std::localtime(&now); + + std::stringstream is; + is << std::setw(2) << std::setfill('0') + << 1900+tm->tm_year << "-" + << std::setw(2) << std::setfill('0') << tm->tm_mon+1 << "-" + << std::setw(2) << std::setfill('0') << tm->tm_mday << "T" + << std::setw(2) << std::setfill('0') << tm->tm_hour << ":" + << std::setw(2) << std::setfill('0') << tm->tm_min << ":" + << std::setw(2) << std::setfill('0') << tm->tm_sec << "Z"; + return is.str(); +} From 3c3cf08a1ac55cd23c98a51ac673c29f80cf936d Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Thu, 15 Apr 2021 21:54:54 +0400 Subject: [PATCH 05/26] Serving /catalog/v2/root.xml Note: This commit somewhat relaxes validation of non variable `` elements in the OPDS feed - the contents of any `` element is replaced with the YYYY-MM-DDThh:mm:ssZ placeholder. --- src/server/internalServer.cpp | 58 ++++++++++++++++++++++++++++++++ src/server/internalServer.h | 2 ++ static/catalog_v2_root.xml | 31 +++++++++++++++++ static/resources_list.txt | 1 + test/server.cpp | 63 +++++++++++++++++++++++++++++------ 5 files changed, 145 insertions(+), 10 deletions(-) create mode 100644 static/catalog_v2_root.xml diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index c626da4e7..460d92765 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -76,6 +76,23 @@ extern "C" { namespace kiwix { +namespace +{ + +inline std::string gen_uuid(const std::string& s) +{ + return to_string(zim::Uuid::generate(s)); +} + +inline std::string normalizeRootUrl(const std::string& rootUrl) +{ + return (rootUrl.empty() || rootUrl[0] == '/') + ? rootUrl + : "/" + rootUrl; +} + +} // unnamed namespace + static IdNameMapper defaultNameMapper; static MHD_Result staticHandlerCallback(void* cls, @@ -606,6 +623,10 @@ std::unique_ptr InternalServer::handle_catalog(const RequestContext& r return Response::build_404(*this, request, ""); } + if (url == "v2") { + return handle_catalog_v2(request); + } + if (url != "searchdescription.xml" && url != "root.xml" && url != "search") { return Response::build_404(*this, request, ""); } @@ -686,6 +707,43 @@ InternalServer::search_catalog(const RequestContext& request, return bookIdsToDump; } +std::unique_ptr InternalServer::handle_catalog_v2(const RequestContext& request) +{ + if (m_verbose.load()) { + printf("** running handle_catalog_v2"); + } + + std::string url; + try { + url = request.get_url_part(2); + } catch (const std::out_of_range&) { + return Response::build_404(*this, request, ""); + } + + if (url == "root.xml") { + return handle_catalog_v2_root(request); + } else { + return Response::build_404(*this, request, ""); + } +} + +std::unique_ptr InternalServer::handle_catalog_v2_root(const RequestContext& request) +{ + const std::string root_url = normalizeRootUrl(m_root); + return ContentResponse::build( + *this, + RESOURCE::catalog_v2_root_xml, + kainjow::mustache::object{ + {"date", gen_date_str()}, + {"endpoint_root", root_url + "/catalog/v2"}, + {"feed_id", gen_uuid(m_server_id)}, + {"all_entries_feed_id", gen_uuid(m_server_id + "/entries")}, + {"category_list_feed_id", gen_uuid(m_server_id + "/categories")} + }, + "application/atom+xml;profile=opds-catalog;kind=navigation" + ); +} + namespace { diff --git a/src/server/internalServer.h b/src/server/internalServer.h index c37c736f2..7e98638a3 100644 --- a/src/server/internalServer.h +++ b/src/server/internalServer.h @@ -73,6 +73,8 @@ class InternalServer { std::unique_ptr build_homepage(const RequestContext& request); std::unique_ptr handle_skin(const RequestContext& request); std::unique_ptr handle_catalog(const RequestContext& request); + std::unique_ptr handle_catalog_v2(const RequestContext& request); + std::unique_ptr handle_catalog_v2_root(const RequestContext& request); std::unique_ptr handle_meta(const RequestContext& request); std::unique_ptr handle_search(const RequestContext& request); std::unique_ptr handle_suggest(const RequestContext& request); diff --git a/static/catalog_v2_root.xml b/static/catalog_v2_root.xml new file mode 100644 index 000000000..928f069bf --- /dev/null +++ b/static/catalog_v2_root.xml @@ -0,0 +1,31 @@ + + + {{feed_id}} + + + OPDS Catalog Root + {{date}} + + + All entries + + {{date}} + {{all_entries_feed_id}} + All entries from this catalog. + + + List of categories + + {{date}} + {{category_list_feed_id}} + List of all categories in this catalog. + + diff --git a/static/resources_list.txt b/static/resources_list.txt index 383c8228e..5b6341a37 100644 --- a/static/resources_list.txt +++ b/static/resources_list.txt @@ -37,3 +37,4 @@ templates/taskbar_part.html templates/external_blocker_part.html templates/captured_external.html opensearchdescription.xml +catalog_v2_root.xml diff --git a/test/server.cpp b/test/server.cpp index 68b18fef5..9c2575023 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -571,18 +571,21 @@ protected: } }; -// Returns a copy of 'text' with every line that fully matches 'pattern' -// replaced with the fixed string 'replacement' +// Returns a copy of 'text' where every line that fully matches 'pattern' +// preceded by optional whitespace is replaced with the fixed string +// 'replacement' preserving the leading whitespace std::string replaceLines(const std::string& text, const std::string& pattern, const std::string& replacement) { - std::regex regex("^" + pattern + "$"); + std::regex regex("^ *" + pattern + "$"); std::ostringstream oss; std::istringstream iss(text); std::string line; while ( std::getline(iss, line) ) { if ( std::regex_match(line, regex) ) { + for ( size_t i = 0; i < line.size() && line[i] == ' '; ++i ) + oss << ' '; oss << replacement << "\n"; } else { oss << line << "\n"; @@ -593,10 +596,10 @@ std::string replaceLines(const std::string& text, std::string maskVariableOPDSFeedData(std::string s) { - s = replaceLines(s, " .+", - " YYYY-MM-DDThh:mm:ssZ"); - s = replaceLines(s, " .+", - " 12345678-90ab-cdef-1234-567890abcdef"); + s = replaceLines(s, R"(\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ)", + "YYYY-MM-DDThh:mm:ssZ"); + s = replaceLines(s, "[[:xdigit:]]{8}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{12}", + "12345678-90ab-cdef-1234-567890abcdef"); return s; } @@ -616,7 +619,7 @@ std::string maskVariableOPDSFeedData(std::string s) " Charles, Ray\n" \ " Wikipedia articles about Ray Charles\n" \ " eng\n" \ - " 2020-03-31T00:00:00Z\n" \ + " YYYY-MM-DDThh:mm:ssZ\n" \ " wikipedia_en_ray_charles\n" \ " \n" \ " jazz\n" \ @@ -640,7 +643,7 @@ std::string maskVariableOPDSFeedData(std::string s) " Ray Charles\n" \ " Wikipedia articles about Ray Charles\n" \ " eng\n" \ - " 2020-03-31T00:00:00Z\n" \ + " YYYY-MM-DDThh:mm:ssZ\n" \ " wikipedia_en_ray_charles\n" \ " \n" \ " wikipedia\n" \ @@ -664,7 +667,7 @@ std::string maskVariableOPDSFeedData(std::string s) " Ray (uncategorized) Charles\n" \ " No category is assigned to this library entry.\n" \ " eng\n" \ - " 2020-03-31T00:00:00Z\n" \ + " YYYY-MM-DDThh:mm:ssZ\n" \ " wikipedia_en_ray_charles\n" \ " \n" \ " \n" \ @@ -898,3 +901,43 @@ TEST_F(LibraryServerTest, catalog_search_results_pagination) ); } } + +TEST_F(LibraryServerTest, catalog_v2_root) +{ + const auto r = zfs1_->GET("/catalog/v2/root.xml"); + EXPECT_EQ(r->status, 200); + EXPECT_EQ(maskVariableOPDSFeedData(r->body), +R"( + + 12345678-90ab-cdef-1234-567890abcdef + + + OPDS Catalog Root + YYYY-MM-DDThh:mm:ssZ + + + All entries + + YYYY-MM-DDThh:mm:ssZ + 12345678-90ab-cdef-1234-567890abcdef + All entries from this catalog. + + + List of categories + + YYYY-MM-DDThh:mm:ssZ + 12345678-90ab-cdef-1234-567890abcdef + List of all categories in this catalog. + + +)" + ); +} From b259afa408e3adb209ec6b2115200e491f9dddce Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Sat, 17 Apr 2021 00:32:04 +0400 Subject: [PATCH 06/26] Library::getBooksCategories() Note: no unit test added --- include/library.h | 9 ++++++++- src/library.cpp | 15 +++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/include/library.h b/include/library.h index f358e6746..95d22e4bd 100644 --- a/include/library.h +++ b/include/library.h @@ -233,12 +233,19 @@ class Library unsigned int getBookCount(const bool localBooks, const bool remoteBooks) const; /** - * Get all langagues of the books in the library. + * Get all languagues of the books in the library. * * @return A list of languages. */ std::vector getBooksLanguages() const; + /** + * Get all categories of the books in the library. + * + * @return A list of categories. + */ + std::vector getBooksCategories() const; + /** * Get all book creators of the books in the library. * diff --git a/src/library.cpp b/src/library.cpp index d8307c9ae..1147282df 100644 --- a/src/library.cpp +++ b/src/library.cpp @@ -201,6 +201,21 @@ std::vector Library::getBooksLanguages() const return booksLanguages; } +std::vector Library::getBooksCategories() const +{ + std::set categories; + + for (const auto& pair: m_books) { + const auto& book = pair.second; + const auto& c = book.getCategory(); + if ( !c.empty() ) { + categories.insert(c); + } + } + + return std::vector(categories.begin(), categories.end()); +} + std::vector Library::getBooksCreators() const { std::vector booksCreators; From 2e53b51696168104db2e6cebb5faedb8535ad86d Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Sat, 17 Apr 2021 00:33:12 +0400 Subject: [PATCH 07/26] Serving /catalog/v2/categories --- src/server/internalServer.cpp | 30 ++++++++++++++++++++++++ src/server/internalServer.h | 1 + static/catalog_v2_categories.xml | 24 +++++++++++++++++++ static/resources_list.txt | 1 + test/server.cpp | 40 ++++++++++++++++++++++++++++++++ 5 files changed, 96 insertions(+) create mode 100644 static/catalog_v2_categories.xml diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index 460d92765..dc95add3d 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -722,6 +722,8 @@ std::unique_ptr InternalServer::handle_catalog_v2(const RequestContext if (url == "root.xml") { return handle_catalog_v2_root(request); + } else if (url == "categories") { + return handle_catalog_v2_categories(request); } else { return Response::build_404(*this, request, ""); } @@ -744,6 +746,34 @@ std::unique_ptr InternalServer::handle_catalog_v2_root(const RequestCo ); } +std::unique_ptr InternalServer::handle_catalog_v2_categories(const RequestContext& request) +{ + const std::string root_url = normalizeRootUrl(m_root); + const auto now = gen_date_str(); + kainjow::mustache::list categoryData; + for ( const auto& category : mp_library->getBooksCategories() ) { + const auto urlencodedCategoryName = urlEncode(category); + categoryData.push_back(kainjow::mustache::object{ + {"name", category}, + {"urlencoded_name", urlencodedCategoryName}, + {"updated", now}, + {"id", gen_uuid(m_server_id + "/categories/" + urlencodedCategoryName)} + }); + } + + return ContentResponse::build( + *this, + RESOURCE::catalog_v2_categories_xml, + kainjow::mustache::object{ + {"date", now}, + {"endpoint_root", root_url + "/catalog/v2"}, + {"feed_id", gen_uuid(m_server_id + "/categories")}, + {"categories", categoryData } + }, + "application/atom+xml;profile=opds-catalog;kind=navigation" + ); +} + namespace { diff --git a/src/server/internalServer.h b/src/server/internalServer.h index 7e98638a3..d57ac6ed9 100644 --- a/src/server/internalServer.h +++ b/src/server/internalServer.h @@ -75,6 +75,7 @@ class InternalServer { std::unique_ptr handle_catalog(const RequestContext& request); std::unique_ptr handle_catalog_v2(const RequestContext& request); std::unique_ptr handle_catalog_v2_root(const RequestContext& request); + std::unique_ptr handle_catalog_v2_categories(const RequestContext& request); std::unique_ptr handle_meta(const RequestContext& request); std::unique_ptr handle_search(const RequestContext& request); std::unique_ptr handle_suggest(const RequestContext& request); diff --git a/static/catalog_v2_categories.xml b/static/catalog_v2_categories.xml new file mode 100644 index 000000000..5592849e6 --- /dev/null +++ b/static/catalog_v2_categories.xml @@ -0,0 +1,24 @@ + + + {{feed_id}} + + + List of categories + {{date}} + + {{#categories}} + + {{name}} + + {{updated}} + {{id}} + All entries with category of '{{name}}'. + + {{/categories}} + diff --git a/static/resources_list.txt b/static/resources_list.txt index 5b6341a37..9691e80e5 100644 --- a/static/resources_list.txt +++ b/static/resources_list.txt @@ -38,3 +38,4 @@ templates/external_blocker_part.html templates/captured_external.html opensearchdescription.xml catalog_v2_root.xml +catalog_v2_categories.xml diff --git a/test/server.cpp b/test/server.cpp index 9c2575023..bcc2e7e05 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -941,3 +941,43 @@ R"( )" ); } + +TEST_F(LibraryServerTest, catalog_v2_categories) +{ + const auto r = zfs1_->GET("/catalog/v2/categories"); + EXPECT_EQ(r->status, 200); + EXPECT_EQ(maskVariableOPDSFeedData(r->body), +R"( + + 12345678-90ab-cdef-1234-567890abcdef + + + List of categories + YYYY-MM-DDThh:mm:ssZ + + + jazz + + YYYY-MM-DDThh:mm:ssZ + 12345678-90ab-cdef-1234-567890abcdef + All entries with category of 'jazz'. + + + wikipedia + + YYYY-MM-DDThh:mm:ssZ + 12345678-90ab-cdef-1234-567890abcdef + All entries with category of 'wikipedia'. + + +)" + ); +} From a1520ce7f159397880f555139648f4e9999bc5bb Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Sun, 18 Apr 2021 00:05:08 +0400 Subject: [PATCH 08/26] Fixing the xenial build Under Ubuntu 16.04/xenial, ccache seems to have issues with multiline raw string literals used inside macros. --- test/server.cpp | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/test/server.cpp b/test/server.cpp index bcc2e7e05..5df6b6bbe 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -906,8 +906,7 @@ TEST_F(LibraryServerTest, catalog_v2_root) { const auto r = zfs1_->GET("/catalog/v2/root.xml"); EXPECT_EQ(r->status, 200); - EXPECT_EQ(maskVariableOPDSFeedData(r->body), -R"( + const char expected_output[] = R"( 12345678-90ab-cdef-1234-567890abcdef List of all categories in this catalog. -)" - ); +)"; + EXPECT_EQ(maskVariableOPDSFeedData(r->body), expected_output); } TEST_F(LibraryServerTest, catalog_v2_categories) { const auto r = zfs1_->GET("/catalog/v2/categories"); EXPECT_EQ(r->status, 200); - EXPECT_EQ(maskVariableOPDSFeedData(r->body), -R"( + const char expected_output[] = R"( 12345678-90ab-cdef-1234-567890abcdef All entries with category of 'wikipedia'. -)" - ); +)"; + EXPECT_EQ(maskVariableOPDSFeedData(r->body), expected_output); } From feeb9f206e4e8b93083a5ea80ec7456ddadb6998 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Sun, 18 Apr 2021 14:30:16 +0400 Subject: [PATCH 09/26] /catalog/v2/* XMLs are OPDS 1.2 --- static/catalog_v2_categories.xml | 3 ++- static/catalog_v2_root.xml | 3 ++- test/server.cpp | 6 ++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/static/catalog_v2_categories.xml b/static/catalog_v2_categories.xml index 5592849e6..4955b52aa 100644 --- a/static/catalog_v2_categories.xml +++ b/static/catalog_v2_categories.xml @@ -1,5 +1,6 @@ - + {{feed_id}} - + {{feed_id}} GET("/catalog/v2/root.xml"); EXPECT_EQ(r->status, 200); const char expected_output[] = R"( - + 12345678-90ab-cdef-1234-567890abcdef GET("/catalog/v2/categories"); EXPECT_EQ(r->status, 200); const char expected_output[] = R"( - + 12345678-90ab-cdef-1234-567890abcdef Date: Sun, 18 Apr 2021 14:59:07 +0400 Subject: [PATCH 10/26] Enter InternalServer::m_library_id The new field is intended to serve as a seed for generating semi-stable OPDS feed ids that only need to change when the library is updated. --- src/server/internalServer.cpp | 11 ++++++----- src/server/internalServer.h | 1 + 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index dc95add3d..b94f215cd 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -170,6 +170,7 @@ bool InternalServer::start() { } auto server_start_time = std::chrono::system_clock::now().time_since_epoch(); m_server_id = kiwix::to_string(server_start_time.count()); + m_library_id = m_server_id; return true; } @@ -738,9 +739,9 @@ std::unique_ptr InternalServer::handle_catalog_v2_root(const RequestCo kainjow::mustache::object{ {"date", gen_date_str()}, {"endpoint_root", root_url + "/catalog/v2"}, - {"feed_id", gen_uuid(m_server_id)}, - {"all_entries_feed_id", gen_uuid(m_server_id + "/entries")}, - {"category_list_feed_id", gen_uuid(m_server_id + "/categories")} + {"feed_id", gen_uuid(m_library_id)}, + {"all_entries_feed_id", gen_uuid(m_library_id + "/entries")}, + {"category_list_feed_id", gen_uuid(m_library_id + "/categories")} }, "application/atom+xml;profile=opds-catalog;kind=navigation" ); @@ -757,7 +758,7 @@ std::unique_ptr InternalServer::handle_catalog_v2_categories(const Req {"name", category}, {"urlencoded_name", urlencodedCategoryName}, {"updated", now}, - {"id", gen_uuid(m_server_id + "/categories/" + urlencodedCategoryName)} + {"id", gen_uuid(m_library_id + "/categories/" + urlencodedCategoryName)} }); } @@ -767,7 +768,7 @@ std::unique_ptr InternalServer::handle_catalog_v2_categories(const Req kainjow::mustache::object{ {"date", now}, {"endpoint_root", root_url + "/catalog/v2"}, - {"feed_id", gen_uuid(m_server_id + "/categories")}, + {"feed_id", gen_uuid(m_library_id + "/categories")}, {"categories", categoryData } }, "application/atom+xml;profile=opds-catalog;kind=navigation" diff --git a/src/server/internalServer.h b/src/server/internalServer.h index d57ac6ed9..af28856f0 100644 --- a/src/server/internalServer.h +++ b/src/server/internalServer.h @@ -107,6 +107,7 @@ class InternalServer { NameMapper* mp_nameMapper; std::string m_server_id; + std::string m_library_id; friend std::unique_ptr Response::build(const InternalServer& server); friend std::unique_ptr ContentResponse::build(const InternalServer& server, const std::string& content, const std::string& mimetype, bool isHomePage); From 19b59fd72f5d4c628d4baf93bb01cfb6e344653e Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Sun, 18 Apr 2021 15:01:43 +0400 Subject: [PATCH 11/26] Serving /catalog/v2/entries /catalog/v2/entries is intended to play the combined role of /catalog/root.xml and /catalog/search of the old OPDS API. Currently, the latter role is not yet implemented. Implementation note: instead of tweaking and reusing `OPDSDumper::dumpOPDSFeed()`, the generation of the OPDS feed is done via `mustache` and a new template `static/catalog_v2_entries.xml`. --- src/server/internalServer.cpp | 46 +++++++++++++++++++++++++++++++++++ src/server/internalServer.h | 1 + static/catalog_v2_entries.xml | 45 ++++++++++++++++++++++++++++++++++ static/resources_list.txt | 1 + test/server.cpp | 30 +++++++++++++++++++++++ 5 files changed, 123 insertions(+) create mode 100644 static/catalog_v2_entries.xml diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index b94f215cd..8f628942b 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -723,6 +723,8 @@ std::unique_ptr InternalServer::handle_catalog_v2(const RequestContext if (url == "root.xml") { return handle_catalog_v2_root(request); + } else if (url == "entries") { + return handle_catalog_v2_entries(request); } else if (url == "categories") { return handle_catalog_v2_categories(request); } else { @@ -747,6 +749,50 @@ std::unique_ptr InternalServer::handle_catalog_v2_root(const RequestCo ); } +std::unique_ptr InternalServer::handle_catalog_v2_entries(const RequestContext& request) +{ + const std::string root_url = normalizeRootUrl(m_root); + + const auto now = gen_date_str(); + kainjow::mustache::list bookData; + for ( const auto& bookId : mp_library->getBooksIds() ) { + const Book& book = mp_library->getBookById(bookId); + const MustacheData bookUrl = book.getUrl().empty() + ? MustacheData(false) + : MustacheData(book.getUrl()); + bookData.push_back(kainjow::mustache::object{ + {"id", "urn:uuid:"+book.getId()}, + {"name", book.getName()}, + {"title", book.getTitle()}, + {"description", book.getDescription()}, + {"language", book.getLanguage()}, + {"content_id", book.getHumanReadableIdFromPath()}, + {"updated", book.getDate() + "T00:00:00Z"}, + {"category", book.getCategory()}, + {"flavour", book.getFlavour()}, + {"tags", book.getTags()}, + {"article_count", to_string(book.getArticleCount())}, + {"media_count", to_string(book.getMediaCount())}, + {"author_name", book.getCreator()}, + {"publisher_name", book.getPublisher()}, + {"url", bookUrl}, + {"size", to_string(book.getSize())}, + }); + } + + return ContentResponse::build( + *this, + RESOURCE::catalog_v2_entries_xml, + kainjow::mustache::object{ + {"date", now}, + {"endpoint_root", root_url + "/catalog/v2"}, + {"feed_id", gen_uuid(m_library_id + "/entries")}, + {"books", bookData } + }, + "application/atom+xml;profile=opds-catalog;kind=acquisition" + ); +} + std::unique_ptr InternalServer::handle_catalog_v2_categories(const RequestContext& request) { const std::string root_url = normalizeRootUrl(m_root); diff --git a/src/server/internalServer.h b/src/server/internalServer.h index af28856f0..142d3fc37 100644 --- a/src/server/internalServer.h +++ b/src/server/internalServer.h @@ -75,6 +75,7 @@ class InternalServer { std::unique_ptr handle_catalog(const RequestContext& request); std::unique_ptr handle_catalog_v2(const RequestContext& request); std::unique_ptr handle_catalog_v2_root(const RequestContext& request); + std::unique_ptr handle_catalog_v2_entries(const RequestContext& request); std::unique_ptr handle_catalog_v2_categories(const RequestContext& request); std::unique_ptr handle_meta(const RequestContext& request); std::unique_ptr handle_search(const RequestContext& request); diff --git a/static/catalog_v2_entries.xml b/static/catalog_v2_entries.xml new file mode 100644 index 000000000..72a698cb7 --- /dev/null +++ b/static/catalog_v2_entries.xml @@ -0,0 +1,45 @@ + + + {{feed_id}} + + + + + + All Entries + {{date}} + + {{#books}} + + {{id}} + {{title}} + {{description}} + {{language}} + {{updated}} + {{name}} + {{flavour}} + {{category}} + {{tags}} + {{article_count}} + {{media_count}} + /meta?name=favicon&content={{{content_id}}} + + + {{author_name}} + + + {{publisher_name}} + + {{#url}} + + {{/url}} + + {{/books}} + diff --git a/static/resources_list.txt b/static/resources_list.txt index 9691e80e5..9b9923bcf 100644 --- a/static/resources_list.txt +++ b/static/resources_list.txt @@ -38,4 +38,5 @@ templates/external_blocker_part.html templates/captured_external.html opensearchdescription.xml catalog_v2_root.xml +catalog_v2_entries.xml catalog_v2_categories.xml diff --git a/test/server.cpp b/test/server.cpp index 89b9f9b10..7a9d14f54 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -981,3 +981,33 @@ TEST_F(LibraryServerTest, catalog_v2_categories) )"; EXPECT_EQ(maskVariableOPDSFeedData(r->body), expected_output); } + +TEST_F(LibraryServerTest, catalog_v2_entries) +{ + const auto r = zfs1_->GET("/catalog/v2/entries"); + EXPECT_EQ(r->status, 200); + EXPECT_EQ(maskVariableOPDSFeedData(r->body), + "\n" + "\n" + " 12345678-90ab-cdef-1234-567890abcdef\n" + "\n" + " \n" + " \n" + " \n" + "\n" + " All Entries\n" + " YYYY-MM-DDThh:mm:ssZ\n" + "\n" + CHARLES_RAY_CATALOG_ENTRY + RAY_CHARLES_CATALOG_ENTRY + UNCATEGORIZED_RAY_CHARLES_CATALOG_ENTRY + "\n" + ); +} From 208dece7e37ddcd81039189f5fcdb49906aff87a Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Mon, 26 Apr 2021 01:29:53 +0400 Subject: [PATCH 12/26] Reordered several statements Reordered several statements so that the next couple of commits are a little simpler. --- src/server/internalServer.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index 8f628942b..99be804ae 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -666,8 +666,6 @@ InternalServer::search_catalog(const RequestContext& request, { auto filter = kiwix::Filter().valid(true).local(true); string query(""); - size_t count(10); - size_t startIndex(0); try { query = request.get_argument("q"); filter.query(query); @@ -684,12 +682,6 @@ InternalServer::search_catalog(const RequestContext& request, try { filter.lang(request.get_argument("lang")); } catch (const std::out_of_range&) {} - try { - count = extractFromString(request.get_argument("count")); - } catch (...) {} - try { - startIndex = extractFromString(request.get_argument("start")); - } catch (...) {} try { filter.acceptTags(kiwix::split(request.get_argument("tag"), ";")); } catch (...) {} @@ -699,6 +691,14 @@ InternalServer::search_catalog(const RequestContext& request, opdsDumper.setTitle("Search result for " + query); std::vector bookIdsToDump = mp_library->filter(filter); const auto totalResults = bookIdsToDump.size(); + size_t count(10); + size_t startIndex(0); + try { + count = extractFromString(request.get_argument("count")); + } catch (...) {} + try { + startIndex = extractFromString(request.get_argument("start")); + } catch (...) {} const auto s = std::min(startIndex, totalResults); bookIdsToDump.erase(bookIdsToDump.begin(), bookIdsToDump.begin()+s); if (count>0 && bookIdsToDump.size() > count) { From 4aa3c792aa388436fecf42c16a8e0bb938c4642e Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Tue, 8 Jun 2021 16:05:29 +0400 Subject: [PATCH 13/26] Extracted get_search_filter() --- src/server/internalServer.cpp | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index 99be804ae..8aa6ce782 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -660,15 +660,14 @@ std::unique_ptr InternalServer::handle_catalog(const RequestContext& r return std::move(response); } -std::vector -InternalServer::search_catalog(const RequestContext& request, - kiwix::OPDSDumper& opdsDumper) +namespace +{ + +Filter get_search_filter(const RequestContext& request) { auto filter = kiwix::Filter().valid(true).local(true); - string query(""); try { - query = request.get_argument("q"); - filter.query(query); + filter.query(request.get_argument("q")); } catch (const std::out_of_range&) {} try { filter.maxSize(extractFromString(request.get_argument("maxsize"))); @@ -688,7 +687,20 @@ InternalServer::search_catalog(const RequestContext& request, try { filter.rejectTags(kiwix::split(request.get_argument("notag"), ";")); } catch (...) {} - opdsDumper.setTitle("Search result for " + query); + return filter; +} + +} // unnamed namespace + +std::vector +InternalServer::search_catalog(const RequestContext& request, + kiwix::OPDSDumper& opdsDumper) +{ + const auto filter = get_search_filter(request); + const std::string q = filter.hasQuery() + ? filter.getQuery() + : ""; + opdsDumper.setTitle("Search result for " + q); std::vector bookIdsToDump = mp_library->filter(filter); const auto totalResults = bookIdsToDump.size(); size_t count(10); From 70d42aec9832166b695b123327ef2cd1311be2a4 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Mon, 26 Apr 2021 02:24:07 +0400 Subject: [PATCH 14/26] A small simplification --- src/server/internalServer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index 8aa6ce782..c0148c241 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -670,7 +670,7 @@ Filter get_search_filter(const RequestContext& request) filter.query(request.get_argument("q")); } catch (const std::out_of_range&) {} try { - filter.maxSize(extractFromString(request.get_argument("maxsize"))); + filter.maxSize(request.get_argument("maxsize")); } catch (...) {} try { filter.name(request.get_argument("name")); From b60e3ffb262a896fd599fe6a3804b2ef7f541c57 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Mon, 26 Apr 2021 02:25:37 +0400 Subject: [PATCH 15/26] RequestContext::get_optional_param() --- src/server/internalServer.cpp | 10 ++-------- src/server/request_context.h | 9 +++++++++ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index c0148c241..3634ef79a 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -703,14 +703,8 @@ InternalServer::search_catalog(const RequestContext& request, opdsDumper.setTitle("Search result for " + q); std::vector bookIdsToDump = mp_library->filter(filter); const auto totalResults = bookIdsToDump.size(); - size_t count(10); - size_t startIndex(0); - try { - count = extractFromString(request.get_argument("count")); - } catch (...) {} - try { - startIndex = extractFromString(request.get_argument("start")); - } catch (...) {} + const size_t count = request.get_optional_param("count", 10UL); + const size_t startIndex = request.get_optional_param("start", 0UL); const auto s = std::min(startIndex, totalResults); bookIdsToDump.erase(bookIdsToDump.begin(), bookIdsToDump.begin()+s); if (count>0 && bookIdsToDump.size() > count) { diff --git a/src/server/request_context.h b/src/server/request_context.h index 7cb8f3da5..3dd446af2 100644 --- a/src/server/request_context.h +++ b/src/server/request_context.h @@ -74,6 +74,15 @@ class RequestContext { return v; } + template + T get_optional_param(const std::string& name, T default_value) const + { + try { + return get_argument(name); + } catch (...) {} + return default_value; + } + RequestMethod get_method() const; std::string get_url() const; From 07252a127af39d09edb0f57086c957b27e99cb73 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Mon, 26 Apr 2021 02:58:02 +0400 Subject: [PATCH 16/26] /catalog/v2/entries is also a search endpoint --- src/server/internalServer.cpp | 26 +++++++--- src/server/request_context.cpp | 10 ++++ src/server/request_context.h | 1 + static/catalog_v2_entries.xml | 2 +- test/server.cpp | 93 ++++++++++++++++++++++++++++------ 5 files changed, 109 insertions(+), 23 deletions(-) diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index 3634ef79a..ae77e0871 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -690,6 +690,13 @@ Filter get_search_filter(const RequestContext& request) return filter; } +template +std::vector subrange(const std::vector& v, size_t s, size_t n) +{ + const size_t e = std::min(v.size(), s+n); + return std::vector(v.begin()+std::min(v.size(), s), v.begin()+e); +} + } // unnamed namespace std::vector @@ -705,11 +712,7 @@ InternalServer::search_catalog(const RequestContext& request, const auto totalResults = bookIdsToDump.size(); const size_t count = request.get_optional_param("count", 10UL); const size_t startIndex = request.get_optional_param("start", 0UL); - const auto s = std::min(startIndex, totalResults); - bookIdsToDump.erase(bookIdsToDump.begin(), bookIdsToDump.begin()+s); - if (count>0 && bookIdsToDump.size() > count) { - bookIdsToDump.resize(count); - } + bookIdsToDump = subrange(bookIdsToDump, startIndex, count); opdsDumper.setOpenSearchInfo(totalResults, startIndex, bookIdsToDump.size()); return bookIdsToDump; } @@ -761,7 +764,11 @@ std::unique_ptr InternalServer::handle_catalog_v2_entries(const Reques const auto now = gen_date_str(); kainjow::mustache::list bookData; - for ( const auto& bookId : mp_library->getBooksIds() ) { + const auto filter = get_search_filter(request); + const auto allMatchingEntries = mp_library->filter(filter); + const size_t count = request.get_optional_param("count", 10UL); + const size_t start = request.get_optional_param("start", 0UL); + for ( const auto& bookId : subrange(allMatchingEntries, start, count) ) { const Book& book = mp_library->getBookById(bookId); const MustacheData bookUrl = book.getUrl().empty() ? MustacheData(false) @@ -786,13 +793,18 @@ std::unique_ptr InternalServer::handle_catalog_v2_entries(const Reques }); } + const auto query = request.get_query().empty() + ? MustacheData(false) + : MustacheData(request.get_query()); + return ContentResponse::build( *this, RESOURCE::catalog_v2_entries_xml, kainjow::mustache::object{ {"date", now}, {"endpoint_root", root_url + "/catalog/v2"}, - {"feed_id", gen_uuid(m_library_id + "/entries")}, + {"feed_id", gen_uuid(m_library_id + "/entries?"+request.get_query())}, + {"filter", query}, {"books", bookData } }, "application/atom+xml;profile=opds-catalog;kind=acquisition" diff --git a/src/server/request_context.cpp b/src/server/request_context.cpp index cad39eefc..765d01adf 100644 --- a/src/server/request_context.cpp +++ b/src/server/request_context.cpp @@ -183,4 +183,14 @@ std::string RequestContext::get_header(const std::string& name) const { return headers.at(lcAll(name)); } +std::string RequestContext::get_query() const { + std::string q; + const char* sep = ""; + for ( const auto& a : arguments ) { + q += sep + a.first + '=' + a.second; + sep = "&"; + } + return q; +} + } diff --git a/src/server/request_context.h b/src/server/request_context.h index 3dd446af2..5457ae4bf 100644 --- a/src/server/request_context.h +++ b/src/server/request_context.h @@ -88,6 +88,7 @@ class RequestContext { std::string get_url() const; std::string get_url_part(int part) const; std::string get_full_url() const; + std::string get_query() const; ByteRange get_range() const; diff --git a/static/catalog_v2_entries.xml b/static/catalog_v2_entries.xml index 72a698cb7..a829b53b6 100644 --- a/static/catalog_v2_entries.xml +++ b/static/catalog_v2_entries.xml @@ -13,7 +13,7 @@ href="{{endpoint_root}}/root.xml" type="application/atom+xml;profile=opds-catalog;kind=navigation"/> - All Entries + {{^filter}}All Entries{{/filter}}{{#filter}}Filtered Entries ({{filter}}){{/filter}} {{date}} {{#books}} diff --git a/test/server.cpp b/test/server.cpp index 7a9d14f54..6f1382fef 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -982,26 +982,30 @@ TEST_F(LibraryServerTest, catalog_v2_categories) EXPECT_EQ(maskVariableOPDSFeedData(r->body), expected_output); } +#define CATALOG_V2_ENTRIES_PREAMBLE \ + "\n" \ + "\n" \ + " 12345678-90ab-cdef-1234-567890abcdef\n" \ + "\n" \ + " \n" \ + " \n" \ + " \n" \ + "\n" \ + + TEST_F(LibraryServerTest, catalog_v2_entries) { const auto r = zfs1_->GET("/catalog/v2/entries"); EXPECT_EQ(r->status, 200); EXPECT_EQ(maskVariableOPDSFeedData(r->body), - "\n" - "\n" - " 12345678-90ab-cdef-1234-567890abcdef\n" - "\n" - " \n" - " \n" - " \n" - "\n" + CATALOG_V2_ENTRIES_PREAMBLE " All Entries\n" " YYYY-MM-DDThh:mm:ssZ\n" "\n" @@ -1011,3 +1015,62 @@ TEST_F(LibraryServerTest, catalog_v2_entries) "\n" ); } + +TEST_F(LibraryServerTest, catalog_v2_entries_filtered_by_range) +{ + { + const auto r = zfs1_->GET("/catalog/v2/entries?start=1"); + EXPECT_EQ(r->status, 200); + EXPECT_EQ(maskVariableOPDSFeedData(r->body), + CATALOG_V2_ENTRIES_PREAMBLE + " Filtered Entries (start=1)\n" + " YYYY-MM-DDThh:mm:ssZ\n" + "\n" + RAY_CHARLES_CATALOG_ENTRY + UNCATEGORIZED_RAY_CHARLES_CATALOG_ENTRY + "\n" + ); + } + + { + const auto r = zfs1_->GET("/catalog/v2/entries?count=2"); + EXPECT_EQ(r->status, 200); + EXPECT_EQ(maskVariableOPDSFeedData(r->body), + CATALOG_V2_ENTRIES_PREAMBLE + " Filtered Entries (count=2)\n" + " YYYY-MM-DDThh:mm:ssZ\n" + "\n" + CHARLES_RAY_CATALOG_ENTRY + RAY_CHARLES_CATALOG_ENTRY + "\n" + ); + } + + { + const auto r = zfs1_->GET("/catalog/v2/entries?start=1&count=1"); + EXPECT_EQ(r->status, 200); + EXPECT_EQ(maskVariableOPDSFeedData(r->body), + CATALOG_V2_ENTRIES_PREAMBLE + " Filtered Entries (count=1&start=1)\n" + " YYYY-MM-DDThh:mm:ssZ\n" + "\n" + RAY_CHARLES_CATALOG_ENTRY + "\n" + ); + } +} + +TEST_F(LibraryServerTest, catalog_v2_entries_filtered_by_search_terms) +{ + const auto r = zfs1_->GET("/catalog/v2/entries?q=\"ray%20charles\""); + EXPECT_EQ(r->status, 200); + EXPECT_EQ(maskVariableOPDSFeedData(r->body), + CATALOG_V2_ENTRIES_PREAMBLE + " Filtered Entries (q="ray charles")\n" + " YYYY-MM-DDThh:mm:ssZ\n" + "\n" + RAY_CHARLES_CATALOG_ENTRY + CHARLES_RAY_CATALOG_ENTRY + "\n" + ); +} From dfad1c3815be8c93044aa7210d0fdedf850c6aff Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Mon, 26 Apr 2021 03:57:29 +0400 Subject: [PATCH 17/26] /catalog/v2/searchdescription.xml --- src/server/internalServer.cpp | 7 +++++++ static/catalog_v2_root.xml | 3 +++ static/catalog_v2_searchdescription.xml | 10 ++++++++++ static/resources_list.txt | 1 + test/server.cpp | 21 +++++++++++++++++++++ 5 files changed, 42 insertions(+) create mode 100644 static/catalog_v2_searchdescription.xml diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index ae77e0871..9520af296 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -732,6 +732,13 @@ std::unique_ptr InternalServer::handle_catalog_v2(const RequestContext if (url == "root.xml") { return handle_catalog_v2_root(request); + } else if (url == "searchdescription.xml") { + const std::string endpoint_root = normalizeRootUrl(m_root) + "/catalog/v2"; + return ContentResponse::build(*this, + RESOURCE::catalog_v2_searchdescription_xml, + kainjow::mustache::object({{"endpoint_root", endpoint_root}}), + "application/opensearchdescription+xml" + ); } else if (url == "entries") { return handle_catalog_v2_entries(request); } else if (url == "categories") { diff --git a/static/catalog_v2_root.xml b/static/catalog_v2_root.xml index ee78cdfec..44db61c13 100644 --- a/static/catalog_v2_root.xml +++ b/static/catalog_v2_root.xml @@ -8,6 +8,9 @@ + OPDS Catalog Root {{date}} diff --git a/static/catalog_v2_searchdescription.xml b/static/catalog_v2_searchdescription.xml new file mode 100644 index 000000000..d9c94967a --- /dev/null +++ b/static/catalog_v2_searchdescription.xml @@ -0,0 +1,10 @@ + + + Zim catalog search + Search zim files in the catalog. + + diff --git a/static/resources_list.txt b/static/resources_list.txt index 9b9923bcf..83dea44aa 100644 --- a/static/resources_list.txt +++ b/static/resources_list.txt @@ -40,3 +40,4 @@ opensearchdescription.xml catalog_v2_root.xml catalog_v2_entries.xml catalog_v2_categories.xml +catalog_v2_searchdescription.xml diff --git a/test/server.cpp b/test/server.cpp index 6f1382fef..42401cc74 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -916,6 +916,9 @@ TEST_F(LibraryServerTest, catalog_v2_root) + OPDS Catalog Root YYYY-MM-DDThh:mm:ssZ @@ -942,6 +945,24 @@ TEST_F(LibraryServerTest, catalog_v2_root) EXPECT_EQ(maskVariableOPDSFeedData(r->body), expected_output); } +TEST_F(LibraryServerTest, catalog_v2_searchdescription_xml) +{ + const auto r = zfs1_->GET("/catalog/v2/searchdescription.xml"); + EXPECT_EQ(r->status, 200); + EXPECT_EQ(r->body, + "\n" + "\n" + " Zim catalog search\n" + " Search zim files in the catalog.\n" + " \n" + "\n" + ); +} + TEST_F(LibraryServerTest, catalog_v2_categories) { const auto r = zfs1_->GET("/catalog/v2/categories"); From cdacc0caf183023a4519a307fe3f1a229a6bc18f Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Mon, 24 May 2021 12:17:15 +0400 Subject: [PATCH 18/26] /catalog/v2/entries going through OPDSDumper OPDSDumper sensed threats to its job security, so it lobbied to be involved in handling the /catalog/v2 endpoints, too. --- include/opds_dumper.h | 18 +++++++++++- include/tools/otherTools.h | 4 +++ src/opds_dumper.cpp | 44 +++++++++++++++++++++++++++++ src/server/internalServer.cpp | 53 ++++------------------------------- src/server/response.cpp | 12 +------- src/tools/otherTools.cpp | 18 ++++++++++++ 6 files changed, 90 insertions(+), 59 deletions(-) diff --git a/include/opds_dumper.h b/include/opds_dumper.h index c6cc60d3d..72d30ef4a 100644 --- a/include/opds_dumper.h +++ b/include/opds_dumper.h @@ -51,11 +51,26 @@ class OPDSDumper /** * Dump the OPDS feed. * - * @param id The id of the library. + * @param bookIds the ids of the books to include in the feed * @return The OPDS feed. */ std::string dumpOPDSFeed(const std::vector& bookIds); + /** + * Dump the OPDS feed. + * + * @param bookIds the ids of the books to include in the feed + * @return The OPDS feed. + */ + std::string dumpOPDSFeedV2(const std::vector& bookIds, const std::string& query) const; + + /** + * Set the id of the library. + * + * @param id the id to use. + */ + void setLibraryId(const std::string& id) { this->libraryId = id;} + /** * Set the id of the opds stream. * @@ -103,6 +118,7 @@ class OPDSDumper protected: kiwix::Library* library; std::string id; + std::string libraryId; std::string title; std::string date; std::string rootLocation; diff --git a/include/tools/otherTools.h b/include/tools/otherTools.h index 4b7d2e184..c105d52f5 100644 --- a/include/tools/otherTools.h +++ b/include/tools/otherTools.h @@ -24,6 +24,7 @@ #include #include #include +#include namespace pugi { class xml_node; @@ -47,6 +48,9 @@ namespace kiwix MimeCounterType parseMimetypeCounter(const std::string& counterData); std::string gen_date_str(); + std::string gen_uuid(const std::string& s); + + std::string render_template(const std::string& template_str, kainjow::mustache::data data); } #endif diff --git a/src/opds_dumper.cpp b/src/opds_dumper.cpp index 2147e0866..c20d492e9 100644 --- a/src/opds_dumper.cpp +++ b/src/opds_dumper.cpp @@ -22,6 +22,9 @@ #include "tools/otherTools.h" +#include "kiwixlib-resources.h" +#include + namespace kiwix { /* Constructor */ @@ -135,4 +138,45 @@ string OPDSDumper::dumpOPDSFeed(const std::vector& bookIds) return nodeToString(root_node); } +typedef kainjow::mustache::data MustacheData; + +string OPDSDumper::dumpOPDSFeedV2(const std::vector& bookIds, const std::string& query) const +{ + kainjow::mustache::list bookData; + for ( const auto& bookId : bookIds ) { + const Book& book = library->getBookById(bookId); + const MustacheData bookUrl = book.getUrl().empty() + ? MustacheData(false) + : MustacheData(book.getUrl()); + bookData.push_back(kainjow::mustache::object{ + {"id", "urn:uuid:"+book.getId()}, + {"name", book.getName()}, + {"title", book.getTitle()}, + {"description", book.getDescription()}, + {"language", book.getLanguage()}, + {"content_id", book.getHumanReadableIdFromPath()}, + {"updated", book.getDate() + "T00:00:00Z"}, + {"category", book.getCategory()}, + {"flavour", book.getFlavour()}, + {"tags", book.getTags()}, + {"article_count", to_string(book.getArticleCount())}, + {"media_count", to_string(book.getMediaCount())}, + {"author_name", book.getCreator()}, + {"publisher_name", book.getPublisher()}, + {"url", bookUrl}, + {"size", to_string(book.getSize())}, + }); + } + + const kainjow::mustache::object template_data{ + {"date", gen_date_str()}, + {"endpoint_root", rootLocation + "/catalog/v2"}, + {"feed_id", gen_uuid(libraryId + "/entries?"+query)}, + {"filter", query.empty() ? MustacheData(false) : MustacheData(query)}, + {"books", bookData } + }; + + return render_template(RESOURCE::catalog_v2_entries_xml, template_data); +} + } diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index 9520af296..77724e669 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -79,11 +79,6 @@ namespace kiwix { namespace { -inline std::string gen_uuid(const std::string& s) -{ - return to_string(zim::Uuid::generate(s)); -} - inline std::string normalizeRootUrl(const std::string& rootUrl) { return (rootUrl.empty() || rootUrl[0] == '/') @@ -767,53 +762,17 @@ std::unique_ptr InternalServer::handle_catalog_v2_root(const RequestCo std::unique_ptr InternalServer::handle_catalog_v2_entries(const RequestContext& request) { - const std::string root_url = normalizeRootUrl(m_root); - - const auto now = gen_date_str(); - kainjow::mustache::list bookData; const auto filter = get_search_filter(request); - const auto allMatchingEntries = mp_library->filter(filter); const size_t count = request.get_optional_param("count", 10UL); const size_t start = request.get_optional_param("start", 0UL); - for ( const auto& bookId : subrange(allMatchingEntries, start, count) ) { - const Book& book = mp_library->getBookById(bookId); - const MustacheData bookUrl = book.getUrl().empty() - ? MustacheData(false) - : MustacheData(book.getUrl()); - bookData.push_back(kainjow::mustache::object{ - {"id", "urn:uuid:"+book.getId()}, - {"name", book.getName()}, - {"title", book.getTitle()}, - {"description", book.getDescription()}, - {"language", book.getLanguage()}, - {"content_id", book.getHumanReadableIdFromPath()}, - {"updated", book.getDate() + "T00:00:00Z"}, - {"category", book.getCategory()}, - {"flavour", book.getFlavour()}, - {"tags", book.getTags()}, - {"article_count", to_string(book.getArticleCount())}, - {"media_count", to_string(book.getMediaCount())}, - {"author_name", book.getCreator()}, - {"publisher_name", book.getPublisher()}, - {"url", bookUrl}, - {"size", to_string(book.getSize())}, - }); - } - - const auto query = request.get_query().empty() - ? MustacheData(false) - : MustacheData(request.get_query()); - + const auto bookIds = subrange(mp_library->filter(filter), start, count); + OPDSDumper opdsDumper(mp_library); + opdsDumper.setRootLocation(normalizeRootUrl(m_root)); + opdsDumper.setLibraryId(m_library_id); + const auto opdsFeed = opdsDumper.dumpOPDSFeedV2(bookIds, request.get_query()); return ContentResponse::build( *this, - RESOURCE::catalog_v2_entries_xml, - kainjow::mustache::object{ - {"date", now}, - {"endpoint_root", root_url + "/catalog/v2"}, - {"feed_id", gen_uuid(m_library_id + "/entries?"+request.get_query())}, - {"filter", query}, - {"books", bookData } - }, + opdsFeed, "application/atom+xml;profile=opds-catalog;kind=acquisition" ); } diff --git a/src/server/response.cpp b/src/server/response.cpp index 3ebec43d2..1f0a8f401 100644 --- a/src/server/response.cpp +++ b/src/server/response.cpp @@ -24,6 +24,7 @@ #include "tools/regexTools.h" #include "tools/stringTools.h" +#include "tools/otherTools.h" #include "string.h" #include @@ -38,17 +39,6 @@ namespace { // some utilities -std::string render_template(const std::string& template_str, kainjow::mustache::data data) -{ - kainjow::mustache::mustache tmpl(template_str); - kainjow::mustache::data urlencode{kainjow::mustache::lambda2{ - [](const std::string& str,const kainjow::mustache::renderer& r) { return urlEncode(r(str), true); }}}; - data.set("urlencoded", urlencode); - std::stringstream ss; - tmpl.render(data, [&ss](const std::string& str) { ss << str; }); - return ss.str(); -} - std::string get_mime_type(const zim::Item& item) { try { diff --git a/src/tools/otherTools.cpp b/src/tools/otherTools.cpp index a51ed5ff3..eb081efb7 100644 --- a/src/tools/otherTools.cpp +++ b/src/tools/otherTools.cpp @@ -33,6 +33,8 @@ #include #include +#include + static std::map codeisomapping { { "aa", "aar" }, @@ -358,3 +360,19 @@ std::string kiwix::gen_date_str() << std::setw(2) << std::setfill('0') << tm->tm_sec << "Z"; return is.str(); } + +std::string kiwix::gen_uuid(const std::string& s) +{ + return kiwix::to_string(zim::Uuid::generate(s)); +} + +std::string kiwix::render_template(const std::string& template_str, kainjow::mustache::data data) +{ + kainjow::mustache::mustache tmpl(template_str); + kainjow::mustache::data urlencode{kainjow::mustache::lambda2{ + [](const std::string& str,const kainjow::mustache::renderer& r) { return urlEncode(r(str), true); }}}; + data.set("urlencoded", urlencode); + std::stringstream ss; + tmpl.render(data, [&ss](const std::string& str) { ss << str; }); + return ss.str(); +} From 9ca6bd006fb9c0ee4ab41e46bd38ce3b6bebc40c Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Mon, 24 May 2021 12:30:32 +0400 Subject: [PATCH 19/26] /catalog/v2/categories goes through OPDSDumper too --- include/opds_dumper.h | 8 ++++++++ src/opds_dumper.cpp | 25 +++++++++++++++++++++++++ src/server/internalServer.cpp | 24 ++++-------------------- 3 files changed, 37 insertions(+), 20 deletions(-) diff --git a/include/opds_dumper.h b/include/opds_dumper.h index 72d30ef4a..8d8da0f6e 100644 --- a/include/opds_dumper.h +++ b/include/opds_dumper.h @@ -64,6 +64,14 @@ class OPDSDumper */ std::string dumpOPDSFeedV2(const std::vector& bookIds, const std::string& query) const; + /** + * Dump the categories OPDS feed. + * + * @param categories list of category names + * @return The OPDS feed. + */ + std::string categoriesOPDSFeed(const std::vector& categories) const; + /** * Set the id of the library. * diff --git a/src/opds_dumper.cpp b/src/opds_dumper.cpp index c20d492e9..7e91e7989 100644 --- a/src/opds_dumper.cpp +++ b/src/opds_dumper.cpp @@ -179,4 +179,29 @@ string OPDSDumper::dumpOPDSFeedV2(const std::vector& bookIds, const return render_template(RESOURCE::catalog_v2_entries_xml, template_data); } +std::string OPDSDumper::categoriesOPDSFeed(const std::vector& categories) const +{ + const auto now = gen_date_str(); + kainjow::mustache::list categoryData; + for ( const auto& category : categories ) { + const auto urlencodedCategoryName = urlEncode(category); + categoryData.push_back(kainjow::mustache::object{ + {"name", category}, + {"urlencoded_name", urlencodedCategoryName}, + {"updated", now}, + {"id", gen_uuid(libraryId + "/categories/" + urlencodedCategoryName)} + }); + } + + return render_template( + RESOURCE::catalog_v2_categories_xml, + kainjow::mustache::object{ + {"date", now}, + {"endpoint_root", rootLocation + "/catalog/v2"}, + {"feed_id", gen_uuid(libraryId + "/categories")}, + {"categories", categoryData } + } + ); +} + } diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index 77724e669..03e130dff 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -779,28 +779,12 @@ std::unique_ptr InternalServer::handle_catalog_v2_entries(const Reques std::unique_ptr InternalServer::handle_catalog_v2_categories(const RequestContext& request) { - const std::string root_url = normalizeRootUrl(m_root); - const auto now = gen_date_str(); - kainjow::mustache::list categoryData; - for ( const auto& category : mp_library->getBooksCategories() ) { - const auto urlencodedCategoryName = urlEncode(category); - categoryData.push_back(kainjow::mustache::object{ - {"name", category}, - {"urlencoded_name", urlencodedCategoryName}, - {"updated", now}, - {"id", gen_uuid(m_library_id + "/categories/" + urlencodedCategoryName)} - }); - } - + OPDSDumper opdsDumper(mp_library); + opdsDumper.setRootLocation(normalizeRootUrl(m_root)); + opdsDumper.setLibraryId(m_library_id); return ContentResponse::build( *this, - RESOURCE::catalog_v2_categories_xml, - kainjow::mustache::object{ - {"date", now}, - {"endpoint_root", root_url + "/catalog/v2"}, - {"feed_id", gen_uuid(m_library_id + "/categories")}, - {"categories", categoryData } - }, + opdsDumper.categoriesOPDSFeed(mp_library->getBooksCategories()), "application/atom+xml;profile=opds-catalog;kind=navigation" ); } From f886c8c07b7c91ea20b346ac0c4a0cb76c4a557b Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Mon, 24 May 2021 13:00:31 +0400 Subject: [PATCH 20/26] Root url is normalized once in the constructor --- src/server/internalServer.cpp | 22 ++++++++++++---------- static/opensearchdescription.xml | 2 +- test/server.cpp | 2 +- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index 03e130dff..21b02310b 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -79,11 +79,14 @@ namespace kiwix { namespace { -inline std::string normalizeRootUrl(const std::string& rootUrl) +inline std::string normalizeRootUrl(std::string rootUrl) { - return (rootUrl.empty() || rootUrl[0] == '/') - ? rootUrl - : "/" + rootUrl; + while ( !rootUrl.empty() && rootUrl.back() == '/' ) + rootUrl.pop_back(); + + while ( !rootUrl.empty() && rootUrl.front() == '/' ) + rootUrl = rootUrl.substr(1); + return rootUrl.empty() ? rootUrl : "/" + rootUrl; } } // unnamed namespace @@ -112,7 +115,7 @@ InternalServer::InternalServer(Library* library, bool blockExternalLinks) : m_addr(addr), m_port(port), - m_root(root), + m_root(normalizeRootUrl(root)), m_nbThreads(nbThreads), m_verbose(verbose), m_withTaskbar(withTaskbar), @@ -728,7 +731,7 @@ std::unique_ptr InternalServer::handle_catalog_v2(const RequestContext if (url == "root.xml") { return handle_catalog_v2_root(request); } else if (url == "searchdescription.xml") { - const std::string endpoint_root = normalizeRootUrl(m_root) + "/catalog/v2"; + const std::string endpoint_root = m_root + "/catalog/v2"; return ContentResponse::build(*this, RESOURCE::catalog_v2_searchdescription_xml, kainjow::mustache::object({{"endpoint_root", endpoint_root}}), @@ -745,13 +748,12 @@ std::unique_ptr InternalServer::handle_catalog_v2(const RequestContext std::unique_ptr InternalServer::handle_catalog_v2_root(const RequestContext& request) { - const std::string root_url = normalizeRootUrl(m_root); return ContentResponse::build( *this, RESOURCE::catalog_v2_root_xml, kainjow::mustache::object{ {"date", gen_date_str()}, - {"endpoint_root", root_url + "/catalog/v2"}, + {"endpoint_root", m_root + "/catalog/v2"}, {"feed_id", gen_uuid(m_library_id)}, {"all_entries_feed_id", gen_uuid(m_library_id + "/entries")}, {"category_list_feed_id", gen_uuid(m_library_id + "/categories")} @@ -767,7 +769,7 @@ std::unique_ptr InternalServer::handle_catalog_v2_entries(const Reques const size_t start = request.get_optional_param("start", 0UL); const auto bookIds = subrange(mp_library->filter(filter), start, count); OPDSDumper opdsDumper(mp_library); - opdsDumper.setRootLocation(normalizeRootUrl(m_root)); + opdsDumper.setRootLocation(m_root); opdsDumper.setLibraryId(m_library_id); const auto opdsFeed = opdsDumper.dumpOPDSFeedV2(bookIds, request.get_query()); return ContentResponse::build( @@ -780,7 +782,7 @@ std::unique_ptr InternalServer::handle_catalog_v2_entries(const Reques std::unique_ptr InternalServer::handle_catalog_v2_categories(const RequestContext& request) { OPDSDumper opdsDumper(mp_library); - opdsDumper.setRootLocation(normalizeRootUrl(m_root)); + opdsDumper.setRootLocation(m_root); opdsDumper.setLibraryId(m_library_id); return ContentResponse::build( *this, diff --git a/static/opensearchdescription.xml b/static/opensearchdescription.xml index 47fd790b5..702152f8b 100644 --- a/static/opensearchdescription.xml +++ b/static/opensearchdescription.xml @@ -6,5 +6,5 @@ xmlns:atom="http://www.w3.org/2005/Atom" xmlns:k="http://kiwix.org/opensearchextension/1.0" indexOffset="0" - template="/{{root}}/catalog/search?q={searchTerms?}&lang={language?}&name={k:name?}&tag={k:tag?}¬ag={k:notag?}&maxsize={k:maxsize?}&count={count?}&start={startIndex?}"/> + template="{{root}}/catalog/search?q={searchTerms?}&lang={language?}&name={k:name?}&tag={k:tag?}¬ag={k:notag?}&maxsize={k:maxsize?}&count={count?}&start={startIndex?}"/> diff --git a/test/server.cpp b/test/server.cpp index 42401cc74..bce722add 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -715,7 +715,7 @@ TEST_F(LibraryServerTest, catalog_searchdescription_xml) " xmlns:atom=\"http://www.w3.org/2005/Atom\"\n" " xmlns:k=\"http://kiwix.org/opensearchextension/1.0\"\n" " indexOffset=\"0\"\n" - " template=\"//catalog/search?q={searchTerms?}&lang={language?}&name={k:name?}&tag={k:tag?}¬ag={k:notag?}&maxsize={k:maxsize?}&count={count?}&start={startIndex?}\"/>\n" + " template=\"/catalog/search?q={searchTerms?}&lang={language?}&name={k:name?}&tag={k:tag?}¬ag={k:notag?}&maxsize={k:maxsize?}&count={count?}&start={startIndex?}\"/>\n" "\n" ); } From f1797993af164d6403e2e27fc7bf8d9b637db065 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Mon, 24 May 2021 13:14:51 +0400 Subject: [PATCH 21/26] Reused InternalServer::search_catalog() --- src/server/internalServer.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index 21b02310b..6fd4a278a 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -764,13 +764,10 @@ std::unique_ptr InternalServer::handle_catalog_v2_root(const RequestCo std::unique_ptr InternalServer::handle_catalog_v2_entries(const RequestContext& request) { - const auto filter = get_search_filter(request); - const size_t count = request.get_optional_param("count", 10UL); - const size_t start = request.get_optional_param("start", 0UL); - const auto bookIds = subrange(mp_library->filter(filter), start, count); OPDSDumper opdsDumper(mp_library); opdsDumper.setRootLocation(m_root); opdsDumper.setLibraryId(m_library_id); + const auto bookIds = search_catalog(request, opdsDumper); const auto opdsFeed = opdsDumper.dumpOPDSFeedV2(bookIds, request.get_query()); return ContentResponse::build( *this, From fa42cbc48f58fbdb506cd6aea830b6b4ea8f115b Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Mon, 24 May 2021 13:45:36 +0400 Subject: [PATCH 22/26] Pagination info in /catalog/v2/entries --- src/opds_dumper.cpp | 3 +++ static/catalog_v2_entries.xml | 9 +++++++-- test/server.cpp | 19 ++++++++++++++----- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/opds_dumper.cpp b/src/opds_dumper.cpp index 7e91e7989..8f98074b5 100644 --- a/src/opds_dumper.cpp +++ b/src/opds_dumper.cpp @@ -173,6 +173,9 @@ string OPDSDumper::dumpOPDSFeedV2(const std::vector& bookIds, const {"endpoint_root", rootLocation + "/catalog/v2"}, {"feed_id", gen_uuid(libraryId + "/entries?"+query)}, {"filter", query.empty() ? MustacheData(false) : MustacheData(query)}, + {"totalResults", to_string(m_totalResults)}, + {"startIndex", to_string(m_startIndex)}, + {"itemsPerPage", to_string(m_count)}, {"books", bookData } }; diff --git a/static/catalog_v2_entries.xml b/static/catalog_v2_entries.xml index a829b53b6..f227c8f43 100644 --- a/static/catalog_v2_entries.xml +++ b/static/catalog_v2_entries.xml @@ -1,6 +1,7 @@ + xmlns:opds="https://specs.opds.io/opds-1.2" + xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/"> {{feed_id}} {{^filter}}All Entries{{/filter}}{{#filter}}Filtered Entries ({{filter}}){{/filter}} {{date}} - +{{#filter}} + {{totalResults}} + {{startIndex}} + {{itemsPerPage}} +{{/filter}} {{#books}} {{id}} diff --git a/test/server.cpp b/test/server.cpp index bce722add..fa366be95 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -1006,7 +1006,8 @@ TEST_F(LibraryServerTest, catalog_v2_categories) #define CATALOG_V2_ENTRIES_PREAMBLE \ "\n" \ "\n" \ + " xmlns:opds=\"https://specs.opds.io/opds-1.2\"\n" \ + " xmlns:opensearch=\"http://a9.com/-/spec/opensearch/1.1/\">\n" \ " 12345678-90ab-cdef-1234-567890abcdef\n" \ "\n" \ " Filtered Entries (start=1)\n" " YYYY-MM-DDThh:mm:ssZ\n" - "\n" + " 3\n" + " 1\n" + " 2\n" RAY_CHARLES_CATALOG_ENTRY UNCATEGORIZED_RAY_CHARLES_CATALOG_ENTRY "\n" @@ -1060,7 +1063,9 @@ TEST_F(LibraryServerTest, catalog_v2_entries_filtered_by_range) CATALOG_V2_ENTRIES_PREAMBLE " Filtered Entries (count=2)\n" " YYYY-MM-DDThh:mm:ssZ\n" - "\n" + " 3\n" + " 0\n" + " 2\n" CHARLES_RAY_CATALOG_ENTRY RAY_CHARLES_CATALOG_ENTRY "\n" @@ -1074,7 +1079,9 @@ TEST_F(LibraryServerTest, catalog_v2_entries_filtered_by_range) CATALOG_V2_ENTRIES_PREAMBLE " Filtered Entries (count=1&start=1)\n" " YYYY-MM-DDThh:mm:ssZ\n" - "\n" + " 3\n" + " 1\n" + " 1\n" RAY_CHARLES_CATALOG_ENTRY "\n" ); @@ -1089,7 +1096,9 @@ TEST_F(LibraryServerTest, catalog_v2_entries_filtered_by_search_terms) CATALOG_V2_ENTRIES_PREAMBLE " Filtered Entries (q="ray charles")\n" " YYYY-MM-DDThh:mm:ssZ\n" - "\n" + " 2\n" + " 0\n" + " 2\n" RAY_CHARLES_CATALOG_ENTRY CHARLES_RAY_CATALOG_ENTRY "\n" From 312f2cb560f85ff5019f33db08925a985209f510 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Mon, 24 May 2021 13:54:54 +0400 Subject: [PATCH 23/26] Moved handle_catalog_v2*() methods into a new file --- src/meson.build | 3 +- src/server/internalServer.cpp | 73 --------------- src/server/internalServer_catalog_v2.cpp | 109 +++++++++++++++++++++++ 3 files changed, 111 insertions(+), 74 deletions(-) create mode 100644 src/server/internalServer_catalog_v2.cpp diff --git a/src/meson.build b/src/meson.build index 333c3a22f..d4c4f9697 100644 --- a/src/meson.build +++ b/src/meson.build @@ -25,7 +25,8 @@ kiwix_sources = [ 'server/etag.cpp', 'server/request_context.cpp', 'server/response.cpp', - 'server/internalServer.cpp' + 'server/internalServer.cpp', + 'server/internalServer_catalog_v2.cpp' ] kiwix_sources += lib_resources diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index 6fd4a278a..cab52ebae 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -715,79 +715,6 @@ InternalServer::search_catalog(const RequestContext& request, return bookIdsToDump; } -std::unique_ptr InternalServer::handle_catalog_v2(const RequestContext& request) -{ - if (m_verbose.load()) { - printf("** running handle_catalog_v2"); - } - - std::string url; - try { - url = request.get_url_part(2); - } catch (const std::out_of_range&) { - return Response::build_404(*this, request, ""); - } - - if (url == "root.xml") { - return handle_catalog_v2_root(request); - } else if (url == "searchdescription.xml") { - const std::string endpoint_root = m_root + "/catalog/v2"; - return ContentResponse::build(*this, - RESOURCE::catalog_v2_searchdescription_xml, - kainjow::mustache::object({{"endpoint_root", endpoint_root}}), - "application/opensearchdescription+xml" - ); - } else if (url == "entries") { - return handle_catalog_v2_entries(request); - } else if (url == "categories") { - return handle_catalog_v2_categories(request); - } else { - return Response::build_404(*this, request, ""); - } -} - -std::unique_ptr InternalServer::handle_catalog_v2_root(const RequestContext& request) -{ - return ContentResponse::build( - *this, - RESOURCE::catalog_v2_root_xml, - kainjow::mustache::object{ - {"date", gen_date_str()}, - {"endpoint_root", m_root + "/catalog/v2"}, - {"feed_id", gen_uuid(m_library_id)}, - {"all_entries_feed_id", gen_uuid(m_library_id + "/entries")}, - {"category_list_feed_id", gen_uuid(m_library_id + "/categories")} - }, - "application/atom+xml;profile=opds-catalog;kind=navigation" - ); -} - -std::unique_ptr InternalServer::handle_catalog_v2_entries(const RequestContext& request) -{ - OPDSDumper opdsDumper(mp_library); - opdsDumper.setRootLocation(m_root); - opdsDumper.setLibraryId(m_library_id); - const auto bookIds = search_catalog(request, opdsDumper); - const auto opdsFeed = opdsDumper.dumpOPDSFeedV2(bookIds, request.get_query()); - return ContentResponse::build( - *this, - opdsFeed, - "application/atom+xml;profile=opds-catalog;kind=acquisition" - ); -} - -std::unique_ptr InternalServer::handle_catalog_v2_categories(const RequestContext& request) -{ - OPDSDumper opdsDumper(mp_library); - opdsDumper.setRootLocation(m_root); - opdsDumper.setLibraryId(m_library_id); - return ContentResponse::build( - *this, - opdsDumper.categoriesOPDSFeed(mp_library->getBooksCategories()), - "application/atom+xml;profile=opds-catalog;kind=navigation" - ); -} - namespace { diff --git a/src/server/internalServer_catalog_v2.cpp b/src/server/internalServer_catalog_v2.cpp new file mode 100644 index 000000000..5a8827096 --- /dev/null +++ b/src/server/internalServer_catalog_v2.cpp @@ -0,0 +1,109 @@ +/* + * Copyright 2021 Veloman Yunkan + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + * MA 02110-1301, USA. + */ + +#include "internalServer.h" + +#include "library.h" +#include "opds_dumper.h" +#include "request_context.h" +#include "response.h" +#include "tools/otherTools.h" +#include "kiwixlib-resources.h" + +#include + +#include +#include + +namespace kiwix { + +std::unique_ptr InternalServer::handle_catalog_v2(const RequestContext& request) +{ + if (m_verbose.load()) { + printf("** running handle_catalog_v2"); + } + + std::string url; + try { + url = request.get_url_part(2); + } catch (const std::out_of_range&) { + return Response::build_404(*this, request, ""); + } + + if (url == "root.xml") { + return handle_catalog_v2_root(request); + } else if (url == "searchdescription.xml") { + const std::string endpoint_root = m_root + "/catalog/v2"; + return ContentResponse::build(*this, + RESOURCE::catalog_v2_searchdescription_xml, + kainjow::mustache::object({{"endpoint_root", endpoint_root}}), + "application/opensearchdescription+xml" + ); + } else if (url == "entries") { + return handle_catalog_v2_entries(request); + } else if (url == "categories") { + return handle_catalog_v2_categories(request); + } else { + return Response::build_404(*this, request, ""); + } +} + +std::unique_ptr InternalServer::handle_catalog_v2_root(const RequestContext& request) +{ + return ContentResponse::build( + *this, + RESOURCE::catalog_v2_root_xml, + kainjow::mustache::object{ + {"date", gen_date_str()}, + {"endpoint_root", m_root + "/catalog/v2"}, + {"feed_id", gen_uuid(m_library_id)}, + {"all_entries_feed_id", gen_uuid(m_library_id + "/entries")}, + {"category_list_feed_id", gen_uuid(m_library_id + "/categories")} + }, + "application/atom+xml;profile=opds-catalog;kind=navigation" + ); +} + +std::unique_ptr InternalServer::handle_catalog_v2_entries(const RequestContext& request) +{ + OPDSDumper opdsDumper(mp_library); + opdsDumper.setRootLocation(m_root); + opdsDumper.setLibraryId(m_library_id); + const auto bookIds = search_catalog(request, opdsDumper); + const auto opdsFeed = opdsDumper.dumpOPDSFeedV2(bookIds, request.get_query()); + return ContentResponse::build( + *this, + opdsFeed, + "application/atom+xml;profile=opds-catalog;kind=acquisition" + ); +} + +std::unique_ptr InternalServer::handle_catalog_v2_categories(const RequestContext& request) +{ + OPDSDumper opdsDumper(mp_library); + opdsDumper.setRootLocation(m_root); + opdsDumper.setLibraryId(m_library_id); + return ContentResponse::build( + *this, + opdsDumper.categoriesOPDSFeed(mp_library->getBooksCategories()), + "application/atom+xml;profile=opds-catalog;kind=navigation" + ); +} + +} // namespace kiwix From e799f2ff1edce70f22909ba97d584eefd2501f0d Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Thu, 27 May 2021 13:20:57 +0400 Subject: [PATCH 24/26] OPDSDumper::dumpOPDSFeed() works via mustache This changes the output of `/catalog/search` as follows: - Entire search query (rather than only the value of the `q` parameter) is put in the node. - Search performed with an empty query presents itself as "All zims". - The feed id remains stable for identical searches on the same library. --- include/opds_dumper.h | 40 +---------- src/opds_dumper.cpp | 125 +++++++++------------------------- src/server/internalServer.cpp | 10 +-- static/catalog_entries.xml | 38 +++++++++++ static/catalog_v2_entries.xml | 2 +- static/resources_list.txt | 1 + test/server.cpp | 32 +++++---- 7 files changed, 94 insertions(+), 154 deletions(-) create mode 100644 static/catalog_entries.xml diff --git a/include/opds_dumper.h b/include/opds_dumper.h index 8d8da0f6e..230d094a1 100644 --- a/include/opds_dumper.h +++ b/include/opds_dumper.h @@ -52,14 +52,16 @@ class OPDSDumper * Dump the OPDS feed. * * @param bookIds the ids of the books to include in the feed + * @param query the query used to obtain the list of book ids * @return The OPDS feed. */ - std::string dumpOPDSFeed(const std::vector<std::string>& bookIds); + std::string dumpOPDSFeed(const std::vector<std::string>& bookIds, const std::string& query) const; /** * Dump the OPDS feed. * * @param bookIds the ids of the books to include in the feed + * @param query the query used to obtain the list of book ids * @return The OPDS feed. */ std::string dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const std::string& query) const; @@ -79,20 +81,6 @@ class OPDSDumper */ void setLibraryId(const std::string& id) { this->libraryId = id;} - /** - * Set the id of the opds stream. - * - * @param id the id to use. - */ - void setId(const std::string& id) { this->id = id;} - - /** - * Set the title oft the opds stream. - * - * @param title the title to use. - */ - void setTitle(const std::string& title) { this->title = title; } - /** * Set the root location used when generating url. * @@ -100,13 +88,6 @@ class OPDSDumper */ void setRootLocation(const std::string& rootLocation) { this->rootLocation = rootLocation; } - /** - * Set the search url. - * - * @param searchUrl the search url to use. - */ - void setSearchDescriptionUrl(const std::string& searchDescriptionUrl) { this->searchDescriptionUrl = searchDescriptionUrl; } - /** * Set some informations about the search results. * @@ -116,28 +97,13 @@ class OPDSDumper */ void setOpenSearchInfo(int totalResult, int startIndex, int count); - /** - * Set the library to dump. - * - * @param library The library to dump. - */ - void setLibrary(Library* library) { this->library = library; } - protected: kiwix::Library* library; - std::string id; std::string libraryId; - std::string title; - std::string date; std::string rootLocation; - std::string searchDescriptionUrl; int m_totalResults; int m_startIndex; int m_count; - bool m_isSearchResult = false; - - private: - pugi::xml_node handleBook(Book book, pugi::xml_node root_node); }; } diff --git a/src/opds_dumper.cpp b/src/opds_dumper.cpp index 8f98074b5..d4be8207c 100644 --- a/src/opds_dumper.cpp +++ b/src/opds_dumper.cpp @@ -27,6 +27,7 @@ namespace kiwix { + /* Constructor */ OPDSDumper::OPDSDumper(Library* library) : library(library) @@ -37,112 +38,22 @@ OPDSDumper::~OPDSDumper() { } -static std::string gen_date_from_yyyy_mm_dd(const std::string& date) -{ - std::stringstream is; - is << date << "T00:00:00Z"; - return is.str(); -} - void OPDSDumper::setOpenSearchInfo(int totalResults, int startIndex, int count) { m_totalResults = totalResults; m_startIndex = startIndex, m_count = count; - m_isSearchResult = true; } -#define ADD_TEXT_ENTRY(node, child, value) (node).append_child((child)).append_child(pugi::node_pcdata).set_value((value).c_str()) - -pugi::xml_node OPDSDumper::handleBook(Book book, pugi::xml_node root_node) { - auto entry_node = root_node.append_child("entry"); - ADD_TEXT_ENTRY(entry_node, "id", "urn:uuid:"+book.getId()); - ADD_TEXT_ENTRY(entry_node, "title", book.getTitle()); - ADD_TEXT_ENTRY(entry_node, "summary", book.getDescription()); - ADD_TEXT_ENTRY(entry_node, "language", book.getLanguage()); - ADD_TEXT_ENTRY(entry_node, "updated", gen_date_from_yyyy_mm_dd(book.getDate())); - ADD_TEXT_ENTRY(entry_node, "name", book.getName()); - ADD_TEXT_ENTRY(entry_node, "flavour", book.getFlavour()); - ADD_TEXT_ENTRY(entry_node, "category", book.getCategory()); - ADD_TEXT_ENTRY(entry_node, "tags", book.getTags()); - ADD_TEXT_ENTRY(entry_node, "articleCount", to_string(book.getArticleCount())); - ADD_TEXT_ENTRY(entry_node, "mediaCount", to_string(book.getMediaCount())); - ADD_TEXT_ENTRY(entry_node, "icon", rootLocation + "/meta?name=favicon&content=" + book.getHumanReadableIdFromPath()); - - auto content_node = entry_node.append_child("link"); - content_node.append_attribute("type") = "text/html"; - content_node.append_attribute("href") = (rootLocation + "/" + book.getHumanReadableIdFromPath()).c_str(); - - auto author_node = entry_node.append_child("author"); - ADD_TEXT_ENTRY(author_node, "name", book.getCreator()); - - auto publisher_node = entry_node.append_child("publisher"); - ADD_TEXT_ENTRY(publisher_node, "name", book.getPublisher()); - - if (! book.getUrl().empty()) { - auto acquisition_link = entry_node.append_child("link"); - acquisition_link.append_attribute("rel") = "http://opds-spec.org/acquisition/open-access"; - acquisition_link.append_attribute("type") = "application/x-zim"; - acquisition_link.append_attribute("href") = book.getUrl().c_str(); - acquisition_link.append_attribute("length") = to_string(book.getSize()).c_str(); - } - - if (! book.getFaviconMimeType().empty() ) { - auto image_link = entry_node.append_child("link"); - image_link.append_attribute("rel") = "http://opds-spec.org/image/thumbnail"; - image_link.append_attribute("type") = book.getFaviconMimeType().c_str(); - image_link.append_attribute("href") = (rootLocation + "/meta?name=favicon&content=" + book.getHumanReadableIdFromPath()).c_str(); - } - return entry_node; -} - -string OPDSDumper::dumpOPDSFeed(const std::vector<std::string>& bookIds) +namespace { - date = gen_date_str(); - pugi::xml_document doc; - - auto root_node = doc.append_child("feed"); - root_node.append_attribute("xmlns") = "http://www.w3.org/2005/Atom"; - root_node.append_attribute("xmlns:opds") = "http://opds-spec.org/2010/catalog"; - - ADD_TEXT_ENTRY(root_node, "id", id); - - ADD_TEXT_ENTRY(root_node, "title", title); - ADD_TEXT_ENTRY(root_node, "updated", date); - - if (m_isSearchResult) { - ADD_TEXT_ENTRY(root_node, "totalResults", to_string(m_totalResults)); - ADD_TEXT_ENTRY(root_node, "startIndex", to_string(m_startIndex)); - ADD_TEXT_ENTRY(root_node, "itemsPerPage", to_string(m_count)); - } - - auto self_link_node = root_node.append_child("link"); - self_link_node.append_attribute("rel") = "self"; - self_link_node.append_attribute("href") = ""; - self_link_node.append_attribute("type") = "application/atom+xml"; - - - if (!searchDescriptionUrl.empty() ) { - auto search_link = root_node.append_child("link"); - search_link.append_attribute("rel") = "search"; - search_link.append_attribute("type") = "application/opensearchdescription+xml"; - search_link.append_attribute("href") = searchDescriptionUrl.c_str(); - } - - if (library) { - for (auto& bookId: bookIds) { - handleBook(library->getBookById(bookId), root_node); - } - } - - return nodeToString(root_node); -} typedef kainjow::mustache::data MustacheData; +typedef kainjow::mustache::list BookData; -string OPDSDumper::dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const std::string& query) const +BookData getBookData(const Library* library, const std::vector<std::string>& bookIds) { - kainjow::mustache::list bookData; + BookData bookData; for ( const auto& bookId : bookIds ) { const Book& book = library->getBookById(bookId); const MustacheData bookUrl = book.getUrl().empty() @@ -168,6 +79,32 @@ string OPDSDumper::dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const }); } + return bookData; +} + +} // unnamed namespace + +string OPDSDumper::dumpOPDSFeed(const std::vector<std::string>& bookIds, const std::string& query) const +{ + const auto bookData = getBookData(library, bookIds); + const kainjow::mustache::object template_data{ + {"date", gen_date_str()}, + {"root", rootLocation}, + {"feed_id", gen_uuid(libraryId + "/catalog/search?"+query)}, + {"filter", query.empty() ? MustacheData(false) : MustacheData(query)}, + {"totalResults", to_string(m_totalResults)}, + {"startIndex", to_string(m_startIndex)}, + {"itemsPerPage", to_string(m_count)}, + {"books", bookData } + }; + + return render_template(RESOURCE::catalog_entries_xml, template_data); +} + +string OPDSDumper::dumpOPDSFeedV2(const std::vector<std::string>& bookIds, const std::string& query) const +{ + const auto bookData = getBookData(library, bookIds); + const kainjow::mustache::object template_data{ {"date", gen_date_str()}, {"endpoint_root", rootLocation + "/catalog/v2"}, diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index cab52ebae..e395e0dc4 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -636,13 +636,11 @@ std::unique_ptr<Response> InternalServer::handle_catalog(const RequestContext& r } zim::Uuid uuid; - kiwix::OPDSDumper opdsDumper; + kiwix::OPDSDumper opdsDumper(mp_library); opdsDumper.setRootLocation(m_root); - opdsDumper.setSearchDescriptionUrl("catalog/searchdescription.xml"); - opdsDumper.setLibrary(mp_library); + opdsDumper.setLibraryId(m_library_id); std::vector<std::string> bookIdsToDump; if (url == "root.xml") { - opdsDumper.setTitle("All zims"); uuid = zim::Uuid::generate(host); bookIdsToDump = mp_library->filter(kiwix::Filter().valid(true).local(true).remote(true)); } else if (url == "search") { @@ -650,10 +648,9 @@ std::unique_ptr<Response> InternalServer::handle_catalog(const RequestContext& r uuid = zim::Uuid::generate(); } - opdsDumper.setId(kiwix::to_string(uuid)); auto response = ContentResponse::build( *this, - opdsDumper.dumpOPDSFeed(bookIdsToDump), + opdsDumper.dumpOPDSFeed(bookIdsToDump, request.get_query()), "application/atom+xml; profile=opds-catalog; kind=acquisition; charset=utf-8"); return std::move(response); } @@ -705,7 +702,6 @@ InternalServer::search_catalog(const RequestContext& request, const std::string q = filter.hasQuery() ? filter.getQuery() : "<Empty query>"; - opdsDumper.setTitle("Search result for " + q); std::vector<std::string> bookIdsToDump = mp_library->filter(filter); const auto totalResults = bookIdsToDump.size(); const size_t count = request.get_optional_param("count", 10UL); diff --git a/static/catalog_entries.xml b/static/catalog_entries.xml new file mode 100644 index 000000000..32754d5b0 --- /dev/null +++ b/static/catalog_entries.xml @@ -0,0 +1,38 @@ +<feed xmlns="http://www.w3.org/2005/Atom" xmlns:opds="http://opds-spec.org/2010/catalog"> + <id>{{feed_id}}</id> + <title>{{^filter}}All zims{{/filter}}{{#filter}}Filtered zims ({{filter}}){{/filter}} + {{date}} +{{#filter}} + {{totalResults}} + {{startIndex}} + {{itemsPerPage}} +{{/filter}} + + + {{#books}} + + {{id}} + {{title}} + {{description}} + {{language}} + {{updated}} + {{name}} + {{flavour}} + {{category}} + {{tags}} + {{article_count}} + {{media_count}} + /meta?name=favicon&content={{{content_id}}} + + + {{author_name}} + + + {{publisher_name}} + + {{#url}} + + {{/url}} + + {{/books}} + diff --git a/static/catalog_v2_entries.xml b/static/catalog_v2_entries.xml index f227c8f43..edc850f02 100644 --- a/static/catalog_v2_entries.xml +++ b/static/catalog_v2_entries.xml @@ -18,7 +18,7 @@ {{date}} {{#filter}} {{totalResults}} - {{startIndex}} + {{startIndex}} {{itemsPerPage}} {{/filter}} {{#books}} diff --git a/static/resources_list.txt b/static/resources_list.txt index 83dea44aa..dbeb5c5cd 100644 --- a/static/resources_list.txt +++ b/static/resources_list.txt @@ -37,6 +37,7 @@ templates/taskbar_part.html templates/external_blocker_part.html templates/captured_external.html opensearchdescription.xml +catalog_entries.xml catalog_v2_root.xml catalog_v2_entries.xml catalog_v2_categories.xml diff --git a/test/server.cpp b/test/server.cpp index fa366be95..1f7e4b506 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -611,7 +611,7 @@ std::string maskVariableOPDSFeedData(std::string s) " \n" \ " \n" + " href=\"/catalog/searchdescription.xml\" />\n" #define CHARLES_RAY_CATALOG_ENTRY \ " \n" \ @@ -694,6 +694,7 @@ TEST_F(LibraryServerTest, catalog_root_xml) " 12345678-90ab-cdef-1234-567890abcdef\n" " All zims\n" " YYYY-MM-DDThh:mm:ssZ\n" + "\n" CATALOG_LINK_TAGS CHARLES_RAY_CATALOG_ENTRY RAY_CHARLES_CATALOG_ENTRY @@ -727,7 +728,7 @@ TEST_F(LibraryServerTest, catalog_search_by_phrase) EXPECT_EQ(maskVariableOPDSFeedData(r->body), OPDS_FEED_TAG " 12345678-90ab-cdef-1234-567890abcdef\n" - " Search result for \"ray charles\"\n" + " Filtered zims (q="ray charles")\n" " YYYY-MM-DDThh:mm:ssZ\n" " 2\n" " 0\n" @@ -746,7 +747,7 @@ TEST_F(LibraryServerTest, catalog_search_by_words) EXPECT_EQ(maskVariableOPDSFeedData(r->body), OPDS_FEED_TAG " 12345678-90ab-cdef-1234-567890abcdef\n" - " Search result for ray charles\n" + " Filtered zims (q=ray charles)\n" " YYYY-MM-DDThh:mm:ssZ\n" " 3\n" " 0\n" @@ -767,7 +768,7 @@ TEST_F(LibraryServerTest, catalog_prefix_search) EXPECT_EQ(maskVariableOPDSFeedData(r->body), OPDS_FEED_TAG " 12345678-90ab-cdef-1234-567890abcdef\n" - " Search result for description:ray description:charles\n" + " Filtered zims (q=description:ray description:charles)\n" " YYYY-MM-DDThh:mm:ssZ\n" " 2\n" " 0\n" @@ -784,7 +785,7 @@ TEST_F(LibraryServerTest, catalog_prefix_search) EXPECT_EQ(maskVariableOPDSFeedData(r->body), OPDS_FEED_TAG " 12345678-90ab-cdef-1234-567890abcdef\n" - " Search result for title:\"ray charles\"\n" + " Filtered zims (q=title:"ray charles")\n" " YYYY-MM-DDThh:mm:ssZ\n" " 1\n" " 0\n" @@ -803,7 +804,7 @@ TEST_F(LibraryServerTest, catalog_search_with_word_exclusion) EXPECT_EQ(maskVariableOPDSFeedData(r->body), OPDS_FEED_TAG " 12345678-90ab-cdef-1234-567890abcdef\n" - " Search result for ray -uncategorized\n" + " Filtered zims (q=ray -uncategorized)\n" " YYYY-MM-DDThh:mm:ssZ\n" " 2\n" " 0\n" @@ -822,7 +823,7 @@ TEST_F(LibraryServerTest, catalog_search_by_tag) EXPECT_EQ(maskVariableOPDSFeedData(r->body), OPDS_FEED_TAG " 12345678-90ab-cdef-1234-567890abcdef\n" - " Search result for <Empty query>\n" + " Filtered zims (tag=_category:jazz)\n" " YYYY-MM-DDThh:mm:ssZ\n" " 1\n" " 0\n" @@ -840,7 +841,7 @@ TEST_F(LibraryServerTest, catalog_search_by_category) EXPECT_EQ(maskVariableOPDSFeedData(r->body), OPDS_FEED_TAG " 12345678-90ab-cdef-1234-567890abcdef\n" - " Search result for <Empty query>\n" + " Filtered zims (category=jazz)\n" " YYYY-MM-DDThh:mm:ssZ\n" " 1\n" " 0\n" @@ -859,7 +860,7 @@ TEST_F(LibraryServerTest, catalog_search_results_pagination) EXPECT_EQ(maskVariableOPDSFeedData(r->body), OPDS_FEED_TAG " 12345678-90ab-cdef-1234-567890abcdef\n" - " Search result for <Empty query>\n" + " Filtered zims (count=1)\n" " YYYY-MM-DDThh:mm:ssZ\n" " 3\n" " 0\n" @@ -875,7 +876,7 @@ TEST_F(LibraryServerTest, catalog_search_results_pagination) EXPECT_EQ(maskVariableOPDSFeedData(r->body), OPDS_FEED_TAG " 12345678-90ab-cdef-1234-567890abcdef\n" - " Search result for <Empty query>\n" + " Filtered zims (count=1&start=1)\n" " YYYY-MM-DDThh:mm:ssZ\n" " 3\n" " 1\n" @@ -891,12 +892,13 @@ TEST_F(LibraryServerTest, catalog_search_results_pagination) EXPECT_EQ(maskVariableOPDSFeedData(r->body), OPDS_FEED_TAG " 12345678-90ab-cdef-1234-567890abcdef\n" - " Search result for <Empty query>\n" + " Filtered zims (count=10&start=100)\n" " YYYY-MM-DDThh:mm:ssZ\n" " 3\n" " 100\n" " 0\n" CATALOG_LINK_TAGS + " \n" "\n" ); } @@ -1048,7 +1050,7 @@ TEST_F(LibraryServerTest, catalog_v2_entries_filtered_by_range) " Filtered Entries (start=1)\n" " YYYY-MM-DDThh:mm:ssZ\n" " 3\n" - " 1\n" + " 1\n" " 2\n" RAY_CHARLES_CATALOG_ENTRY UNCATEGORIZED_RAY_CHARLES_CATALOG_ENTRY @@ -1064,7 +1066,7 @@ TEST_F(LibraryServerTest, catalog_v2_entries_filtered_by_range) " Filtered Entries (count=2)\n" " YYYY-MM-DDThh:mm:ssZ\n" " 3\n" - " 0\n" + " 0\n" " 2\n" CHARLES_RAY_CATALOG_ENTRY RAY_CHARLES_CATALOG_ENTRY @@ -1080,7 +1082,7 @@ TEST_F(LibraryServerTest, catalog_v2_entries_filtered_by_range) " Filtered Entries (count=1&start=1)\n" " YYYY-MM-DDThh:mm:ssZ\n" " 3\n" - " 1\n" + " 1\n" " 1\n" RAY_CHARLES_CATALOG_ENTRY "\n" @@ -1097,7 +1099,7 @@ TEST_F(LibraryServerTest, catalog_v2_entries_filtered_by_search_terms) " Filtered Entries (q="ray charles")\n" " YYYY-MM-DDThh:mm:ssZ\n" " 2\n" - " 0\n" + " 0\n" " 2\n" RAY_CHARLES_CATALOG_ENTRY CHARLES_RAY_CATALOG_ENTRY From dd60235010153399c9bd812fd49cf80f52a0ff54 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Thu, 27 May 2021 13:48:32 +0400 Subject: [PATCH 25/26] Fixed the self link in the output of /catalog/v2/entries --- src/opds_dumper.cpp | 1 + static/catalog_v2_entries.xml | 2 +- test/server.cpp | 14 +++++++------- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/opds_dumper.cpp b/src/opds_dumper.cpp index d4be8207c..dfba46b70 100644 --- a/src/opds_dumper.cpp +++ b/src/opds_dumper.cpp @@ -110,6 +110,7 @@ string OPDSDumper::dumpOPDSFeedV2(const std::vector& bookIds, const {"endpoint_root", rootLocation + "/catalog/v2"}, {"feed_id", gen_uuid(libraryId + "/entries?"+query)}, {"filter", query.empty() ? MustacheData(false) : MustacheData(query)}, + {"query", query.empty() ? "" : "?" + urlEncode(query)}, {"totalResults", to_string(m_totalResults)}, {"startIndex", to_string(m_startIndex)}, {"itemsPerPage", to_string(m_count)}, diff --git a/static/catalog_v2_entries.xml b/static/catalog_v2_entries.xml index edc850f02..05cbb5054 100644 --- a/static/catalog_v2_entries.xml +++ b/static/catalog_v2_entries.xml @@ -5,7 +5,7 @@ {{feed_id}} body), expected_output); } -#define CATALOG_V2_ENTRIES_PREAMBLE \ +#define CATALOG_V2_ENTRIES_PREAMBLE(q) \ "\n" \ "12345678-90ab-cdef-1234-567890abcdef\n" \ "\n" \ " \n" \ " GET("/catalog/v2/entries"); EXPECT_EQ(r->status, 200); EXPECT_EQ(maskVariableOPDSFeedData(r->body), - CATALOG_V2_ENTRIES_PREAMBLE + CATALOG_V2_ENTRIES_PREAMBLE("") " All Entries\n" " YYYY-MM-DDThh:mm:ssZ\n" "\n" @@ -1046,7 +1046,7 @@ TEST_F(LibraryServerTest, catalog_v2_entries_filtered_by_range) const auto r = zfs1_->GET("/catalog/v2/entries?start=1"); EXPECT_EQ(r->status, 200); EXPECT_EQ(maskVariableOPDSFeedData(r->body), - CATALOG_V2_ENTRIES_PREAMBLE + CATALOG_V2_ENTRIES_PREAMBLE("?start=1") " Filtered Entries (start=1)\n" " YYYY-MM-DDThh:mm:ssZ\n" " 3\n" @@ -1062,7 +1062,7 @@ TEST_F(LibraryServerTest, catalog_v2_entries_filtered_by_range) const auto r = zfs1_->GET("/catalog/v2/entries?count=2"); EXPECT_EQ(r->status, 200); EXPECT_EQ(maskVariableOPDSFeedData(r->body), - CATALOG_V2_ENTRIES_PREAMBLE + CATALOG_V2_ENTRIES_PREAMBLE("?count=2") " Filtered Entries (count=2)\n" " YYYY-MM-DDThh:mm:ssZ\n" " 3\n" @@ -1078,7 +1078,7 @@ TEST_F(LibraryServerTest, catalog_v2_entries_filtered_by_range) const auto r = zfs1_->GET("/catalog/v2/entries?start=1&count=1"); EXPECT_EQ(r->status, 200); EXPECT_EQ(maskVariableOPDSFeedData(r->body), - CATALOG_V2_ENTRIES_PREAMBLE + CATALOG_V2_ENTRIES_PREAMBLE("?count=1&start=1") " Filtered Entries (count=1&start=1)\n" " YYYY-MM-DDThh:mm:ssZ\n" " 3\n" @@ -1095,7 +1095,7 @@ TEST_F(LibraryServerTest, catalog_v2_entries_filtered_by_search_terms) const auto r = zfs1_->GET("/catalog/v2/entries?q=\"ray%20charles\""); EXPECT_EQ(r->status, 200); EXPECT_EQ(maskVariableOPDSFeedData(r->body), - CATALOG_V2_ENTRIES_PREAMBLE + CATALOG_V2_ENTRIES_PREAMBLE("?q=%22ray%20charles%22") " Filtered Entries (q="ray charles")\n" " YYYY-MM-DDThh:mm:ssZ\n" " 2\n" From 78083f1f4af5b40cce7ffb3206e9ccf170e605a9 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Tue, 8 Jun 2021 16:18:28 +0400 Subject: [PATCH 26/26] Moved OPDS templates under static/templates --- src/opds_dumper.cpp | 6 +++--- src/server/internalServer_catalog_v2.cpp | 2 +- static/resources_list.txt | 8 ++++---- static/{ => templates}/catalog_entries.xml | 0 static/{ => templates}/catalog_v2_categories.xml | 0 static/{ => templates}/catalog_v2_entries.xml | 0 static/{ => templates}/catalog_v2_root.xml | 0 7 files changed, 8 insertions(+), 8 deletions(-) rename static/{ => templates}/catalog_entries.xml (100%) rename static/{ => templates}/catalog_v2_categories.xml (100%) rename static/{ => templates}/catalog_v2_entries.xml (100%) rename static/{ => templates}/catalog_v2_root.xml (100%) diff --git a/src/opds_dumper.cpp b/src/opds_dumper.cpp index dfba46b70..9d6fdfd5e 100644 --- a/src/opds_dumper.cpp +++ b/src/opds_dumper.cpp @@ -98,7 +98,7 @@ string OPDSDumper::dumpOPDSFeed(const std::vector& bookIds, const s {"books", bookData } }; - return render_template(RESOURCE::catalog_entries_xml, template_data); + return render_template(RESOURCE::templates::catalog_entries_xml, template_data); } string OPDSDumper::dumpOPDSFeedV2(const std::vector& bookIds, const std::string& query) const @@ -117,7 +117,7 @@ string OPDSDumper::dumpOPDSFeedV2(const std::vector& bookIds, const {"books", bookData } }; - return render_template(RESOURCE::catalog_v2_entries_xml, template_data); + return render_template(RESOURCE::templates::catalog_v2_entries_xml, template_data); } std::string OPDSDumper::categoriesOPDSFeed(const std::vector& categories) const @@ -135,7 +135,7 @@ std::string OPDSDumper::categoriesOPDSFeed(const std::vector& categ } return render_template( - RESOURCE::catalog_v2_categories_xml, + RESOURCE::templates::catalog_v2_categories_xml, kainjow::mustache::object{ {"date", now}, {"endpoint_root", rootLocation + "/catalog/v2"}, diff --git a/src/server/internalServer_catalog_v2.cpp b/src/server/internalServer_catalog_v2.cpp index 5a8827096..e7c05bac6 100644 --- a/src/server/internalServer_catalog_v2.cpp +++ b/src/server/internalServer_catalog_v2.cpp @@ -68,7 +68,7 @@ std::unique_ptr InternalServer::handle_catalog_v2_root(const RequestCo { return ContentResponse::build( *this, - RESOURCE::catalog_v2_root_xml, + RESOURCE::templates::catalog_v2_root_xml, kainjow::mustache::object{ {"date", gen_date_str()}, {"endpoint_root", m_root + "/catalog/v2"}, diff --git a/static/resources_list.txt b/static/resources_list.txt index dbeb5c5cd..025742834 100644 --- a/static/resources_list.txt +++ b/static/resources_list.txt @@ -36,9 +36,9 @@ templates/head_taskbar.html templates/taskbar_part.html templates/external_blocker_part.html templates/captured_external.html +templates/catalog_entries.xml +templates/catalog_v2_root.xml +templates/catalog_v2_entries.xml +templates/catalog_v2_categories.xml opensearchdescription.xml -catalog_entries.xml -catalog_v2_root.xml -catalog_v2_entries.xml -catalog_v2_categories.xml catalog_v2_searchdescription.xml diff --git a/static/catalog_entries.xml b/static/templates/catalog_entries.xml similarity index 100% rename from static/catalog_entries.xml rename to static/templates/catalog_entries.xml diff --git a/static/catalog_v2_categories.xml b/static/templates/catalog_v2_categories.xml similarity index 100% rename from static/catalog_v2_categories.xml rename to static/templates/catalog_v2_categories.xml diff --git a/static/catalog_v2_entries.xml b/static/templates/catalog_v2_entries.xml similarity index 100% rename from static/catalog_v2_entries.xml rename to static/templates/catalog_v2_entries.xml diff --git a/static/catalog_v2_root.xml b/static/templates/catalog_v2_root.xml similarity index 100% rename from static/catalog_v2_root.xml rename to static/templates/catalog_v2_root.xml