From 26331b401ea5ec95f4524d00f0e5e00c194a50e2 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Thu, 4 Mar 2021 20:36:11 +0400 Subject: [PATCH 01/15] Fixed the month in OPDS feed date `tm::tm_mon` varies in the [0, 11] range. --- src/opds_dumper.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/opds_dumper.cpp b/src/opds_dumper.cpp index 115ba4c38..98854cbe1 100644 --- a/src/opds_dumper.cpp +++ b/src/opds_dumper.cpp @@ -43,7 +43,7 @@ std::string gen_date_str() std::stringstream is; is << std::setw(2) << std::setfill('0') << 1900+tm->tm_year << "-" - << std::setw(2) << std::setfill('0') << tm->tm_mon << "-" + << std::setw(2) << std::setfill('0') << tm->tm_mon+1 << "-" << std::setw(2) << std::setfill('0') << tm->tm_mday << "T" << std::setw(2) << std::setfill('0') << tm->tm_hour << ":" << std::setw(2) << std::setfill('0') << tm->tm_min << ":" From ae32ff40c036bf59dd794a123e0e5399c48c01c4 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Thu, 4 Mar 2021 20:46:16 +0400 Subject: [PATCH 02/15] Dropped an extra colon from book dates --- src/opds_dumper.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/opds_dumper.cpp b/src/opds_dumper.cpp index 98854cbe1..ef4c8dd03 100644 --- a/src/opds_dumper.cpp +++ b/src/opds_dumper.cpp @@ -54,7 +54,7 @@ std::string gen_date_str() static std::string gen_date_from_yyyy_mm_dd(const std::string& date) { std::stringstream is; - is << date << "T00:00::00:Z"; + is << date << "T00:00::00Z"; return is.str(); } From c5c40cb189c0334475c306d7f417fd42d64637c5 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Thu, 4 Mar 2021 21:08:26 +0400 Subject: [PATCH 03/15] New unit-test LibraryServerTest.catalog_root_xml --- test/data/library.xml | 34 ++++++++++++++ test/meson.build | 3 +- test/server.cpp | 104 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 test/data/library.xml diff --git a/test/data/library.xml b/test/data/library.xml new file mode 100644 index 000000000..e8fd2e345 --- /dev/null +++ b/test/data/library.xml @@ -0,0 +1,34 @@ + + + + diff --git a/test/meson.build b/test/meson.build index 2cd9055e6..15d878e55 100644 --- a/test/meson.build +++ b/test/meson.build @@ -25,7 +25,8 @@ if gtest_dep.found() and not meson.is_cross_build() data_files = [ 'example.zim', 'zimfile.zim', - 'corner_cases.zim' + 'corner_cases.zim', + 'library.xml' ] foreach file : data_files # configure_file(input : 'data/' + file, diff --git a/test/server.cpp b/test/server.cpp index 2e18657c0..bb963b7b2 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -53,6 +53,7 @@ public: // types typedef std::vector FilePathCollection; public: // functions + ZimFileServer(int serverPort, std::string libraryFilePath); ZimFileServer(int serverPort, const FilePathCollection& zimpaths); ~ZimFileServer(); @@ -66,6 +67,9 @@ public: // functions return client->Head(path, headers); } +private: + void run(int serverPort); + private: // data kiwix::Library library; kiwix::Manager manager; @@ -74,6 +78,16 @@ private: // data std::unique_ptr client; }; +ZimFileServer::ZimFileServer(int serverPort, std::string libraryFilePath) +: manager(&this->library) +{ + if ( isRelativePath(libraryFilePath) ) + libraryFilePath = computeAbsolutePath(getCurrentDirectory(), libraryFilePath); + manager.readFile(libraryFilePath, true, true); + + run(serverPort); +} + ZimFileServer::ZimFileServer(int serverPort, const FilePathCollection& zimpaths) : manager(&this->library) { @@ -82,6 +96,11 @@ ZimFileServer::ZimFileServer(int serverPort, const FilePathCollection& zimpaths) throw std::runtime_error("Unable to add the ZIM file '" + zimpath + "'"); } + run(serverPort); +} + +void ZimFileServer::run(int serverPort) +{ const std::string address = "127.0.0.1"; nameMapper.reset(new kiwix::HumanReadableNameMapper(library, false)); server.reset(new kiwix::Server(&library, nameMapper.get())); @@ -546,3 +565,88 @@ TEST_F(ServerTest, RangeHeaderIsCaseInsensitive) EXPECT_EQ(r0->body, r->body); } } + +//////////////////////////////////////////////////////////////////////////////// +// Testing of the library-related functionality of the server +//////////////////////////////////////////////////////////////////////////////// + +class LibraryServerTest : public ::testing::Test +{ +protected: + std::unique_ptr zfs1_; + + const int PORT = 8002; + +protected: + void SetUp() override { + zfs1_.reset(new ZimFileServer(PORT, "./test/library.xml")); + } + + void TearDown() override { + zfs1_.reset(); + } +}; + +std::string maskVariableOPDSFeedData(const std::string& s) +{ + const auto p = s.find(""); + const std::string u("YYYY-MM-DDThh:mm:ssZ"); + return s.substr(0, p) + u + s.substr(p + u.size()); +} + +TEST_F(LibraryServerTest, catalog_root_xml) +{ + const auto r = zfs1_->GET("/catalog/root.xml"); + EXPECT_EQ(r->status, 200); + EXPECT_EQ(maskVariableOPDSFeedData(r->body), + "\n" + " 5e2e6fa3-14d5-f2c2-6c16-8ceaedd82237\n" + " All zims\n" + " YYYY-MM-DDThh:mm:ssZ\n" + " \n" + " \n" + " \n" + " urn:uuid:charlesray\n" + " Charles, Ray\n" + " Wikipedia articles about Charles, Ray\n" + " eng\n" + " 2020-03-31T00:00::00Z\n" + " wikipedia_en_ray_charles\n" + " \n" + " unittest;wikipedia;_category:jazz;_pictures:no;_videos:no;_details:no;_ftindex:yes\n" + " 284\n" + " 2\n" + " /meta?name=favicon&content=zimfile\n" + " \n" + " \n" + " Wikipedia\n" + " \n" + " \n" + " Kiwix\n" + " \n" + " \n" + " \n" + " \n" + " urn:uuid:raycharles\n" + " Ray Charles\n" + " Wikipedia articles about Ray Charles\n" + " eng\n" + " 2020-03-31T00:00::00Z\n" + " wikipedia_en_ray_charles\n" + " \n" + " unittest;wikipedia;_category:wikipedia;_pictures:no;_videos:no;_details:no;_ftindex:yes\n" + " 284\n" + " 2\n" + " /meta?name=favicon&content=zimfile\n" + " \n" + " \n" + " Wikipedia\n" + " \n" + " \n" + " Kiwix\n" + " \n" + " \n" + " \n" + "\n" + ); +} From 9913f748e20b93b88ec879a4a003fc71912a19e6 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Thu, 4 Mar 2021 21:15:06 +0400 Subject: [PATCH 04/15] LibraryServerTest.catalog_searchdescription_xml --- test/server.cpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/server.cpp b/test/server.cpp index bb963b7b2..9c4d73075 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -650,3 +650,21 @@ TEST_F(LibraryServerTest, catalog_root_xml) "\n" ); } + +TEST_F(LibraryServerTest, catalog_searchdescription_xml) +{ + const auto r = zfs1_->GET("/catalog/searchdescription.xml"); + EXPECT_EQ(r->status, 200); + EXPECT_EQ(r->body, + "\n" + "\n" + " Zim catalog search\n" + " Search zim files in the catalog.\n" + " \n" + "\n" + ); +} From 0b1740e6c5814c9fd6e54d87e8c4b458b24f49a6 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Fri, 5 Mar 2021 00:10:44 +0400 Subject: [PATCH 05/15] LibraryServerTest.catalog_search_by_tag --- test/server.cpp | 71 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 66 insertions(+), 5 deletions(-) diff --git a/test/server.cpp b/test/server.cpp index 9c4d73075..28fffc72d 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -587,11 +587,33 @@ protected: } }; -std::string maskVariableOPDSFeedData(const std::string& s) +// Returns a copy of 'text' with every line that fully matches 'pattern' +// replaced with the fixed string 'replacement' +std::string replaceLines(const std::string& text, + const std::string& pattern, + const std::string& replacement) { - const auto p = s.find(""); - const std::string u("YYYY-MM-DDThh:mm:ssZ"); - return s.substr(0, p) + u + s.substr(p + u.size()); + std::regex regex("^" + pattern + "$"); + std::ostringstream oss; + std::istringstream iss(text); + std::string line; + while ( std::getline(iss, line) ) { + if ( std::regex_match(line, regex) ) { + oss << replacement << "\n"; + } else { + oss << line << "\n"; + } + } + return oss.str(); +} + +std::string maskVariableOPDSFeedData(std::string s) +{ + s = replaceLines(s, " .+", + " YYYY-MM-DDThh:mm:ssZ"); + s = replaceLines(s, " .+", + " 12345678-90ab-cdef-1234-567890abcdef"); + return s; } TEST_F(LibraryServerTest, catalog_root_xml) @@ -600,7 +622,7 @@ TEST_F(LibraryServerTest, catalog_root_xml) EXPECT_EQ(r->status, 200); EXPECT_EQ(maskVariableOPDSFeedData(r->body), "\n" - " 5e2e6fa3-14d5-f2c2-6c16-8ceaedd82237\n" + " 12345678-90ab-cdef-1234-567890abcdef\n" " All zims\n" " YYYY-MM-DDThh:mm:ssZ\n" " \n" @@ -668,3 +690,42 @@ TEST_F(LibraryServerTest, catalog_searchdescription_xml) "\n" ); } + +TEST_F(LibraryServerTest, catalog_search_by_tag) +{ + const auto r = zfs1_->GET("/catalog/search?tag=_category:jazz"); + EXPECT_EQ(r->status, 200); + EXPECT_EQ(maskVariableOPDSFeedData(r->body), + "\n" + " 12345678-90ab-cdef-1234-567890abcdef\n" + " Search result for <Empty query>\n" + " YYYY-MM-DDThh:mm:ssZ\n" + " 1\n" + " 0\n" + " 1\n" + " \n" + " \n" + " \n" + " urn:uuid:charlesray\n" + " Charles, Ray\n" + " Wikipedia articles about Charles, Ray\n" + " eng\n" + " 2020-03-31T00:00::00Z\n" + " wikipedia_en_ray_charles\n" + " \n" + " unittest;wikipedia;_category:jazz;_pictures:no;_videos:no;_details:no;_ftindex:yes\n" + " 284\n" + " 2\n" + " /meta?name=favicon&content=zimfile\n" + " \n" + " \n" + " Wikipedia\n" + " \n" + " \n" + " Kiwix\n" + " \n" + " \n" + " \n" + "\n" + ); +} From 071d2bedd3e24ef0829dc685d63d70524726645c Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Fri, 5 Mar 2021 00:21:49 +0400 Subject: [PATCH 06/15] LibraryServerTest.catalog_search_by_text --- test/server.cpp | 131 +++++++++++++++++++++++++----------------------- 1 file changed, 68 insertions(+), 63 deletions(-) diff --git a/test/server.cpp b/test/server.cpp index 28fffc72d..ee3421e6a 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -616,6 +616,52 @@ std::string maskVariableOPDSFeedData(std::string s) return s; } +#define CHARLES_RAY_CATALOG_ENTRY \ + " \n" \ + " urn:uuid:charlesray\n" \ + " Charles, Ray\n" \ + " Wikipedia articles about Charles, Ray\n" \ + " eng\n" \ + " 2020-03-31T00:00::00Z\n" \ + " wikipedia_en_ray_charles\n" \ + " \n" \ + " unittest;wikipedia;_category:jazz;_pictures:no;_videos:no;_details:no;_ftindex:yes\n" \ + " 284\n" \ + " 2\n" \ + " /meta?name=favicon&content=zimfile\n" \ + " \n" \ + " \n" \ + " Wikipedia\n" \ + " \n" \ + " \n" \ + " Kiwix\n" \ + " \n" \ + " \n" \ + " \n" + +#define RAY_CHARLES_CATALOG_ENTRY \ + " \n" \ + " urn:uuid:raycharles\n" \ + " Ray Charles\n" \ + " Wikipedia articles about Ray Charles\n" \ + " eng\n" \ + " 2020-03-31T00:00::00Z\n" \ + " wikipedia_en_ray_charles\n" \ + " \n" \ + " unittest;wikipedia;_category:wikipedia;_pictures:no;_videos:no;_details:no;_ftindex:yes\n" \ + " 284\n" \ + " 2\n" \ + " /meta?name=favicon&content=zimfile\n" \ + " \n" \ + " \n" \ + " Wikipedia\n" \ + " \n" \ + " \n" \ + " Kiwix\n" \ + " \n" \ + " \n" \ + " \n" + TEST_F(LibraryServerTest, catalog_root_xml) { const auto r = zfs1_->GET("/catalog/root.xml"); @@ -627,48 +673,8 @@ TEST_F(LibraryServerTest, catalog_root_xml) " YYYY-MM-DDThh:mm:ssZ\n" " \n" " \n" - " \n" - " urn:uuid:charlesray\n" - " Charles, Ray\n" - " Wikipedia articles about Charles, Ray\n" - " eng\n" - " 2020-03-31T00:00::00Z\n" - " wikipedia_en_ray_charles\n" - " \n" - " unittest;wikipedia;_category:jazz;_pictures:no;_videos:no;_details:no;_ftindex:yes\n" - " 284\n" - " 2\n" - " /meta?name=favicon&content=zimfile\n" - " \n" - " \n" - " Wikipedia\n" - " \n" - " \n" - " Kiwix\n" - " \n" - " \n" - " \n" - " \n" - " urn:uuid:raycharles\n" - " Ray Charles\n" - " Wikipedia articles about Ray Charles\n" - " eng\n" - " 2020-03-31T00:00::00Z\n" - " wikipedia_en_ray_charles\n" - " \n" - " unittest;wikipedia;_category:wikipedia;_pictures:no;_videos:no;_details:no;_ftindex:yes\n" - " 284\n" - " 2\n" - " /meta?name=favicon&content=zimfile\n" - " \n" - " \n" - " Wikipedia\n" - " \n" - " \n" - " Kiwix\n" - " \n" - " \n" - " \n" + CHARLES_RAY_CATALOG_ENTRY + RAY_CHARLES_CATALOG_ENTRY "\n" ); } @@ -691,6 +697,25 @@ TEST_F(LibraryServerTest, catalog_searchdescription_xml) ); } +TEST_F(LibraryServerTest, catalog_search_by_text) +{ + const auto r = zfs1_->GET("/catalog/search?q=ray%20charles"); + EXPECT_EQ(r->status, 200); + EXPECT_EQ(maskVariableOPDSFeedData(r->body), + "\n" + " 12345678-90ab-cdef-1234-567890abcdef\n" + " Search result for ray charles\n" + " YYYY-MM-DDThh:mm:ssZ\n" + " 1\n" + " 0\n" + " 1\n" + " \n" + " \n" + RAY_CHARLES_CATALOG_ENTRY + "\n" + ); +} + TEST_F(LibraryServerTest, catalog_search_by_tag) { const auto r = zfs1_->GET("/catalog/search?tag=_category:jazz"); @@ -705,27 +730,7 @@ TEST_F(LibraryServerTest, catalog_search_by_tag) " 1\n" " \n" " \n" - " \n" - " urn:uuid:charlesray\n" - " Charles, Ray\n" - " Wikipedia articles about Charles, Ray\n" - " eng\n" - " 2020-03-31T00:00::00Z\n" - " wikipedia_en_ray_charles\n" - " \n" - " unittest;wikipedia;_category:jazz;_pictures:no;_videos:no;_details:no;_ftindex:yes\n" - " 284\n" - " 2\n" - " /meta?name=favicon&content=zimfile\n" - " \n" - " \n" - " Wikipedia\n" - " \n" - " \n" - " Kiwix\n" - " \n" - " \n" - " \n" + CHARLES_RAY_CATALOG_ENTRY "\n" ); } From 6d43fd065fd92fac60e7c127bac43e67e5fb8830 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Fri, 5 Mar 2021 00:33:49 +0400 Subject: [PATCH 07/15] Less boilerplate in LibraryServerTest unit-tests --- test/server.cpp | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/test/server.cpp b/test/server.cpp index ee3421e6a..5e1d52e91 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -616,6 +616,16 @@ std::string maskVariableOPDSFeedData(std::string s) return s; } +#define OPDS_FEED_TAG \ + "\n" + +#define CATALOG_LINK_TAGS \ + " \n" \ + " \n" + #define CHARLES_RAY_CATALOG_ENTRY \ " \n" \ " urn:uuid:charlesray\n" \ @@ -667,12 +677,11 @@ TEST_F(LibraryServerTest, catalog_root_xml) const auto r = zfs1_->GET("/catalog/root.xml"); EXPECT_EQ(r->status, 200); EXPECT_EQ(maskVariableOPDSFeedData(r->body), - "\n" + OPDS_FEED_TAG " 12345678-90ab-cdef-1234-567890abcdef\n" " All zims\n" " YYYY-MM-DDThh:mm:ssZ\n" - " \n" - " \n" + CATALOG_LINK_TAGS CHARLES_RAY_CATALOG_ENTRY RAY_CHARLES_CATALOG_ENTRY "\n" @@ -702,15 +711,14 @@ TEST_F(LibraryServerTest, catalog_search_by_text) const auto r = zfs1_->GET("/catalog/search?q=ray%20charles"); EXPECT_EQ(r->status, 200); EXPECT_EQ(maskVariableOPDSFeedData(r->body), - "\n" + OPDS_FEED_TAG " 12345678-90ab-cdef-1234-567890abcdef\n" " Search result for ray charles\n" " YYYY-MM-DDThh:mm:ssZ\n" " 1\n" " 0\n" " 1\n" - " \n" - " \n" + CATALOG_LINK_TAGS RAY_CHARLES_CATALOG_ENTRY "\n" ); @@ -721,15 +729,14 @@ TEST_F(LibraryServerTest, catalog_search_by_tag) const auto r = zfs1_->GET("/catalog/search?tag=_category:jazz"); EXPECT_EQ(r->status, 200); EXPECT_EQ(maskVariableOPDSFeedData(r->body), - "\n" + OPDS_FEED_TAG " 12345678-90ab-cdef-1234-567890abcdef\n" " Search result for <Empty query>\n" " YYYY-MM-DDThh:mm:ssZ\n" " 1\n" " 0\n" " 1\n" - " \n" - " \n" + CATALOG_LINK_TAGS CHARLES_RAY_CATALOG_ENTRY "\n" ); From 58186ffb264df5753361a466b41c16cc26c1446b Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Fri, 5 Mar 2021 19:10:01 +0400 Subject: [PATCH 08/15] kiwix::Book::getCategory() --- include/book.h | 1 + src/book.cpp | 12 ++++++++++++ src/opds_dumper.cpp | 1 + src/wrapper/java/book.cpp | 1 + src/wrapper/java/org/kiwix/kiwixlib/Book.java | 1 + test/server.cpp | 2 ++ 6 files changed, 18 insertions(+) diff --git a/include/book.h b/include/book.h index 2baeab921..eff8a037f 100644 --- a/include/book.h +++ b/include/book.h @@ -59,6 +59,7 @@ class Book const std::string& getDate() const { return m_date; } const std::string& getUrl() const { return m_url; } const std::string& getName() const { return m_name; } + std::string getCategory() const; const std::string& getTags() const { return m_tags; } std::string getTagStr(const std::string& tagName) const; bool getTagBool(const std::string& tagName) const; diff --git a/src/book.cpp b/src/book.cpp index beb3b5eba..f7f284036 100644 --- a/src/book.cpp +++ b/src/book.cpp @@ -220,4 +220,16 @@ bool Book::getTagBool(const std::string& tagName) const { return convertStrToBool(getTagStr(tagName)); } +std::string Book::getCategory() const +{ + try + { + return getTagStr("category"); + } + catch ( const std::out_of_range& ) + { + return ""; + } +} + } diff --git a/src/opds_dumper.cpp b/src/opds_dumper.cpp index ef4c8dd03..8f14e0949 100644 --- a/src/opds_dumper.cpp +++ b/src/opds_dumper.cpp @@ -77,6 +77,7 @@ pugi::xml_node OPDSDumper::handleBook(Book book, pugi::xml_node root_node) { ADD_TEXT_ENTRY(entry_node, "updated", gen_date_from_yyyy_mm_dd(book.getDate())); ADD_TEXT_ENTRY(entry_node, "name", book.getName()); ADD_TEXT_ENTRY(entry_node, "flavour", book.getFlavour()); + ADD_TEXT_ENTRY(entry_node, "category", book.getCategory()); ADD_TEXT_ENTRY(entry_node, "tags", book.getTags()); ADD_TEXT_ENTRY(entry_node, "articleCount", to_string(book.getArticleCount())); ADD_TEXT_ENTRY(entry_node, "mediaCount", to_string(book.getMediaCount())); diff --git a/src/wrapper/java/book.cpp b/src/wrapper/java/book.cpp index 2d555f3b7..157cc5251 100644 --- a/src/wrapper/java/book.cpp +++ b/src/wrapper/java/book.cpp @@ -69,6 +69,7 @@ GETTER(jstring, getDate) GETTER(jstring, getUrl) GETTER(jstring, getName) GETTER(jstring, getFlavour) +GETTER(jstring, getCategory) GETTER(jstring, getTags) GETTER(jlong, getArticleCount) GETTER(jlong, getMediaCount) diff --git a/src/wrapper/java/org/kiwix/kiwixlib/Book.java b/src/wrapper/java/org/kiwix/kiwixlib/Book.java index c40cd1670..693e83375 100644 --- a/src/wrapper/java/org/kiwix/kiwixlib/Book.java +++ b/src/wrapper/java/org/kiwix/kiwixlib/Book.java @@ -24,6 +24,7 @@ public class Book public native String getUrl(); public native String getName(); public native String getFlavour(); + public native String getCategory(); public native String getTags(); /** * Return the value associated to the tag tagName diff --git a/test/server.cpp b/test/server.cpp index 5e1d52e91..dc09c28b6 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -635,6 +635,7 @@ std::string maskVariableOPDSFeedData(std::string s) " 2020-03-31T00:00::00Z\n" \ " wikipedia_en_ray_charles\n" \ " \n" \ + " jazz\n" \ " unittest;wikipedia;_category:jazz;_pictures:no;_videos:no;_details:no;_ftindex:yes\n" \ " 284\n" \ " 2\n" \ @@ -658,6 +659,7 @@ std::string maskVariableOPDSFeedData(std::string s) " 2020-03-31T00:00::00Z\n" \ " wikipedia_en_ray_charles\n" \ " \n" \ + " wikipedia\n" \ " unittest;wikipedia;_category:wikipedia;_pictures:no;_videos:no;_details:no;_ftindex:yes\n" \ " 284\n" \ " 2\n" \ From f270724b1febed837e73ff80ed65b7d97d520a40 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Fri, 5 Mar 2021 19:17:17 +0400 Subject: [PATCH 09/15] Testing of a library entry without a category --- test/data/library.xml | 16 ++++++++++++++++ test/server.cpp | 30 ++++++++++++++++++++++++++++-- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/test/data/library.xml b/test/data/library.xml index e8fd2e345..130f3aa48 100644 --- a/test/data/library.xml +++ b/test/data/library.xml @@ -15,6 +15,22 @@ mediaCount="2" size="556" > + \n" \ " \n" +#define UNCATEGORIZED_RAY_CHARLES_CATALOG_ENTRY \ + " \n" \ + " urn:uuid:raycharles_uncategorized\n" \ + " Ray Charles\n" \ + " Wikipedia articles about Ray Charles\n" \ + " eng\n" \ + " 2020-03-31T00:00::00Z\n" \ + " wikipedia_en_ray_charles\n" \ + " \n" \ + " \n" \ + " unittest;wikipedia;_pictures:no;_videos:no;_details:no;_ftindex:yes\n" \ + " 284\n" \ + " 2\n" \ + " /meta?name=favicon&content=zimfile\n" \ + " \n" \ + " \n" \ + " Wikipedia\n" \ + " \n" \ + " \n" \ + " Kiwix\n" \ + " \n" \ + " \n" \ + " \n" + TEST_F(LibraryServerTest, catalog_root_xml) { const auto r = zfs1_->GET("/catalog/root.xml"); @@ -686,6 +710,7 @@ TEST_F(LibraryServerTest, catalog_root_xml) CATALOG_LINK_TAGS CHARLES_RAY_CATALOG_ENTRY RAY_CHARLES_CATALOG_ENTRY + UNCATEGORIZED_RAY_CHARLES_CATALOG_ENTRY "\n" ); } @@ -717,11 +742,12 @@ TEST_F(LibraryServerTest, catalog_search_by_text) " 12345678-90ab-cdef-1234-567890abcdef\n" " Search result for ray charles\n" " YYYY-MM-DDThh:mm:ssZ\n" - " 1\n" + " 2\n" " 0\n" - " 1\n" + " 2\n" CATALOG_LINK_TAGS RAY_CHARLES_CATALOG_ENTRY + UNCATEGORIZED_RAY_CHARLES_CATALOG_ENTRY "\n" ); } From 80d4f7e349bcc95c4d4e560b97fe301b07c03337 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Wed, 3 Mar 2021 14:36:38 +0400 Subject: [PATCH 10/15] Extracted InternalServer::search_catalog() --- src/server/internalServer.cpp | 28 ++++++++++++++++++---------- src/server/internalServer.h | 4 ++++ 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index 52b5960e6..4dd343274 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -668,6 +668,22 @@ std::unique_ptr InternalServer::handle_catalog(const RequestContext& r uuid = zim::Uuid::generate(host); bookIdsToDump = mp_library->filter(kiwix::Filter().valid(true).local(true).remote(true)); } else if (url == "search") { + bookIdsToDump = search_catalog(request, opdsDumper); + uuid = zim::Uuid::generate(); + } + + opdsDumper.setId(kiwix::to_string(uuid)); + auto response = ContentResponse::build( + *this, + opdsDumper.dumpOPDSFeed(bookIdsToDump), + "application/atom+xml; profile=opds-catalog; kind=acquisition; charset=utf-8"); + return std::move(response); +} + +std::vector +InternalServer::search_catalog(const RequestContext& request, + kiwix::OPDSDumper& opdsDumper) +{ auto filter = kiwix::Filter().valid(true).local(true).remote(true); string query(""); size_t count(10); @@ -698,22 +714,14 @@ std::unique_ptr InternalServer::handle_catalog(const RequestContext& r filter.rejectTags(kiwix::split(request.get_argument("notag"), ";")); } catch (...) {} opdsDumper.setTitle("Search result for " + query); - uuid = zim::Uuid::generate(); - bookIdsToDump = mp_library->filter(filter); + std::vector bookIdsToDump = mp_library->filter(filter); auto totalResults = bookIdsToDump.size(); bookIdsToDump.erase(bookIdsToDump.begin(), bookIdsToDump.begin()+startIndex); if (count>0 && bookIdsToDump.size() > count) { bookIdsToDump.resize(count); } opdsDumper.setOpenSearchInfo(totalResults, startIndex, bookIdsToDump.size()); - } - - opdsDumper.setId(kiwix::to_string(uuid)); - auto response = ContentResponse::build( - *this, - opdsDumper.dumpOPDSFeed(bookIdsToDump), - "application/atom+xml; profile=opds-catalog; kind=acquisition; charset=utf-8"); - return std::move(response); + return bookIdsToDump; } namespace diff --git a/src/server/internalServer.h b/src/server/internalServer.h index 19720df41..318f4d426 100644 --- a/src/server/internalServer.h +++ b/src/server/internalServer.h @@ -41,6 +41,7 @@ namespace kiwix { typedef kainjow::mustache::data MustacheData; class Entry; +class OPDSDumper; class InternalServer { public: @@ -79,6 +80,9 @@ class InternalServer { std::unique_ptr handle_captured_external(const RequestContext& request); std::unique_ptr handle_content(const RequestContext& request); + std::vector search_catalog(const RequestContext& request, + kiwix::OPDSDumper& opdsDumper); + MustacheData get_default_data() const; MustacheData homepage_data() const; From e55bf514e8036d8492aa306ac3071e3cce005218 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Fri, 5 Mar 2021 19:24:59 +0400 Subject: [PATCH 11/15] Dedicated 'category' parameter in catalog search --- include/library.h | 2 ++ src/library.cpp | 9 +++++++++ src/server/internalServer.cpp | 3 +++ test/server.cpp | 18 ++++++++++++++++++ 4 files changed, 32 insertions(+) diff --git a/include/library.h b/include/library.h index 53dec415a..86dfafaee 100644 --- a/include/library.h +++ b/include/library.h @@ -52,6 +52,7 @@ class Filter { uint64_t activeFilters; std::vector _acceptTags; std::vector _rejectTags; + std::string _category; std::string _lang; std::string _publisher; std::string _creator; @@ -96,6 +97,7 @@ class Filter { Filter& acceptTags(std::vector tags); Filter& rejectTags(std::vector tags); + Filter& category(std::string category); Filter& lang(std::string lang); Filter& publisher(std::string publisher); Filter& creator(std::string creator); diff --git a/src/library.cpp b/src/library.cpp index 0dad74794..c9855bd94 100644 --- a/src/library.cpp +++ b/src/library.cpp @@ -391,6 +391,7 @@ enum filterTypes { MAXSIZE = FLAG(11), QUERY = FLAG(12), NAME = FLAG(13), + CATEGORY = FLAG(14), }; Filter& Filter::local(bool accept) @@ -443,6 +444,13 @@ Filter& Filter::rejectTags(std::vector tags) return *this; } +Filter& Filter::category(std::string category) +{ + _category = category; + activeFilters |= CATEGORY; + return *this; +} + Filter& Filter::lang(std::string lang) { _lang = lang; @@ -502,6 +510,7 @@ bool Filter::accept(const Book& book) const FILTER(_NOREMOTE, !remote) FILTER(MAXSIZE, book.getSize() <= _maxSize) + FILTER(CATEGORY, book.getCategory() == _category) FILTER(LANG, book.getLanguage() == _lang) FILTER(_PUBLISHER, book.getPublisher() == _publisher) FILTER(_CREATOR, book.getCreator() == _creator) diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index 4dd343274..c2ccfd742 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -698,6 +698,9 @@ InternalServer::search_catalog(const RequestContext& request, try { filter.name(request.get_argument("name")); } catch (const std::out_of_range&) {} + try { + filter.category(request.get_argument("category")); + } catch (const std::out_of_range&) {} try { filter.lang(request.get_argument("lang")); } catch (const std::out_of_range&) {} diff --git a/test/server.cpp b/test/server.cpp index c1c240157..d4bd59de1 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -769,3 +769,21 @@ TEST_F(LibraryServerTest, catalog_search_by_tag) "\n" ); } + +TEST_F(LibraryServerTest, catalog_search_by_category) +{ + const auto r = zfs1_->GET("/catalog/search?category=jazz"); + EXPECT_EQ(r->status, 200); + EXPECT_EQ(maskVariableOPDSFeedData(r->body), + OPDS_FEED_TAG + " 12345678-90ab-cdef-1234-567890abcdef\n" + " Search result for <Empty query>\n" + " YYYY-MM-DDThh:mm:ssZ\n" + " 1\n" + " 0\n" + " 1\n" + CATALOG_LINK_TAGS + CHARLES_RAY_CATALOG_ENTRY + "\n" + ); +} From 6b2067c236fa34a159a9606c6c435c4eb5ba00ec Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Wed, 10 Mar 2021 21:26:28 +0400 Subject: [PATCH 12/15] Reading category element from OPDS stream --- include/book.h | 4 ++++ src/book.cpp | 8 ++++++++ test/library.cpp | 18 +++++++++++++++--- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/include/book.h b/include/book.h index eff8a037f..f9e685f5f 100644 --- a/include/book.h +++ b/include/book.h @@ -95,6 +95,9 @@ class Book void setFaviconMimeType(const std::string& faviconMimeType) { m_faviconMimeType = faviconMimeType; } void setDownloadId(const std::string& downloadId) { m_downloadId = downloadId; } + private: + std::string getCategoryFromTags() const; + protected: std::string m_id; std::string m_downloadId; @@ -102,6 +105,7 @@ class Book bool m_pathValid = false; std::string m_title; std::string m_description; + std::string m_category; std::string m_language; std::string m_creator; std::string m_publisher; diff --git a/src/book.cpp b/src/book.cpp index f7f284036..a2f349384 100644 --- a/src/book.cpp +++ b/src/book.cpp @@ -127,6 +127,7 @@ void Book::updateFromXml(const pugi::xml_node& node, const std::string& baseDir) try { m_downloadId = ATTR("downloadId"); } catch(...) {} + m_category = getCategoryFromTags(); } #undef ATTR @@ -156,6 +157,8 @@ void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost m_name = VALUE("name"); m_flavour = VALUE("flavour"); m_tags = VALUE("tags"); + const auto catnode = node.child("category"); + m_category = catnode.empty() ? getCategoryFromTags() : catnode.child_value(); m_articleCount = strtoull(VALUE("articleCount"), 0, 0); m_mediaCount = strtoull(VALUE("mediaCount"), 0, 0); for(auto linkNode = node.child("link"); linkNode; @@ -221,6 +224,11 @@ bool Book::getTagBool(const std::string& tagName) const { } std::string Book::getCategory() const +{ + return m_category; +} + +std::string Book::getCategoryFromTags() const { try { diff --git a/test/library.cpp b/test/library.cpp index 322890f7f..7f34e008b 100644 --- a/test/library.cpp +++ b/test/library.cpp @@ -46,7 +46,7 @@ const char * sampleOpdsStream = R"( 2018-06-23T00:00::00:Z fra Tania Louis videos - youtube + youtube;_category:category_defined_via_tags_only Tania Louis @@ -61,6 +61,7 @@ const char * sampleOpdsStream = R"( 2019-06-05T00:00::00:Z fra Une page de Wikiquote, le recueil des citations libres. + category_defined_via_category_element_only wikiquote;nopic @@ -76,7 +77,8 @@ const char * sampleOpdsStream = R"( 2019-06-02T00:00::00:Z Une sélection d'articles de Wikipédia sur la géographie fra - wikipedia;nopic + category_element_overrides_tags + wikipedia;nopic;_category:tags_override_category_element Wikipedia @@ -91,7 +93,8 @@ const char * sampleOpdsStream = R"( 2019-05-13T00:00::00:Z fra Une - wikipedia;nopic + wikipedia;nopic;_category:tags_override_category_element + category_element_overrides_tags Wikipedia @@ -228,6 +231,15 @@ TEST_F(LibraryTest, sanityCheck) EXPECT_EQ(lib.getBooksPublishers().size(), 1U); } +TEST_F(LibraryTest, categoryHandling) +{ + EXPECT_EQ("", lib.getBookById("0c45160e-f917-760a-9159-dfe3c53cdcdd").getCategory()); + EXPECT_EQ("category_defined_via_tags_only", lib.getBookById("0d0bcd57-d3f6-cb22-44cc-a723ccb4e1b2").getCategory()); + EXPECT_EQ("category_defined_via_category_element_only", lib.getBookById("0ea1cde6-441d-6c58-f2c7-21c2838e659f").getCategory()); + EXPECT_EQ("category_element_overrides_tags", lib.getBookById("1123e574-6eef-6d54-28fc-13e4caeae474").getCategory()); + EXPECT_EQ("category_element_overrides_tags", lib.getBookById("14829621-c490-c376-0792-9de558b57efa").getCategory()); +} + TEST_F(LibraryTest, filterCheck) { auto bookIds = lib.filter(kiwix::Filter()); From 4abc4f85187b26a593f581a965d223b6b3005082 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Sat, 13 Mar 2021 00:54:06 +0400 Subject: [PATCH 13/15] Support for book category attribute in library.xml --- src/book.cpp | 3 +- test/book.cpp | 108 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 1 deletion(-) diff --git a/src/book.cpp b/src/book.cpp index a2f349384..e8c4e0f47 100644 --- a/src/book.cpp +++ b/src/book.cpp @@ -127,7 +127,8 @@ void Book::updateFromXml(const pugi::xml_node& node, const std::string& baseDir) try { m_downloadId = ATTR("downloadId"); } catch(...) {} - m_category = getCategoryFromTags(); + const auto catattr = node.attribute("category"); + m_category = catattr.empty() ? getCategoryFromTags() : catattr.value(); } #undef ATTR diff --git a/test/book.cpp b/test/book.cpp index 3298ee953..261ac9c59 100644 --- a/test/book.cpp +++ b/test/book.cpp @@ -1,5 +1,6 @@ #include "gtest/gtest.h" #include "../include/book.h" +#include TEST(BookTest, updateTest) { @@ -31,3 +32,110 @@ TEST(BookTest, updateTest) EXPECT_EQ(newBook.getFavicon(), book.getFavicon()); EXPECT_EQ(newBook.getFaviconMimeType(), book.getFaviconMimeType()); } + +namespace +{ + +struct XMLDoc : pugi::xml_document +{ + explicit XMLDoc(const std::string& xml) + { + load_buffer(xml.c_str(), xml.size()); + } +}; + +} // unnamed namespace + +TEST(BookTest, updateFromXMLTest) +{ + const XMLDoc xml(R"( + + + )"); + + kiwix::Book book; + book.updateFromXml(xml.child("book"), "/data/zim"); + + EXPECT_EQ(book.getPath(), "/data/zim/zara.zim"); + EXPECT_EQ(book.getUrl(), "https://who.org/zara.zim"); + EXPECT_EQ(book.getTitle(), "Catch an infection in 24 hours"); + EXPECT_EQ(book.getDescription(), "Complete guide to contagious diseases"); + EXPECT_EQ(book.getTags(), "unittest;_category:medicine;_pictures:yes"); + EXPECT_EQ(book.getName(), "who_contagious_diseases_en"); + EXPECT_EQ(book.getCategory(), "medicine"); + EXPECT_EQ(book.getArticleCount(), 123456U); + EXPECT_EQ(book.getMediaCount(), 234567U); + EXPECT_EQ(book.getSize(), 345678U*1024U); +} + +TEST(BookTest, updateFromXMLCategoryHandlingTest) +{ + { + const XMLDoc xml(R"( + + + )"); + + kiwix::Book book; + book.updateFromXml(xml.child("book"), ""); + + EXPECT_EQ(book.getCategory(), "category_defined_via_tags_only"); + } + { + const XMLDoc xml(R"( + + + )"); + + kiwix::Book book; + book.updateFromXml(xml.child("book"), ""); + + EXPECT_EQ(book.getCategory(), "category_defined_via_attribute_only"); + } + { + const XMLDoc xml(R"( + + + )"); + + kiwix::Book book; + book.updateFromXml(xml.child("book"), ""); + + EXPECT_EQ(book.getCategory(), "category_attribute_overrides_tags"); + } + { + const XMLDoc xml(R"( + + + )"); + + kiwix::Book book; + book.updateFromXml(xml.child("book"), ""); + + EXPECT_EQ(book.getCategory(), "category_attribute_overrides_tags"); + } +} From a870e056218d86984f1847af7b77a559ddf6ff7a Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Mon, 15 Mar 2021 17:50:26 +0400 Subject: [PATCH 14/15] Slight enhancement of BookTest.updateTest --- test/book.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/book.cpp b/test/book.cpp index 261ac9c59..3c4a67790 100644 --- a/test/book.cpp +++ b/test/book.cpp @@ -6,6 +6,7 @@ TEST(BookTest, updateTest) { kiwix::Book book; + book.setId("xyz"); book.setReadOnly(false); book.setPath("/home/user/Downloads/skin-of-color-society_en_all_2019-11.zim"); book.setPathValid(true); @@ -21,6 +22,9 @@ TEST(BookTest, updateTest) EXPECT_FALSE(newBook.update(book)); newBook.setReadOnly(false); + EXPECT_FALSE(newBook.update(book)); + + newBook.setId("xyz"); EXPECT_TRUE(newBook.update(book)); EXPECT_EQ(newBook.readOnly(), book.readOnly()); @@ -28,6 +32,7 @@ TEST(BookTest, updateTest) EXPECT_EQ(newBook.isPathValid(), book.isPathValid()); EXPECT_EQ(newBook.getUrl(), book.getUrl()); EXPECT_EQ(newBook.getTags(), book.getTags()); + EXPECT_EQ(newBook.getCategory(), book.getCategory()); EXPECT_EQ(newBook.getName(), book.getName()); EXPECT_EQ(newBook.getFavicon(), book.getFavicon()); EXPECT_EQ(newBook.getFaviconMimeType(), book.getFaviconMimeType()); From b7b0bdbdd8d08453c95b37a81d7c5bddd317bbfe Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Mon, 15 Mar 2021 17:56:15 +0400 Subject: [PATCH 15/15] Both Book::update() methods update the category --- src/book.cpp | 2 ++ test/book.cpp | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/src/book.cpp b/src/book.cpp index e8c4e0f47..02c82a735 100644 --- a/src/book.cpp +++ b/src/book.cpp @@ -61,6 +61,7 @@ bool Book::update(const kiwix::Book& other) m_name = other.m_name; m_flavour = other.m_flavour; m_tags = other.m_tags; + m_category = other.m_category; m_origId = other.m_origId; m_articleCount = other.m_articleCount; m_mediaCount = other.m_mediaCount; @@ -88,6 +89,7 @@ void Book::update(const kiwix::Reader& reader) m_name = reader.getName(); m_flavour = reader.getFlavour(); m_tags = reader.getTags(); + m_category = getCategoryFromTags(); m_origId = reader.getOrigId(); m_articleCount = reader.getArticleCount(); m_mediaCount = reader.getMediaCount(); diff --git a/test/book.cpp b/test/book.cpp index 3c4a67790..22eca5428 100644 --- a/test/book.cpp +++ b/test/book.cpp @@ -144,3 +144,25 @@ TEST(BookTest, updateFromXMLCategoryHandlingTest) EXPECT_EQ(book.getCategory(), "category_attribute_overrides_tags"); } } + +TEST(BookTest, setTagsDoesntAffectCategory) +{ + kiwix::Book book; + + book.setTags("_category:youtube"); + ASSERT_EQ("", book.getCategory()); +} + +TEST(BookTest, updateCopiesCategory) +{ + const XMLDoc xml(R"()"); + + kiwix::Book book; + book.updateFromXml(xml.child("book"), ""); + + kiwix::Book newBook; + newBook.setId("abcd"); + EXPECT_EQ(newBook.getCategory(), ""); + newBook.update(book); + EXPECT_EQ(newBook.getCategory(), "ted"); +}