#define CPPHTTPLIB_ZLIB_SUPPORT 1 #include "./httplib.h" #include "gtest/gtest.h" #include "../include/manager.h" #include "../include/server.h" #include "../include/name_mapper.h" #include "../include/tools.h" using TestContextImpl = std::vector >; struct TestContext : TestContextImpl { TestContext(const std::initializer_list& il) : TestContextImpl(il) {} }; std::ostream& operator<<(std::ostream& out, const TestContext& ctx) { out << "Test context:\n"; for ( const auto& kv : ctx ) out << "\t" << kv.first << ": " << kv.second << "\n"; out << std::endl; return out; } bool is_valid_etag(const std::string& etag) { return etag.size() >= 2 && etag.front() == '"' && etag.back() == '"'; } template T1 concat(T1 a, const T2& b) { a.insert(a.end(), b.begin(), b.end()); return a; } typedef httplib::Headers Headers; Headers invariantHeaders(Headers headers) { headers.erase("Date"); return headers; } std::string replace(std::string s, std::string pattern, std::string replacement) { return std::regex_replace(s, std::regex(pattern), replacement); } // Output generated via mustache templates sometimes contains end-of-line // whitespace. This complicates representing the expected output of a unit-test // as C++ raw strings in editors that are configured to delete EOL whitespace. // A workaround is to put special markers (//EOLWHITESPACEMARKER) at the end // of such lines in the expected output string and remove them at runtime. // This is exactly what this function is for. std::string removeEOLWhitespaceMarkers(const std::string& s) { const std::regex pattern("//EOLWHITESPACEMARKER"); return std::regex_replace(s, pattern, ""); } class ZimFileServer { public: // types typedef std::shared_ptr Response; typedef std::vector FilePathCollection; public: // functions ZimFileServer(int serverPort, std::string libraryFilePath); ZimFileServer(int serverPort, bool withTaskbar, const FilePathCollection& zimpaths, std::string indexTemplateString = ""); ~ZimFileServer(); Response GET(const char* path, const Headers& headers = Headers()) { return client->Get(path, headers); } Response HEAD(const char* path, const Headers& headers = Headers()) { return client->Head(path, headers); } private: void run(int serverPort, std::string indexTemplateString = ""); private: // data kiwix::Library library; kiwix::Manager manager; std::unique_ptr nameMapper; std::unique_ptr server; std::unique_ptr client; const bool withTaskbar = true; }; ZimFileServer::ZimFileServer(int serverPort, std::string libraryFilePath) : manager(&this->library) { if ( kiwix::isRelativePath(libraryFilePath) ) libraryFilePath = kiwix::computeAbsolutePath(kiwix::getCurrentDirectory(), libraryFilePath); manager.readFile(libraryFilePath, true, true); run(serverPort); } ZimFileServer::ZimFileServer(int serverPort, bool _withTaskbar, const FilePathCollection& zimpaths, std::string indexTemplateString) : manager(&this->library) , withTaskbar(_withTaskbar) { for ( const auto& zimpath : zimpaths ) { if (!manager.addBookFromPath(zimpath, zimpath, "", false)) throw std::runtime_error("Unable to add the ZIM file '" + zimpath + "'"); } run(serverPort, indexTemplateString); } void ZimFileServer::run(int serverPort, std::string indexTemplateString) { const std::string address = "127.0.0.1"; nameMapper.reset(new kiwix::HumanReadableNameMapper(library, false)); server.reset(new kiwix::Server(&library, nameMapper.get())); server->setRoot("ROOT"); server->setAddress(address); server->setPort(serverPort); server->setNbThreads(2); server->setVerbose(false); server->setTaskbar(withTaskbar, withTaskbar); if (!indexTemplateString.empty()) { server->setIndexTemplateString(indexTemplateString); } if ( !server->start() ) throw std::runtime_error("ZimFileServer failed to start"); client.reset(new httplib::Client(address, serverPort)); } ZimFileServer::~ZimFileServer() { server->stop(); } class ServerTest : public ::testing::Test { protected: std::unique_ptr zfs1_; const int PORT = 8001; const ZimFileServer::FilePathCollection ZIMFILES { "./test/zimfile.zim", "./test/poor.zim", "./test/corner_cases.zim" }; protected: void SetUp() override { zfs1_.reset(new ZimFileServer(PORT, /*withTaskbar=*/true, ZIMFILES)); } void TearDown() override { zfs1_.reset(); } }; class TaskbarlessServerTest : public ServerTest { protected: void SetUp() override { zfs1_.reset(new ZimFileServer(PORT, /*withTaskbar=*/false, ZIMFILES)); } }; const bool WITH_ETAG = true; const bool NO_ETAG = false; struct Resource { bool etag_expected; const char* url; }; std::ostream& operator<<(std::ostream& out, const Resource& r) { out << "url: " << r.url; return out; } typedef std::vector ResourceCollection; const ResourceCollection resources200Compressible{ { WITH_ETAG, "/ROOT/" }, { WITH_ETAG, "/ROOT/skin/jquery-ui/jquery-ui.structure.min.css" }, { WITH_ETAG, "/ROOT/skin/jquery-ui/jquery-ui.min.js" }, { WITH_ETAG, "/ROOT/skin/jquery-ui/external/jquery/jquery.js" }, { WITH_ETAG, "/ROOT/skin/jquery-ui/jquery-ui.theme.min.css" }, { WITH_ETAG, "/ROOT/skin/jquery-ui/jquery-ui.min.css" }, { WITH_ETAG, "/ROOT/skin/taskbar.js" }, { WITH_ETAG, "/ROOT/skin/taskbar.css" }, { WITH_ETAG, "/ROOT/skin/block_external.js" }, { NO_ETAG, "/ROOT/catalog/search" }, { NO_ETAG, "/ROOT/search?content=zimfile&pattern=a" }, { NO_ETAG, "/ROOT/suggest?content=zimfile&term=ray" }, { NO_ETAG, "/ROOT/catch/external?source=www.example.com" }, { WITH_ETAG, "/ROOT/zimfile/A/index" }, { WITH_ETAG, "/ROOT/zimfile/A/Ray_Charles" }, { WITH_ETAG, "/ROOT/raw/zimfile/content/A/index" }, { WITH_ETAG, "/ROOT/raw/zimfile/content/A/Ray_Charles" }, }; const ResourceCollection resources200Uncompressible{ { WITH_ETAG, "/ROOT/skin/jquery-ui/images/animated-overlay.gif" }, { WITH_ETAG, "/ROOT/skin/caret.png" }, { WITH_ETAG, "/ROOT/raw/zimfile/meta/Title" }, { WITH_ETAG, "/ROOT/raw/zimfile/meta/Description" }, { WITH_ETAG, "/ROOT/raw/zimfile/meta/Language" }, { WITH_ETAG, "/ROOT/raw/zimfile/meta/Name" }, { WITH_ETAG, "/ROOT/raw/zimfile/meta/Tags" }, { WITH_ETAG, "/ROOT/raw/zimfile/meta/Date" }, { WITH_ETAG, "/ROOT/raw/zimfile/meta/Creator" }, { WITH_ETAG, "/ROOT/raw/zimfile/meta/Publisher" }, { NO_ETAG, "/ROOT/catalog/v2/illustration/zimfile?size=48" }, { WITH_ETAG, "/ROOT/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg" }, { WITH_ETAG, "/ROOT/corner_cases/A/empty.html" }, { WITH_ETAG, "/ROOT/corner_cases/-/empty.css" }, { WITH_ETAG, "/ROOT/corner_cases/-/empty.js" }, // The following url's responses are too small to be compressed { NO_ETAG, "/ROOT/catalog/root.xml" }, { NO_ETAG, "/ROOT/catalog/searchdescription.xml" }, { NO_ETAG, "/ROOT/suggest?content=zimfile" }, { WITH_ETAG, "/ROOT/raw/zimfile/meta/Creator" }, { WITH_ETAG, "/ROOT/raw/zimfile/meta/Title" }, }; ResourceCollection all200Resources() { return concat(resources200Compressible, resources200Uncompressible); } TEST(indexTemplateStringTest, emptyIndexTemplate) { const int PORT = 8001; const ZimFileServer::FilePathCollection ZIMFILES { "./test/zimfile.zim", "./test/corner_cases.zim" }; ZimFileServer zfs(PORT, /*withTaskbar=*/true, ZIMFILES, ""); EXPECT_EQ(200, zfs.GET("/ROOT/")->status); } TEST(indexTemplateStringTest, indexTemplateCheck) { const int PORT = 8001; const ZimFileServer::FilePathCollection ZIMFILES { "./test/zimfile.zim", "./test/corner_cases.zim" }; ZimFileServer zfs(PORT, /*withTaskbar=*/true, ZIMFILES, "" "Welcome to kiwix library" "" ""); EXPECT_EQ("" "Welcome to kiwix library" "" "" "", zfs.GET("/ROOT/")->body); } TEST_F(ServerTest, 200) { for ( const Resource& res : all200Resources() ) EXPECT_EQ(200, zfs1_->GET(res.url)->status) << "res.url: " << res.url; } TEST_F(ServerTest, CompressibleContentIsCompressedIfAcceptable) { for ( const Resource& res : resources200Compressible ) { const auto x = zfs1_->GET(res.url, { {"Accept-Encoding", "gzip"} }); EXPECT_EQ(200, x->status) << res; EXPECT_EQ("gzip", x->get_header_value("Content-Encoding")) << res; EXPECT_EQ("Accept-Encoding", x->get_header_value("Vary")) << res; } } TEST_F(ServerTest, UncompressibleContentIsNotCompressed) { for ( const Resource& res : resources200Uncompressible ) { const auto x = zfs1_->GET(res.url, { {"Accept-Encoding", "gzip"} }); EXPECT_EQ(200, x->status) << res; EXPECT_EQ("", x->get_header_value("Content-Encoding")) << res; } } // Selects from text only the lines containing the specified (fixed string) // pattern std::string fgrep(const std::string& pattern, const std::string& text) { std::istringstream iss(text); std::string line; std::string result; while ( getline(iss, line) ) { if ( line.find(pattern) != std::string::npos ) { result += line + "\n"; } } return result; } TEST_F(ServerTest, CacheIdsOfStaticResources) { typedef std::pair UrlAndExpectedResult; const std::vector testData{ { /* url */ "/ROOT/", R"EXPECTEDRESULT( src="/ROOT/skin/jquery-ui/external/jquery/jquery.js?cacheid=1d85f0f3" src="/ROOT/skin/jquery-ui/jquery-ui.min.js?cacheid=d927c2ff" href="/ROOT/skin/jquery-ui/jquery-ui.min.css?cacheid=e1de77b3" href="/ROOT/skin/jquery-ui/jquery-ui.theme.min.css?cacheid=2a5841f9" href="/ROOT/skin/index.css?cacheid=1aca980a" src: url("/ROOT/skin/fonts/Poppins.ttf?cacheid=af705837") format("truetype"); src: url("/ROOT/skin/fonts/Roboto.ttf?cacheid=84d10248") format("truetype"); )EXPECTEDRESULT" }, { /* url */ "/ROOT/skin/index.js", R"EXPECTEDRESULT( direct download download hash download magnet download torrent )EXPECTEDRESULT" }, { /* url */ "/ROOT/zimfile/A/index", R"EXPECTEDRESULT( )EXPECTEDRESULT" }, { // Searching in a ZIM file without a full-text index returns // a page rendered from static/templates/no_search_result_html /* url */ "/ROOT/search?content=poor&pattern=whatever", R"EXPECTEDRESULT( )EXPECTEDRESULT" }, }; for ( const auto& urlAndExpectedResult : testData ) { const std::string url = urlAndExpectedResult.first; const std::string expectedResult = urlAndExpectedResult.second; const TestContext ctx{ {"url", url} }; const auto r = zfs1_->GET(url.c_str()); EXPECT_EQ(r->body.find("KIWIXCACHEID"), std::string::npos) << ctx; EXPECT_EQ(fgrep("/skin/", r->body), expectedResult) << ctx; } } const char* urls400[] = { "/ROOT/search", "/ROOT/search?content=zimfile", "/ROOT/search?content=non-existing-book&pattern=asdfqwerty", "/ROOT/search?content=non-existing-book&pattern=asdGET(url)->status) << "url: " << url; } } const char* urls404[] = { "/", "/zimfile", "/ROOT/non-existent-item", "/ROOT/skin/non-existent-skin-resource", "/ROOT/catalog", "/ROOT/catalog/non-existent-item", "/ROOT/catalogBLABLABLA/root.xml", "/ROOT/catalog/v2/illustration/zimfile?size=96", "/ROOT/meta", "/ROOT/meta?content=zimfile", "/ROOT/meta?content=zimfile&name=non-existent-item", "/ROOT/meta?content=non-existent-book&name=title", "/ROOT/random", "/ROOT/random?content=non-existent-book", "/ROOT/suggest", "/ROOT/suggest?content=non-existent-book&term=abcd", "/ROOT/catch/external", "/ROOT/zimfile/A/non-existent-article", "/ROOT/raw/non-existent-book/meta/Title", "/ROOT/raw/zimfile/wrong-kind/Foo", // zimfile has no Favicon nor Illustration_48x48@1 meta item "/ROOT/raw/zimfile/meta/Favicon", "/ROOT/raw/zimfile/meta/Illustration_48x48@1", }; TEST_F(ServerTest, 404) { for ( const char* url : urls404 ) { EXPECT_EQ(404, zfs1_->GET(url)->status) << "url: " << url; } } namespace TestingOfHtmlResponses { struct ExpectedResponseData { const std::string expectedPageTitle; const std::string expectedCssUrl; const std::string bookName; const std::string bookTitle; const std::string expectedBody; }; enum ExpectedResponseDataType { expected_page_title, expected_css_url, book_name, book_title, expected_body }; // Operator overloading is used as a means of defining a mini-DSL for // defining test data in a concise way (see usage in // TEST_F(ServerTest, 404WithBodyTesting)) ExpectedResponseData operator==(ExpectedResponseDataType t, std::string s) { switch (t) { case expected_page_title: return ExpectedResponseData{s, "", "", "", ""}; case expected_css_url: return ExpectedResponseData{"", s, "", "", ""}; case book_name: return ExpectedResponseData{"", "", s, "", ""}; case book_title: return ExpectedResponseData{"", "", "", s, ""}; case expected_body: return ExpectedResponseData{"", "", "", "", s}; default: assert(false); return ExpectedResponseData{}; } } std::string selectNonEmpty(const std::string& a, const std::string& b) { if ( a.empty() ) return b; assert(b.empty()); return a; } ExpectedResponseData operator&&(const ExpectedResponseData& a, const ExpectedResponseData& b) { return ExpectedResponseData{ selectNonEmpty(a.expectedPageTitle, b.expectedPageTitle), selectNonEmpty(a.expectedCssUrl, b.expectedCssUrl), selectNonEmpty(a.bookName, b.bookName), selectNonEmpty(a.bookTitle, b.bookTitle), selectNonEmpty(a.expectedBody, b.expectedBody) }; } class TestContentIn404HtmlResponse : public ExpectedResponseData { public: TestContentIn404HtmlResponse(const std::string& url, const ExpectedResponseData& erd) : ExpectedResponseData(erd) , url(url) {} virtual ~TestContentIn404HtmlResponse() = default; const std::string url; std::string expectedResponse() const; private: bool isTranslatedVersion() const; virtual std::string pageTitle() const; std::string pageCssLink() const; std::string hiddenBookNameInput() const; std::string searchPatternInput() const; std::string taskbarLinks() const; std::string goToWelcomePageText() const; }; std::string TestContentIn404HtmlResponse::expectedResponse() const { const std::string frag[] = { R"FRAG( )FRAG", R"FRAG( )FRAG", R"FRAG(
)FRAG", R"FRAG( )FRAG", R"FRAG(
)FRAG", R"FRAG(
)FRAG", R"FRAG( )FRAG" }; return frag[0] + pageTitle() + frag[1] + pageCssLink() + frag[2] + hiddenBookNameInput() + frag[3] + searchPatternInput() + frag[4] + goToWelcomePageText() + frag[5] + goToWelcomePageText() + frag[6] + taskbarLinks() + frag[7] + expectedBody + frag[8]; } std::string TestContentIn404HtmlResponse::pageTitle() const { return expectedPageTitle.empty() ? "Content not found" : expectedPageTitle; } std::string TestContentIn404HtmlResponse::pageCssLink() const { if ( expectedCssUrl.empty() ) return ""; return R"( )"; } std::string TestContentIn404HtmlResponse::hiddenBookNameInput() const { return bookName.empty() ? "" : R"()"; } std::string TestContentIn404HtmlResponse::searchPatternInput() const { const std::string searchboxTooltip = isTranslatedVersion() ? "Որոնել '" + bookTitle + "'֊ում" : "Search '" + bookTitle + "'"; return R"( )"; } std::string TestContentIn404HtmlResponse::taskbarLinks() const { if ( bookName.empty() ) return ""; const auto goToMainPageOfBook = isTranslatedVersion() ? "Դեպի '" + bookTitle + "'֊ի գլխավոր էջը" : "Go to the main page of '" + bookTitle + "'"; const std::string goToRandomPage = isTranslatedVersion() ? "Բացել պատահական էջ" : "Go to a randomly selected page"; return R"( )"; } bool TestContentIn404HtmlResponse::isTranslatedVersion() const { return url.find("userlang=hy") != std::string::npos; } std::string TestContentIn404HtmlResponse::goToWelcomePageText() const { return isTranslatedVersion() ? "Գրադարանի էջ" : "Go to welcome page"; } class TestContentIn400HtmlResponse : public TestContentIn404HtmlResponse { public: TestContentIn400HtmlResponse(const std::string& url, const ExpectedResponseData& erd) : TestContentIn404HtmlResponse(url, erd) {} private: std::string pageTitle() const; }; std::string TestContentIn400HtmlResponse::pageTitle() const { return expectedPageTitle.empty() ? "Invalid request" : expectedPageTitle; } } // namespace TestingOfHtmlResponses TEST_F(ServerTest, 404WithBodyTesting) { using namespace TestingOfHtmlResponses; const std::vector testData{ { /* url */ "/ROOT/random?content=non-existent-book", expected_body==R"(

Not Found

No such book: non-existent-book

)" }, { /* url */ "/ROOT/random?content=non-existent-book&userlang=hy", expected_page_title=="Սխալ հասցե" && expected_body==R"(

Սխալ հասցե

Գիրքը բացակայում է՝ non-existent-book

)" }, { /* url */ "/ROOT/suggest?content=no-such-book&term=whatever", expected_body==R"(

Not Found

No such book: no-such-book

)" }, { /* url */ "/ROOT/catalog/", expected_body==R"(

Not Found

The requested URL "/ROOT/catalog/" was not found on this server.

)" }, { /* url */ "/ROOT/catalog/?userlang=hy", expected_page_title=="Սխալ հասցե" && expected_body==R"(

Սխալ հասցե

Սխալ հասցե՝ /ROOT/catalog/

)" }, { /* url */ "/ROOT/catalog/invalid_endpoint", expected_body==R"(

Not Found

The requested URL "/ROOT/catalog/invalid_endpoint" was not found on this server.

)" }, { /* url */ "/ROOT/catalog/invalid_endpoint?userlang=hy", expected_page_title=="Սխալ հասցե" && expected_body==R"(

Սխալ հասցե

Սխալ հասցե՝ /ROOT/catalog/invalid_endpoint

)" }, { /* url */ "/ROOT/invalid-book/whatever", expected_body==R"(

Not Found

The requested URL "/ROOT/invalid-book/whatever" was not found on this server.

Make a full text search for whatever

)" }, { /* url */ "/ROOT/zimfile/invalid-article", book_name=="zimfile" && book_title=="Ray Charles" && expected_body==R"(

Not Found

The requested URL "/ROOT/zimfile/invalid-article" was not found on this server.

Make a full text search for invalid-article

)" }, { /* url */ R"(/ROOT/">)", expected_body==R"(

Not Found

The requested URL "/ROOT/"><svg onload=alert(1)>" was not found on this server.

Make a full text search for "><svg onload=alert(1)>

)" }, { /* url */ R"(/ROOT/zimfile/">)", book_name=="zimfile" && book_title=="Ray Charles" && expected_body==R"(

Not Found

The requested URL "/ROOT/zimfile/"><svg onload=alert(1)>" was not found on this server.

Make a full text search for "><svg onload=alert(1)>

)" }, { /* url */ "/ROOT/zimfile/invalid-article?userlang=hy", expected_page_title=="Սխալ հասցե" && book_name=="zimfile" && book_title=="Ray Charles" && expected_body==R"(

Սխալ հասցե

Սխալ հասցե՝ /ROOT/zimfile/invalid-article

Որոնել invalid-article

)" }, { /* url */ "/ROOT/raw/no-such-book/meta/Title", expected_body==R"(

Not Found

The requested URL "/ROOT/raw/no-such-book/meta/Title" was not found on this server.

No such book: no-such-book

)" }, { /* url */ "/ROOT/raw/zimfile/XYZ", expected_body==R"(

Not Found

The requested URL "/ROOT/raw/zimfile/XYZ" was not found on this server.

XYZ is not a valid request for raw content.

)" }, { /* url */ "/ROOT/raw/zimfile/meta/invalid-metadata", expected_body==R"(

Not Found

The requested URL "/ROOT/raw/zimfile/meta/invalid-metadata" was not found on this server.

Cannot find meta entry invalid-metadata

)" }, { /* url */ "/ROOT/raw/zimfile/content/invalid-article", expected_body==R"(

Not Found

The requested URL "/ROOT/raw/zimfile/content/invalid-article" was not found on this server.

Cannot find content entry invalid-article

)" }, { /* url */ "/ROOT/search?content=poor&pattern=whatever", expected_page_title=="Fulltext search unavailable" && expected_css_url=="/ROOT/skin/search_results.css?cacheid=76d39c84" && book_name=="poor" && book_title=="poor" && expected_body==R"(

Not Found

The fulltext search engine is not available for this content.

)" }, }; for ( const auto& t : testData ) { const TestContext ctx{ {"url", t.url} }; const auto r = zfs1_->GET(t.url.c_str()); EXPECT_EQ(r->status, 404) << ctx; EXPECT_EQ(r->body, t.expectedResponse()) << ctx; } } TEST_F(ServerTest, 400WithBodyTesting) { using namespace TestingOfHtmlResponses; const std::vector testData{ { /* url */ "/ROOT/search", expected_body== R"(

Invalid request

The requested URL "/ROOT/search" is not a valid request.

No query provided.

)" }, { /* url */ "/ROOT/search?content=zimfile", expected_body==R"(

Invalid request

The requested URL "/ROOT/search?content=zimfile" is not a valid request.

No query provided.

)" }, { /* url */ "/ROOT/search?content=non-existing-book&pattern=asdfqwerty", expected_body==R"(

Invalid request

The requested URL "/ROOT/search?content=non-existing-book&pattern=asdfqwerty" is not a valid request.

The requested book doesn't exist.

)" }, { /* url */ "/ROOT/search?content=non-existing-book&pattern=a\"