From 54718190211d6f84cd706ed6e21223415ae21799 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Sun, 9 Oct 2022 15:33:50 +0400 Subject: [PATCH 01/16] Finer categorization of URLs in the server unit-test Preparing the server unit-test for the more elaborate handling of HTTP caching. --- test/server.cpp | 88 ++++++++++++++++++++++++++----------------------- 1 file changed, 47 insertions(+), 41 deletions(-) diff --git a/test/server.cpp b/test/server.cpp index 2d1e1068a..3b5d273a5 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -23,13 +23,19 @@ T1 concat(T1 a, const T2& b) return a; } -const bool WITH_ETAG = true; -const bool NO_ETAG = false; +enum ResourceKind +{ + ZIM_CONTENT, + STATIC_CONTENT, + DYNAMIC_CONTENT, +}; struct Resource { - bool etag_expected; + ResourceKind kind; const char* url; + + bool etag_expected() const { return kind != DYNAMIC_CONTENT; } }; std::ostream& operator<<(std::ostream& out, const Resource& r) @@ -41,55 +47,55 @@ std::ostream& operator<<(std::ostream& out, const Resource& r) typedef std::vector ResourceCollection; const ResourceCollection resources200Compressible{ - { WITH_ETAG, "/ROOT/" }, + { STATIC_CONTENT, "/ROOT/" }, - { WITH_ETAG, "/ROOT/skin/autoComplete.min.js" }, - { WITH_ETAG, "/ROOT/skin/css/autoComplete.css" }, - { WITH_ETAG, "/ROOT/skin/taskbar.css" }, + { STATIC_CONTENT, "/ROOT/skin/autoComplete.min.js" }, + { STATIC_CONTENT, "/ROOT/skin/css/autoComplete.css" }, + { STATIC_CONTENT, "/ROOT/skin/taskbar.css" }, - { NO_ETAG, "/ROOT/catalog/search" }, + { DYNAMIC_CONTENT, "/ROOT/catalog/search" }, - { NO_ETAG, "/ROOT/search?content=zimfile&pattern=a" }, + { DYNAMIC_CONTENT, "/ROOT/search?content=zimfile&pattern=a" }, - { NO_ETAG, "/ROOT/suggest?content=zimfile&term=ray" }, + { DYNAMIC_CONTENT, "/ROOT/suggest?content=zimfile&term=ray" }, - { WITH_ETAG, "/ROOT/content/zimfile/A/index" }, - { WITH_ETAG, "/ROOT/content/zimfile/A/Ray_Charles" }, + { ZIM_CONTENT, "/ROOT/content/zimfile/A/index" }, + { ZIM_CONTENT, "/ROOT/content/zimfile/A/Ray_Charles" }, - { WITH_ETAG, "/ROOT/raw/zimfile/content/A/index" }, - { WITH_ETAG, "/ROOT/raw/zimfile/content/A/Ray_Charles" }, + { ZIM_CONTENT, "/ROOT/raw/zimfile/content/A/index" }, + { ZIM_CONTENT, "/ROOT/raw/zimfile/content/A/Ray_Charles" }, }; const ResourceCollection resources200Uncompressible{ - { WITH_ETAG, "/ROOT/skin/caret.png" }, - { WITH_ETAG, "/ROOT/skin/css/images/search.svg" }, + { STATIC_CONTENT, "/ROOT/skin/caret.png" }, + { STATIC_CONTENT, "/ROOT/skin/css/images/search.svg" }, - { WITH_ETAG, "/ROOT/raw/zimfile/meta/Title" }, - { WITH_ETAG, "/ROOT/raw/zimfile/meta/Description" }, - { WITH_ETAG, "/ROOT/raw/zimfile/meta/Language" }, - { WITH_ETAG, "/ROOT/raw/zimfile/meta/Name" }, - { WITH_ETAG, "/ROOT/raw/zimfile/meta/Tags" }, - { WITH_ETAG, "/ROOT/raw/zimfile/meta/Date" }, - { WITH_ETAG, "/ROOT/raw/zimfile/meta/Creator" }, - { WITH_ETAG, "/ROOT/raw/zimfile/meta/Publisher" }, + { ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Title" }, + { ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Description" }, + { ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Language" }, + { ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Name" }, + { ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Tags" }, + { ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Date" }, + { ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Creator" }, + { ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Publisher" }, - { NO_ETAG, "/ROOT/catalog/v2/illustration/6f1d19d0-633f-087b-fb55-7ac324ff9baf?size=48" }, + { DYNAMIC_CONTENT, "/ROOT/catalog/v2/illustration/6f1d19d0-633f-087b-fb55-7ac324ff9baf?size=48" }, - { NO_ETAG, "/ROOT/catch/external?source=www.example.com" }, + { DYNAMIC_CONTENT, "/ROOT/catch/external?source=www.example.com" }, - { WITH_ETAG, "/ROOT/content/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg" }, + { ZIM_CONTENT, "/ROOT/content/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg" }, - { WITH_ETAG, "/ROOT/content/corner_cases/A/empty.html" }, - { WITH_ETAG, "/ROOT/content/corner_cases/-/empty.css" }, - { WITH_ETAG, "/ROOT/content/corner_cases/-/empty.js" }, + { ZIM_CONTENT, "/ROOT/content/corner_cases/A/empty.html" }, + { ZIM_CONTENT, "/ROOT/content/corner_cases/-/empty.css" }, + { ZIM_CONTENT, "/ROOT/content/corner_cases/-/empty.js" }, // The following url's responses are too small to be compressed - { NO_ETAG, "/ROOT/catalog/root.xml" }, - { NO_ETAG, "/ROOT/catalog/searchdescription.xml" }, - { NO_ETAG, "/ROOT/suggest?content=zimfile" }, - { WITH_ETAG, "/ROOT/raw/zimfile/meta/Creator" }, - { WITH_ETAG, "/ROOT/raw/zimfile/meta/Title" }, + { DYNAMIC_CONTENT, "/ROOT/catalog/root.xml" }, + { DYNAMIC_CONTENT, "/ROOT/catalog/searchdescription.xml" }, + { DYNAMIC_CONTENT, "/ROOT/suggest?content=zimfile" }, + { ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Creator" }, + { ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Title" }, }; ResourceCollection all200Resources() @@ -1063,8 +1069,8 @@ TEST_F(ServerTest, ETagHeaderIsSetAsNeeded) { for ( const Resource& res : all200Resources() ) { const auto responseToGet = zfs1_->GET(res.url); - EXPECT_EQ(res.etag_expected, responseToGet->has_header("ETag")) << res; - if ( res.etag_expected ) { + EXPECT_EQ(res.etag_expected(), responseToGet->has_header("ETag")) << res; + if ( res.etag_expected() ) { EXPECT_TRUE(is_valid_etag(responseToGet->get_header_value("ETag"))); } } @@ -1092,7 +1098,7 @@ TEST_F(ServerTest, DifferentServerInstancesProduceDifferentETags) { ZimFileServer zfs2(SERVER_PORT + 1, ZimFileServer::DEFAULT_OPTIONS, ZIMFILES); for ( const Resource& res : all200Resources() ) { - if ( !res.etag_expected ) continue; + if ( !res.etag_expected() ) continue; const auto h1 = zfs1_->HEAD(res.url); const auto h2 = zfs2.HEAD(res.url); EXPECT_NE(h1->get_header_value("ETag"), h2->get_header_value("ETag")); @@ -1102,7 +1108,7 @@ TEST_F(ServerTest, DifferentServerInstancesProduceDifferentETags) TEST_F(ServerTest, CompressionInfluencesETag) { for ( const Resource& res : resources200Compressible ) { - if ( ! res.etag_expected ) continue; + if ( ! res.etag_expected() ) continue; const auto g1 = zfs1_->GET(res.url); const auto g2 = zfs1_->GET(res.url, { {"Accept-Encoding", ""} } ); const auto g3 = zfs1_->GET(res.url, { {"Accept-Encoding", "gzip"} } ); @@ -1115,7 +1121,7 @@ TEST_F(ServerTest, CompressionInfluencesETag) TEST_F(ServerTest, ETagOfUncompressibleContentIsNotAffectedByAcceptEncoding) { for ( const Resource& res : resources200Uncompressible ) { - if ( ! res.etag_expected ) continue; + if ( ! res.etag_expected() ) continue; const auto g1 = zfs1_->GET(res.url); const auto g2 = zfs1_->GET(res.url, { {"Accept-Encoding", ""} } ); const auto g3 = zfs1_->GET(res.url, { {"Accept-Encoding", "gzip"} } ); @@ -1160,7 +1166,7 @@ TEST_F(ServerTest, IfNoneMatchRequestsWithMatchingETagResultIn304Responses) const char* const encodings[] = { "", "gzip" }; for ( const Resource& res : all200Resources() ) { for ( const char* enc: encodings ) { - if ( ! res.etag_expected ) continue; + if ( ! res.etag_expected() ) continue; const TestContext ctx{ {"url", res.url}, {"encoding", enc} }; const auto g = zfs1_->GET(res.url, { {"Accept-Encoding", enc} }); From 190156e0951c5f3327f0023a79852bb185675950 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Sun, 9 Oct 2022 16:28:58 +0400 Subject: [PATCH 02/16] Setting Cache-Control: for three types of content At this point the ETag value for ZIM content is still generated from the timestamp of the server start-up time. --- src/server/etag.cpp | 2 +- src/server/etag.h | 2 +- src/server/internalServer.cpp | 9 +++------ src/server/response.cpp | 26 ++++++++++++++++++++++---- src/server/response.h | 11 ++++++++++- test/server.cpp | 4 ++-- 6 files changed, 39 insertions(+), 15 deletions(-) diff --git a/src/server/etag.cpp b/src/server/etag.cpp index 1ba89470f..e031ce46c 100644 --- a/src/server/etag.cpp +++ b/src/server/etag.cpp @@ -37,7 +37,7 @@ namespace { // into the ETag for ETag::Option opt. // IMPORTANT: The characters in all_options must come in sorted order (so that // IMPORTANT: isValidOptionsString() works correctly). -const char all_options[] = "cz"; +const char all_options[] = "Zz"; static_assert(ETag::OPTION_COUNT == sizeof(all_options) - 1, ""); diff --git a/src/server/etag.h b/src/server/etag.h index 49ff0e724..f8da02360 100644 --- a/src/server/etag.h +++ b/src/server/etag.h @@ -51,7 +51,7 @@ class ETag { public: // types enum Option { - CACHEABLE_ENTITY, + ZIM_CONTENT, COMPRESSED_CONTENT, OPTION_COUNT }; diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index d6e6a91a4..772b056e5 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -606,11 +606,8 @@ MustacheData InternalServer::get_default_data() const bool InternalServer::etag_not_needed(const RequestContext& request) const { const std::string url = request.get_url(); - return kiwix::startsWith(url, "/catalog") - || url == "/search" - || url == "/suggest" - || url == "/random" - || url == "/catch/external"; + return kiwix::startsWith(url, "/skin") + || url == "/random"; } ETag @@ -761,7 +758,7 @@ std::unique_ptr InternalServer::handle_skin(const RequestContext& requ *this, getResource(resourceName), getMimeTypeForFile(resourceName)); - response->set_cacheable(); + response->set_kind(Response::STATIC_RESOURCE); return std::move(response); } catch (const ResourceNotFound& e) { return HTTP404Response(*this, request) diff --git a/src/server/response.cpp b/src/server/response.cpp index 7067ff207..2c751e7b0 100644 --- a/src/server/response.cpp +++ b/src/server/response.cpp @@ -102,6 +102,14 @@ bool compress(std::string &content) { } +const char* getCacheControlHeader(Response::Kind k) +{ + switch(k) { + case Response::STATIC_RESOURCE: return "max-age=31536000, immutable"; + case Response::ZIM_CONTENT: return "max-age=3600, must-revalidate"; + default: return "max-age=0, must-revalidate"; + } +} } // unnamed namespace @@ -112,6 +120,13 @@ Response::Response(bool verbose) add_header(MHD_HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN, "*"); } +void Response::set_kind(Kind k) +{ + m_kind = k; + if ( k == ZIM_CONTENT ) + m_etag.set_option(ETag::ZIM_CONTENT); +} + std::unique_ptr Response::build(const InternalServer& server) { return std::unique_ptr(new Response(server.m_verbose.load())); @@ -122,6 +137,9 @@ std::unique_ptr Response::build_304(const InternalServer& server, cons auto response = Response::build(server); response->set_code(MHD_HTTP_NOT_MODIFIED); response->m_etag = etag; + if ( etag.get_option(ETag::ZIM_CONTENT) ) { + response->set_kind(Response::ZIM_CONTENT); + } if ( etag.get_option(ETag::COMPRESSED_CONTENT) ) { response->add_header(MHD_HTTP_HEADER_VARY, "Accept-Encoding"); } @@ -355,7 +373,7 @@ MHD_Result Response::send(const RequestContext& request, MHD_Connection* connect MHD_Response* response = create_mhd_response(request); MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL, - m_etag.get_option(ETag::CACHEABLE_ENTITY) ? "max-age=2723040, public" : "no-cache, no-store, must-revalidate"); + getCacheControlHeader(m_kind)); const std::string etag = m_etag.get_etag(); if ( ! etag.empty() ) MHD_add_response_header(response, MHD_HTTP_HEADER_ETAG, etag.c_str()); @@ -411,7 +429,7 @@ ItemResponse::ItemResponse(bool verbose, const zim::Item& item, const std::strin m_mimeType(mimetype) { m_byteRange = byterange; - set_cacheable(); + set_kind(Response::ZIM_CONTENT); add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType); } @@ -423,14 +441,14 @@ std::unique_ptr ItemResponse::build(const InternalServer& server, cons if (noRange && is_compressible_mime_type(mimetype)) { // Return a contentResponse auto response = ContentResponse::build(server, item.getData(), mimetype); - response->set_cacheable(); + response->set_kind(Response::ZIM_CONTENT); response->m_byteRange = byteRange; return std::move(response); } if (byteRange.kind() == ByteRange::RESOLVED_UNSATISFIABLE) { auto response = Response::build_416(server, item.getSize()); - response->set_cacheable(); + response->set_kind(Response::ZIM_CONTENT); return response; } diff --git a/src/server/response.h b/src/server/response.h index 55c8fde4a..d1aa90ffc 100644 --- a/src/server/response.h +++ b/src/server/response.h @@ -45,6 +45,14 @@ class InternalServer; class RequestContext; class Response { + public: + enum Kind + { + STATIC_RESOURCE, + ZIM_CONTENT, + DYNAMIC_CONTENT + }; + public: Response(bool verbose); virtual ~Response() = default; @@ -57,7 +65,7 @@ class Response { MHD_Result send(const RequestContext& request, MHD_Connection* connection); void set_code(int code) { m_returnCode = code; } - void set_cacheable() { m_etag.set_option(ETag::CACHEABLE_ENTITY); } + void set_kind(Kind k); void set_server_id(const std::string& id) { m_etag.set_server_id(id); } void add_header(const std::string& name, const std::string& value) { m_customHeaders[name] = value; } @@ -68,6 +76,7 @@ class Response { MHD_Response* create_error_response(const RequestContext& request) const; protected: // data + Kind m_kind = DYNAMIC_CONTENT; bool m_verbose; int m_returnCode; ByteRange m_byteRange; diff --git a/test/server.cpp b/test/server.cpp index 3b5d273a5..bd67a3b78 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -35,7 +35,7 @@ struct Resource ResourceKind kind; const char* url; - bool etag_expected() const { return kind != DYNAMIC_CONTENT; } + bool etag_expected() const { return kind != STATIC_CONTENT; } }; std::ostream& operator<<(std::ostream& out, const Resource& r) @@ -47,7 +47,7 @@ std::ostream& operator<<(std::ostream& out, const Resource& r) typedef std::vector ResourceCollection; const ResourceCollection resources200Compressible{ - { STATIC_CONTENT, "/ROOT/" }, + { DYNAMIC_CONTENT, "/ROOT/" }, { STATIC_CONTENT, "/ROOT/skin/autoComplete.min.js" }, { STATIC_CONTENT, "/ROOT/skin/css/autoComplete.css" }, From 43c8da9b04e6dd11f5c2859f0719bd5c623bbf9e Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Sun, 9 Oct 2022 20:46:40 +0400 Subject: [PATCH 03/16] Testing of cache control --- test/server.cpp | 46 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/test/server.cpp b/test/server.cpp index bd67a3b78..904c2361d 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -316,6 +316,11 @@ std::string getHeaderValue(const Headers& headers, const std::string& name) return er.first->second; } +std::string getCacheControlHeader(const httplib::Response& r) +{ + return getHeaderValue(r.headers, "Cache-Control"); +} + TEST_F(CustomizedServerTest, NewResourcesCanBeAdded) { // ServerTest.404 verifies that "/ROOT/non-existent-item" doesn't exist @@ -958,6 +963,8 @@ TEST_F(ServerTest, RandomPageRedirectsToAnExistingArticle) ASSERT_EQ(302, g->status); ASSERT_TRUE(g->has_header("Location")); ASSERT_TRUE(kiwix::startsWith(g->get_header_value("Location"), "/ROOT/content/zimfile/A/")); + ASSERT_EQ(getCacheControlHeader(*g), "max-age=0, must-revalidate"); + ASSERT_FALSE(g->has_header("ETag")); } TEST_F(ServerTest, NonEndpointUrlsAreRedirectedToContentUrls) @@ -1001,6 +1008,8 @@ TEST_F(ServerTest, NonEndpointUrlsAreRedirectedToContentUrls) ASSERT_EQ(302, g->status) << ctx; ASSERT_TRUE(g->has_header("Location")) << ctx; ASSERT_EQ("/ROOT/content" + p, g->get_header_value("Location")) << ctx; + ASSERT_EQ(getCacheControlHeader(*g), "max-age=0, must-revalidate"); + ASSERT_FALSE(g->has_header("ETag")); } } @@ -1065,6 +1074,39 @@ TEST_F(ServerTest, HeadersAreTheSameInResponsesToHeadAndGetRequests) } } +TEST_F(ServerTest, CacheControlOfZimContent) +{ + for ( const Resource& res : all200Resources() ) { + if ( res.kind == ZIM_CONTENT ) { + const auto g = zfs1_->GET(res.url); + EXPECT_EQ(getCacheControlHeader(*g), "max-age=3600, must-revalidate") << res; + EXPECT_TRUE(g->has_header("ETag")) << res; + } + } +} + +TEST_F(ServerTest, CacheControlOfStaticContent) +{ + for ( const Resource& res : all200Resources() ) { + if ( res.kind == STATIC_CONTENT ) { + const auto g = zfs1_->GET(res.url); + EXPECT_EQ(getCacheControlHeader(*g), "max-age=31536000, immutable") << res; + EXPECT_FALSE(g->has_header("ETag")) << res; + } + } +} + +TEST_F(ServerTest, CacheControlOfDynamicContent) +{ + for ( const Resource& res : all200Resources() ) { + if ( res.kind == DYNAMIC_CONTENT ) { + const auto g = zfs1_->GET(res.url); + EXPECT_EQ(getCacheControlHeader(*g), "max-age=0, must-revalidate") << res; + EXPECT_TRUE(g->has_header("ETag")) << res; + } + } +} + TEST_F(ServerTest, ETagHeaderIsSetAsNeeded) { for ( const Resource& res : all200Resources() ) { @@ -1193,8 +1235,8 @@ TEST_F(ServerTest, IfNoneMatchRequestsWithMismatchingETagResultIn200Responses) const auto etag2 = etag.substr(0, etag.size() - 1) + "x\""; const auto h = zfs1_->HEAD(res.url, { {"If-None-Match", etag2} } ); const auto g2 = zfs1_->GET(res.url, { {"If-None-Match", etag2} } ); - EXPECT_EQ(200, h->status); - EXPECT_EQ(200, g2->status); + EXPECT_EQ(200, h->status) << res; + EXPECT_EQ(200, g2->status) << res; } } From a31ccb658816bcd7a2b03b697b613da72c28f3d8 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Sun, 9 Oct 2022 21:29:38 +0400 Subject: [PATCH 04/16] Decoupling ETags from the server id --- src/server/etag.cpp | 16 ++++++++-------- src/server/etag.h | 15 ++++++++------- src/server/response.h | 2 +- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/server/etag.cpp b/src/server/etag.cpp index e031ce46c..0492bdde1 100644 --- a/src/server/etag.cpp +++ b/src/server/etag.cpp @@ -41,7 +41,7 @@ const char all_options[] = "Zz"; static_assert(ETag::OPTION_COUNT == sizeof(all_options) - 1, ""); -bool isValidServerId(const std::string& s) +bool isValidETagBody(const std::string& s) { return !s.empty() && s.find_first_of("\"/") == std::string::npos; } @@ -83,17 +83,17 @@ bool ETag::get_option(Option opt) const std::string ETag::get_etag() const { - if ( m_serverId.empty() ) + if ( m_body.empty() ) return std::string(); - return "\"" + m_serverId + "/" + m_options + "\""; + return "\"" + m_body + "/" + m_options + "\""; } -ETag::ETag(const std::string& serverId, const std::string& options) +ETag::ETag(const std::string& body, const std::string& options) { - if ( isValidServerId(serverId) && isValidOptionsString(options) ) + if ( isValidETagBody(body) && isValidOptionsString(options) ) { - m_serverId = serverId; + m_body = body; m_options = options; } } @@ -115,7 +115,7 @@ ETag ETag::parse(std::string s) return ETag(s.substr(0, i), s.substr(i+1)); } -ETag ETag::match(const std::string& etags, const std::string& server_id) +ETag ETag::match(const std::string& etags, const std::string& body) { std::istringstream ss(etags); std::string etag_str; @@ -125,7 +125,7 @@ ETag ETag::match(const std::string& etags, const std::string& server_id) etag_str.pop_back(); const ETag etag = parse(etag_str); - if ( etag && etag.m_serverId == server_id ) + if ( etag && etag.m_body == body ) return etag; } diff --git a/src/server/etag.h b/src/server/etag.h index f8da02360..b6346e7e6 100644 --- a/src/server/etag.h +++ b/src/server/etag.h @@ -28,10 +28,11 @@ namespace kiwix { // The ETag string used by Kiwix server (more precisely, its value inside the // double quotes) consists of two parts: // -// 1. ServerId - The string obtained on server start up +// 1. Body - A string uniquely identifying the object or state from which +// the resource has been obtained. // -// 2. Options - Zero or more characters encoding the values of some of the -// headers of the response +// 2. Options - Zero or more characters encoding the type of the ETag and/or +// the values of some of the headers of the response // // The two parts are separated with a slash (/) symbol (which is always present, // even when the the options part is empty). Neither portion of a Kiwix ETag @@ -40,7 +41,7 @@ namespace kiwix { // // "abcdefghijklmn/" // "1234567890/z" -// "1234567890/cz" +// "6f1d19d0-633f-087b-fb55-7ac324ff9baf/Zz" // // The options part of the Kiwix ETag allows to correctly set the required // headers when responding to a conditional If-None-Match request with a 304 @@ -59,10 +60,10 @@ class ETag public: // functions ETag() {} - void set_server_id(const std::string& id) { m_serverId = id; } + void set_body(const std::string& s) { m_body = s; } void set_option(Option opt); - explicit operator bool() const { return !m_serverId.empty(); } + explicit operator bool() const { return !m_body.empty(); } bool get_option(Option opt) const; std::string get_etag() const; @@ -76,7 +77,7 @@ class ETag static ETag parse(std::string s); private: // data - std::string m_serverId; + std::string m_body; std::string m_options; }; diff --git a/src/server/response.h b/src/server/response.h index d1aa90ffc..31bcfc0da 100644 --- a/src/server/response.h +++ b/src/server/response.h @@ -66,7 +66,7 @@ class Response { void set_code(int code) { m_returnCode = code; } void set_kind(Kind k); - void set_server_id(const std::string& id) { m_etag.set_server_id(id); } + void set_server_id(const std::string& id) { m_etag.set_body(id); } void add_header(const std::string& name, const std::string& value) { m_customHeaders[name] = value; } int getReturnCode() const { return m_returnCode; } From b249edee60d0a505e45bba47410d63c324fdaa9a Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Sun, 9 Oct 2022 22:15:08 +0400 Subject: [PATCH 05/16] ETags for ZIM content use the ZIM file UUID --- src/server/internalServer.cpp | 31 ++++++++++++++++++++++++------- src/server/internalServer.h | 2 +- src/server/response.h | 3 ++- test/server.cpp | 15 +++++++++++++-- 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index 772b056e5..0583ba016 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -511,8 +511,10 @@ MHD_Result InternalServer::handlerCallback(struct MHD_Connection* connection, } } - if (response->getReturnCode() == MHD_HTTP_OK && !etag_not_needed(request)) - response->set_server_id(m_server_id); + if (response->getReturnCode() == MHD_HTTP_OK + && response->get_kind() == Response::DYNAMIC_CONTENT + && !etag_not_needed(request)) + response->set_etag_body(m_server_id); auto ret = response->send(request, connection); auto end_time = std::chrono::steady_clock::now(); @@ -542,7 +544,7 @@ std::unique_ptr InternalServer::handle_request(const RequestContext& r + urlNotFoundMsg; } - const ETag etag = get_matching_if_none_match_etag(request); + const ETag etag = get_matching_if_none_match_etag(request, m_server_id); if ( etag ) return Response::build_304(*this, etag); @@ -611,11 +613,11 @@ bool InternalServer::etag_not_needed(const RequestContext& request) const } ETag -InternalServer::get_matching_if_none_match_etag(const RequestContext& r) const +InternalServer::get_matching_if_none_match_etag(const RequestContext& r, const std::string& etagBody) const { try { const std::string etag_list = r.get_header(MHD_HTTP_HEADER_IF_NONE_MATCH); - return ETag::match(etag_list, m_server_id); + return ETag::match(etag_list, etagBody); } catch (const std::out_of_range&) { return ETag(); } @@ -1049,6 +1051,11 @@ std::unique_ptr InternalServer::handle_content(const RequestContext& r + suggestSearchMsg(searchURL, kiwix::urlDecode(pattern)); } + const std::string archiveUuid(archive->getUuid()); + const ETag etag = get_matching_if_none_match_etag(request, archiveUuid); + if ( etag ) + return Response::build_304(*this, etag); + auto urlStr = url.substr(prefixLength + bookName.size()); if (urlStr[0] == '/') { urlStr = urlStr.substr(1); @@ -1067,6 +1074,7 @@ std::unique_ptr InternalServer::handle_content(const RequestContext& r return build_redirect(bookName, getFinalItem(*archive, entry)); } auto response = ItemResponse::build(*this, request, entry.getItem()); + response->set_etag_body(archiveUuid); if (m_verbose.load()) { printf("Found %s\n", entry.getPath().c_str()); @@ -1120,6 +1128,11 @@ std::unique_ptr InternalServer::handle_raw(const RequestContext& reque + noSuchBookErrorMsg(bookName); } + const std::string archiveUuid(archive->getUuid()); + const ETag etag = get_matching_if_none_match_etag(request, archiveUuid); + if ( etag ) + return Response::build_304(*this, etag); + // Remove the beggining of the path: // /raw///foo // ^^^^^ ^ ^ @@ -1129,13 +1142,17 @@ std::unique_ptr InternalServer::handle_raw(const RequestContext& reque try { if (kind == "meta") { auto item = archive->getMetadataItem(itemPath); - return ItemResponse::build(*this, request, item); + auto response = ItemResponse::build(*this, request, item); + response->set_etag_body(archiveUuid); + return response; } else { auto entry = archive->getEntryByPath(itemPath); if (entry.isRedirect()) { return build_redirect(bookName, entry.getItem(true)); } - return ItemResponse::build(*this, request, entry.getItem()); + auto response = ItemResponse::build(*this, request, entry.getItem()); + response->set_etag_body(archiveUuid); + return response; } } catch (zim::EntryNotFound& e ) { if (m_verbose.load()) { diff --git a/src/server/internalServer.h b/src/server/internalServer.h index 6f523336e..48fab9c39 100644 --- a/src/server/internalServer.h +++ b/src/server/internalServer.h @@ -148,7 +148,7 @@ class InternalServer { MustacheData get_default_data() const; bool etag_not_needed(const RequestContext& r) const; - ETag get_matching_if_none_match_etag(const RequestContext& request) const; + ETag get_matching_if_none_match_etag(const RequestContext& request, const std::string& etagBody) const; std::pair selectBooks(const RequestContext& r) const; SearchInfo getSearchInfo(const RequestContext& r) const; diff --git a/src/server/response.h b/src/server/response.h index 31bcfc0da..4ed07e628 100644 --- a/src/server/response.h +++ b/src/server/response.h @@ -66,7 +66,8 @@ class Response { void set_code(int code) { m_returnCode = code; } void set_kind(Kind k); - void set_server_id(const std::string& id) { m_etag.set_body(id); } + Kind get_kind() const { return m_kind; } + void set_etag_body(const std::string& id) { m_etag.set_body(id); } void add_header(const std::string& name, const std::string& value) { m_customHeaders[name] = value; } int getReturnCode() const { return m_returnCode; } diff --git a/test/server.cpp b/test/server.cpp index 904c2361d..99fd3716a 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -1136,17 +1136,28 @@ TEST_F(ServerTest, ETagIsTheSameAcrossHeadAndGet) } } -TEST_F(ServerTest, DifferentServerInstancesProduceDifferentETags) +TEST_F(ServerTest, DifferentServerInstancesProduceDifferentETagsForDynamicContent) { ZimFileServer zfs2(SERVER_PORT + 1, ZimFileServer::DEFAULT_OPTIONS, ZIMFILES); for ( const Resource& res : all200Resources() ) { - if ( !res.etag_expected() ) continue; + if ( res.kind != DYNAMIC_CONTENT ) continue; const auto h1 = zfs1_->HEAD(res.url); const auto h2 = zfs2.HEAD(res.url); EXPECT_NE(h1->get_header_value("ETag"), h2->get_header_value("ETag")); } } +TEST_F(ServerTest, DifferentServerInstancesProduceIdenticalETagsForZimContent) +{ + ZimFileServer zfs2(SERVER_PORT + 1, ZimFileServer::DEFAULT_OPTIONS, ZIMFILES); + for ( const Resource& res : all200Resources() ) { + if ( res.kind != ZIM_CONTENT ) continue; + const auto h1 = zfs1_->HEAD(res.url); + const auto h2 = zfs2.HEAD(res.url); + EXPECT_EQ(h1->get_header_value("ETag"), h2->get_header_value("ETag")); + } +} + TEST_F(ServerTest, CompressionInfluencesETag) { for ( const Resource& res : resources200Compressible ) { From c91df1cb269f4dde3520cc5e41fbccbeebd761de Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Sun, 9 Oct 2022 22:18:14 +0400 Subject: [PATCH 06/16] Two private funcs of InternalServer became free --- src/server/internalServer.cpp | 36 +++++++++++++++++------------------ src/server/internalServer.h | 2 -- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index 0583ba016..50606db4c 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -218,6 +218,24 @@ struct CustomizedResourceData std::string resourceFilePath; }; +bool etag_not_needed(const RequestContext& request) +{ + const std::string url = request.get_url(); + return kiwix::startsWith(url, "/skin") + || url == "/random"; +} + +ETag +get_matching_if_none_match_etag(const RequestContext& r, const std::string& etagBody) +{ + try { + const std::string etag_list = r.get_header(MHD_HTTP_HEADER_IF_NONE_MATCH); + return ETag::match(etag_list, etagBody); + } catch (const std::out_of_range&) { + return ETag(); + } +} + } // unnamed namespace std::pair InternalServer::selectBooks(const RequestContext& request) const @@ -605,24 +623,6 @@ MustacheData InternalServer::get_default_data() const return data; } -bool InternalServer::etag_not_needed(const RequestContext& request) const -{ - const std::string url = request.get_url(); - return kiwix::startsWith(url, "/skin") - || url == "/random"; -} - -ETag -InternalServer::get_matching_if_none_match_etag(const RequestContext& r, const std::string& etagBody) const -{ - try { - const std::string etag_list = r.get_header(MHD_HTTP_HEADER_IF_NONE_MATCH); - return ETag::match(etag_list, etagBody); - } catch (const std::out_of_range&) { - return ETag(); - } -} - std::unique_ptr InternalServer::build_homepage(const RequestContext& request) { return ContentResponse::build(*this, m_indexTemplateString, get_default_data(), "text/html; charset=utf-8"); diff --git a/src/server/internalServer.h b/src/server/internalServer.h index 48fab9c39..f6b7c3116 100644 --- a/src/server/internalServer.h +++ b/src/server/internalServer.h @@ -147,8 +147,6 @@ class InternalServer { MustacheData get_default_data() const; - bool etag_not_needed(const RequestContext& r) const; - ETag get_matching_if_none_match_etag(const RequestContext& request, const std::string& etagBody) const; std::pair selectBooks(const RequestContext& r) const; SearchInfo getSearchInfo(const RequestContext& r) const; From 6b8d6232f03d4f64ca0052e755ebb72fc79f9e0d Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Sun, 9 Oct 2022 22:25:21 +0400 Subject: [PATCH 07/16] InternalServer::getLibraryId() --- src/server/internalServer.cpp | 12 ++++++++---- src/server/internalServer.h | 3 ++- src/server/internalServer_catalog_v2.cpp | 19 ++++++++++--------- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index 50606db4c..c28306a82 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -461,7 +461,6 @@ bool InternalServer::start() { } auto server_start_time = std::chrono::system_clock::now().time_since_epoch(); m_server_id = kiwix::to_string(server_start_time.count()); - m_library_id = m_server_id; return true; } @@ -532,7 +531,7 @@ MHD_Result InternalServer::handlerCallback(struct MHD_Connection* connection, if (response->getReturnCode() == MHD_HTTP_OK && response->get_kind() == Response::DYNAMIC_CONTENT && !etag_not_needed(request)) - response->set_etag_body(m_server_id); + response->set_etag_body(getLibraryId()); auto ret = response->send(request, connection); auto end_time = std::chrono::steady_clock::now(); @@ -554,6 +553,11 @@ bool isEndpointUrl(const std::string& url, const std::string& endpoint) } // unnamed namespace +std::string InternalServer::getLibraryId() const +{ + return m_server_id + "." + kiwix::to_string(mp_library->getRevision()); +} + std::unique_ptr InternalServer::handle_request(const RequestContext& request) { try { @@ -562,7 +566,7 @@ std::unique_ptr InternalServer::handle_request(const RequestContext& r + urlNotFoundMsg; } - const ETag etag = get_matching_if_none_match_etag(request, m_server_id); + const ETag etag = get_matching_if_none_match_etag(request, getLibraryId()); if ( etag ) return Response::build_304(*this, etag); @@ -968,7 +972,7 @@ std::unique_ptr InternalServer::handle_catalog(const RequestContext& r zim::Uuid uuid; kiwix::OPDSDumper opdsDumper(mp_library); opdsDumper.setRootLocation(m_root); - opdsDumper.setLibraryId(m_library_id); + opdsDumper.setLibraryId(getLibraryId()); std::vector bookIdsToDump; if (url == "root.xml") { uuid = zim::Uuid::generate(host); diff --git a/src/server/internalServer.h b/src/server/internalServer.h index f6b7c3116..02360443a 100644 --- a/src/server/internalServer.h +++ b/src/server/internalServer.h @@ -152,6 +152,8 @@ class InternalServer { bool isLocallyCustomizedResource(const std::string& url) const; + std::string getLibraryId() const; + private: // types class LockableSuggestionSearcher; typedef ConcurrentCache> SearchCache; @@ -178,7 +180,6 @@ class InternalServer { SuggestionSearcherCache suggestionSearcherCache; std::string m_server_id; - std::string m_library_id; class CustomizedResources; std::unique_ptr m_customizedResources; diff --git a/src/server/internalServer_catalog_v2.cpp b/src/server/internalServer_catalog_v2.cpp index b082dd1c1..802d81cda 100644 --- a/src/server/internalServer_catalog_v2.cpp +++ b/src/server/internalServer_catalog_v2.cpp @@ -77,17 +77,18 @@ std::unique_ptr InternalServer::handle_catalog_v2(const RequestContext std::unique_ptr InternalServer::handle_catalog_v2_root(const RequestContext& request) { + const std::string libraryId = getLibraryId(); return ContentResponse::build( *this, RESOURCE::templates::catalog_v2_root_xml, kainjow::mustache::object{ {"date", gen_date_str()}, {"endpoint_root", m_root + "/catalog/v2"}, - {"feed_id", gen_uuid(m_library_id)}, - {"all_entries_feed_id", gen_uuid(m_library_id + "/entries")}, - {"partial_entries_feed_id", gen_uuid(m_library_id + "/partial_entries")}, - {"category_list_feed_id", gen_uuid(m_library_id + "/categories")}, - {"language_list_feed_id", gen_uuid(m_library_id + "/languages")} + {"feed_id", gen_uuid(libraryId)}, + {"all_entries_feed_id", gen_uuid(libraryId + "/entries")}, + {"partial_entries_feed_id", gen_uuid(libraryId + "/partial_entries")}, + {"category_list_feed_id", gen_uuid(libraryId + "/categories")}, + {"language_list_feed_id", gen_uuid(libraryId + "/languages")} }, "application/atom+xml;profile=opds-catalog;kind=navigation" ); @@ -97,7 +98,7 @@ std::unique_ptr InternalServer::handle_catalog_v2_entries(const Reques { OPDSDumper opdsDumper(mp_library); opdsDumper.setRootLocation(m_root); - opdsDumper.setLibraryId(m_library_id); + opdsDumper.setLibraryId(getLibraryId()); const auto bookIds = search_catalog(request, opdsDumper); const auto opdsFeed = opdsDumper.dumpOPDSFeedV2(bookIds, request.get_query(), partial); return ContentResponse::build( @@ -118,7 +119,7 @@ std::unique_ptr InternalServer::handle_catalog_v2_complete_entry(const OPDSDumper opdsDumper(mp_library); opdsDumper.setRootLocation(m_root); - opdsDumper.setLibraryId(m_library_id); + opdsDumper.setLibraryId(getLibraryId()); const auto opdsFeed = opdsDumper.dumpOPDSCompleteEntry(entryId); return ContentResponse::build( *this, @@ -131,7 +132,7 @@ std::unique_ptr InternalServer::handle_catalog_v2_categories(const Req { OPDSDumper opdsDumper(mp_library); opdsDumper.setRootLocation(m_root); - opdsDumper.setLibraryId(m_library_id); + opdsDumper.setLibraryId(getLibraryId()); return ContentResponse::build( *this, opdsDumper.categoriesOPDSFeed(), @@ -143,7 +144,7 @@ std::unique_ptr InternalServer::handle_catalog_v2_languages(const Requ { OPDSDumper opdsDumper(mp_library); opdsDumper.setRootLocation(m_root); - opdsDumper.setLibraryId(m_library_id); + opdsDumper.setLibraryId(getLibraryId()); return ContentResponse::build( *this, opdsDumper.languagesOPDSFeed(), From 9fd1423100c773c2c9a15e0b4e673df05aa00ac9 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Sun, 9 Oct 2022 22:46:31 +0400 Subject: [PATCH 08/16] Small clean-up --- src/server/internalServer.cpp | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index c28306a82..2f65586b5 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -218,11 +218,11 @@ struct CustomizedResourceData std::string resourceFilePath; }; -bool etag_not_needed(const RequestContext& request) +bool responseMustBeETaggedWithLibraryId(const Response& response, const RequestContext& request) { - const std::string url = request.get_url(); - return kiwix::startsWith(url, "/skin") - || url == "/random"; + return response.getReturnCode() == MHD_HTTP_OK + && response.get_kind() == Response::DYNAMIC_CONTENT + && request.get_url() != "/random"; } ETag @@ -528,10 +528,9 @@ MHD_Result InternalServer::handlerCallback(struct MHD_Connection* connection, } } - if (response->getReturnCode() == MHD_HTTP_OK - && response->get_kind() == Response::DYNAMIC_CONTENT - && !etag_not_needed(request)) + if ( responseMustBeETaggedWithLibraryId(*response, request) ) { response->set_etag_body(getLibraryId()); + } auto ret = response->send(request, connection); auto end_time = std::chrono::steady_clock::now(); From ce8b2bf9d91ceea02fb9edac4f194c9847ca6962 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Sun, 9 Oct 2022 22:58:19 +0400 Subject: [PATCH 09/16] Library::removeBookById() updates the revision --- include/library.h | 4 ++-- src/library.cpp | 6 +++++- test/library.cpp | 6 ++++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/include/library.h b/include/library.h index 87b0315ea..856a3a9ef 100644 --- a/include/library.h +++ b/include/library.h @@ -332,8 +332,8 @@ class Library /** * Return the current revision of the library. * - * The revision of the library is updated (incremented by one) only by - * the addBook() operation. + * The revision of the library is updated (incremented by one) by + * the addBook() and removeBookById() operations. * * @return Current revision of the library. */ diff --git a/src/library.cpp b/src/library.cpp index 07eb428cb..508d7c8e1 100644 --- a/src/library.cpp +++ b/src/library.cpp @@ -221,7 +221,11 @@ bool Library::removeBookById(const std::string& id) // Having a too big cache is not a problem here (or it would have been before) // (And setMaxSize doesn't actually reduce the cache size, extra cached items // will be removed in put or getOrPut). - return mp_impl->m_books.erase(id) == 1; + const bool bookWasRemoved = mp_impl->m_books.erase(id) == 1; + if ( bookWasRemoved ) { + ++mp_impl->m_revision; + } + return bookWasRemoved; } Library::Revision Library::getRevision() const diff --git a/test/library.cpp b/test/library.cpp index ef41c4d7c..60762cb51 100644 --- a/test/library.cpp +++ b/test/library.cpp @@ -801,8 +801,14 @@ TEST_F(LibraryTest, removeBooksNotUpdatedSince) lib.addBook(lib.getBookByIdThreadSafe(id)); } + EXPECT_GT(lib.getRevision(), rev); + + const uint64_t rev2 = lib.getRevision(); + EXPECT_EQ(9u, lib.removeBooksNotUpdatedSince(rev)); + EXPECT_GT(lib.getRevision(), rev2); + EXPECT_FILTER_RESULTS(kiwix::Filter(), "Islam Stack Exchange", "Movies & TV Stack Exchange", From 6bc7e0178d192e579dd9a8428b39c449b6f6c77c Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Mon, 10 Oct 2022 12:35:14 +0400 Subject: [PATCH 10/16] Added all static resources to the server unit-test --- test/server.cpp | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/test/server.cpp b/test/server.cpp index 99fd3716a..544d47fc2 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -49,9 +49,17 @@ typedef std::vector ResourceCollection; const ResourceCollection resources200Compressible{ { DYNAMIC_CONTENT, "/ROOT/" }, + { STATIC_CONTENT, "/ROOT/viewer" }, + { STATIC_CONTENT, "/ROOT/skin/autoComplete.min.js" }, { STATIC_CONTENT, "/ROOT/skin/css/autoComplete.css" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/favicon.ico" }, + { STATIC_CONTENT, "/ROOT/skin/index.css" }, + { STATIC_CONTENT, "/ROOT/skin/index.js" }, + { STATIC_CONTENT, "/ROOT/skin/iso6391To3.js" }, + { STATIC_CONTENT, "/ROOT/skin/isotope.pkgd.min.js" }, { STATIC_CONTENT, "/ROOT/skin/taskbar.css" }, + { STATIC_CONTENT, "/ROOT/skin/viewer.js" }, { DYNAMIC_CONTENT, "/ROOT/catalog/search" }, @@ -67,8 +75,30 @@ const ResourceCollection resources200Compressible{ }; const ResourceCollection resources200Uncompressible{ + { STATIC_CONTENT, "/ROOT/skin/bittorrent.png" }, + { STATIC_CONTENT, "/ROOT/skin/blank.html" }, { STATIC_CONTENT, "/ROOT/skin/caret.png" }, { STATIC_CONTENT, "/ROOT/skin/css/images/search.svg" }, + { STATIC_CONTENT, "/ROOT/skin/download.png" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/android-chrome-192x192.png" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/android-chrome-512x512.png" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/apple-touch-icon.png" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/browserconfig.xml" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/favicon-16x16.png" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/favicon-32x32.png" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/mstile-144x144.png" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/mstile-150x150.png" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/mstile-310x150.png" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/mstile-310x310.png" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/mstile-70x70.png" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/safari-pinned-tab.svg" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/site.webmanifest" }, + { STATIC_CONTENT, "/ROOT/skin/fonts/Poppins.ttf" }, + { STATIC_CONTENT, "/ROOT/skin/fonts/Roboto.ttf" }, + { STATIC_CONTENT, "/ROOT/skin/hash.png" }, + { STATIC_CONTENT, "/ROOT/skin/magnet.png" }, + { STATIC_CONTENT, "/ROOT/skin/search-icon.svg" }, + { STATIC_CONTENT, "/ROOT/skin/search_results.css" }, { ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Title" }, { ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Description" }, From b62486c2f9ad3f903603036582fcebef456b9851 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Mon, 10 Oct 2022 12:59:16 +0400 Subject: [PATCH 11/16] Added /catalog URLs to general purpose server tests --- test/server.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/server.cpp b/test/server.cpp index 544d47fc2..eee1dc643 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -63,6 +63,11 @@ const ResourceCollection resources200Compressible{ { DYNAMIC_CONTENT, "/ROOT/catalog/search" }, + { DYNAMIC_CONTENT, "/ROOT/catalog/v2/root.xml" }, + { DYNAMIC_CONTENT, "/ROOT/catalog/v2/languages" }, + { DYNAMIC_CONTENT, "/ROOT/catalog/v2/entries" }, + { DYNAMIC_CONTENT, "/ROOT/catalog/v2/partial_entries" }, + { DYNAMIC_CONTENT, "/ROOT/search?content=zimfile&pattern=a" }, { DYNAMIC_CONTENT, "/ROOT/suggest?content=zimfile&term=ray" }, @@ -109,6 +114,11 @@ const ResourceCollection resources200Uncompressible{ { ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Creator" }, { ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Publisher" }, + { DYNAMIC_CONTENT, "/ROOT/catalog/root.xml" }, + { DYNAMIC_CONTENT, "/ROOT/catalog/searchdescription.xml" }, + + { DYNAMIC_CONTENT, "/ROOT/catalog/v2/categories" }, + { DYNAMIC_CONTENT, "/ROOT/catalog/v2/searchdescription.xml" }, { DYNAMIC_CONTENT, "/ROOT/catalog/v2/illustration/6f1d19d0-633f-087b-fb55-7ac324ff9baf?size=48" }, { DYNAMIC_CONTENT, "/ROOT/catch/external?source=www.example.com" }, From 12a638750e47b747570a2f8fdcea9f013594ecf3 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Mon, 10 Oct 2022 13:37:56 +0400 Subject: [PATCH 12/16] Fixed URLs to static resources without cacheids One (hopefully, last) remaining relative URL to a static resource is the reference to ./search-icon.svg found in skin/index.css to which KIWIXCACHEID could not be applied because of the limitations of the resource preprocessing script `kiwix-resources`. --- static/templates/index.html | 2 +- static/viewer.html | 4 ++-- test/server.cpp | 5 +++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/static/templates/index.html b/static/templates/index.html index 42bbc6a1b..2bd067a23 100644 --- a/static/templates/index.html +++ b/static/templates/index.html @@ -13,7 +13,7 @@ - + diff --git a/static/viewer.html b/static/viewer.html index 74ab8814f..65f25d9a3 100644 --- a/static/viewer.html +++ b/static/viewer.html @@ -16,7 +16,7 @@ } const root = getRootLocation(); - const blankPageUrl = `${root}/skin/blank.html`; + const blankPageUrl = root + "/skin/blank.html?KIWIXCACHEID"; if ( location.hash == '' ) { location.href = root + '/'; @@ -58,7 +58,7 @@ diff --git a/test/server.cpp b/test/server.cpp index eee1dc643..d909f78da 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -222,7 +222,7 @@ R"EXPECTEDRESULT( href="/ROOT/skin/index.css?cacheid=3b470cee" - + @@ -247,8 +247,9 @@ R"EXPECTEDRESULT( - const blankPageUrl = `${root}/skin/blank.html`; + const blankPageUrl = root + "/skin/blank.html?cacheid=6b1fa032"; + src="./skin/blank.html?cacheid=6b1fa032" title="ZIM content" width="100%" )EXPECTEDRESULT" }, { From b9f60ecfe96e952be115495295910b9bde41f5fe Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Tue, 11 Oct 2022 14:21:08 +0400 Subject: [PATCH 13/16] Handling of cacheid when serving static resources During static resource preprocessing and compilation their cacheid values are embedded into libkiwix and can be accessed at runtime. If a static resource is requsted without specifying any cacheid it is served as dynamic content (with short TTL and the library id used for the ETag, though using the cacheid for the ETag would be better). If a cacheid is supplied in the request it must match the cacheid of the resource (otherwise a 404 Not Found error is returned) whereupon the resource is served as immutable content. Known issues: - One issue is caused by the fact that some static resources don't get a cacheid; this is resolved in the next commit. - Interaction of this change with the support for dynamically customizing static resources (via KIWIX_SERVE_CUSTOMIZED_RESOURCES env var) was not addressed. --- scripts/kiwix-compile-resources | 32 ++++++++-- scripts/kiwix-resources | 15 +++-- src/server/internalServer.cpp | 25 +++++++- test/server.cpp | 103 +++++++++++++++++++++----------- 4 files changed, 129 insertions(+), 46 deletions(-) diff --git a/scripts/kiwix-compile-resources b/scripts/kiwix-compile-resources index 7c1bf3a8a..7d308afc3 100755 --- a/scripts/kiwix-compile-resources +++ b/scripts/kiwix-compile-resources @@ -52,15 +52,21 @@ resource_getter_template = """ return RESOURCE::{identifier}; """ +resource_cacheid_getter_template = """ + if (name == "{common_name}") + return "{cacheid}"; +""" + resource_decl_template = """{namespaces_open} extern const std::string {identifier}; {namespaces_close}""" class Resource: - def __init__(self, base_dirs, filename): - filename = filename.strip() + def __init__(self, base_dirs, filename, cacheid=None): + filename = filename self.filename = filename self.identifier = full_identifier(filename) + self.cacheid = cacheid found = False for base_dir in base_dirs: try: @@ -71,7 +77,7 @@ class Resource: except FileNotFoundError: continue if not found: - raise Exception("Impossible to found {}".format(filename)) + raise Exception("Resource not found: {}".format(filename)) def dump_impl(self): nb_row = len(self.data)//16 + (1 if len(self.data) % 16 else 0) @@ -93,6 +99,12 @@ class Resource: identifier="::".join(self.identifier) ) + def dump_cacheid_getter(self): + return resource_cacheid_getter_template.format( + common_name=self.filename, + cacheid=self.cacheid + ) + def dump_decl(self): return resource_decl_template.format( namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]), @@ -123,7 +135,12 @@ static std::string init_resource(const char* name, const unsigned char* content, const std::string& getResource_{basename}(const std::string& name) {{ {RESOURCES_GETTER} - throw ResourceNotFound("Resource not found."); + throw ResourceNotFound("Resource not found: " + name); +}} + +const char* getResourceCacheId_{basename}(const std::string& name) {{ +{RESOURCE_CACHEID_GETTER} + return nullptr; }} {RESOURCES} @@ -134,6 +151,7 @@ def gen_c_file(resources, basename): return master_c_template.format( RESOURCES="\n\n".join(r.dump_impl() for r in resources), RESOURCES_GETTER="\n\n".join(r.dump_getter() for r in resources), + RESOURCE_CACHEID_GETTER="\n\n".join(r.dump_cacheid_getter() for r in resources if r.cacheid is not None), include_file=basename, basename=to_identifier(basename) ) @@ -159,8 +177,10 @@ class ResourceNotFound : public std::runtime_error {{ }}; const std::string& getResource_{basename}(const std::string& name); +const char* getResourceCacheId_{basename}(const std::string& name); #define getResource(a) (getResource_{basename}(a)) +#define getResourceCacheId(a) (getResourceCacheId_{basename}(a)) #endif // KIWIX_{BASENAME} @@ -189,8 +209,8 @@ if __name__ == "__main__": base_dir = os.path.dirname(os.path.realpath(args.resource_file)) source_dir = args.source_dir or [] with open(args.resource_file, 'r') as f: - resources = [Resource([base_dir]+source_dir, filename) - for filename in f.readlines()] + resources = [Resource([base_dir]+source_dir, *line.strip().split()) + for line in f.readlines()] h_identifier = to_identifier(os.path.basename(args.hfile)) with open(args.hfile, 'w') as f: diff --git a/scripts/kiwix-resources b/scripts/kiwix-resources index 481b03f06..5fb7b8db2 100755 --- a/scripts/kiwix-resources +++ b/scripts/kiwix-resources @@ -99,16 +99,21 @@ def preprocess_resource(resource_path): print(preprocessed_content, end='', file=target) -def copy_file(src_path, dst_path): - with open(src_path, 'rb') as src: - with open(dst_path, 'wb') as dst: - dst.write(src.read()) +def copy_resource_list_file(src_path, dst_path): + with open(src_path, 'r') as src: + with open(dst_path, 'w') as dst: + for line in src: + res = line.strip() + if line.startswith("skin/") and res in resource_revisions: + dst.write(res + " " + resource_revisions[res] + "\n") + else: + dst.write(line) def preprocess_resources(resource_file_path): resource_filename = os.path.basename(resource_file_path) for resource in read_resource_file(resource_file_path): preprocess_resource(resource) - copy_file(resource_file_path, os.path.join(OUT_DIR, resource_filename)) + copy_resource_list_file(resource_file_path, os.path.join(OUT_DIR, resource_filename)) if __name__ == "__main__": parser = argparse.ArgumentParser() diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index 2f65586b5..eeadd15d2 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -748,6 +748,25 @@ std::unique_ptr InternalServer::handle_viewer_settings(const RequestCo return ContentResponse::build(*this, RESOURCE::templates::viewer_settings_js, data, "application/javascript; charset=utf-8"); } +namespace +{ + +Response::Kind staticResourceAccessType(const RequestContext& req, const char* expectedCacheid) +{ + if ( expectedCacheid == nullptr ) + return Response::DYNAMIC_CONTENT; + + try { + if ( expectedCacheid != req.get_argument("cacheid") ) + throw ResourceNotFound("Wrong cacheid"); + return Response::STATIC_RESOURCE; + } catch( const std::out_of_range& ) { + return Response::DYNAMIC_CONTENT; + } +} + +} // unnamed namespace + std::unique_ptr InternalServer::handle_skin(const RequestContext& request) { if (m_verbose.load()) { @@ -758,12 +777,16 @@ std::unique_ptr InternalServer::handle_skin(const RequestContext& requ auto resourceName = isRequestForViewer ? "viewer.html" : request.get_url().substr(1); + + const char* const resourceCacheId = getResourceCacheId(resourceName); + try { + const auto accessType = staticResourceAccessType(request, resourceCacheId); auto response = ContentResponse::build( *this, getResource(resourceName), getMimeTypeForFile(resourceName)); - response->set_kind(Response::STATIC_RESOURCE); + response->set_kind(accessType); return std::move(response); } catch (const ResourceNotFound& e) { return HTTP404Response(*this, request) diff --git a/test/server.cpp b/test/server.cpp index d909f78da..1f646574c 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -49,17 +49,27 @@ typedef std::vector ResourceCollection; const ResourceCollection resources200Compressible{ { DYNAMIC_CONTENT, "/ROOT/" }, - { STATIC_CONTENT, "/ROOT/viewer" }, + { DYNAMIC_CONTENT, "/ROOT/viewer" }, + { DYNAMIC_CONTENT, "/ROOT/viewer?cacheid=whatever" }, - { STATIC_CONTENT, "/ROOT/skin/autoComplete.min.js" }, - { STATIC_CONTENT, "/ROOT/skin/css/autoComplete.css" }, - { STATIC_CONTENT, "/ROOT/skin/favicon/favicon.ico" }, - { STATIC_CONTENT, "/ROOT/skin/index.css" }, - { STATIC_CONTENT, "/ROOT/skin/index.js" }, - { STATIC_CONTENT, "/ROOT/skin/iso6391To3.js" }, - { STATIC_CONTENT, "/ROOT/skin/isotope.pkgd.min.js" }, - { STATIC_CONTENT, "/ROOT/skin/taskbar.css" }, - { STATIC_CONTENT, "/ROOT/skin/viewer.js" }, + { DYNAMIC_CONTENT, "/ROOT/skin/autoComplete.min.js" }, + { STATIC_CONTENT, "/ROOT/skin/autoComplete.min.js?cacheid=1191aaaf" }, + { DYNAMIC_CONTENT, "/ROOT/skin/css/autoComplete.css" }, + { STATIC_CONTENT, "/ROOT/skin/css/autoComplete.css?cacheid=08951e06" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/favicon.ico" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/favicon.ico?cacheid=fba03a27" }, + { DYNAMIC_CONTENT, "/ROOT/skin/index.css" }, + { STATIC_CONTENT, "/ROOT/skin/index.css?cacheid=3b470cee" }, + { DYNAMIC_CONTENT, "/ROOT/skin/index.js" }, + { STATIC_CONTENT, "/ROOT/skin/index.js?cacheid=2f5a81ac" }, + { DYNAMIC_CONTENT, "/ROOT/skin/iso6391To3.js" }, + { STATIC_CONTENT, "/ROOT/skin/iso6391To3.js?cacheid=ecde2bb3" }, + { DYNAMIC_CONTENT, "/ROOT/skin/isotope.pkgd.min.js" }, + { STATIC_CONTENT, "/ROOT/skin/isotope.pkgd.min.js?cacheid=2e48d392" }, + { DYNAMIC_CONTENT, "/ROOT/skin/taskbar.css" }, + { STATIC_CONTENT, "/ROOT/skin/taskbar.css?cacheid=216d6b5d" }, + { DYNAMIC_CONTENT, "/ROOT/skin/viewer.js" }, + { STATIC_CONTENT, "/ROOT/skin/viewer.js?cacheid=51e745c2" }, { DYNAMIC_CONTENT, "/ROOT/catalog/search" }, @@ -80,30 +90,54 @@ const ResourceCollection resources200Compressible{ }; const ResourceCollection resources200Uncompressible{ - { STATIC_CONTENT, "/ROOT/skin/bittorrent.png" }, - { STATIC_CONTENT, "/ROOT/skin/blank.html" }, - { STATIC_CONTENT, "/ROOT/skin/caret.png" }, - { STATIC_CONTENT, "/ROOT/skin/css/images/search.svg" }, - { STATIC_CONTENT, "/ROOT/skin/download.png" }, - { STATIC_CONTENT, "/ROOT/skin/favicon/android-chrome-192x192.png" }, - { STATIC_CONTENT, "/ROOT/skin/favicon/android-chrome-512x512.png" }, - { STATIC_CONTENT, "/ROOT/skin/favicon/apple-touch-icon.png" }, - { STATIC_CONTENT, "/ROOT/skin/favicon/browserconfig.xml" }, - { STATIC_CONTENT, "/ROOT/skin/favicon/favicon-16x16.png" }, - { STATIC_CONTENT, "/ROOT/skin/favicon/favicon-32x32.png" }, - { STATIC_CONTENT, "/ROOT/skin/favicon/mstile-144x144.png" }, - { STATIC_CONTENT, "/ROOT/skin/favicon/mstile-150x150.png" }, - { STATIC_CONTENT, "/ROOT/skin/favicon/mstile-310x150.png" }, - { STATIC_CONTENT, "/ROOT/skin/favicon/mstile-310x310.png" }, - { STATIC_CONTENT, "/ROOT/skin/favicon/mstile-70x70.png" }, - { STATIC_CONTENT, "/ROOT/skin/favicon/safari-pinned-tab.svg" }, - { STATIC_CONTENT, "/ROOT/skin/favicon/site.webmanifest" }, - { STATIC_CONTENT, "/ROOT/skin/fonts/Poppins.ttf" }, - { STATIC_CONTENT, "/ROOT/skin/fonts/Roboto.ttf" }, - { STATIC_CONTENT, "/ROOT/skin/hash.png" }, - { STATIC_CONTENT, "/ROOT/skin/magnet.png" }, - { STATIC_CONTENT, "/ROOT/skin/search-icon.svg" }, - { STATIC_CONTENT, "/ROOT/skin/search_results.css" }, + { DYNAMIC_CONTENT, "/ROOT/skin/bittorrent.png" }, + { STATIC_CONTENT, "/ROOT/skin/bittorrent.png?cacheid=4f5c6882" }, + { DYNAMIC_CONTENT, "/ROOT/skin/blank.html" }, + { STATIC_CONTENT, "/ROOT/skin/blank.html?cacheid=6b1fa032" }, + { DYNAMIC_CONTENT, "/ROOT/skin/caret.png" }, + { STATIC_CONTENT, "/ROOT/skin/caret.png?cacheid=22b942b4" }, + { DYNAMIC_CONTENT, "/ROOT/skin/css/images/search.svg" }, + //{ STATIC_CONTENT, "/ROOT/skin/css/images/search.svg?cacheid=XXXX" }, + { DYNAMIC_CONTENT, "/ROOT/skin/download.png" }, + { STATIC_CONTENT, "/ROOT/skin/download.png?cacheid=a39aa502" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/android-chrome-192x192.png" }, + //{ STATIC_CONTENT, "/ROOT/skin/favicon/android-chrome-192x192.png?cacheid=" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/android-chrome-512x512.png" }, + //{ STATIC_CONTENT, "/ROOT/skin/favicon/android-chrome-512x512.png?cacheid=" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/apple-touch-icon.png" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/apple-touch-icon.png?cacheid=f86f8df3" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/browserconfig.xml" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/browserconfig.xml?cacheid=f29a7c4a" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/favicon-16x16.png" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/favicon-16x16.png?cacheid=a986fedc" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/favicon-32x32.png" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/favicon-32x32.png?cacheid=79ded625" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-144x144.png" }, + //{ STATIC_CONTENT, "/ROOT/skin/favicon/mstile-144x144.png?cacheid=" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-150x150.png" }, + //{ STATIC_CONTENT, "/ROOT/skin/favicon/mstile-150x150.png?cacheid=" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-310x150.png" }, + //{ STATIC_CONTENT, "/ROOT/skin/favicon/mstile-310x150.png?cacheid=" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-310x310.png" }, + //{ STATIC_CONTENT, "/ROOT/skin/favicon/mstile-310x310.png?cacheid=" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-70x70.png" }, + //{ STATIC_CONTENT, "/ROOT/skin/favicon/mstile-70x70.png?cacheid=" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/safari-pinned-tab.svg" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/safari-pinned-tab.svg?cacheid=8d487e95" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/site.webmanifest" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/site.webmanifest?cacheid=bc396efb" }, + { DYNAMIC_CONTENT, "/ROOT/skin/fonts/Poppins.ttf" }, + { STATIC_CONTENT, "/ROOT/skin/fonts/Poppins.ttf?cacheid=af705837" }, + { DYNAMIC_CONTENT, "/ROOT/skin/fonts/Roboto.ttf" }, + { STATIC_CONTENT, "/ROOT/skin/fonts/Roboto.ttf?cacheid=84d10248" }, + { DYNAMIC_CONTENT, "/ROOT/skin/hash.png" }, + { STATIC_CONTENT, "/ROOT/skin/hash.png?cacheid=f836e872" }, + { DYNAMIC_CONTENT, "/ROOT/skin/magnet.png" }, + { STATIC_CONTENT, "/ROOT/skin/magnet.png?cacheid=73b6bddf" }, + { DYNAMIC_CONTENT, "/ROOT/skin/search-icon.svg" }, + //{ STATIC_CONTENT, "/ROOT/skin/search-icon.svg?cacheid=" }, + { DYNAMIC_CONTENT, "/ROOT/skin/search_results.css" }, + { STATIC_CONTENT, "/ROOT/skin/search_results.css?cacheid=76d39c84" }, { ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Title" }, { ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Description" }, @@ -299,6 +333,7 @@ const char* urls404[] = { "/", "/zimfile", "/ROOT/skin/non-existent-skin-resource", + "/ROOT/skin/autoComplete.min.js?cacheid=wrongcacheid", "/ROOT/catalog", "/ROOT/catalog/", "/ROOT/catalog/non-existent-item", From 415ec4109989ef2d2b7443f3dbcd54081b934115 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Tue, 11 Oct 2022 14:32:57 +0400 Subject: [PATCH 14/16] Cacheids are computed for all static resources Before this change cacheids were computed only for those static resources that were referenced from other resources via KIWIXCACHEID. A few static resources without such references existed. Now all resources under skin/ have their cacheids computed. --- scripts/kiwix-resources | 5 ++++- test/server.cpp | 18 +++++++++--------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/scripts/kiwix-resources b/scripts/kiwix-resources index 5fb7b8db2..569c25e41 100755 --- a/scripts/kiwix-resources +++ b/scripts/kiwix-resources @@ -112,7 +112,10 @@ def copy_resource_list_file(src_path, dst_path): def preprocess_resources(resource_file_path): resource_filename = os.path.basename(resource_file_path) for resource in read_resource_file(resource_file_path): - preprocess_resource(resource) + if resource.startswith('skin/'): + get_resource_revision(resource) + else: + preprocess_resource(resource) copy_resource_list_file(resource_file_path, os.path.join(OUT_DIR, resource_filename)) if __name__ == "__main__": diff --git a/test/server.cpp b/test/server.cpp index 1f646574c..ceabaa887 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -97,13 +97,13 @@ const ResourceCollection resources200Uncompressible{ { DYNAMIC_CONTENT, "/ROOT/skin/caret.png" }, { STATIC_CONTENT, "/ROOT/skin/caret.png?cacheid=22b942b4" }, { DYNAMIC_CONTENT, "/ROOT/skin/css/images/search.svg" }, - //{ STATIC_CONTENT, "/ROOT/skin/css/images/search.svg?cacheid=XXXX" }, + { STATIC_CONTENT, "/ROOT/skin/css/images/search.svg?cacheid=f0bbdb80" }, { DYNAMIC_CONTENT, "/ROOT/skin/download.png" }, { STATIC_CONTENT, "/ROOT/skin/download.png?cacheid=a39aa502" }, { DYNAMIC_CONTENT, "/ROOT/skin/favicon/android-chrome-192x192.png" }, - //{ STATIC_CONTENT, "/ROOT/skin/favicon/android-chrome-192x192.png?cacheid=" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/android-chrome-192x192.png?cacheid=bfac158b" }, { DYNAMIC_CONTENT, "/ROOT/skin/favicon/android-chrome-512x512.png" }, - //{ STATIC_CONTENT, "/ROOT/skin/favicon/android-chrome-512x512.png?cacheid=" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/android-chrome-512x512.png?cacheid=380c3653" }, { DYNAMIC_CONTENT, "/ROOT/skin/favicon/apple-touch-icon.png" }, { STATIC_CONTENT, "/ROOT/skin/favicon/apple-touch-icon.png?cacheid=f86f8df3" }, { DYNAMIC_CONTENT, "/ROOT/skin/favicon/browserconfig.xml" }, @@ -113,15 +113,15 @@ const ResourceCollection resources200Uncompressible{ { DYNAMIC_CONTENT, "/ROOT/skin/favicon/favicon-32x32.png" }, { STATIC_CONTENT, "/ROOT/skin/favicon/favicon-32x32.png?cacheid=79ded625" }, { DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-144x144.png" }, - //{ STATIC_CONTENT, "/ROOT/skin/favicon/mstile-144x144.png?cacheid=" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/mstile-144x144.png?cacheid=c25a7641" }, { DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-150x150.png" }, - //{ STATIC_CONTENT, "/ROOT/skin/favicon/mstile-150x150.png?cacheid=" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/mstile-150x150.png?cacheid=6fa6f467" }, { DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-310x150.png" }, - //{ STATIC_CONTENT, "/ROOT/skin/favicon/mstile-310x150.png?cacheid=" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/mstile-310x150.png?cacheid=e0ed9032" }, { DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-310x310.png" }, - //{ STATIC_CONTENT, "/ROOT/skin/favicon/mstile-310x310.png?cacheid=" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/mstile-310x310.png?cacheid=26b20530" }, { DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-70x70.png" }, - //{ STATIC_CONTENT, "/ROOT/skin/favicon/mstile-70x70.png?cacheid=" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/mstile-70x70.png?cacheid=64ffd9dc" }, { DYNAMIC_CONTENT, "/ROOT/skin/favicon/safari-pinned-tab.svg" }, { STATIC_CONTENT, "/ROOT/skin/favicon/safari-pinned-tab.svg?cacheid=8d487e95" }, { DYNAMIC_CONTENT, "/ROOT/skin/favicon/site.webmanifest" }, @@ -135,7 +135,7 @@ const ResourceCollection resources200Uncompressible{ { DYNAMIC_CONTENT, "/ROOT/skin/magnet.png" }, { STATIC_CONTENT, "/ROOT/skin/magnet.png?cacheid=73b6bddf" }, { DYNAMIC_CONTENT, "/ROOT/skin/search-icon.svg" }, - //{ STATIC_CONTENT, "/ROOT/skin/search-icon.svg?cacheid=" }, + { STATIC_CONTENT, "/ROOT/skin/search-icon.svg?cacheid=b10ae7ed" }, { DYNAMIC_CONTENT, "/ROOT/skin/search_results.css" }, { STATIC_CONTENT, "/ROOT/skin/search_results.css?cacheid=76d39c84" }, From 602c20f16038be95cf46837512291f53d1e03901 Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Thu, 13 Oct 2022 13:58:32 +0400 Subject: [PATCH 15/16] Removed unused resource skin/css/images/search.svg --- static/resources_list.txt | 1 - static/skin/css/images/search.svg | 8 -------- test/server.cpp | 2 -- 3 files changed, 11 deletions(-) delete mode 100644 static/skin/css/images/search.svg diff --git a/static/resources_list.txt b/static/resources_list.txt index 2730f1b20..47b60bffb 100644 --- a/static/resources_list.txt +++ b/static/resources_list.txt @@ -36,7 +36,6 @@ opensearchdescription.xml ft_opensearchdescription.xml catalog_v2_searchdescription.xml skin/css/autoComplete.css -skin/css/images/search.svg skin/favicon/android-chrome-192x192.png skin/favicon/android-chrome-512x512.png skin/favicon/apple-touch-icon.png diff --git a/static/skin/css/images/search.svg b/static/skin/css/images/search.svg deleted file mode 100644 index 8063ea104..000000000 --- a/static/skin/css/images/search.svg +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/test/server.cpp b/test/server.cpp index ceabaa887..57b4f9e78 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -96,8 +96,6 @@ const ResourceCollection resources200Uncompressible{ { STATIC_CONTENT, "/ROOT/skin/blank.html?cacheid=6b1fa032" }, { DYNAMIC_CONTENT, "/ROOT/skin/caret.png" }, { STATIC_CONTENT, "/ROOT/skin/caret.png?cacheid=22b942b4" }, - { DYNAMIC_CONTENT, "/ROOT/skin/css/images/search.svg" }, - { STATIC_CONTENT, "/ROOT/skin/css/images/search.svg?cacheid=f0bbdb80" }, { DYNAMIC_CONTENT, "/ROOT/skin/download.png" }, { STATIC_CONTENT, "/ROOT/skin/download.png?cacheid=a39aa502" }, { DYNAMIC_CONTENT, "/ROOT/skin/favicon/android-chrome-192x192.png" }, From 18a18c17a9bd48dfca476af64147339aa132dd1a Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Thu, 13 Oct 2022 14:06:46 +0400 Subject: [PATCH 16/16] Applied KIWIXCACHEID to skin/search-icon.svg --- static/skin/index.css | 2 +- test/server.cpp | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/static/skin/index.css b/static/skin/index.css index 4ed8425eb..f6e16be20 100644 --- a/static/skin/index.css +++ b/static/skin/index.css @@ -105,7 +105,7 @@ body { border-radius: 10px; border: solid 1px #b5b2b2; padding: 10px; - background-image: url('./search-icon.svg'); + background-image: url('../skin/search-icon.svg?KIWIXCACHEID'); background-repeat: no-repeat; background-position: right center; background-origin: content-box; diff --git a/test/server.cpp b/test/server.cpp index 57b4f9e78..0bb97f76a 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -59,7 +59,7 @@ const ResourceCollection resources200Compressible{ { DYNAMIC_CONTENT, "/ROOT/skin/favicon/favicon.ico" }, { STATIC_CONTENT, "/ROOT/skin/favicon/favicon.ico?cacheid=fba03a27" }, { DYNAMIC_CONTENT, "/ROOT/skin/index.css" }, - { STATIC_CONTENT, "/ROOT/skin/index.css?cacheid=3b470cee" }, + { STATIC_CONTENT, "/ROOT/skin/index.css?cacheid=0f9ba34e" }, { DYNAMIC_CONTENT, "/ROOT/skin/index.js" }, { STATIC_CONTENT, "/ROOT/skin/index.js?cacheid=2f5a81ac" }, { DYNAMIC_CONTENT, "/ROOT/skin/iso6391To3.js" }, @@ -250,7 +250,7 @@ TEST_F(ServerTest, CacheIdsOfStaticResources) const std::vector testData{ { /* url */ "/ROOT/", -R"EXPECTEDRESULT( href="/ROOT/skin/index.css?cacheid=3b470cee" +R"EXPECTEDRESULT( href="/ROOT/skin/index.css?cacheid=0f9ba34e" @@ -263,6 +263,11 @@ R"EXPECTEDRESULT( href="/ROOT/skin/index.css?cacheid=3b470cee" +)EXPECTEDRESULT" + }, + { + /* url */ "/ROOT/skin/index.css", +R"EXPECTEDRESULT( background-image: url('../skin/search-icon.svg?cacheid=b10ae7ed'); )EXPECTEDRESULT" }, {