diff --git a/include/library.h b/include/library.h index 87b0315ea..856a3a9ef 100644 --- a/include/library.h +++ b/include/library.h @@ -332,8 +332,8 @@ class Library /** * Return the current revision of the library. * - * The revision of the library is updated (incremented by one) only by - * the addBook() operation. + * The revision of the library is updated (incremented by one) by + * the addBook() and removeBookById() operations. * * @return Current revision of the library. */ diff --git a/scripts/kiwix-compile-resources b/scripts/kiwix-compile-resources index 7c1bf3a8a..7d308afc3 100755 --- a/scripts/kiwix-compile-resources +++ b/scripts/kiwix-compile-resources @@ -52,15 +52,21 @@ resource_getter_template = """ return RESOURCE::{identifier}; """ +resource_cacheid_getter_template = """ + if (name == "{common_name}") + return "{cacheid}"; +""" + resource_decl_template = """{namespaces_open} extern const std::string {identifier}; {namespaces_close}""" class Resource: - def __init__(self, base_dirs, filename): - filename = filename.strip() + def __init__(self, base_dirs, filename, cacheid=None): + filename = filename self.filename = filename self.identifier = full_identifier(filename) + self.cacheid = cacheid found = False for base_dir in base_dirs: try: @@ -71,7 +77,7 @@ class Resource: except FileNotFoundError: continue if not found: - raise Exception("Impossible to found {}".format(filename)) + raise Exception("Resource not found: {}".format(filename)) def dump_impl(self): nb_row = len(self.data)//16 + (1 if len(self.data) % 16 else 0) @@ -93,6 +99,12 @@ class Resource: identifier="::".join(self.identifier) ) + def dump_cacheid_getter(self): + return resource_cacheid_getter_template.format( + common_name=self.filename, + cacheid=self.cacheid + ) + def dump_decl(self): return resource_decl_template.format( namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]), @@ -123,7 +135,12 @@ static std::string init_resource(const char* name, const unsigned char* content, const std::string& getResource_{basename}(const std::string& name) {{ {RESOURCES_GETTER} - throw ResourceNotFound("Resource not found."); + throw ResourceNotFound("Resource not found: " + name); +}} + +const char* getResourceCacheId_{basename}(const std::string& name) {{ +{RESOURCE_CACHEID_GETTER} + return nullptr; }} {RESOURCES} @@ -134,6 +151,7 @@ def gen_c_file(resources, basename): return master_c_template.format( RESOURCES="\n\n".join(r.dump_impl() for r in resources), RESOURCES_GETTER="\n\n".join(r.dump_getter() for r in resources), + RESOURCE_CACHEID_GETTER="\n\n".join(r.dump_cacheid_getter() for r in resources if r.cacheid is not None), include_file=basename, basename=to_identifier(basename) ) @@ -159,8 +177,10 @@ class ResourceNotFound : public std::runtime_error {{ }}; const std::string& getResource_{basename}(const std::string& name); +const char* getResourceCacheId_{basename}(const std::string& name); #define getResource(a) (getResource_{basename}(a)) +#define getResourceCacheId(a) (getResourceCacheId_{basename}(a)) #endif // KIWIX_{BASENAME} @@ -189,8 +209,8 @@ if __name__ == "__main__": base_dir = os.path.dirname(os.path.realpath(args.resource_file)) source_dir = args.source_dir or [] with open(args.resource_file, 'r') as f: - resources = [Resource([base_dir]+source_dir, filename) - for filename in f.readlines()] + resources = [Resource([base_dir]+source_dir, *line.strip().split()) + for line in f.readlines()] h_identifier = to_identifier(os.path.basename(args.hfile)) with open(args.hfile, 'w') as f: diff --git a/scripts/kiwix-resources b/scripts/kiwix-resources index 481b03f06..569c25e41 100755 --- a/scripts/kiwix-resources +++ b/scripts/kiwix-resources @@ -99,16 +99,24 @@ def preprocess_resource(resource_path): print(preprocessed_content, end='', file=target) -def copy_file(src_path, dst_path): - with open(src_path, 'rb') as src: - with open(dst_path, 'wb') as dst: - dst.write(src.read()) +def copy_resource_list_file(src_path, dst_path): + with open(src_path, 'r') as src: + with open(dst_path, 'w') as dst: + for line in src: + res = line.strip() + if line.startswith("skin/") and res in resource_revisions: + dst.write(res + " " + resource_revisions[res] + "\n") + else: + dst.write(line) def preprocess_resources(resource_file_path): resource_filename = os.path.basename(resource_file_path) for resource in read_resource_file(resource_file_path): - preprocess_resource(resource) - copy_file(resource_file_path, os.path.join(OUT_DIR, resource_filename)) + if resource.startswith('skin/'): + get_resource_revision(resource) + else: + preprocess_resource(resource) + copy_resource_list_file(resource_file_path, os.path.join(OUT_DIR, resource_filename)) if __name__ == "__main__": parser = argparse.ArgumentParser() diff --git a/src/library.cpp b/src/library.cpp index 07eb428cb..508d7c8e1 100644 --- a/src/library.cpp +++ b/src/library.cpp @@ -221,7 +221,11 @@ bool Library::removeBookById(const std::string& id) // Having a too big cache is not a problem here (or it would have been before) // (And setMaxSize doesn't actually reduce the cache size, extra cached items // will be removed in put or getOrPut). - return mp_impl->m_books.erase(id) == 1; + const bool bookWasRemoved = mp_impl->m_books.erase(id) == 1; + if ( bookWasRemoved ) { + ++mp_impl->m_revision; + } + return bookWasRemoved; } Library::Revision Library::getRevision() const diff --git a/src/server/etag.cpp b/src/server/etag.cpp index 1ba89470f..0492bdde1 100644 --- a/src/server/etag.cpp +++ b/src/server/etag.cpp @@ -37,11 +37,11 @@ namespace { // into the ETag for ETag::Option opt. // IMPORTANT: The characters in all_options must come in sorted order (so that // IMPORTANT: isValidOptionsString() works correctly). -const char all_options[] = "cz"; +const char all_options[] = "Zz"; static_assert(ETag::OPTION_COUNT == sizeof(all_options) - 1, ""); -bool isValidServerId(const std::string& s) +bool isValidETagBody(const std::string& s) { return !s.empty() && s.find_first_of("\"/") == std::string::npos; } @@ -83,17 +83,17 @@ bool ETag::get_option(Option opt) const std::string ETag::get_etag() const { - if ( m_serverId.empty() ) + if ( m_body.empty() ) return std::string(); - return "\"" + m_serverId + "/" + m_options + "\""; + return "\"" + m_body + "/" + m_options + "\""; } -ETag::ETag(const std::string& serverId, const std::string& options) +ETag::ETag(const std::string& body, const std::string& options) { - if ( isValidServerId(serverId) && isValidOptionsString(options) ) + if ( isValidETagBody(body) && isValidOptionsString(options) ) { - m_serverId = serverId; + m_body = body; m_options = options; } } @@ -115,7 +115,7 @@ ETag ETag::parse(std::string s) return ETag(s.substr(0, i), s.substr(i+1)); } -ETag ETag::match(const std::string& etags, const std::string& server_id) +ETag ETag::match(const std::string& etags, const std::string& body) { std::istringstream ss(etags); std::string etag_str; @@ -125,7 +125,7 @@ ETag ETag::match(const std::string& etags, const std::string& server_id) etag_str.pop_back(); const ETag etag = parse(etag_str); - if ( etag && etag.m_serverId == server_id ) + if ( etag && etag.m_body == body ) return etag; } diff --git a/src/server/etag.h b/src/server/etag.h index 49ff0e724..b6346e7e6 100644 --- a/src/server/etag.h +++ b/src/server/etag.h @@ -28,10 +28,11 @@ namespace kiwix { // The ETag string used by Kiwix server (more precisely, its value inside the // double quotes) consists of two parts: // -// 1. ServerId - The string obtained on server start up +// 1. Body - A string uniquely identifying the object or state from which +// the resource has been obtained. // -// 2. Options - Zero or more characters encoding the values of some of the -// headers of the response +// 2. Options - Zero or more characters encoding the type of the ETag and/or +// the values of some of the headers of the response // // The two parts are separated with a slash (/) symbol (which is always present, // even when the the options part is empty). Neither portion of a Kiwix ETag @@ -40,7 +41,7 @@ namespace kiwix { // // "abcdefghijklmn/" // "1234567890/z" -// "1234567890/cz" +// "6f1d19d0-633f-087b-fb55-7ac324ff9baf/Zz" // // The options part of the Kiwix ETag allows to correctly set the required // headers when responding to a conditional If-None-Match request with a 304 @@ -51,7 +52,7 @@ class ETag { public: // types enum Option { - CACHEABLE_ENTITY, + ZIM_CONTENT, COMPRESSED_CONTENT, OPTION_COUNT }; @@ -59,10 +60,10 @@ class ETag public: // functions ETag() {} - void set_server_id(const std::string& id) { m_serverId = id; } + void set_body(const std::string& s) { m_body = s; } void set_option(Option opt); - explicit operator bool() const { return !m_serverId.empty(); } + explicit operator bool() const { return !m_body.empty(); } bool get_option(Option opt) const; std::string get_etag() const; @@ -76,7 +77,7 @@ class ETag static ETag parse(std::string s); private: // data - std::string m_serverId; + std::string m_body; std::string m_options; }; diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index d6e6a91a4..eeadd15d2 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -218,6 +218,24 @@ struct CustomizedResourceData std::string resourceFilePath; }; +bool responseMustBeETaggedWithLibraryId(const Response& response, const RequestContext& request) +{ + return response.getReturnCode() == MHD_HTTP_OK + && response.get_kind() == Response::DYNAMIC_CONTENT + && request.get_url() != "/random"; +} + +ETag +get_matching_if_none_match_etag(const RequestContext& r, const std::string& etagBody) +{ + try { + const std::string etag_list = r.get_header(MHD_HTTP_HEADER_IF_NONE_MATCH); + return ETag::match(etag_list, etagBody); + } catch (const std::out_of_range&) { + return ETag(); + } +} + } // unnamed namespace std::pair InternalServer::selectBooks(const RequestContext& request) const @@ -443,7 +461,6 @@ bool InternalServer::start() { } auto server_start_time = std::chrono::system_clock::now().time_since_epoch(); m_server_id = kiwix::to_string(server_start_time.count()); - m_library_id = m_server_id; return true; } @@ -511,8 +528,9 @@ MHD_Result InternalServer::handlerCallback(struct MHD_Connection* connection, } } - if (response->getReturnCode() == MHD_HTTP_OK && !etag_not_needed(request)) - response->set_server_id(m_server_id); + if ( responseMustBeETaggedWithLibraryId(*response, request) ) { + response->set_etag_body(getLibraryId()); + } auto ret = response->send(request, connection); auto end_time = std::chrono::steady_clock::now(); @@ -534,6 +552,11 @@ bool isEndpointUrl(const std::string& url, const std::string& endpoint) } // unnamed namespace +std::string InternalServer::getLibraryId() const +{ + return m_server_id + "." + kiwix::to_string(mp_library->getRevision()); +} + std::unique_ptr InternalServer::handle_request(const RequestContext& request) { try { @@ -542,7 +565,7 @@ std::unique_ptr InternalServer::handle_request(const RequestContext& r + urlNotFoundMsg; } - const ETag etag = get_matching_if_none_match_etag(request); + const ETag etag = get_matching_if_none_match_etag(request, getLibraryId()); if ( etag ) return Response::build_304(*this, etag); @@ -603,27 +626,6 @@ MustacheData InternalServer::get_default_data() const return data; } -bool InternalServer::etag_not_needed(const RequestContext& request) const -{ - const std::string url = request.get_url(); - return kiwix::startsWith(url, "/catalog") - || url == "/search" - || url == "/suggest" - || url == "/random" - || url == "/catch/external"; -} - -ETag -InternalServer::get_matching_if_none_match_etag(const RequestContext& r) const -{ - try { - const std::string etag_list = r.get_header(MHD_HTTP_HEADER_IF_NONE_MATCH); - return ETag::match(etag_list, m_server_id); - } catch (const std::out_of_range&) { - return ETag(); - } -} - std::unique_ptr InternalServer::build_homepage(const RequestContext& request) { return ContentResponse::build(*this, m_indexTemplateString, get_default_data(), "text/html; charset=utf-8"); @@ -746,6 +748,25 @@ std::unique_ptr InternalServer::handle_viewer_settings(const RequestCo return ContentResponse::build(*this, RESOURCE::templates::viewer_settings_js, data, "application/javascript; charset=utf-8"); } +namespace +{ + +Response::Kind staticResourceAccessType(const RequestContext& req, const char* expectedCacheid) +{ + if ( expectedCacheid == nullptr ) + return Response::DYNAMIC_CONTENT; + + try { + if ( expectedCacheid != req.get_argument("cacheid") ) + throw ResourceNotFound("Wrong cacheid"); + return Response::STATIC_RESOURCE; + } catch( const std::out_of_range& ) { + return Response::DYNAMIC_CONTENT; + } +} + +} // unnamed namespace + std::unique_ptr InternalServer::handle_skin(const RequestContext& request) { if (m_verbose.load()) { @@ -756,12 +777,16 @@ std::unique_ptr InternalServer::handle_skin(const RequestContext& requ auto resourceName = isRequestForViewer ? "viewer.html" : request.get_url().substr(1); + + const char* const resourceCacheId = getResourceCacheId(resourceName); + try { + const auto accessType = staticResourceAccessType(request, resourceCacheId); auto response = ContentResponse::build( *this, getResource(resourceName), getMimeTypeForFile(resourceName)); - response->set_cacheable(); + response->set_kind(accessType); return std::move(response); } catch (const ResourceNotFound& e) { return HTTP404Response(*this, request) @@ -969,7 +994,7 @@ std::unique_ptr InternalServer::handle_catalog(const RequestContext& r zim::Uuid uuid; kiwix::OPDSDumper opdsDumper(mp_library); opdsDumper.setRootLocation(m_root); - opdsDumper.setLibraryId(m_library_id); + opdsDumper.setLibraryId(getLibraryId()); std::vector bookIdsToDump; if (url == "root.xml") { uuid = zim::Uuid::generate(host); @@ -1052,6 +1077,11 @@ std::unique_ptr InternalServer::handle_content(const RequestContext& r + suggestSearchMsg(searchURL, kiwix::urlDecode(pattern)); } + const std::string archiveUuid(archive->getUuid()); + const ETag etag = get_matching_if_none_match_etag(request, archiveUuid); + if ( etag ) + return Response::build_304(*this, etag); + auto urlStr = url.substr(prefixLength + bookName.size()); if (urlStr[0] == '/') { urlStr = urlStr.substr(1); @@ -1070,6 +1100,7 @@ std::unique_ptr InternalServer::handle_content(const RequestContext& r return build_redirect(bookName, getFinalItem(*archive, entry)); } auto response = ItemResponse::build(*this, request, entry.getItem()); + response->set_etag_body(archiveUuid); if (m_verbose.load()) { printf("Found %s\n", entry.getPath().c_str()); @@ -1123,6 +1154,11 @@ std::unique_ptr InternalServer::handle_raw(const RequestContext& reque + noSuchBookErrorMsg(bookName); } + const std::string archiveUuid(archive->getUuid()); + const ETag etag = get_matching_if_none_match_etag(request, archiveUuid); + if ( etag ) + return Response::build_304(*this, etag); + // Remove the beggining of the path: // /raw///foo // ^^^^^ ^ ^ @@ -1132,13 +1168,17 @@ std::unique_ptr InternalServer::handle_raw(const RequestContext& reque try { if (kind == "meta") { auto item = archive->getMetadataItem(itemPath); - return ItemResponse::build(*this, request, item); + auto response = ItemResponse::build(*this, request, item); + response->set_etag_body(archiveUuid); + return response; } else { auto entry = archive->getEntryByPath(itemPath); if (entry.isRedirect()) { return build_redirect(bookName, entry.getItem(true)); } - return ItemResponse::build(*this, request, entry.getItem()); + auto response = ItemResponse::build(*this, request, entry.getItem()); + response->set_etag_body(archiveUuid); + return response; } } catch (zim::EntryNotFound& e ) { if (m_verbose.load()) { diff --git a/src/server/internalServer.h b/src/server/internalServer.h index 6f523336e..02360443a 100644 --- a/src/server/internalServer.h +++ b/src/server/internalServer.h @@ -147,13 +147,13 @@ class InternalServer { MustacheData get_default_data() const; - bool etag_not_needed(const RequestContext& r) const; - ETag get_matching_if_none_match_etag(const RequestContext& request) const; std::pair selectBooks(const RequestContext& r) const; SearchInfo getSearchInfo(const RequestContext& r) const; bool isLocallyCustomizedResource(const std::string& url) const; + std::string getLibraryId() const; + private: // types class LockableSuggestionSearcher; typedef ConcurrentCache> SearchCache; @@ -180,7 +180,6 @@ class InternalServer { SuggestionSearcherCache suggestionSearcherCache; std::string m_server_id; - std::string m_library_id; class CustomizedResources; std::unique_ptr m_customizedResources; diff --git a/src/server/internalServer_catalog_v2.cpp b/src/server/internalServer_catalog_v2.cpp index b082dd1c1..802d81cda 100644 --- a/src/server/internalServer_catalog_v2.cpp +++ b/src/server/internalServer_catalog_v2.cpp @@ -77,17 +77,18 @@ std::unique_ptr InternalServer::handle_catalog_v2(const RequestContext std::unique_ptr InternalServer::handle_catalog_v2_root(const RequestContext& request) { + const std::string libraryId = getLibraryId(); return ContentResponse::build( *this, RESOURCE::templates::catalog_v2_root_xml, kainjow::mustache::object{ {"date", gen_date_str()}, {"endpoint_root", m_root + "/catalog/v2"}, - {"feed_id", gen_uuid(m_library_id)}, - {"all_entries_feed_id", gen_uuid(m_library_id + "/entries")}, - {"partial_entries_feed_id", gen_uuid(m_library_id + "/partial_entries")}, - {"category_list_feed_id", gen_uuid(m_library_id + "/categories")}, - {"language_list_feed_id", gen_uuid(m_library_id + "/languages")} + {"feed_id", gen_uuid(libraryId)}, + {"all_entries_feed_id", gen_uuid(libraryId + "/entries")}, + {"partial_entries_feed_id", gen_uuid(libraryId + "/partial_entries")}, + {"category_list_feed_id", gen_uuid(libraryId + "/categories")}, + {"language_list_feed_id", gen_uuid(libraryId + "/languages")} }, "application/atom+xml;profile=opds-catalog;kind=navigation" ); @@ -97,7 +98,7 @@ std::unique_ptr InternalServer::handle_catalog_v2_entries(const Reques { OPDSDumper opdsDumper(mp_library); opdsDumper.setRootLocation(m_root); - opdsDumper.setLibraryId(m_library_id); + opdsDumper.setLibraryId(getLibraryId()); const auto bookIds = search_catalog(request, opdsDumper); const auto opdsFeed = opdsDumper.dumpOPDSFeedV2(bookIds, request.get_query(), partial); return ContentResponse::build( @@ -118,7 +119,7 @@ std::unique_ptr InternalServer::handle_catalog_v2_complete_entry(const OPDSDumper opdsDumper(mp_library); opdsDumper.setRootLocation(m_root); - opdsDumper.setLibraryId(m_library_id); + opdsDumper.setLibraryId(getLibraryId()); const auto opdsFeed = opdsDumper.dumpOPDSCompleteEntry(entryId); return ContentResponse::build( *this, @@ -131,7 +132,7 @@ std::unique_ptr InternalServer::handle_catalog_v2_categories(const Req { OPDSDumper opdsDumper(mp_library); opdsDumper.setRootLocation(m_root); - opdsDumper.setLibraryId(m_library_id); + opdsDumper.setLibraryId(getLibraryId()); return ContentResponse::build( *this, opdsDumper.categoriesOPDSFeed(), @@ -143,7 +144,7 @@ std::unique_ptr InternalServer::handle_catalog_v2_languages(const Requ { OPDSDumper opdsDumper(mp_library); opdsDumper.setRootLocation(m_root); - opdsDumper.setLibraryId(m_library_id); + opdsDumper.setLibraryId(getLibraryId()); return ContentResponse::build( *this, opdsDumper.languagesOPDSFeed(), diff --git a/src/server/response.cpp b/src/server/response.cpp index 7067ff207..2c751e7b0 100644 --- a/src/server/response.cpp +++ b/src/server/response.cpp @@ -102,6 +102,14 @@ bool compress(std::string &content) { } +const char* getCacheControlHeader(Response::Kind k) +{ + switch(k) { + case Response::STATIC_RESOURCE: return "max-age=31536000, immutable"; + case Response::ZIM_CONTENT: return "max-age=3600, must-revalidate"; + default: return "max-age=0, must-revalidate"; + } +} } // unnamed namespace @@ -112,6 +120,13 @@ Response::Response(bool verbose) add_header(MHD_HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN, "*"); } +void Response::set_kind(Kind k) +{ + m_kind = k; + if ( k == ZIM_CONTENT ) + m_etag.set_option(ETag::ZIM_CONTENT); +} + std::unique_ptr Response::build(const InternalServer& server) { return std::unique_ptr(new Response(server.m_verbose.load())); @@ -122,6 +137,9 @@ std::unique_ptr Response::build_304(const InternalServer& server, cons auto response = Response::build(server); response->set_code(MHD_HTTP_NOT_MODIFIED); response->m_etag = etag; + if ( etag.get_option(ETag::ZIM_CONTENT) ) { + response->set_kind(Response::ZIM_CONTENT); + } if ( etag.get_option(ETag::COMPRESSED_CONTENT) ) { response->add_header(MHD_HTTP_HEADER_VARY, "Accept-Encoding"); } @@ -355,7 +373,7 @@ MHD_Result Response::send(const RequestContext& request, MHD_Connection* connect MHD_Response* response = create_mhd_response(request); MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL, - m_etag.get_option(ETag::CACHEABLE_ENTITY) ? "max-age=2723040, public" : "no-cache, no-store, must-revalidate"); + getCacheControlHeader(m_kind)); const std::string etag = m_etag.get_etag(); if ( ! etag.empty() ) MHD_add_response_header(response, MHD_HTTP_HEADER_ETAG, etag.c_str()); @@ -411,7 +429,7 @@ ItemResponse::ItemResponse(bool verbose, const zim::Item& item, const std::strin m_mimeType(mimetype) { m_byteRange = byterange; - set_cacheable(); + set_kind(Response::ZIM_CONTENT); add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType); } @@ -423,14 +441,14 @@ std::unique_ptr ItemResponse::build(const InternalServer& server, cons if (noRange && is_compressible_mime_type(mimetype)) { // Return a contentResponse auto response = ContentResponse::build(server, item.getData(), mimetype); - response->set_cacheable(); + response->set_kind(Response::ZIM_CONTENT); response->m_byteRange = byteRange; return std::move(response); } if (byteRange.kind() == ByteRange::RESOLVED_UNSATISFIABLE) { auto response = Response::build_416(server, item.getSize()); - response->set_cacheable(); + response->set_kind(Response::ZIM_CONTENT); return response; } diff --git a/src/server/response.h b/src/server/response.h index 55c8fde4a..4ed07e628 100644 --- a/src/server/response.h +++ b/src/server/response.h @@ -45,6 +45,14 @@ class InternalServer; class RequestContext; class Response { + public: + enum Kind + { + STATIC_RESOURCE, + ZIM_CONTENT, + DYNAMIC_CONTENT + }; + public: Response(bool verbose); virtual ~Response() = default; @@ -57,8 +65,9 @@ class Response { MHD_Result send(const RequestContext& request, MHD_Connection* connection); void set_code(int code) { m_returnCode = code; } - void set_cacheable() { m_etag.set_option(ETag::CACHEABLE_ENTITY); } - void set_server_id(const std::string& id) { m_etag.set_server_id(id); } + void set_kind(Kind k); + Kind get_kind() const { return m_kind; } + void set_etag_body(const std::string& id) { m_etag.set_body(id); } void add_header(const std::string& name, const std::string& value) { m_customHeaders[name] = value; } int getReturnCode() const { return m_returnCode; } @@ -68,6 +77,7 @@ class Response { MHD_Response* create_error_response(const RequestContext& request) const; protected: // data + Kind m_kind = DYNAMIC_CONTENT; bool m_verbose; int m_returnCode; ByteRange m_byteRange; diff --git a/static/resources_list.txt b/static/resources_list.txt index 2730f1b20..47b60bffb 100644 --- a/static/resources_list.txt +++ b/static/resources_list.txt @@ -36,7 +36,6 @@ opensearchdescription.xml ft_opensearchdescription.xml catalog_v2_searchdescription.xml skin/css/autoComplete.css -skin/css/images/search.svg skin/favicon/android-chrome-192x192.png skin/favicon/android-chrome-512x512.png skin/favicon/apple-touch-icon.png diff --git a/static/skin/css/images/search.svg b/static/skin/css/images/search.svg deleted file mode 100644 index 8063ea104..000000000 --- a/static/skin/css/images/search.svg +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/static/skin/index.css b/static/skin/index.css index 4ed8425eb..f6e16be20 100644 --- a/static/skin/index.css +++ b/static/skin/index.css @@ -105,7 +105,7 @@ body { border-radius: 10px; border: solid 1px #b5b2b2; padding: 10px; - background-image: url('./search-icon.svg'); + background-image: url('../skin/search-icon.svg?KIWIXCACHEID'); background-repeat: no-repeat; background-position: right center; background-origin: content-box; diff --git a/static/templates/index.html b/static/templates/index.html index 42bbc6a1b..2bd067a23 100644 --- a/static/templates/index.html +++ b/static/templates/index.html @@ -13,7 +13,7 @@ - + diff --git a/static/viewer.html b/static/viewer.html index 74ab8814f..65f25d9a3 100644 --- a/static/viewer.html +++ b/static/viewer.html @@ -16,7 +16,7 @@ } const root = getRootLocation(); - const blankPageUrl = `${root}/skin/blank.html`; + const blankPageUrl = root + "/skin/blank.html?KIWIXCACHEID"; if ( location.hash == '' ) { location.href = root + '/'; @@ -58,7 +58,7 @@ diff --git a/test/library.cpp b/test/library.cpp index ef41c4d7c..60762cb51 100644 --- a/test/library.cpp +++ b/test/library.cpp @@ -801,8 +801,14 @@ TEST_F(LibraryTest, removeBooksNotUpdatedSince) lib.addBook(lib.getBookByIdThreadSafe(id)); } + EXPECT_GT(lib.getRevision(), rev); + + const uint64_t rev2 = lib.getRevision(); + EXPECT_EQ(9u, lib.removeBooksNotUpdatedSince(rev)); + EXPECT_GT(lib.getRevision(), rev2); + EXPECT_FILTER_RESULTS(kiwix::Filter(), "Islam Stack Exchange", "Movies & TV Stack Exchange", diff --git a/test/server.cpp b/test/server.cpp index 2d1e1068a..0bb97f76a 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -23,13 +23,19 @@ T1 concat(T1 a, const T2& b) return a; } -const bool WITH_ETAG = true; -const bool NO_ETAG = false; +enum ResourceKind +{ + ZIM_CONTENT, + STATIC_CONTENT, + DYNAMIC_CONTENT, +}; struct Resource { - bool etag_expected; + ResourceKind kind; const char* url; + + bool etag_expected() const { return kind != STATIC_CONTENT; } }; std::ostream& operator<<(std::ostream& out, const Resource& r) @@ -41,55 +47,127 @@ std::ostream& operator<<(std::ostream& out, const Resource& r) typedef std::vector ResourceCollection; const ResourceCollection resources200Compressible{ - { WITH_ETAG, "/ROOT/" }, + { DYNAMIC_CONTENT, "/ROOT/" }, - { WITH_ETAG, "/ROOT/skin/autoComplete.min.js" }, - { WITH_ETAG, "/ROOT/skin/css/autoComplete.css" }, - { WITH_ETAG, "/ROOT/skin/taskbar.css" }, + { DYNAMIC_CONTENT, "/ROOT/viewer" }, + { DYNAMIC_CONTENT, "/ROOT/viewer?cacheid=whatever" }, - { NO_ETAG, "/ROOT/catalog/search" }, + { DYNAMIC_CONTENT, "/ROOT/skin/autoComplete.min.js" }, + { STATIC_CONTENT, "/ROOT/skin/autoComplete.min.js?cacheid=1191aaaf" }, + { DYNAMIC_CONTENT, "/ROOT/skin/css/autoComplete.css" }, + { STATIC_CONTENT, "/ROOT/skin/css/autoComplete.css?cacheid=08951e06" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/favicon.ico" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/favicon.ico?cacheid=fba03a27" }, + { DYNAMIC_CONTENT, "/ROOT/skin/index.css" }, + { STATIC_CONTENT, "/ROOT/skin/index.css?cacheid=0f9ba34e" }, + { DYNAMIC_CONTENT, "/ROOT/skin/index.js" }, + { STATIC_CONTENT, "/ROOT/skin/index.js?cacheid=2f5a81ac" }, + { DYNAMIC_CONTENT, "/ROOT/skin/iso6391To3.js" }, + { STATIC_CONTENT, "/ROOT/skin/iso6391To3.js?cacheid=ecde2bb3" }, + { DYNAMIC_CONTENT, "/ROOT/skin/isotope.pkgd.min.js" }, + { STATIC_CONTENT, "/ROOT/skin/isotope.pkgd.min.js?cacheid=2e48d392" }, + { DYNAMIC_CONTENT, "/ROOT/skin/taskbar.css" }, + { STATIC_CONTENT, "/ROOT/skin/taskbar.css?cacheid=216d6b5d" }, + { DYNAMIC_CONTENT, "/ROOT/skin/viewer.js" }, + { STATIC_CONTENT, "/ROOT/skin/viewer.js?cacheid=51e745c2" }, - { NO_ETAG, "/ROOT/search?content=zimfile&pattern=a" }, + { DYNAMIC_CONTENT, "/ROOT/catalog/search" }, - { NO_ETAG, "/ROOT/suggest?content=zimfile&term=ray" }, + { DYNAMIC_CONTENT, "/ROOT/catalog/v2/root.xml" }, + { DYNAMIC_CONTENT, "/ROOT/catalog/v2/languages" }, + { DYNAMIC_CONTENT, "/ROOT/catalog/v2/entries" }, + { DYNAMIC_CONTENT, "/ROOT/catalog/v2/partial_entries" }, - { WITH_ETAG, "/ROOT/content/zimfile/A/index" }, - { WITH_ETAG, "/ROOT/content/zimfile/A/Ray_Charles" }, + { DYNAMIC_CONTENT, "/ROOT/search?content=zimfile&pattern=a" }, - { WITH_ETAG, "/ROOT/raw/zimfile/content/A/index" }, - { WITH_ETAG, "/ROOT/raw/zimfile/content/A/Ray_Charles" }, + { DYNAMIC_CONTENT, "/ROOT/suggest?content=zimfile&term=ray" }, + + { ZIM_CONTENT, "/ROOT/content/zimfile/A/index" }, + { ZIM_CONTENT, "/ROOT/content/zimfile/A/Ray_Charles" }, + + { ZIM_CONTENT, "/ROOT/raw/zimfile/content/A/index" }, + { ZIM_CONTENT, "/ROOT/raw/zimfile/content/A/Ray_Charles" }, }; const ResourceCollection resources200Uncompressible{ - { WITH_ETAG, "/ROOT/skin/caret.png" }, - { WITH_ETAG, "/ROOT/skin/css/images/search.svg" }, + { DYNAMIC_CONTENT, "/ROOT/skin/bittorrent.png" }, + { STATIC_CONTENT, "/ROOT/skin/bittorrent.png?cacheid=4f5c6882" }, + { DYNAMIC_CONTENT, "/ROOT/skin/blank.html" }, + { STATIC_CONTENT, "/ROOT/skin/blank.html?cacheid=6b1fa032" }, + { DYNAMIC_CONTENT, "/ROOT/skin/caret.png" }, + { STATIC_CONTENT, "/ROOT/skin/caret.png?cacheid=22b942b4" }, + { DYNAMIC_CONTENT, "/ROOT/skin/download.png" }, + { STATIC_CONTENT, "/ROOT/skin/download.png?cacheid=a39aa502" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/android-chrome-192x192.png" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/android-chrome-192x192.png?cacheid=bfac158b" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/android-chrome-512x512.png" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/android-chrome-512x512.png?cacheid=380c3653" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/apple-touch-icon.png" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/apple-touch-icon.png?cacheid=f86f8df3" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/browserconfig.xml" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/browserconfig.xml?cacheid=f29a7c4a" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/favicon-16x16.png" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/favicon-16x16.png?cacheid=a986fedc" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/favicon-32x32.png" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/favicon-32x32.png?cacheid=79ded625" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-144x144.png" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/mstile-144x144.png?cacheid=c25a7641" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-150x150.png" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/mstile-150x150.png?cacheid=6fa6f467" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-310x150.png" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/mstile-310x150.png?cacheid=e0ed9032" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-310x310.png" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/mstile-310x310.png?cacheid=26b20530" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-70x70.png" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/mstile-70x70.png?cacheid=64ffd9dc" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/safari-pinned-tab.svg" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/safari-pinned-tab.svg?cacheid=8d487e95" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/site.webmanifest" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/site.webmanifest?cacheid=bc396efb" }, + { DYNAMIC_CONTENT, "/ROOT/skin/fonts/Poppins.ttf" }, + { STATIC_CONTENT, "/ROOT/skin/fonts/Poppins.ttf?cacheid=af705837" }, + { DYNAMIC_CONTENT, "/ROOT/skin/fonts/Roboto.ttf" }, + { STATIC_CONTENT, "/ROOT/skin/fonts/Roboto.ttf?cacheid=84d10248" }, + { DYNAMIC_CONTENT, "/ROOT/skin/hash.png" }, + { STATIC_CONTENT, "/ROOT/skin/hash.png?cacheid=f836e872" }, + { DYNAMIC_CONTENT, "/ROOT/skin/magnet.png" }, + { STATIC_CONTENT, "/ROOT/skin/magnet.png?cacheid=73b6bddf" }, + { DYNAMIC_CONTENT, "/ROOT/skin/search-icon.svg" }, + { STATIC_CONTENT, "/ROOT/skin/search-icon.svg?cacheid=b10ae7ed" }, + { DYNAMIC_CONTENT, "/ROOT/skin/search_results.css" }, + { STATIC_CONTENT, "/ROOT/skin/search_results.css?cacheid=76d39c84" }, - { WITH_ETAG, "/ROOT/raw/zimfile/meta/Title" }, - { WITH_ETAG, "/ROOT/raw/zimfile/meta/Description" }, - { WITH_ETAG, "/ROOT/raw/zimfile/meta/Language" }, - { WITH_ETAG, "/ROOT/raw/zimfile/meta/Name" }, - { WITH_ETAG, "/ROOT/raw/zimfile/meta/Tags" }, - { WITH_ETAG, "/ROOT/raw/zimfile/meta/Date" }, - { WITH_ETAG, "/ROOT/raw/zimfile/meta/Creator" }, - { WITH_ETAG, "/ROOT/raw/zimfile/meta/Publisher" }, + { ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Title" }, + { ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Description" }, + { ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Language" }, + { ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Name" }, + { ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Tags" }, + { ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Date" }, + { ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Creator" }, + { ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Publisher" }, - { NO_ETAG, "/ROOT/catalog/v2/illustration/6f1d19d0-633f-087b-fb55-7ac324ff9baf?size=48" }, + { DYNAMIC_CONTENT, "/ROOT/catalog/root.xml" }, + { DYNAMIC_CONTENT, "/ROOT/catalog/searchdescription.xml" }, - { NO_ETAG, "/ROOT/catch/external?source=www.example.com" }, + { DYNAMIC_CONTENT, "/ROOT/catalog/v2/categories" }, + { DYNAMIC_CONTENT, "/ROOT/catalog/v2/searchdescription.xml" }, + { DYNAMIC_CONTENT, "/ROOT/catalog/v2/illustration/6f1d19d0-633f-087b-fb55-7ac324ff9baf?size=48" }, - { WITH_ETAG, "/ROOT/content/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg" }, + { DYNAMIC_CONTENT, "/ROOT/catch/external?source=www.example.com" }, - { WITH_ETAG, "/ROOT/content/corner_cases/A/empty.html" }, - { WITH_ETAG, "/ROOT/content/corner_cases/-/empty.css" }, - { WITH_ETAG, "/ROOT/content/corner_cases/-/empty.js" }, + { ZIM_CONTENT, "/ROOT/content/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg" }, + + { ZIM_CONTENT, "/ROOT/content/corner_cases/A/empty.html" }, + { ZIM_CONTENT, "/ROOT/content/corner_cases/-/empty.css" }, + { ZIM_CONTENT, "/ROOT/content/corner_cases/-/empty.js" }, // The following url's responses are too small to be compressed - { NO_ETAG, "/ROOT/catalog/root.xml" }, - { NO_ETAG, "/ROOT/catalog/searchdescription.xml" }, - { NO_ETAG, "/ROOT/suggest?content=zimfile" }, - { WITH_ETAG, "/ROOT/raw/zimfile/meta/Creator" }, - { WITH_ETAG, "/ROOT/raw/zimfile/meta/Title" }, + { DYNAMIC_CONTENT, "/ROOT/catalog/root.xml" }, + { DYNAMIC_CONTENT, "/ROOT/catalog/searchdescription.xml" }, + { DYNAMIC_CONTENT, "/ROOT/suggest?content=zimfile" }, + { ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Creator" }, + { ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Title" }, }; ResourceCollection all200Resources() @@ -172,11 +250,11 @@ TEST_F(ServerTest, CacheIdsOfStaticResources) const std::vector testData{ { /* url */ "/ROOT/", -R"EXPECTEDRESULT( href="/ROOT/skin/index.css?cacheid=3b470cee" +R"EXPECTEDRESULT( href="/ROOT/skin/index.css?cacheid=0f9ba34e" - + @@ -185,6 +263,11 @@ R"EXPECTEDRESULT( href="/ROOT/skin/index.css?cacheid=3b470cee" +)EXPECTEDRESULT" + }, + { + /* url */ "/ROOT/skin/index.css", +R"EXPECTEDRESULT( background-image: url('../skin/search-icon.svg?cacheid=b10ae7ed'); )EXPECTEDRESULT" }, { @@ -201,8 +284,9 @@ R"EXPECTEDRESULT( - const blankPageUrl = `${root}/skin/blank.html`; + const blankPageUrl = root + "/skin/blank.html?cacheid=6b1fa032"; + src="./skin/blank.html?cacheid=6b1fa032" title="ZIM content" width="100%" )EXPECTEDRESULT" }, { @@ -252,6 +336,7 @@ const char* urls404[] = { "/", "/zimfile", "/ROOT/skin/non-existent-skin-resource", + "/ROOT/skin/autoComplete.min.js?cacheid=wrongcacheid", "/ROOT/catalog", "/ROOT/catalog/", "/ROOT/catalog/non-existent-item", @@ -310,6 +395,11 @@ std::string getHeaderValue(const Headers& headers, const std::string& name) return er.first->second; } +std::string getCacheControlHeader(const httplib::Response& r) +{ + return getHeaderValue(r.headers, "Cache-Control"); +} + TEST_F(CustomizedServerTest, NewResourcesCanBeAdded) { // ServerTest.404 verifies that "/ROOT/non-existent-item" doesn't exist @@ -952,6 +1042,8 @@ TEST_F(ServerTest, RandomPageRedirectsToAnExistingArticle) ASSERT_EQ(302, g->status); ASSERT_TRUE(g->has_header("Location")); ASSERT_TRUE(kiwix::startsWith(g->get_header_value("Location"), "/ROOT/content/zimfile/A/")); + ASSERT_EQ(getCacheControlHeader(*g), "max-age=0, must-revalidate"); + ASSERT_FALSE(g->has_header("ETag")); } TEST_F(ServerTest, NonEndpointUrlsAreRedirectedToContentUrls) @@ -995,6 +1087,8 @@ TEST_F(ServerTest, NonEndpointUrlsAreRedirectedToContentUrls) ASSERT_EQ(302, g->status) << ctx; ASSERT_TRUE(g->has_header("Location")) << ctx; ASSERT_EQ("/ROOT/content" + p, g->get_header_value("Location")) << ctx; + ASSERT_EQ(getCacheControlHeader(*g), "max-age=0, must-revalidate"); + ASSERT_FALSE(g->has_header("ETag")); } } @@ -1059,12 +1153,45 @@ TEST_F(ServerTest, HeadersAreTheSameInResponsesToHeadAndGetRequests) } } +TEST_F(ServerTest, CacheControlOfZimContent) +{ + for ( const Resource& res : all200Resources() ) { + if ( res.kind == ZIM_CONTENT ) { + const auto g = zfs1_->GET(res.url); + EXPECT_EQ(getCacheControlHeader(*g), "max-age=3600, must-revalidate") << res; + EXPECT_TRUE(g->has_header("ETag")) << res; + } + } +} + +TEST_F(ServerTest, CacheControlOfStaticContent) +{ + for ( const Resource& res : all200Resources() ) { + if ( res.kind == STATIC_CONTENT ) { + const auto g = zfs1_->GET(res.url); + EXPECT_EQ(getCacheControlHeader(*g), "max-age=31536000, immutable") << res; + EXPECT_FALSE(g->has_header("ETag")) << res; + } + } +} + +TEST_F(ServerTest, CacheControlOfDynamicContent) +{ + for ( const Resource& res : all200Resources() ) { + if ( res.kind == DYNAMIC_CONTENT ) { + const auto g = zfs1_->GET(res.url); + EXPECT_EQ(getCacheControlHeader(*g), "max-age=0, must-revalidate") << res; + EXPECT_TRUE(g->has_header("ETag")) << res; + } + } +} + TEST_F(ServerTest, ETagHeaderIsSetAsNeeded) { for ( const Resource& res : all200Resources() ) { const auto responseToGet = zfs1_->GET(res.url); - EXPECT_EQ(res.etag_expected, responseToGet->has_header("ETag")) << res; - if ( res.etag_expected ) { + EXPECT_EQ(res.etag_expected(), responseToGet->has_header("ETag")) << res; + if ( res.etag_expected() ) { EXPECT_TRUE(is_valid_etag(responseToGet->get_header_value("ETag"))); } } @@ -1088,21 +1215,32 @@ TEST_F(ServerTest, ETagIsTheSameAcrossHeadAndGet) } } -TEST_F(ServerTest, DifferentServerInstancesProduceDifferentETags) +TEST_F(ServerTest, DifferentServerInstancesProduceDifferentETagsForDynamicContent) { ZimFileServer zfs2(SERVER_PORT + 1, ZimFileServer::DEFAULT_OPTIONS, ZIMFILES); for ( const Resource& res : all200Resources() ) { - if ( !res.etag_expected ) continue; + if ( res.kind != DYNAMIC_CONTENT ) continue; const auto h1 = zfs1_->HEAD(res.url); const auto h2 = zfs2.HEAD(res.url); EXPECT_NE(h1->get_header_value("ETag"), h2->get_header_value("ETag")); } } +TEST_F(ServerTest, DifferentServerInstancesProduceIdenticalETagsForZimContent) +{ + ZimFileServer zfs2(SERVER_PORT + 1, ZimFileServer::DEFAULT_OPTIONS, ZIMFILES); + for ( const Resource& res : all200Resources() ) { + if ( res.kind != ZIM_CONTENT ) continue; + const auto h1 = zfs1_->HEAD(res.url); + const auto h2 = zfs2.HEAD(res.url); + EXPECT_EQ(h1->get_header_value("ETag"), h2->get_header_value("ETag")); + } +} + TEST_F(ServerTest, CompressionInfluencesETag) { for ( const Resource& res : resources200Compressible ) { - if ( ! res.etag_expected ) continue; + if ( ! res.etag_expected() ) continue; const auto g1 = zfs1_->GET(res.url); const auto g2 = zfs1_->GET(res.url, { {"Accept-Encoding", ""} } ); const auto g3 = zfs1_->GET(res.url, { {"Accept-Encoding", "gzip"} } ); @@ -1115,7 +1253,7 @@ TEST_F(ServerTest, CompressionInfluencesETag) TEST_F(ServerTest, ETagOfUncompressibleContentIsNotAffectedByAcceptEncoding) { for ( const Resource& res : resources200Uncompressible ) { - if ( ! res.etag_expected ) continue; + if ( ! res.etag_expected() ) continue; const auto g1 = zfs1_->GET(res.url); const auto g2 = zfs1_->GET(res.url, { {"Accept-Encoding", ""} } ); const auto g3 = zfs1_->GET(res.url, { {"Accept-Encoding", "gzip"} } ); @@ -1160,7 +1298,7 @@ TEST_F(ServerTest, IfNoneMatchRequestsWithMatchingETagResultIn304Responses) const char* const encodings[] = { "", "gzip" }; for ( const Resource& res : all200Resources() ) { for ( const char* enc: encodings ) { - if ( ! res.etag_expected ) continue; + if ( ! res.etag_expected() ) continue; const TestContext ctx{ {"url", res.url}, {"encoding", enc} }; const auto g = zfs1_->GET(res.url, { {"Accept-Encoding", enc} }); @@ -1187,8 +1325,8 @@ TEST_F(ServerTest, IfNoneMatchRequestsWithMismatchingETagResultIn200Responses) const auto etag2 = etag.substr(0, etag.size() - 1) + "x\""; const auto h = zfs1_->HEAD(res.url, { {"If-None-Match", etag2} } ); const auto g2 = zfs1_->GET(res.url, { {"If-None-Match", etag2} } ); - EXPECT_EQ(200, h->status); - EXPECT_EQ(200, g2->status); + EXPECT_EQ(200, h->status) << res; + EXPECT_EQ(200, g2->status) << res; } }