Merge pull request #833 from kiwix/http_caching

include/library.h

@@ -332,8 +332,8 @@ class Library
   /**
    * Return the current revision of the library.
    *
-   * The revision of the library is updated (incremented by one) only by
+   * The revision of the library is updated (incremented by one) by
-   * the addBook() operation.
+   * the addBook() and removeBookById() operations.
    *
    * @return Current revision of the library.
    */

scripts/kiwix-compile-resources

@@ -52,15 +52,21 @@ resource_getter_template = """
         return RESOURCE::{identifier};
 """
 
+resource_cacheid_getter_template = """
+    if (name == "{common_name}")
+        return "{cacheid}";
+"""
+
 resource_decl_template = """{namespaces_open}
 extern const std::string {identifier};
 {namespaces_close}"""
 
 class Resource:
-    def __init__(self, base_dirs, filename):
+    def __init__(self, base_dirs, filename, cacheid=None):
-        filename = filename.strip()
+        filename = filename
         self.filename = filename
         self.identifier = full_identifier(filename)
+        self.cacheid = cacheid
         found = False
         for base_dir in base_dirs:
             try:
@@ -71,7 +77,7 @@ class Resource:
             except FileNotFoundError:
                 continue
         if not found:
-            raise Exception("Impossible to found {}".format(filename))
+            raise Exception("Resource not found: {}".format(filename))
 
     def dump_impl(self):
         nb_row = len(self.data)//16 + (1 if len(self.data) % 16 else 0)
@@ -93,6 +99,12 @@ class Resource:
             identifier="::".join(self.identifier)
         )
 
+    def dump_cacheid_getter(self):
+        return resource_cacheid_getter_template.format(
+            common_name=self.filename,
+            cacheid=self.cacheid
+        )
+
     def dump_decl(self):
         return resource_decl_template.format(
             namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]),
@@ -123,7 +135,12 @@ static std::string init_resource(const char* name, const unsigned char* content,
 
 const std::string& getResource_{basename}(const std::string& name) {{
 {RESOURCES_GETTER}
-    throw ResourceNotFound("Resource not found.");
+    throw ResourceNotFound("Resource not found: " + name);
+}}
+
+const char* getResourceCacheId_{basename}(const std::string& name) {{
+{RESOURCE_CACHEID_GETTER}
+    return nullptr;
 }}
 
 {RESOURCES}
@@ -134,6 +151,7 @@ def gen_c_file(resources, basename):
     return master_c_template.format(
        RESOURCES="\n\n".join(r.dump_impl() for r in resources),
        RESOURCES_GETTER="\n\n".join(r.dump_getter() for r in resources),
+       RESOURCE_CACHEID_GETTER="\n\n".join(r.dump_cacheid_getter() for r in resources if r.cacheid is not None),
        include_file=basename,
        basename=to_identifier(basename)
     )
@@ -159,8 +177,10 @@ class ResourceNotFound : public std::runtime_error {{
 }};
 
 const std::string& getResource_{basename}(const std::string& name);
+const char* getResourceCacheId_{basename}(const std::string& name);
 
 #define getResource(a) (getResource_{basename}(a))
+#define getResourceCacheId(a) (getResourceCacheId_{basename}(a))
 
 #endif // KIWIX_{BASENAME}
 
@@ -189,8 +209,8 @@ if __name__ == "__main__":
     base_dir = os.path.dirname(os.path.realpath(args.resource_file))
     source_dir = args.source_dir or []
     with open(args.resource_file, 'r') as f:
-        resources = [Resource([base_dir]+source_dir, filename)
+        resources = [Resource([base_dir]+source_dir, *line.strip().split())
-                        for filename in f.readlines()]
+                        for line in f.readlines()]
 
     h_identifier = to_identifier(os.path.basename(args.hfile))
     with open(args.hfile, 'w') as f:

scripts/kiwix-resources

@@ -99,16 +99,24 @@ def preprocess_resource(resource_path):
             print(preprocessed_content, end='', file=target)
 
 
-def copy_file(src_path, dst_path):
+def copy_resource_list_file(src_path, dst_path):
-    with open(src_path, 'rb') as src:
+    with open(src_path, 'r') as src:
-        with open(dst_path, 'wb') as dst:
+        with open(dst_path, 'w') as dst:
-            dst.write(src.read())
+            for line in src:
+                res = line.strip()
+                if line.startswith("skin/") and res in resource_revisions:
+                    dst.write(res + " " + resource_revisions[res] + "\n")
+                else:
+                    dst.write(line)
 
 def preprocess_resources(resource_file_path):
     resource_filename = os.path.basename(resource_file_path)
     for resource in read_resource_file(resource_file_path):
-        preprocess_resource(resource)
+        if resource.startswith('skin/'):
-    copy_file(resource_file_path, os.path.join(OUT_DIR, resource_filename))
+            get_resource_revision(resource)
+        else:
+            preprocess_resource(resource)
+    copy_resource_list_file(resource_file_path, os.path.join(OUT_DIR, resource_filename))
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()

src/library.cpp

@@ -221,7 +221,11 @@ bool Library::removeBookById(const std::string& id)
   // Having a too big cache is not a problem here (or it would have been before)
   // (And setMaxSize doesn't actually reduce the cache size, extra cached items
   //  will be removed in put or getOrPut).
-  return mp_impl->m_books.erase(id) == 1;
+  const bool bookWasRemoved = mp_impl->m_books.erase(id) == 1;
+  if ( bookWasRemoved ) {
+    ++mp_impl->m_revision;
+  }
+  return bookWasRemoved;
 }
 
 Library::Revision Library::getRevision() const

src/server/etag.cpp

@@ -37,11 +37,11 @@ namespace {
 // into the ETag for ETag::Option opt.
 // IMPORTANT: The characters in all_options must come in sorted order (so that
 // IMPORTANT: isValidOptionsString() works correctly).
-const char all_options[] = "cz";
+const char all_options[] = "Zz";
 
 static_assert(ETag::OPTION_COUNT == sizeof(all_options) - 1, "");
 
-bool isValidServerId(const std::string& s)
+bool isValidETagBody(const std::string& s)
 {
   return !s.empty() && s.find_first_of("\"/") == std::string::npos;
 }
@@ -83,17 +83,17 @@ bool ETag::get_option(Option opt) const
 
 std::string ETag::get_etag() const
 {
-  if ( m_serverId.empty() )
+  if ( m_body.empty() )
     return std::string();
 
-  return "\"" + m_serverId + "/" + m_options + "\"";
+  return "\"" + m_body + "/" + m_options + "\"";
 }
 
-ETag::ETag(const std::string& serverId, const std::string& options)
+ETag::ETag(const std::string& body, const std::string& options)
 {
-  if ( isValidServerId(serverId) && isValidOptionsString(options) )
+  if ( isValidETagBody(body) && isValidOptionsString(options) )
   {
-    m_serverId = serverId;
+    m_body = body;
     m_options = options;
   }
 }
@@ -115,7 +115,7 @@ ETag ETag::parse(std::string s)
   return ETag(s.substr(0, i), s.substr(i+1));
 }
 
-ETag ETag::match(const std::string& etags, const std::string& server_id)
+ETag ETag::match(const std::string& etags, const std::string& body)
 {
   std::istringstream ss(etags);
   std::string etag_str;
@@ -125,7 +125,7 @@ ETag ETag::match(const std::string& etags, const std::string& server_id)
       etag_str.pop_back();
 
     const ETag etag = parse(etag_str);
-    if ( etag && etag.m_serverId == server_id )
+    if ( etag && etag.m_body == body )
       return etag;
   }
 

src/server/etag.h

@@ -28,10 +28,11 @@ namespace kiwix {
 // The ETag string used by Kiwix server (more precisely, its value inside the
 // double quotes) consists of two parts:
 //
-// 1. ServerId - The string obtained on server start up
+// 1. Body    -  A string uniquely identifying the object or state from which
+//               the resource has been obtained.
 //
-// 2. Options -  Zero or more characters encoding the values of some of the
+// 2. Options -  Zero or more characters encoding the type of the ETag and/or
-//               headers of the response
+//               the values of some of the headers of the response
 //
 // The two parts are separated with a slash (/) symbol (which is always present,
 // even when the the options part is empty). Neither portion of a Kiwix ETag
@@ -40,7 +41,7 @@ namespace kiwix {
 //
 //   "abcdefghijklmn/"
 //   "1234567890/z"
-//   "1234567890/cz"
+//   "6f1d19d0-633f-087b-fb55-7ac324ff9baf/Zz"
 //
 // The options part of the Kiwix ETag allows to correctly set the required
 // headers when responding to a conditional If-None-Match request with a 304
@@ -51,7 +52,7 @@ class ETag
 {
   public: // types
     enum Option {
-      CACHEABLE_ENTITY,
+      ZIM_CONTENT,
       COMPRESSED_CONTENT,
       OPTION_COUNT
     };
@@ -59,10 +60,10 @@ class ETag
   public: // functions
     ETag() {}
 
-    void set_server_id(const std::string& id) { m_serverId = id; }
+    void set_body(const std::string& s) { m_body = s; }
     void set_option(Option opt);
 
-    explicit operator bool() const { return !m_serverId.empty(); }
+    explicit operator bool() const { return !m_body.empty(); }
 
     bool get_option(Option opt) const;
     std::string get_etag() const;
@@ -76,7 +77,7 @@ class ETag
     static ETag parse(std::string s);
 
   private: // data
-    std::string m_serverId;
+    std::string m_body;
     std::string m_options;
 };
 

src/server/internalServer.cpp

@@ -218,6 +218,24 @@ struct CustomizedResourceData
   std::string resourceFilePath;
 };
 
+bool responseMustBeETaggedWithLibraryId(const Response& response, const RequestContext& request)
+{
+  return response.getReturnCode() == MHD_HTTP_OK
+      && response.get_kind() == Response::DYNAMIC_CONTENT
+      && request.get_url() != "/random";
+}
+
+ETag
+get_matching_if_none_match_etag(const RequestContext& r, const std::string& etagBody)
+{
+  try {
+    const std::string etag_list = r.get_header(MHD_HTTP_HEADER_IF_NONE_MATCH);
+    return ETag::match(etag_list, etagBody);
+  } catch (const std::out_of_range&) {
+    return ETag();
+  }
+}
+
 } // unnamed namespace
 
 std::pair<std::string, Library::BookIdSet> InternalServer::selectBooks(const RequestContext& request) const
@@ -443,7 +461,6 @@ bool InternalServer::start() {
   }
   auto server_start_time = std::chrono::system_clock::now().time_since_epoch();
   m_server_id = kiwix::to_string(server_start_time.count());
-  m_library_id = m_server_id;
   return true;
 }
 
@@ -511,8 +528,9 @@ MHD_Result InternalServer::handlerCallback(struct MHD_Connection* connection,
     }
   }
 
-  if (response->getReturnCode() == MHD_HTTP_OK && !etag_not_needed(request))
+  if ( responseMustBeETaggedWithLibraryId(*response, request) ) {
-    response->set_server_id(m_server_id);
+    response->set_etag_body(getLibraryId());
+  }
 
   auto ret = response->send(request, connection);
   auto end_time = std::chrono::steady_clock::now();
@@ -534,6 +552,11 @@ bool isEndpointUrl(const std::string& url, const std::string& endpoint)
 
 } // unnamed namespace
 
+std::string InternalServer::getLibraryId() const
+{
+  return m_server_id + "." + kiwix::to_string(mp_library->getRevision());
+}
+
 std::unique_ptr<Response> InternalServer::handle_request(const RequestContext& request)
 {
   try {
@@ -542,7 +565,7 @@ std::unique_ptr<Response> InternalServer::handle_request(const RequestContext& r
              + urlNotFoundMsg;
     }
 
-    const ETag etag = get_matching_if_none_match_etag(request);
+    const ETag etag = get_matching_if_none_match_etag(request, getLibraryId());
     if ( etag )
       return Response::build_304(*this, etag);
 
@@ -603,27 +626,6 @@ MustacheData InternalServer::get_default_data() const
   return data;
 }
 
-bool InternalServer::etag_not_needed(const RequestContext& request) const
-{
-  const std::string url = request.get_url();
-  return kiwix::startsWith(url, "/catalog")
-      || url == "/search"
-      || url == "/suggest"
-      || url == "/random"
-      || url == "/catch/external";
-}
-
-ETag
-InternalServer::get_matching_if_none_match_etag(const RequestContext& r) const
-{
-  try {
-    const std::string etag_list = r.get_header(MHD_HTTP_HEADER_IF_NONE_MATCH);
-    return ETag::match(etag_list, m_server_id);
-  } catch (const std::out_of_range&) {
-    return ETag();
-  }
-}
-
 std::unique_ptr<Response> InternalServer::build_homepage(const RequestContext& request)
 {
   return ContentResponse::build(*this, m_indexTemplateString, get_default_data(), "text/html; charset=utf-8");
@@ -746,6 +748,25 @@ std::unique_ptr<Response> InternalServer::handle_viewer_settings(const RequestCo
   return ContentResponse::build(*this, RESOURCE::templates::viewer_settings_js, data, "application/javascript; charset=utf-8");
 }
 
+namespace
+{
+
+Response::Kind staticResourceAccessType(const RequestContext& req, const char* expectedCacheid)
+{
+  if ( expectedCacheid == nullptr )
+    return Response::DYNAMIC_CONTENT;
+
+  try {
+    if ( expectedCacheid != req.get_argument("cacheid") )
+      throw ResourceNotFound("Wrong cacheid");
+    return Response::STATIC_RESOURCE;
+  } catch( const std::out_of_range& ) {
+    return Response::DYNAMIC_CONTENT;
+  }
+}
+
+} // unnamed namespace
+
 std::unique_ptr<Response> InternalServer::handle_skin(const RequestContext& request)
 {
   if (m_verbose.load()) {
@@ -756,12 +777,16 @@ std::unique_ptr<Response> InternalServer::handle_skin(const RequestContext& requ
   auto resourceName = isRequestForViewer
                     ? "viewer.html"
                     : request.get_url().substr(1);
+
+  const char* const resourceCacheId = getResourceCacheId(resourceName);
+
   try {
+    const auto accessType = staticResourceAccessType(request, resourceCacheId);
     auto response = ContentResponse::build(
         *this,
         getResource(resourceName),
         getMimeTypeForFile(resourceName));
-    response->set_cacheable();
+    response->set_kind(accessType);
     return std::move(response);
   } catch (const ResourceNotFound& e) {
     return HTTP404Response(*this, request)
@@ -969,7 +994,7 @@ std::unique_ptr<Response> InternalServer::handle_catalog(const RequestContext& r
   zim::Uuid uuid;
   kiwix::OPDSDumper opdsDumper(mp_library);
   opdsDumper.setRootLocation(m_root);
-  opdsDumper.setLibraryId(m_library_id);
+  opdsDumper.setLibraryId(getLibraryId());
   std::vector<std::string> bookIdsToDump;
   if (url == "root.xml") {
     uuid = zim::Uuid::generate(host);
@@ -1052,6 +1077,11 @@ std::unique_ptr<Response> InternalServer::handle_content(const RequestContext& r
            + suggestSearchMsg(searchURL, kiwix::urlDecode(pattern));
   }
 
+  const std::string archiveUuid(archive->getUuid());
+  const ETag etag = get_matching_if_none_match_etag(request, archiveUuid);
+  if ( etag )
+    return Response::build_304(*this, etag);
+
   auto urlStr = url.substr(prefixLength + bookName.size());
   if (urlStr[0] == '/') {
     urlStr = urlStr.substr(1);
@@ -1070,6 +1100,7 @@ std::unique_ptr<Response> InternalServer::handle_content(const RequestContext& r
       return build_redirect(bookName, getFinalItem(*archive, entry));
     }
     auto response = ItemResponse::build(*this, request, entry.getItem());
+    response->set_etag_body(archiveUuid);
 
     if (m_verbose.load()) {
       printf("Found %s\n", entry.getPath().c_str());
@@ -1123,6 +1154,11 @@ std::unique_ptr<Response> InternalServer::handle_raw(const RequestContext& reque
            + noSuchBookErrorMsg(bookName);
   }
 
+  const std::string archiveUuid(archive->getUuid());
+  const ETag etag = get_matching_if_none_match_etag(request, archiveUuid);
+  if ( etag )
+    return Response::build_304(*this, etag);
+
   // Remove the beggining of the path:
   // /raw/<bookName>/<kind>/foo
   // ^^^^^          ^      ^
@@ -1132,13 +1168,17 @@ std::unique_ptr<Response> InternalServer::handle_raw(const RequestContext& reque
   try {
     if (kind == "meta") {
       auto item = archive->getMetadataItem(itemPath);
-      return ItemResponse::build(*this, request, item);
+      auto response = ItemResponse::build(*this, request, item);
+      response->set_etag_body(archiveUuid);
+      return response;
     } else {
       auto entry = archive->getEntryByPath(itemPath);
       if (entry.isRedirect()) {
         return build_redirect(bookName, entry.getItem(true));
       }
-      return ItemResponse::build(*this, request, entry.getItem());
+      auto response = ItemResponse::build(*this, request, entry.getItem());
+      response->set_etag_body(archiveUuid);
+      return response;
     }
   } catch (zim::EntryNotFound& e ) {
     if (m_verbose.load()) {

src/server/internalServer.h

@@ -147,13 +147,13 @@ class InternalServer {
 
     MustacheData get_default_data() const;
 
-    bool etag_not_needed(const RequestContext& r) const;
-    ETag get_matching_if_none_match_etag(const RequestContext& request) const;
     std::pair<std::string, Library::BookIdSet> selectBooks(const RequestContext& r) const;
     SearchInfo getSearchInfo(const RequestContext& r) const;
 
     bool isLocallyCustomizedResource(const std::string& url) const;
 
+    std::string getLibraryId() const;
+
   private: // types
     class LockableSuggestionSearcher;
     typedef ConcurrentCache<SearchInfo, std::shared_ptr<zim::Search>> SearchCache;
@@ -180,7 +180,6 @@ class InternalServer {
     SuggestionSearcherCache suggestionSearcherCache;
 
     std::string m_server_id;
-    std::string m_library_id;
 
     class CustomizedResources;
     std::unique_ptr<CustomizedResources> m_customizedResources;

src/server/internalServer_catalog_v2.cpp

@@ -77,17 +77,18 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2(const RequestContext
 
 std::unique_ptr<Response> InternalServer::handle_catalog_v2_root(const RequestContext& request)
 {
+  const std::string libraryId = getLibraryId();
   return ContentResponse::build(
              *this,
              RESOURCE::templates::catalog_v2_root_xml,
              kainjow::mustache::object{
                {"date", gen_date_str()},
                {"endpoint_root", m_root + "/catalog/v2"},
-               {"feed_id", gen_uuid(m_library_id)},
+               {"feed_id", gen_uuid(libraryId)},
-               {"all_entries_feed_id", gen_uuid(m_library_id + "/entries")},
+               {"all_entries_feed_id", gen_uuid(libraryId + "/entries")},
-               {"partial_entries_feed_id", gen_uuid(m_library_id + "/partial_entries")},
+               {"partial_entries_feed_id", gen_uuid(libraryId + "/partial_entries")},
-               {"category_list_feed_id", gen_uuid(m_library_id + "/categories")},
+               {"category_list_feed_id", gen_uuid(libraryId + "/categories")},
-               {"language_list_feed_id", gen_uuid(m_library_id + "/languages")}
+               {"language_list_feed_id", gen_uuid(libraryId + "/languages")}
              },
              "application/atom+xml;profile=opds-catalog;kind=navigation"
   );
@@ -97,7 +98,7 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_entries(const Reques
 {
   OPDSDumper opdsDumper(mp_library);
   opdsDumper.setRootLocation(m_root);
-  opdsDumper.setLibraryId(m_library_id);
+  opdsDumper.setLibraryId(getLibraryId());
   const auto bookIds = search_catalog(request, opdsDumper);
   const auto opdsFeed = opdsDumper.dumpOPDSFeedV2(bookIds, request.get_query(), partial);
   return ContentResponse::build(
@@ -118,7 +119,7 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_complete_entry(const
 
   OPDSDumper opdsDumper(mp_library);
   opdsDumper.setRootLocation(m_root);
-  opdsDumper.setLibraryId(m_library_id);
+  opdsDumper.setLibraryId(getLibraryId());
   const auto opdsFeed = opdsDumper.dumpOPDSCompleteEntry(entryId);
   return ContentResponse::build(
              *this,
@@ -131,7 +132,7 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_categories(const Req
 {
   OPDSDumper opdsDumper(mp_library);
   opdsDumper.setRootLocation(m_root);
-  opdsDumper.setLibraryId(m_library_id);
+  opdsDumper.setLibraryId(getLibraryId());
   return ContentResponse::build(
              *this,
              opdsDumper.categoriesOPDSFeed(),
@@ -143,7 +144,7 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_languages(const Requ
 {
   OPDSDumper opdsDumper(mp_library);
   opdsDumper.setRootLocation(m_root);
-  opdsDumper.setLibraryId(m_library_id);
+  opdsDumper.setLibraryId(getLibraryId());
   return ContentResponse::build(
              *this,
              opdsDumper.languagesOPDSFeed(),

src/server/response.cpp

@@ -102,6 +102,14 @@ bool compress(std::string &content) {
 }
 
 
+const char* getCacheControlHeader(Response::Kind k)
+{
+  switch(k) {
+    case Response::STATIC_RESOURCE: return "max-age=31536000, immutable";
+    case Response::ZIM_CONTENT:     return "max-age=3600, must-revalidate";
+    default:                        return "max-age=0, must-revalidate";
+  }
+}
 
 } // unnamed namespace
 
@@ -112,6 +120,13 @@ Response::Response(bool verbose)
   add_header(MHD_HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN, "*");
 }
 
+void Response::set_kind(Kind k)
+{
+  m_kind = k;
+  if ( k == ZIM_CONTENT )
+    m_etag.set_option(ETag::ZIM_CONTENT);
+}
+
 std::unique_ptr<Response> Response::build(const InternalServer& server)
 {
   return std::unique_ptr<Response>(new Response(server.m_verbose.load()));
@@ -122,6 +137,9 @@ std::unique_ptr<Response> Response::build_304(const InternalServer& server, cons
   auto response = Response::build(server);
   response->set_code(MHD_HTTP_NOT_MODIFIED);
   response->m_etag = etag;
+  if ( etag.get_option(ETag::ZIM_CONTENT) ) {
+    response->set_kind(Response::ZIM_CONTENT);
+  }
   if ( etag.get_option(ETag::COMPRESSED_CONTENT) ) {
     response->add_header(MHD_HTTP_HEADER_VARY, "Accept-Encoding");
   }
@@ -355,7 +373,7 @@ MHD_Result Response::send(const RequestContext& request, MHD_Connection* connect
   MHD_Response* response = create_mhd_response(request);
 
   MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL,
-    m_etag.get_option(ETag::CACHEABLE_ENTITY) ? "max-age=2723040, public" : "no-cache, no-store, must-revalidate");
+                          getCacheControlHeader(m_kind));
   const std::string etag = m_etag.get_etag();
   if ( ! etag.empty() )
     MHD_add_response_header(response, MHD_HTTP_HEADER_ETAG, etag.c_str());
@@ -411,7 +429,7 @@ ItemResponse::ItemResponse(bool verbose, const zim::Item& item, const std::strin
   m_mimeType(mimetype)
 {
   m_byteRange = byterange;
-  set_cacheable();
+  set_kind(Response::ZIM_CONTENT);
   add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType);
 }
 
@@ -423,14 +441,14 @@ std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, cons
   if (noRange && is_compressible_mime_type(mimetype)) {
     // Return a contentResponse
     auto response = ContentResponse::build(server, item.getData(), mimetype);
-    response->set_cacheable();
+    response->set_kind(Response::ZIM_CONTENT);
     response->m_byteRange = byteRange;
     return std::move(response);
   }
 
   if (byteRange.kind() == ByteRange::RESOLVED_UNSATISFIABLE) {
     auto response = Response::build_416(server, item.getSize());
-    response->set_cacheable();
+    response->set_kind(Response::ZIM_CONTENT);
     return response;
   }
 

src/server/response.h

@@ -45,6 +45,14 @@ class InternalServer;
 class RequestContext;
 
 class Response {
+  public:
+    enum Kind
+    {
+      STATIC_RESOURCE,
+      ZIM_CONTENT,
+      DYNAMIC_CONTENT
+    };
+
   public:
     Response(bool verbose);
     virtual ~Response() = default;
@@ -57,8 +65,9 @@ class Response {
     MHD_Result send(const RequestContext& request, MHD_Connection* connection);
 
     void set_code(int code) { m_returnCode = code; }
-    void set_cacheable() { m_etag.set_option(ETag::CACHEABLE_ENTITY); }
+    void set_kind(Kind k);
-    void set_server_id(const std::string& id) { m_etag.set_server_id(id); }
+    Kind get_kind() const { return m_kind; }
+    void set_etag_body(const std::string& id) { m_etag.set_body(id); }
     void add_header(const std::string& name, const std::string& value) { m_customHeaders[name] = value; }
 
     int getReturnCode() const { return m_returnCode; }
@@ -68,6 +77,7 @@ class Response {
     MHD_Response* create_error_response(const RequestContext& request) const;
 
   protected: // data
+    Kind m_kind = DYNAMIC_CONTENT;
     bool m_verbose;
     int m_returnCode;
     ByteRange m_byteRange;

static/resources_list.txt

@@ -36,7 +36,6 @@ opensearchdescription.xml
 ft_opensearchdescription.xml
 catalog_v2_searchdescription.xml
 skin/css/autoComplete.css
-skin/css/images/search.svg
 skin/favicon/android-chrome-192x192.png
 skin/favicon/android-chrome-512x512.png
 skin/favicon/apple-touch-icon.png

static/skin/css/images/search.svg

@@ -1,8 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" focusable="false" x="0px" y="0px" width="30" height="30" viewBox="0 0 171 171" style=" fill:#000000;">
-    <g fill="none" fill-rule="nonzero" stroke="none" stroke-width="1" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="10" stroke-dasharray="" stroke-dashoffset="0" font-family="none" font-weight="none" font-size="none" text-anchor="none" style="mix-blend-mode: normal">
-        <path d="M0,171.99609v-171.99609h171.99609v171.99609z" fill="none"></path>
-        <g fill="#ff7a7a">
-            <path d="M74.1,17.1c-31.41272,0 -57,25.58728 -57,57c0,31.41272 25.58728,57 57,57c13.6601,0 26.20509,-4.85078 36.03692,-12.90293l34.03301,34.03301c1.42965,1.48907 3.55262,2.08891 5.55014,1.56818c1.99752,-0.52073 3.55746,-2.08067 4.07819,-4.07819c0.52073,-1.99752 -0.0791,-4.12049 -1.56818,-5.55014l-34.03301,-34.03301c8.05215,-9.83182 12.90293,-22.37682 12.90293,-36.03692c0,-31.41272 -25.58728,-57 -57,-57zM74.1,28.5c25.2517,0 45.6,20.3483 45.6,45.6c0,25.2517 -20.3483,45.6 -45.6,45.6c-25.2517,0 -45.6,-20.3483 -45.6,-45.6c0,-25.2517 20.3483,-45.6 45.6,-45.6z"></path>
-        </g>
-    </g>
-</svg>

static/skin/index.css

@@ -105,7 +105,7 @@ body {
     border-radius: 10px;
     border: solid 1px #b5b2b2;
     padding: 10px;
-    background-image: url('./search-icon.svg');
+    background-image: url('../skin/search-icon.svg?KIWIXCACHEID');
     background-repeat: no-repeat;
     background-position: right center;
     background-origin: content-box;

static/templates/index.html

@@ -13,7 +13,7 @@
     <link rel="apple-touch-icon" sizes="180x180" href="{{root}}/skin/favicon/apple-touch-icon.png?KIWIXCACHEID">
     <link rel="icon" type="image/png" sizes="32x32" href="{{root}}/skin/favicon/favicon-32x32.png?KIWIXCACHEID">
     <link rel="icon" type="image/png" sizes="16x16" href="{{root}}/skin/favicon/favicon-16x16.png?KIWIXCACHEID">
-    <link rel="manifest" href="{{root}}/skin/favicon/site.webmanifest">
+    <link rel="manifest" href="{{root}}/skin/favicon/site.webmanifest?KIWIXCACHEID">
     <link rel="mask-icon" href="{{root}}/skin/favicon/safari-pinned-tab.svg?KIWIXCACHEID" color="#5bbad5">
     <link rel="shortcut icon" href="{{root}}/skin/favicon/favicon.ico?KIWIXCACHEID">
     <meta name="msapplication-TileColor" content="#da532c">

static/viewer.html

@@ -16,7 +16,7 @@
       }
 
       const root = getRootLocation();
-      const blankPageUrl = `${root}/skin/blank.html`;
+      const blankPageUrl = root + "/skin/blank.html?KIWIXCACHEID";
 
       if ( location.hash == '' ) {
         location.href = root + '/';
@@ -58,7 +58,7 @@
     <iframe id="content_iframe"
             referrerpolicy="same-origin"
             onload="on_content_load()"
-            src="skin/blank.html" title="ZIM content" width="100%"
+            src="./skin/blank.html?KIWIXCACHEID" title="ZIM content" width="100%"
             style="border:0px">
     </iframe>
 

test/library.cpp

@@ -801,8 +801,14 @@ TEST_F(LibraryTest, removeBooksNotUpdatedSince)
     lib.addBook(lib.getBookByIdThreadSafe(id));
   }
 
+  EXPECT_GT(lib.getRevision(), rev);
+
+  const uint64_t rev2 = lib.getRevision();
+
   EXPECT_EQ(9u, lib.removeBooksNotUpdatedSince(rev));
 
+  EXPECT_GT(lib.getRevision(), rev2);
+
   EXPECT_FILTER_RESULTS(kiwix::Filter(),
     "Islam Stack Exchange",
     "Movies & TV Stack Exchange",

test/server.cpp

@@ -23,13 +23,19 @@ T1 concat(T1 a, const T2& b)
   return a;
 }
 
-const bool WITH_ETAG = true;
+enum ResourceKind
-const bool NO_ETAG = false;
+{
+  ZIM_CONTENT,
+  STATIC_CONTENT,
+  DYNAMIC_CONTENT,
+};
 
 struct Resource
 {
-  bool etag_expected;
+  ResourceKind kind;
   const char* url;
+
+  bool etag_expected() const { return kind != STATIC_CONTENT; }
 };
 
 std::ostream& operator<<(std::ostream& out, const Resource& r)
@@ -41,55 +47,127 @@ std::ostream& operator<<(std::ostream& out, const Resource& r)
 typedef std::vector<Resource> ResourceCollection;
 
 const ResourceCollection resources200Compressible{
-  { WITH_ETAG, "/ROOT/" },
+  { DYNAMIC_CONTENT, "/ROOT/" },
 
-  { WITH_ETAG, "/ROOT/skin/autoComplete.min.js" },
+  { DYNAMIC_CONTENT, "/ROOT/viewer" },
-  { WITH_ETAG, "/ROOT/skin/css/autoComplete.css" },
+  { DYNAMIC_CONTENT, "/ROOT/viewer?cacheid=whatever" },
-  { WITH_ETAG, "/ROOT/skin/taskbar.css" },
 
-  { NO_ETAG,   "/ROOT/catalog/search" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/autoComplete.min.js" },
+  { STATIC_CONTENT,  "/ROOT/skin/autoComplete.min.js?cacheid=1191aaaf" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/css/autoComplete.css" },
+  { STATIC_CONTENT,  "/ROOT/skin/css/autoComplete.css?cacheid=08951e06" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/favicon/favicon.ico" },
+  { STATIC_CONTENT,  "/ROOT/skin/favicon/favicon.ico?cacheid=fba03a27" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/index.css" },
+  { STATIC_CONTENT,  "/ROOT/skin/index.css?cacheid=0f9ba34e" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/index.js" },
+  { STATIC_CONTENT,  "/ROOT/skin/index.js?cacheid=2f5a81ac" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/iso6391To3.js" },
+  { STATIC_CONTENT,  "/ROOT/skin/iso6391To3.js?cacheid=ecde2bb3" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/isotope.pkgd.min.js" },
+  { STATIC_CONTENT,  "/ROOT/skin/isotope.pkgd.min.js?cacheid=2e48d392" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/taskbar.css" },
+  { STATIC_CONTENT,  "/ROOT/skin/taskbar.css?cacheid=216d6b5d" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/viewer.js" },
+  { STATIC_CONTENT,  "/ROOT/skin/viewer.js?cacheid=51e745c2" },
 
-  { NO_ETAG,   "/ROOT/search?content=zimfile&pattern=a" },
+  { DYNAMIC_CONTENT, "/ROOT/catalog/search" },
 
-  { NO_ETAG,   "/ROOT/suggest?content=zimfile&term=ray" },
+  { DYNAMIC_CONTENT, "/ROOT/catalog/v2/root.xml" },
+  { DYNAMIC_CONTENT, "/ROOT/catalog/v2/languages" },
+  { DYNAMIC_CONTENT, "/ROOT/catalog/v2/entries" },
+  { DYNAMIC_CONTENT, "/ROOT/catalog/v2/partial_entries" },
 
-  { WITH_ETAG, "/ROOT/content/zimfile/A/index" },
+  { DYNAMIC_CONTENT, "/ROOT/search?content=zimfile&pattern=a" },
-  { WITH_ETAG, "/ROOT/content/zimfile/A/Ray_Charles" },
 
-  { WITH_ETAG, "/ROOT/raw/zimfile/content/A/index" },
+  { DYNAMIC_CONTENT, "/ROOT/suggest?content=zimfile&term=ray" },
-  { WITH_ETAG, "/ROOT/raw/zimfile/content/A/Ray_Charles" },
+
+  { ZIM_CONTENT,     "/ROOT/content/zimfile/A/index" },
+  { ZIM_CONTENT,     "/ROOT/content/zimfile/A/Ray_Charles" },
+
+  { ZIM_CONTENT,     "/ROOT/raw/zimfile/content/A/index" },
+  { ZIM_CONTENT,     "/ROOT/raw/zimfile/content/A/Ray_Charles" },
 };
 
 const ResourceCollection resources200Uncompressible{
-  { WITH_ETAG, "/ROOT/skin/caret.png" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/bittorrent.png" },
-  { WITH_ETAG, "/ROOT/skin/css/images/search.svg" },
+  { STATIC_CONTENT,  "/ROOT/skin/bittorrent.png?cacheid=4f5c6882" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/blank.html" },
+  { STATIC_CONTENT,  "/ROOT/skin/blank.html?cacheid=6b1fa032" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/caret.png" },
+  { STATIC_CONTENT,  "/ROOT/skin/caret.png?cacheid=22b942b4" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/download.png" },
+  { STATIC_CONTENT,  "/ROOT/skin/download.png?cacheid=a39aa502" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/favicon/android-chrome-192x192.png" },
+  { STATIC_CONTENT,  "/ROOT/skin/favicon/android-chrome-192x192.png?cacheid=bfac158b" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/favicon/android-chrome-512x512.png" },
+  { STATIC_CONTENT,  "/ROOT/skin/favicon/android-chrome-512x512.png?cacheid=380c3653" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/favicon/apple-touch-icon.png" },
+  { STATIC_CONTENT,  "/ROOT/skin/favicon/apple-touch-icon.png?cacheid=f86f8df3" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/favicon/browserconfig.xml" },
+  { STATIC_CONTENT,  "/ROOT/skin/favicon/browserconfig.xml?cacheid=f29a7c4a" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/favicon/favicon-16x16.png" },
+  { STATIC_CONTENT,  "/ROOT/skin/favicon/favicon-16x16.png?cacheid=a986fedc" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/favicon/favicon-32x32.png" },
+  { STATIC_CONTENT,  "/ROOT/skin/favicon/favicon-32x32.png?cacheid=79ded625" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-144x144.png" },
+  { STATIC_CONTENT,  "/ROOT/skin/favicon/mstile-144x144.png?cacheid=c25a7641" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-150x150.png" },
+  { STATIC_CONTENT,  "/ROOT/skin/favicon/mstile-150x150.png?cacheid=6fa6f467" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-310x150.png" },
+  { STATIC_CONTENT,  "/ROOT/skin/favicon/mstile-310x150.png?cacheid=e0ed9032" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-310x310.png" },
+  { STATIC_CONTENT,  "/ROOT/skin/favicon/mstile-310x310.png?cacheid=26b20530" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-70x70.png" },
+  { STATIC_CONTENT,  "/ROOT/skin/favicon/mstile-70x70.png?cacheid=64ffd9dc" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/favicon/safari-pinned-tab.svg" },
+  { STATIC_CONTENT,  "/ROOT/skin/favicon/safari-pinned-tab.svg?cacheid=8d487e95" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/favicon/site.webmanifest" },
+  { STATIC_CONTENT,  "/ROOT/skin/favicon/site.webmanifest?cacheid=bc396efb" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/fonts/Poppins.ttf" },
+  { STATIC_CONTENT,  "/ROOT/skin/fonts/Poppins.ttf?cacheid=af705837" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/fonts/Roboto.ttf" },
+  { STATIC_CONTENT,  "/ROOT/skin/fonts/Roboto.ttf?cacheid=84d10248" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/hash.png" },
+  { STATIC_CONTENT,  "/ROOT/skin/hash.png?cacheid=f836e872" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/magnet.png" },
+  { STATIC_CONTENT,  "/ROOT/skin/magnet.png?cacheid=73b6bddf" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/search-icon.svg" },
+  { STATIC_CONTENT,  "/ROOT/skin/search-icon.svg?cacheid=b10ae7ed" },
+  { DYNAMIC_CONTENT, "/ROOT/skin/search_results.css" },
+  { STATIC_CONTENT,  "/ROOT/skin/search_results.css?cacheid=76d39c84" },
 
-  { WITH_ETAG, "/ROOT/raw/zimfile/meta/Title" },
+  { ZIM_CONTENT,     "/ROOT/raw/zimfile/meta/Title" },
-  { WITH_ETAG, "/ROOT/raw/zimfile/meta/Description" },
+  { ZIM_CONTENT,     "/ROOT/raw/zimfile/meta/Description" },
-  { WITH_ETAG, "/ROOT/raw/zimfile/meta/Language" },
+  { ZIM_CONTENT,     "/ROOT/raw/zimfile/meta/Language" },
-  { WITH_ETAG, "/ROOT/raw/zimfile/meta/Name" },
+  { ZIM_CONTENT,     "/ROOT/raw/zimfile/meta/Name" },
-  { WITH_ETAG, "/ROOT/raw/zimfile/meta/Tags" },
+  { ZIM_CONTENT,     "/ROOT/raw/zimfile/meta/Tags" },
-  { WITH_ETAG, "/ROOT/raw/zimfile/meta/Date" },
+  { ZIM_CONTENT,     "/ROOT/raw/zimfile/meta/Date" },
-  { WITH_ETAG, "/ROOT/raw/zimfile/meta/Creator" },
+  { ZIM_CONTENT,     "/ROOT/raw/zimfile/meta/Creator" },
-  { WITH_ETAG, "/ROOT/raw/zimfile/meta/Publisher" },
+  { ZIM_CONTENT,     "/ROOT/raw/zimfile/meta/Publisher" },
 
-  { NO_ETAG, "/ROOT/catalog/v2/illustration/6f1d19d0-633f-087b-fb55-7ac324ff9baf?size=48" },
+  { DYNAMIC_CONTENT, "/ROOT/catalog/root.xml" },
+  { DYNAMIC_CONTENT, "/ROOT/catalog/searchdescription.xml" },
 
-  { NO_ETAG,   "/ROOT/catch/external?source=www.example.com" },
+  { DYNAMIC_CONTENT, "/ROOT/catalog/v2/categories" },
+  { DYNAMIC_CONTENT, "/ROOT/catalog/v2/searchdescription.xml" },
+  { DYNAMIC_CONTENT, "/ROOT/catalog/v2/illustration/6f1d19d0-633f-087b-fb55-7ac324ff9baf?size=48" },
 
-  { WITH_ETAG, "/ROOT/content/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg" },
+  { DYNAMIC_CONTENT, "/ROOT/catch/external?source=www.example.com" },
 
-  { WITH_ETAG, "/ROOT/content/corner_cases/A/empty.html" },
+  { ZIM_CONTENT,     "/ROOT/content/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg" },
-  { WITH_ETAG, "/ROOT/content/corner_cases/-/empty.css" },
+
-  { WITH_ETAG, "/ROOT/content/corner_cases/-/empty.js" },
+  { ZIM_CONTENT,     "/ROOT/content/corner_cases/A/empty.html" },
+  { ZIM_CONTENT,     "/ROOT/content/corner_cases/-/empty.css" },
+  { ZIM_CONTENT,     "/ROOT/content/corner_cases/-/empty.js" },
 
 
   // The following url's responses are too small to be compressed
-  { NO_ETAG,   "/ROOT/catalog/root.xml" },
+  { DYNAMIC_CONTENT, "/ROOT/catalog/root.xml" },
-  { NO_ETAG,   "/ROOT/catalog/searchdescription.xml" },
+  { DYNAMIC_CONTENT, "/ROOT/catalog/searchdescription.xml" },
-  { NO_ETAG,   "/ROOT/suggest?content=zimfile" },
+  { DYNAMIC_CONTENT, "/ROOT/suggest?content=zimfile" },
-  { WITH_ETAG, "/ROOT/raw/zimfile/meta/Creator" },
+  { ZIM_CONTENT,     "/ROOT/raw/zimfile/meta/Creator" },
-  { WITH_ETAG, "/ROOT/raw/zimfile/meta/Title" },
+  { ZIM_CONTENT,     "/ROOT/raw/zimfile/meta/Title" },
 };
 
 ResourceCollection all200Resources()
@@ -172,11 +250,11 @@ TEST_F(ServerTest, CacheIdsOfStaticResources)
   const std::vector<UrlAndExpectedResult> testData{
     {
       /* url */ "/ROOT/",
-R"EXPECTEDRESULT(      href="/ROOT/skin/index.css?cacheid=3b470cee"
+R"EXPECTEDRESULT(      href="/ROOT/skin/index.css?cacheid=0f9ba34e"
     <link rel="apple-touch-icon" sizes="180x180" href="/ROOT/skin/favicon/apple-touch-icon.png?cacheid=f86f8df3">
     <link rel="icon" type="image/png" sizes="32x32" href="/ROOT/skin/favicon/favicon-32x32.png?cacheid=79ded625">
     <link rel="icon" type="image/png" sizes="16x16" href="/ROOT/skin/favicon/favicon-16x16.png?cacheid=a986fedc">
-    <link rel="manifest" href="/ROOT/skin/favicon/site.webmanifest">
+    <link rel="manifest" href="/ROOT/skin/favicon/site.webmanifest?cacheid=bc396efb">
     <link rel="mask-icon" href="/ROOT/skin/favicon/safari-pinned-tab.svg?cacheid=8d487e95" color="#5bbad5">
     <link rel="shortcut icon" href="/ROOT/skin/favicon/favicon.ico?cacheid=fba03a27">
     <meta name="msapplication-config" content="/ROOT/skin/favicon/browserconfig.xml?cacheid=f29a7c4a">
@@ -185,6 +263,11 @@ R"EXPECTEDRESULT(      href="/ROOT/skin/index.css?cacheid=3b470cee"
     <script src="/ROOT/skin/isotope.pkgd.min.js?cacheid=2e48d392" defer></script>
     <script src="/ROOT/skin/iso6391To3.js?cacheid=ecde2bb3"></script>
     <script type="text/javascript" src="/ROOT/skin/index.js?cacheid=2f5a81ac" defer></script>
+)EXPECTEDRESULT"
+    },
+    {
+      /* url */ "/ROOT/skin/index.css",
+R"EXPECTEDRESULT(    background-image: url('../skin/search-icon.svg?cacheid=b10ae7ed');
 )EXPECTEDRESULT"
     },
     {
@@ -201,8 +284,9 @@ R"EXPECTEDRESULT(    <link type="text/css" href="./skin/taskbar.css?cacheid=216d
     <link type="text/css" href="./skin/css/autoComplete.css?cacheid=08951e06" rel="Stylesheet" />
     <script type="text/javascript" src="./skin/viewer.js?cacheid=51e745c2" defer></script>
     <script type="text/javascript" src="./skin/autoComplete.min.js?cacheid=1191aaaf"></script>
-      const blankPageUrl = `${root}/skin/blank.html`;
+      const blankPageUrl = root + "/skin/blank.html?cacheid=6b1fa032";
             <label for="kiwix_button_show_toggle"><img src="./skin/caret.png?cacheid=22b942b4" alt=""></label>
+            src="./skin/blank.html?cacheid=6b1fa032" title="ZIM content" width="100%"
 )EXPECTEDRESULT"
     },
     {
@@ -252,6 +336,7 @@ const char* urls404[] = {
   "/",
   "/zimfile",
   "/ROOT/skin/non-existent-skin-resource",
+  "/ROOT/skin/autoComplete.min.js?cacheid=wrongcacheid",
   "/ROOT/catalog",
   "/ROOT/catalog/",
   "/ROOT/catalog/non-existent-item",
@@ -310,6 +395,11 @@ std::string getHeaderValue(const Headers& headers, const std::string& name)
   return er.first->second;
 }
 
+std::string getCacheControlHeader(const httplib::Response& r)
+{
+  return getHeaderValue(r.headers, "Cache-Control");
+}
+
 TEST_F(CustomizedServerTest, NewResourcesCanBeAdded)
 {
   // ServerTest.404 verifies that "/ROOT/non-existent-item" doesn't exist
@@ -952,6 +1042,8 @@ TEST_F(ServerTest, RandomPageRedirectsToAnExistingArticle)
   ASSERT_EQ(302, g->status);
   ASSERT_TRUE(g->has_header("Location"));
   ASSERT_TRUE(kiwix::startsWith(g->get_header_value("Location"), "/ROOT/content/zimfile/A/"));
+  ASSERT_EQ(getCacheControlHeader(*g), "max-age=0, must-revalidate");
+  ASSERT_FALSE(g->has_header("ETag"));
 }
 
 TEST_F(ServerTest, NonEndpointUrlsAreRedirectedToContentUrls)
@@ -995,6 +1087,8 @@ TEST_F(ServerTest, NonEndpointUrlsAreRedirectedToContentUrls)
     ASSERT_EQ(302, g->status) << ctx;
     ASSERT_TRUE(g->has_header("Location")) << ctx;
     ASSERT_EQ("/ROOT/content" + p, g->get_header_value("Location")) << ctx;
+    ASSERT_EQ(getCacheControlHeader(*g), "max-age=0, must-revalidate");
+    ASSERT_FALSE(g->has_header("ETag"));
   }
 }
 
@@ -1059,12 +1153,45 @@ TEST_F(ServerTest, HeadersAreTheSameInResponsesToHeadAndGetRequests)
   }
 }
 
+TEST_F(ServerTest, CacheControlOfZimContent)
+{
+  for ( const Resource& res : all200Resources() ) {
+    if ( res.kind == ZIM_CONTENT ) {
+      const auto g = zfs1_->GET(res.url);
+      EXPECT_EQ(getCacheControlHeader(*g), "max-age=3600, must-revalidate") << res;
+      EXPECT_TRUE(g->has_header("ETag")) << res;
+    }
+  }
+}
+
+TEST_F(ServerTest, CacheControlOfStaticContent)
+{
+  for ( const Resource& res : all200Resources() ) {
+    if ( res.kind == STATIC_CONTENT ) {
+      const auto g = zfs1_->GET(res.url);
+      EXPECT_EQ(getCacheControlHeader(*g), "max-age=31536000, immutable") << res;
+      EXPECT_FALSE(g->has_header("ETag")) << res;
+    }
+  }
+}
+
+TEST_F(ServerTest, CacheControlOfDynamicContent)
+{
+  for ( const Resource& res : all200Resources() ) {
+    if ( res.kind == DYNAMIC_CONTENT ) {
+      const auto g = zfs1_->GET(res.url);
+      EXPECT_EQ(getCacheControlHeader(*g), "max-age=0, must-revalidate") << res;
+      EXPECT_TRUE(g->has_header("ETag")) << res;
+    }
+  }
+}
+
 TEST_F(ServerTest, ETagHeaderIsSetAsNeeded)
 {
   for ( const Resource& res : all200Resources() ) {
     const auto responseToGet = zfs1_->GET(res.url);
-    EXPECT_EQ(res.etag_expected, responseToGet->has_header("ETag")) << res;
+    EXPECT_EQ(res.etag_expected(), responseToGet->has_header("ETag")) << res;
-    if ( res.etag_expected ) {
+    if ( res.etag_expected() ) {
       EXPECT_TRUE(is_valid_etag(responseToGet->get_header_value("ETag")));
     }
   }
@@ -1088,21 +1215,32 @@ TEST_F(ServerTest, ETagIsTheSameAcrossHeadAndGet)
   }
 }
 
-TEST_F(ServerTest, DifferentServerInstancesProduceDifferentETags)
+TEST_F(ServerTest, DifferentServerInstancesProduceDifferentETagsForDynamicContent)
 {
   ZimFileServer zfs2(SERVER_PORT + 1, ZimFileServer::DEFAULT_OPTIONS, ZIMFILES);
   for ( const Resource& res : all200Resources() ) {
-    if ( !res.etag_expected ) continue;
+    if ( res.kind != DYNAMIC_CONTENT ) continue;
     const auto h1 = zfs1_->HEAD(res.url);
     const auto h2 = zfs2.HEAD(res.url);
     EXPECT_NE(h1->get_header_value("ETag"), h2->get_header_value("ETag"));
   }
 }
 
+TEST_F(ServerTest, DifferentServerInstancesProduceIdenticalETagsForZimContent)
+{
+  ZimFileServer zfs2(SERVER_PORT + 1, ZimFileServer::DEFAULT_OPTIONS, ZIMFILES);
+  for ( const Resource& res : all200Resources() ) {
+    if ( res.kind != ZIM_CONTENT ) continue;
+    const auto h1 = zfs1_->HEAD(res.url);
+    const auto h2 = zfs2.HEAD(res.url);
+    EXPECT_EQ(h1->get_header_value("ETag"), h2->get_header_value("ETag"));
+  }
+}
+
 TEST_F(ServerTest, CompressionInfluencesETag)
 {
   for ( const Resource& res : resources200Compressible ) {
-    if ( ! res.etag_expected ) continue;
+    if ( ! res.etag_expected() ) continue;
     const auto g1 = zfs1_->GET(res.url);
     const auto g2 = zfs1_->GET(res.url, { {"Accept-Encoding", ""} } );
     const auto g3 = zfs1_->GET(res.url, { {"Accept-Encoding", "gzip"} } );
@@ -1115,7 +1253,7 @@ TEST_F(ServerTest, CompressionInfluencesETag)
 TEST_F(ServerTest, ETagOfUncompressibleContentIsNotAffectedByAcceptEncoding)
 {
   for ( const Resource& res : resources200Uncompressible ) {
-    if ( ! res.etag_expected ) continue;
+    if ( ! res.etag_expected() ) continue;
     const auto g1 = zfs1_->GET(res.url);
     const auto g2 = zfs1_->GET(res.url, { {"Accept-Encoding", ""} } );
     const auto g3 = zfs1_->GET(res.url, { {"Accept-Encoding", "gzip"} } );
@@ -1160,7 +1298,7 @@ TEST_F(ServerTest, IfNoneMatchRequestsWithMatchingETagResultIn304Responses)
   const char* const encodings[] = { "", "gzip" };
   for ( const Resource& res : all200Resources() ) {
     for ( const char* enc: encodings ) {
-      if ( ! res.etag_expected ) continue;
+      if ( ! res.etag_expected() ) continue;
       const TestContext ctx{ {"url", res.url}, {"encoding", enc} };
 
       const auto g = zfs1_->GET(res.url, { {"Accept-Encoding", enc} });
@@ -1187,8 +1325,8 @@ TEST_F(ServerTest, IfNoneMatchRequestsWithMismatchingETagResultIn200Responses)
     const auto etag2 = etag.substr(0, etag.size() - 1) + "x\"";
     const auto h = zfs1_->HEAD(res.url, { {"If-None-Match", etag2} } );
     const auto g2 = zfs1_->GET(res.url, { {"If-None-Match", etag2} } );
-    EXPECT_EQ(200, h->status);
+    EXPECT_EQ(200, h->status) << res;
-    EXPECT_EQ(200, g2->status);
+    EXPECT_EQ(200, g2->status) << res;
   }
 }