From b9f60ecfe96e952be115495295910b9bde41f5fe Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Tue, 11 Oct 2022 14:21:08 +0400 Subject: [PATCH] Handling of cacheid when serving static resources During static resource preprocessing and compilation their cacheid values are embedded into libkiwix and can be accessed at runtime. If a static resource is requsted without specifying any cacheid it is served as dynamic content (with short TTL and the library id used for the ETag, though using the cacheid for the ETag would be better). If a cacheid is supplied in the request it must match the cacheid of the resource (otherwise a 404 Not Found error is returned) whereupon the resource is served as immutable content. Known issues: - One issue is caused by the fact that some static resources don't get a cacheid; this is resolved in the next commit. - Interaction of this change with the support for dynamically customizing static resources (via KIWIX_SERVE_CUSTOMIZED_RESOURCES env var) was not addressed. --- scripts/kiwix-compile-resources | 32 ++++++++-- scripts/kiwix-resources | 15 +++-- src/server/internalServer.cpp | 25 +++++++- test/server.cpp | 103 +++++++++++++++++++++----------- 4 files changed, 129 insertions(+), 46 deletions(-) diff --git a/scripts/kiwix-compile-resources b/scripts/kiwix-compile-resources index 7c1bf3a8a..7d308afc3 100755 --- a/scripts/kiwix-compile-resources +++ b/scripts/kiwix-compile-resources @@ -52,15 +52,21 @@ resource_getter_template = """ return RESOURCE::{identifier}; """ +resource_cacheid_getter_template = """ + if (name == "{common_name}") + return "{cacheid}"; +""" + resource_decl_template = """{namespaces_open} extern const std::string {identifier}; {namespaces_close}""" class Resource: - def __init__(self, base_dirs, filename): - filename = filename.strip() + def __init__(self, base_dirs, filename, cacheid=None): + filename = filename self.filename = filename self.identifier = full_identifier(filename) + self.cacheid = cacheid found = False for base_dir in base_dirs: try: @@ -71,7 +77,7 @@ class Resource: except FileNotFoundError: continue if not found: - raise Exception("Impossible to found {}".format(filename)) + raise Exception("Resource not found: {}".format(filename)) def dump_impl(self): nb_row = len(self.data)//16 + (1 if len(self.data) % 16 else 0) @@ -93,6 +99,12 @@ class Resource: identifier="::".join(self.identifier) ) + def dump_cacheid_getter(self): + return resource_cacheid_getter_template.format( + common_name=self.filename, + cacheid=self.cacheid + ) + def dump_decl(self): return resource_decl_template.format( namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]), @@ -123,7 +135,12 @@ static std::string init_resource(const char* name, const unsigned char* content, const std::string& getResource_{basename}(const std::string& name) {{ {RESOURCES_GETTER} - throw ResourceNotFound("Resource not found."); + throw ResourceNotFound("Resource not found: " + name); +}} + +const char* getResourceCacheId_{basename}(const std::string& name) {{ +{RESOURCE_CACHEID_GETTER} + return nullptr; }} {RESOURCES} @@ -134,6 +151,7 @@ def gen_c_file(resources, basename): return master_c_template.format( RESOURCES="\n\n".join(r.dump_impl() for r in resources), RESOURCES_GETTER="\n\n".join(r.dump_getter() for r in resources), + RESOURCE_CACHEID_GETTER="\n\n".join(r.dump_cacheid_getter() for r in resources if r.cacheid is not None), include_file=basename, basename=to_identifier(basename) ) @@ -159,8 +177,10 @@ class ResourceNotFound : public std::runtime_error {{ }}; const std::string& getResource_{basename}(const std::string& name); +const char* getResourceCacheId_{basename}(const std::string& name); #define getResource(a) (getResource_{basename}(a)) +#define getResourceCacheId(a) (getResourceCacheId_{basename}(a)) #endif // KIWIX_{BASENAME} @@ -189,8 +209,8 @@ if __name__ == "__main__": base_dir = os.path.dirname(os.path.realpath(args.resource_file)) source_dir = args.source_dir or [] with open(args.resource_file, 'r') as f: - resources = [Resource([base_dir]+source_dir, filename) - for filename in f.readlines()] + resources = [Resource([base_dir]+source_dir, *line.strip().split()) + for line in f.readlines()] h_identifier = to_identifier(os.path.basename(args.hfile)) with open(args.hfile, 'w') as f: diff --git a/scripts/kiwix-resources b/scripts/kiwix-resources index 481b03f06..5fb7b8db2 100755 --- a/scripts/kiwix-resources +++ b/scripts/kiwix-resources @@ -99,16 +99,21 @@ def preprocess_resource(resource_path): print(preprocessed_content, end='', file=target) -def copy_file(src_path, dst_path): - with open(src_path, 'rb') as src: - with open(dst_path, 'wb') as dst: - dst.write(src.read()) +def copy_resource_list_file(src_path, dst_path): + with open(src_path, 'r') as src: + with open(dst_path, 'w') as dst: + for line in src: + res = line.strip() + if line.startswith("skin/") and res in resource_revisions: + dst.write(res + " " + resource_revisions[res] + "\n") + else: + dst.write(line) def preprocess_resources(resource_file_path): resource_filename = os.path.basename(resource_file_path) for resource in read_resource_file(resource_file_path): preprocess_resource(resource) - copy_file(resource_file_path, os.path.join(OUT_DIR, resource_filename)) + copy_resource_list_file(resource_file_path, os.path.join(OUT_DIR, resource_filename)) if __name__ == "__main__": parser = argparse.ArgumentParser() diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp index 2f65586b5..eeadd15d2 100644 --- a/src/server/internalServer.cpp +++ b/src/server/internalServer.cpp @@ -748,6 +748,25 @@ std::unique_ptr InternalServer::handle_viewer_settings(const RequestCo return ContentResponse::build(*this, RESOURCE::templates::viewer_settings_js, data, "application/javascript; charset=utf-8"); } +namespace +{ + +Response::Kind staticResourceAccessType(const RequestContext& req, const char* expectedCacheid) +{ + if ( expectedCacheid == nullptr ) + return Response::DYNAMIC_CONTENT; + + try { + if ( expectedCacheid != req.get_argument("cacheid") ) + throw ResourceNotFound("Wrong cacheid"); + return Response::STATIC_RESOURCE; + } catch( const std::out_of_range& ) { + return Response::DYNAMIC_CONTENT; + } +} + +} // unnamed namespace + std::unique_ptr InternalServer::handle_skin(const RequestContext& request) { if (m_verbose.load()) { @@ -758,12 +777,16 @@ std::unique_ptr InternalServer::handle_skin(const RequestContext& requ auto resourceName = isRequestForViewer ? "viewer.html" : request.get_url().substr(1); + + const char* const resourceCacheId = getResourceCacheId(resourceName); + try { + const auto accessType = staticResourceAccessType(request, resourceCacheId); auto response = ContentResponse::build( *this, getResource(resourceName), getMimeTypeForFile(resourceName)); - response->set_kind(Response::STATIC_RESOURCE); + response->set_kind(accessType); return std::move(response); } catch (const ResourceNotFound& e) { return HTTP404Response(*this, request) diff --git a/test/server.cpp b/test/server.cpp index d909f78da..1f646574c 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -49,17 +49,27 @@ typedef std::vector ResourceCollection; const ResourceCollection resources200Compressible{ { DYNAMIC_CONTENT, "/ROOT/" }, - { STATIC_CONTENT, "/ROOT/viewer" }, + { DYNAMIC_CONTENT, "/ROOT/viewer" }, + { DYNAMIC_CONTENT, "/ROOT/viewer?cacheid=whatever" }, - { STATIC_CONTENT, "/ROOT/skin/autoComplete.min.js" }, - { STATIC_CONTENT, "/ROOT/skin/css/autoComplete.css" }, - { STATIC_CONTENT, "/ROOT/skin/favicon/favicon.ico" }, - { STATIC_CONTENT, "/ROOT/skin/index.css" }, - { STATIC_CONTENT, "/ROOT/skin/index.js" }, - { STATIC_CONTENT, "/ROOT/skin/iso6391To3.js" }, - { STATIC_CONTENT, "/ROOT/skin/isotope.pkgd.min.js" }, - { STATIC_CONTENT, "/ROOT/skin/taskbar.css" }, - { STATIC_CONTENT, "/ROOT/skin/viewer.js" }, + { DYNAMIC_CONTENT, "/ROOT/skin/autoComplete.min.js" }, + { STATIC_CONTENT, "/ROOT/skin/autoComplete.min.js?cacheid=1191aaaf" }, + { DYNAMIC_CONTENT, "/ROOT/skin/css/autoComplete.css" }, + { STATIC_CONTENT, "/ROOT/skin/css/autoComplete.css?cacheid=08951e06" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/favicon.ico" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/favicon.ico?cacheid=fba03a27" }, + { DYNAMIC_CONTENT, "/ROOT/skin/index.css" }, + { STATIC_CONTENT, "/ROOT/skin/index.css?cacheid=3b470cee" }, + { DYNAMIC_CONTENT, "/ROOT/skin/index.js" }, + { STATIC_CONTENT, "/ROOT/skin/index.js?cacheid=2f5a81ac" }, + { DYNAMIC_CONTENT, "/ROOT/skin/iso6391To3.js" }, + { STATIC_CONTENT, "/ROOT/skin/iso6391To3.js?cacheid=ecde2bb3" }, + { DYNAMIC_CONTENT, "/ROOT/skin/isotope.pkgd.min.js" }, + { STATIC_CONTENT, "/ROOT/skin/isotope.pkgd.min.js?cacheid=2e48d392" }, + { DYNAMIC_CONTENT, "/ROOT/skin/taskbar.css" }, + { STATIC_CONTENT, "/ROOT/skin/taskbar.css?cacheid=216d6b5d" }, + { DYNAMIC_CONTENT, "/ROOT/skin/viewer.js" }, + { STATIC_CONTENT, "/ROOT/skin/viewer.js?cacheid=51e745c2" }, { DYNAMIC_CONTENT, "/ROOT/catalog/search" }, @@ -80,30 +90,54 @@ const ResourceCollection resources200Compressible{ }; const ResourceCollection resources200Uncompressible{ - { STATIC_CONTENT, "/ROOT/skin/bittorrent.png" }, - { STATIC_CONTENT, "/ROOT/skin/blank.html" }, - { STATIC_CONTENT, "/ROOT/skin/caret.png" }, - { STATIC_CONTENT, "/ROOT/skin/css/images/search.svg" }, - { STATIC_CONTENT, "/ROOT/skin/download.png" }, - { STATIC_CONTENT, "/ROOT/skin/favicon/android-chrome-192x192.png" }, - { STATIC_CONTENT, "/ROOT/skin/favicon/android-chrome-512x512.png" }, - { STATIC_CONTENT, "/ROOT/skin/favicon/apple-touch-icon.png" }, - { STATIC_CONTENT, "/ROOT/skin/favicon/browserconfig.xml" }, - { STATIC_CONTENT, "/ROOT/skin/favicon/favicon-16x16.png" }, - { STATIC_CONTENT, "/ROOT/skin/favicon/favicon-32x32.png" }, - { STATIC_CONTENT, "/ROOT/skin/favicon/mstile-144x144.png" }, - { STATIC_CONTENT, "/ROOT/skin/favicon/mstile-150x150.png" }, - { STATIC_CONTENT, "/ROOT/skin/favicon/mstile-310x150.png" }, - { STATIC_CONTENT, "/ROOT/skin/favicon/mstile-310x310.png" }, - { STATIC_CONTENT, "/ROOT/skin/favicon/mstile-70x70.png" }, - { STATIC_CONTENT, "/ROOT/skin/favicon/safari-pinned-tab.svg" }, - { STATIC_CONTENT, "/ROOT/skin/favicon/site.webmanifest" }, - { STATIC_CONTENT, "/ROOT/skin/fonts/Poppins.ttf" }, - { STATIC_CONTENT, "/ROOT/skin/fonts/Roboto.ttf" }, - { STATIC_CONTENT, "/ROOT/skin/hash.png" }, - { STATIC_CONTENT, "/ROOT/skin/magnet.png" }, - { STATIC_CONTENT, "/ROOT/skin/search-icon.svg" }, - { STATIC_CONTENT, "/ROOT/skin/search_results.css" }, + { DYNAMIC_CONTENT, "/ROOT/skin/bittorrent.png" }, + { STATIC_CONTENT, "/ROOT/skin/bittorrent.png?cacheid=4f5c6882" }, + { DYNAMIC_CONTENT, "/ROOT/skin/blank.html" }, + { STATIC_CONTENT, "/ROOT/skin/blank.html?cacheid=6b1fa032" }, + { DYNAMIC_CONTENT, "/ROOT/skin/caret.png" }, + { STATIC_CONTENT, "/ROOT/skin/caret.png?cacheid=22b942b4" }, + { DYNAMIC_CONTENT, "/ROOT/skin/css/images/search.svg" }, + //{ STATIC_CONTENT, "/ROOT/skin/css/images/search.svg?cacheid=XXXX" }, + { DYNAMIC_CONTENT, "/ROOT/skin/download.png" }, + { STATIC_CONTENT, "/ROOT/skin/download.png?cacheid=a39aa502" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/android-chrome-192x192.png" }, + //{ STATIC_CONTENT, "/ROOT/skin/favicon/android-chrome-192x192.png?cacheid=" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/android-chrome-512x512.png" }, + //{ STATIC_CONTENT, "/ROOT/skin/favicon/android-chrome-512x512.png?cacheid=" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/apple-touch-icon.png" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/apple-touch-icon.png?cacheid=f86f8df3" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/browserconfig.xml" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/browserconfig.xml?cacheid=f29a7c4a" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/favicon-16x16.png" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/favicon-16x16.png?cacheid=a986fedc" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/favicon-32x32.png" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/favicon-32x32.png?cacheid=79ded625" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-144x144.png" }, + //{ STATIC_CONTENT, "/ROOT/skin/favicon/mstile-144x144.png?cacheid=" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-150x150.png" }, + //{ STATIC_CONTENT, "/ROOT/skin/favicon/mstile-150x150.png?cacheid=" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-310x150.png" }, + //{ STATIC_CONTENT, "/ROOT/skin/favicon/mstile-310x150.png?cacheid=" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-310x310.png" }, + //{ STATIC_CONTENT, "/ROOT/skin/favicon/mstile-310x310.png?cacheid=" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-70x70.png" }, + //{ STATIC_CONTENT, "/ROOT/skin/favicon/mstile-70x70.png?cacheid=" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/safari-pinned-tab.svg" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/safari-pinned-tab.svg?cacheid=8d487e95" }, + { DYNAMIC_CONTENT, "/ROOT/skin/favicon/site.webmanifest" }, + { STATIC_CONTENT, "/ROOT/skin/favicon/site.webmanifest?cacheid=bc396efb" }, + { DYNAMIC_CONTENT, "/ROOT/skin/fonts/Poppins.ttf" }, + { STATIC_CONTENT, "/ROOT/skin/fonts/Poppins.ttf?cacheid=af705837" }, + { DYNAMIC_CONTENT, "/ROOT/skin/fonts/Roboto.ttf" }, + { STATIC_CONTENT, "/ROOT/skin/fonts/Roboto.ttf?cacheid=84d10248" }, + { DYNAMIC_CONTENT, "/ROOT/skin/hash.png" }, + { STATIC_CONTENT, "/ROOT/skin/hash.png?cacheid=f836e872" }, + { DYNAMIC_CONTENT, "/ROOT/skin/magnet.png" }, + { STATIC_CONTENT, "/ROOT/skin/magnet.png?cacheid=73b6bddf" }, + { DYNAMIC_CONTENT, "/ROOT/skin/search-icon.svg" }, + //{ STATIC_CONTENT, "/ROOT/skin/search-icon.svg?cacheid=" }, + { DYNAMIC_CONTENT, "/ROOT/skin/search_results.css" }, + { STATIC_CONTENT, "/ROOT/skin/search_results.css?cacheid=76d39c84" }, { ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Title" }, { ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Description" }, @@ -299,6 +333,7 @@ const char* urls404[] = { "/", "/zimfile", "/ROOT/skin/non-existent-skin-resource", + "/ROOT/skin/autoComplete.min.js?cacheid=wrongcacheid", "/ROOT/catalog", "/ROOT/catalog/", "/ROOT/catalog/non-existent-item",