smallsolar
/
libkiwix
mirror of https://github.com/kiwix/libkiwix.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Handling of cacheid when serving static resources 

During static resource preprocessing and compilation their cacheid
values are embedded into libkiwix and can be accessed at runtime.

If a static resource is requsted without specifying any cacheid
it is served as dynamic content (with short TTL and the library id
used for the ETag, though using the cacheid for the ETag would
be better).

If a cacheid is supplied in the request it must match the cacheid of the
resource (otherwise a 404 Not Found error is returned) whereupon the
resource is served as immutable content.

Known issues:

- One issue is caused by the fact that some static resources don't get a
  cacheid; this is resolved in the next commit.

- Interaction of this change with the support for dynamically customizing
  static resources (via KIWIX_SERVE_CUSTOMIZED_RESOURCES env var) was
  not addressed.
This commit is contained in:
Veloman Yunkan 2022-10-11 14:21:08 +04:00
parent 12a638750e
commit b9f60ecfe9
4 changed files with 129 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
scripts/kiwix-compile-resources
View File

@ -52,15 +52,21 @@ resource_getter_template = """
return RESOURCE::{identifier}; return RESOURCE::{identifier};
""" """
resource_cacheid_getter_template = """
if (name == "{common_name}")
return "{cacheid}";
"""
resource_decl_template = """{namespaces_open} resource_decl_template = """{namespaces_open}
extern const std::string {identifier}; extern const std::string {identifier};
{namespaces_close}""" {namespaces_close}"""
class Resource: class Resource:
def __init__(self, base_dirs, filename): def __init__(self, base_dirs, filename, cacheid=None):
filename = filename.strip() filename = filename
self.filename = filename self.filename = filename
self.identifier = full_identifier(filename) self.identifier = full_identifier(filename)
self.cacheid = cacheid
found = False found = False
for base_dir in base_dirs: for base_dir in base_dirs:
try: try:
@ -71,7 +77,7 @@ class Resource:
except FileNotFoundError: except FileNotFoundError:
continue continue
if not found: if not found:
raise Exception("Impossible to found {}".format(filename)) raise Exception("Resource not found: {}".format(filename))
def dump_impl(self): def dump_impl(self):
nb_row = len(self.data)//16 + (1 if len(self.data) % 16 else 0) nb_row = len(self.data)//16 + (1 if len(self.data) % 16 else 0)
@ -93,6 +99,12 @@ class Resource:
identifier="::".join(self.identifier) identifier="::".join(self.identifier)
) )
def dump_cacheid_getter(self):
return resource_cacheid_getter_template.format(
common_name=self.filename,
cacheid=self.cacheid
)
def dump_decl(self): def dump_decl(self):
return resource_decl_template.format( return resource_decl_template.format(
namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]), namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]),
@ -123,7 +135,12 @@ static std::string init_resource(const char* name, const unsigned char* content,
const std::string& getResource_{basename}(const std::string& name) {{ const std::string& getResource_{basename}(const std::string& name) {{
{RESOURCES_GETTER} {RESOURCES_GETTER}
throw ResourceNotFound("Resource not found."); throw ResourceNotFound("Resource not found: " + name);
}}
const char* getResourceCacheId_{basename}(const std::string& name) {{
{RESOURCE_CACHEID_GETTER}
return nullptr;
}} }}
{RESOURCES} {RESOURCES}
@ -134,6 +151,7 @@ def gen_c_file(resources, basename):
return master_c_template.format( return master_c_template.format(
RESOURCES="\n\n".join(r.dump_impl() for r in resources), RESOURCES="\n\n".join(r.dump_impl() for r in resources),
RESOURCES_GETTER="\n\n".join(r.dump_getter() for r in resources), RESOURCES_GETTER="\n\n".join(r.dump_getter() for r in resources),
RESOURCE_CACHEID_GETTER="\n\n".join(r.dump_cacheid_getter() for r in resources if r.cacheid is not None),
include_file=basename, include_file=basename,
basename=to_identifier(basename) basename=to_identifier(basename)
) )
@ -159,8 +177,10 @@ class ResourceNotFound : public std::runtime_error {{
}}; }};
const std::string& getResource_{basename}(const std::string& name); const std::string& getResource_{basename}(const std::string& name);
const char* getResourceCacheId_{basename}(const std::string& name);
#define getResource(a) (getResource_{basename}(a)) #define getResource(a) (getResource_{basename}(a))
#define getResourceCacheId(a) (getResourceCacheId_{basename}(a))
#endif // KIWIX_{BASENAME} #endif // KIWIX_{BASENAME}
@ -189,8 +209,8 @@ if __name__ == "__main__":
base_dir = os.path.dirname(os.path.realpath(args.resource_file)) base_dir = os.path.dirname(os.path.realpath(args.resource_file))
source_dir = args.source_dir or [] source_dir = args.source_dir or []
with open(args.resource_file, 'r') as f: with open(args.resource_file, 'r') as f:
resources = [Resource([base_dir]+source_dir, filename) resources = [Resource([base_dir]+source_dir, *line.strip().split())
for filename in f.readlines()] for line in f.readlines()]
h_identifier = to_identifier(os.path.basename(args.hfile)) h_identifier = to_identifier(os.path.basename(args.hfile))
with open(args.hfile, 'w') as f: with open(args.hfile, 'w') as f:

15
scripts/kiwix-resources
View File

@ -99,16 +99,21 @@ def preprocess_resource(resource_path):
print(preprocessed_content, end='', file=target) print(preprocessed_content, end='', file=target)
def copy_file(src_path, dst_path): def copy_resource_list_file(src_path, dst_path):
with open(src_path, 'rb') as src: with open(src_path, 'r') as src:
with open(dst_path, 'wb') as dst: with open(dst_path, 'w') as dst:
dst.write(src.read()) for line in src:
res = line.strip()
if line.startswith("skin/") and res in resource_revisions:
dst.write(res + " " + resource_revisions[res] + "\n")
else:
dst.write(line)
def preprocess_resources(resource_file_path): def preprocess_resources(resource_file_path):
resource_filename = os.path.basename(resource_file_path) resource_filename = os.path.basename(resource_file_path)
for resource in read_resource_file(resource_file_path): for resource in read_resource_file(resource_file_path):
preprocess_resource(resource) preprocess_resource(resource)
copy_file(resource_file_path, os.path.join(OUT_DIR, resource_filename)) copy_resource_list_file(resource_file_path, os.path.join(OUT_DIR, resource_filename))
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()

25
src/server/internalServer.cpp
View File

@ -748,6 +748,25 @@ std::unique_ptr<Response> InternalServer::handle_viewer_settings(const RequestCo
return ContentResponse::build(*this, RESOURCE::templates::viewer_settings_js, data, "application/javascript; charset=utf-8"); return ContentResponse::build(*this, RESOURCE::templates::viewer_settings_js, data, "application/javascript; charset=utf-8");
} }
namespace
{
Response::Kind staticResourceAccessType(const RequestContext& req, const char* expectedCacheid)
{
if ( expectedCacheid == nullptr )
return Response::DYNAMIC_CONTENT;
try {
if ( expectedCacheid != req.get_argument("cacheid") )
throw ResourceNotFound("Wrong cacheid");
return Response::STATIC_RESOURCE;
} catch( const std::out_of_range& ) {
return Response::DYNAMIC_CONTENT;
}
}
} // unnamed namespace
std::unique_ptr<Response> InternalServer::handle_skin(const RequestContext& request) std::unique_ptr<Response> InternalServer::handle_skin(const RequestContext& request)
{ {
if (m_verbose.load()) { if (m_verbose.load()) {
@ -758,12 +777,16 @@ std::unique_ptr<Response> InternalServer::handle_skin(const RequestContext& requ
auto resourceName = isRequestForViewer auto resourceName = isRequestForViewer
? "viewer.html" ? "viewer.html"
: request.get_url().substr(1); : request.get_url().substr(1);
const char* const resourceCacheId = getResourceCacheId(resourceName);
try { try {
const auto accessType = staticResourceAccessType(request, resourceCacheId);
auto response = ContentResponse::build( auto response = ContentResponse::build(
*this, *this,
getResource(resourceName), getResource(resourceName),
getMimeTypeForFile(resourceName)); getMimeTypeForFile(resourceName));
response->set_kind(Response::STATIC_RESOURCE); response->set_kind(accessType);
return std::move(response); return std::move(response);
} catch (const ResourceNotFound& e) { } catch (const ResourceNotFound& e) {
return HTTP404Response(*this, request) return HTTP404Response(*this, request)

103
test/server.cpp
View File

@ -49,17 +49,27 @@ typedef std::vector<Resource> ResourceCollection;
const ResourceCollection resources200Compressible{ const ResourceCollection resources200Compressible{
{ DYNAMIC_CONTENT, "/ROOT/" }, { DYNAMIC_CONTENT, "/ROOT/" },
{ STATIC_CONTENT, "/ROOT/viewer" }, { DYNAMIC_CONTENT, "/ROOT/viewer" },
{ DYNAMIC_CONTENT, "/ROOT/viewer?cacheid=whatever" },
{ STATIC_CONTENT, "/ROOT/skin/autoComplete.min.js" }, { DYNAMIC_CONTENT, "/ROOT/skin/autoComplete.min.js" },
{ STATIC_CONTENT, "/ROOT/skin/css/autoComplete.css" }, { STATIC_CONTENT, "/ROOT/skin/autoComplete.min.js?cacheid=1191aaaf" },
{ STATIC_CONTENT, "/ROOT/skin/favicon/favicon.ico" }, { DYNAMIC_CONTENT, "/ROOT/skin/css/autoComplete.css" },
{ STATIC_CONTENT, "/ROOT/skin/index.css" }, { STATIC_CONTENT, "/ROOT/skin/css/autoComplete.css?cacheid=08951e06" },
{ STATIC_CONTENT, "/ROOT/skin/index.js" }, { DYNAMIC_CONTENT, "/ROOT/skin/favicon/favicon.ico" },
{ STATIC_CONTENT, "/ROOT/skin/iso6391To3.js" }, { STATIC_CONTENT, "/ROOT/skin/favicon/favicon.ico?cacheid=fba03a27" },
{ STATIC_CONTENT, "/ROOT/skin/isotope.pkgd.min.js" }, { DYNAMIC_CONTENT, "/ROOT/skin/index.css" },
{ STATIC_CONTENT, "/ROOT/skin/taskbar.css" }, { STATIC_CONTENT, "/ROOT/skin/index.css?cacheid=3b470cee" },
{ STATIC_CONTENT, "/ROOT/skin/viewer.js" }, { DYNAMIC_CONTENT, "/ROOT/skin/index.js" },
{ STATIC_CONTENT, "/ROOT/skin/index.js?cacheid=2f5a81ac" },
{ DYNAMIC_CONTENT, "/ROOT/skin/iso6391To3.js" },
{ STATIC_CONTENT, "/ROOT/skin/iso6391To3.js?cacheid=ecde2bb3" },
{ DYNAMIC_CONTENT, "/ROOT/skin/isotope.pkgd.min.js" },
{ STATIC_CONTENT, "/ROOT/skin/isotope.pkgd.min.js?cacheid=2e48d392" },
{ DYNAMIC_CONTENT, "/ROOT/skin/taskbar.css" },
{ STATIC_CONTENT, "/ROOT/skin/taskbar.css?cacheid=216d6b5d" },
{ DYNAMIC_CONTENT, "/ROOT/skin/viewer.js" },
{ STATIC_CONTENT, "/ROOT/skin/viewer.js?cacheid=51e745c2" },
{ DYNAMIC_CONTENT, "/ROOT/catalog/search" }, { DYNAMIC_CONTENT, "/ROOT/catalog/search" },
@ -80,30 +90,54 @@ const ResourceCollection resources200Compressible{
}; };
const ResourceCollection resources200Uncompressible{ const ResourceCollection resources200Uncompressible{
{ STATIC_CONTENT, "/ROOT/skin/bittorrent.png" }, { DYNAMIC_CONTENT, "/ROOT/skin/bittorrent.png" },
{ STATIC_CONTENT, "/ROOT/skin/blank.html" }, { STATIC_CONTENT, "/ROOT/skin/bittorrent.png?cacheid=4f5c6882" },
{ STATIC_CONTENT, "/ROOT/skin/caret.png" }, { DYNAMIC_CONTENT, "/ROOT/skin/blank.html" },
{ STATIC_CONTENT, "/ROOT/skin/css/images/search.svg" }, { STATIC_CONTENT, "/ROOT/skin/blank.html?cacheid=6b1fa032" },
{ STATIC_CONTENT, "/ROOT/skin/download.png" }, { DYNAMIC_CONTENT, "/ROOT/skin/caret.png" },
{ STATIC_CONTENT, "/ROOT/skin/favicon/android-chrome-192x192.png" }, { STATIC_CONTENT, "/ROOT/skin/caret.png?cacheid=22b942b4" },
{ STATIC_CONTENT, "/ROOT/skin/favicon/android-chrome-512x512.png" }, { DYNAMIC_CONTENT, "/ROOT/skin/css/images/search.svg" },
{ STATIC_CONTENT, "/ROOT/skin/favicon/apple-touch-icon.png" }, //{ STATIC_CONTENT, "/ROOT/skin/css/images/search.svg?cacheid=XXXX" },
{ STATIC_CONTENT, "/ROOT/skin/favicon/browserconfig.xml" }, { DYNAMIC_CONTENT, "/ROOT/skin/download.png" },
{ STATIC_CONTENT, "/ROOT/skin/favicon/favicon-16x16.png" }, { STATIC_CONTENT, "/ROOT/skin/download.png?cacheid=a39aa502" },
{ STATIC_CONTENT, "/ROOT/skin/favicon/favicon-32x32.png" }, { DYNAMIC_CONTENT, "/ROOT/skin/favicon/android-chrome-192x192.png" },
{ STATIC_CONTENT, "/ROOT/skin/favicon/mstile-144x144.png" }, //{ STATIC_CONTENT, "/ROOT/skin/favicon/android-chrome-192x192.png?cacheid=" },
{ STATIC_CONTENT, "/ROOT/skin/favicon/mstile-150x150.png" }, { DYNAMIC_CONTENT, "/ROOT/skin/favicon/android-chrome-512x512.png" },
{ STATIC_CONTENT, "/ROOT/skin/favicon/mstile-310x150.png" }, //{ STATIC_CONTENT, "/ROOT/skin/favicon/android-chrome-512x512.png?cacheid=" },
{ STATIC_CONTENT, "/ROOT/skin/favicon/mstile-310x310.png" }, { DYNAMIC_CONTENT, "/ROOT/skin/favicon/apple-touch-icon.png" },
{ STATIC_CONTENT, "/ROOT/skin/favicon/mstile-70x70.png" }, { STATIC_CONTENT, "/ROOT/skin/favicon/apple-touch-icon.png?cacheid=f86f8df3" },
{ STATIC_CONTENT, "/ROOT/skin/favicon/safari-pinned-tab.svg" }, { DYNAMIC_CONTENT, "/ROOT/skin/favicon/browserconfig.xml" },
{ STATIC_CONTENT, "/ROOT/skin/favicon/site.webmanifest" }, { STATIC_CONTENT, "/ROOT/skin/favicon/browserconfig.xml?cacheid=f29a7c4a" },
{ STATIC_CONTENT, "/ROOT/skin/fonts/Poppins.ttf" }, { DYNAMIC_CONTENT, "/ROOT/skin/favicon/favicon-16x16.png" },
{ STATIC_CONTENT, "/ROOT/skin/fonts/Roboto.ttf" }, { STATIC_CONTENT, "/ROOT/skin/favicon/favicon-16x16.png?cacheid=a986fedc" },
{ STATIC_CONTENT, "/ROOT/skin/hash.png" }, { DYNAMIC_CONTENT, "/ROOT/skin/favicon/favicon-32x32.png" },
{ STATIC_CONTENT, "/ROOT/skin/magnet.png" }, { STATIC_CONTENT, "/ROOT/skin/favicon/favicon-32x32.png?cacheid=79ded625" },
{ STATIC_CONTENT, "/ROOT/skin/search-icon.svg" }, { DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-144x144.png" },
{ STATIC_CONTENT, "/ROOT/skin/search_results.css" }, //{ STATIC_CONTENT, "/ROOT/skin/favicon/mstile-144x144.png?cacheid=" },
{ DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-150x150.png" },
//{ STATIC_CONTENT, "/ROOT/skin/favicon/mstile-150x150.png?cacheid=" },
{ DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-310x150.png" },
//{ STATIC_CONTENT, "/ROOT/skin/favicon/mstile-310x150.png?cacheid=" },
{ DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-310x310.png" },
//{ STATIC_CONTENT, "/ROOT/skin/favicon/mstile-310x310.png?cacheid=" },
{ DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-70x70.png" },
//{ STATIC_CONTENT, "/ROOT/skin/favicon/mstile-70x70.png?cacheid=" },
{ DYNAMIC_CONTENT, "/ROOT/skin/favicon/safari-pinned-tab.svg" },
{ STATIC_CONTENT, "/ROOT/skin/favicon/safari-pinned-tab.svg?cacheid=8d487e95" },
{ DYNAMIC_CONTENT, "/ROOT/skin/favicon/site.webmanifest" },
{ STATIC_CONTENT, "/ROOT/skin/favicon/site.webmanifest?cacheid=bc396efb" },
{ DYNAMIC_CONTENT, "/ROOT/skin/fonts/Poppins.ttf" },
{ STATIC_CONTENT, "/ROOT/skin/fonts/Poppins.ttf?cacheid=af705837" },
{ DYNAMIC_CONTENT, "/ROOT/skin/fonts/Roboto.ttf" },
{ STATIC_CONTENT, "/ROOT/skin/fonts/Roboto.ttf?cacheid=84d10248" },
{ DYNAMIC_CONTENT, "/ROOT/skin/hash.png" },
{ STATIC_CONTENT, "/ROOT/skin/hash.png?cacheid=f836e872" },
{ DYNAMIC_CONTENT, "/ROOT/skin/magnet.png" },
{ STATIC_CONTENT, "/ROOT/skin/magnet.png?cacheid=73b6bddf" },
{ DYNAMIC_CONTENT, "/ROOT/skin/search-icon.svg" },
//{ STATIC_CONTENT, "/ROOT/skin/search-icon.svg?cacheid=" },
{ DYNAMIC_CONTENT, "/ROOT/skin/search_results.css" },
{ STATIC_CONTENT, "/ROOT/skin/search_results.css?cacheid=76d39c84" },
{ ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Title" }, { ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Title" },
{ ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Description" }, { ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Description" },
@ -299,6 +333,7 @@ const char* urls404[] = {
"/", "/",
"/zimfile", "/zimfile",
"/ROOT/skin/non-existent-skin-resource", "/ROOT/skin/non-existent-skin-resource",
"/ROOT/skin/autoComplete.min.js?cacheid=wrongcacheid",
"/ROOT/catalog", "/ROOT/catalog",
"/ROOT/catalog/", "/ROOT/catalog/",
"/ROOT/catalog/non-existent-item", "/ROOT/catalog/non-existent-item",