From fc85215ea0fb1e5f6922491661c32b5545ca077f Mon Sep 17 00:00:00 2001 From: Veloman Yunkan Date: Wed, 6 Apr 2022 19:47:37 +0400 Subject: [PATCH] Preprocessing of template resources In template resources (found under static/templates), strings of the form "PATH/TO/STATIC/RESOURCE?KIWIXCACHEID" are expanded into "PATH/TO/STATIC/RESOURCE?cacheid=CACHEIDVAL" where CACHEIDVAL is a 8-digit hexadecimal hash digest of the file at static/PATH/TO/STATIC/RESOURCE. --- scripts/kiwix-resources | 116 +++++++++++++++++++++++++++++ scripts/meson.build | 1 + static/meson.build | 19 +++-- static/templates/head_taskbar.html | 12 +-- static/templates/index.html | 20 ++--- static/templates/taskbar_part.html | 2 +- test/server.cpp | 62 +++++++-------- 7 files changed, 179 insertions(+), 53 deletions(-) create mode 100755 scripts/kiwix-resources diff --git a/scripts/kiwix-resources b/scripts/kiwix-resources new file mode 100755 index 000000000..72e029a20 --- /dev/null +++ b/scripts/kiwix-resources @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 + +''' +Copyright 2022 Veloman Yunkan + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or any +later version. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +02110-1301, USA. +''' + +import argparse +import hashlib +import os.path +import re + +def read_resource_file(resource_file_path): + with open(resource_file_path, 'r') as f: + return [line.strip() for line in f] + +def list_resources(resource_file_path): + for resource_path in read_resource_file(resource_file_path): + print(resource_path) + +def get_resource_revision(base_dir, resource_path): + with open(os.path.join(base_dir, resource_path), 'rb') as f: + return hashlib.sha1(f.read()).hexdigest()[:8] + +resource_revisions = {} + +def fill_resource_revisions(resource_file_path): + base_dir = os.path.dirname(os.path.realpath(resource_file_path)) + for resource in read_resource_file(resource_file_path): + resource_revisions[resource] = get_resource_revision(base_dir, resource) + +RESOURCE_WITH_CACHEID_URL_PATTERN=r'"([^"?]+)\?KIWIXCACHEID([^"]*)"' + +def set_cacheid(resource_matchobj): + path = resource_matchobj.group(1) + resource = path + root_prefix = '{{root}}/' + if resource.startswith(root_prefix): + resource = resource[len(root_prefix):] + extra_query = resource_matchobj.group(2) + cacheid = 'cacheid=' + resource_revisions[resource] + return f'"{path}?{cacheid}{extra_query}"' + +def preprocess_line(line): + if 'KIWIXCACHEID' in line: + line = re.sub(RESOURCE_WITH_CACHEID_URL_PATTERN, set_cacheid, line) + assert not 'KIWIXCACHEID' in line + return line + +def preprocess_template(srcpath, dstpath): + preprocessed_lines = [] + with open(srcpath, 'r') as source: + for line in source: + preprocessed_lines.append(preprocess_line(line)) + + with open(dstpath, 'w') as target: + print("".join(preprocessed_lines), end='', file=target) + +def symlink_resource(src, resource_path): + if os.path.exists(resource_path): + if os.path.islink(resource_path) and os.readlink(resource_path) == src: + return + os.remove(resource_path) + os.symlink(src, resource_path) + +def preprocess_resource(srcdir, resource_path, outdir): + resource_dir = os.path.dirname(resource_path) + if resource_dir != '': + os.makedirs(os.path.join(outdir, resource_dir), exist_ok=True) + srcpath = os.path.join(srcdir, resource_path) + outpath = os.path.join(outdir, resource_path) + if resource_path.startswith('templates/'): + preprocess_template(srcpath, outpath) + else: + symlink_resource(srcpath, outpath) + +def copy_file(src_path, dst_path): + with open(src_path, 'rb') as src: + with open(dst_path, 'wb') as dst: + dst.write(src.read()) + +def preprocess_resources(resource_file_path, outdir): + base_dir = os.path.dirname(os.path.realpath(resource_file_path)) + resource_filename = os.path.basename(resource_file_path) + for resource in read_resource_file(resource_file_path): + preprocess_resource(base_dir, resource, outdir) + copy_file(resource_file_path, os.path.join(outdir, resource_filename)) + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + commands = parser.add_mutually_exclusive_group() + commands.add_argument('--list-all', action='store_true') + commands.add_argument('--preprocess', action='store_true') + parser.add_argument('--outdir') + parser.add_argument('resource_file') + args = parser.parse_args() + + if args.list_all: + list_resources(args.resource_file) + elif args.preprocess: + fill_resource_revisions(args.resource_file) + preprocess_resources(args.resource_file, args.outdir) diff --git a/scripts/meson.build b/scripts/meson.build index c4ddec873..889a53248 100644 --- a/scripts/meson.build +++ b/scripts/meson.build @@ -1,4 +1,5 @@ +res_manager = find_program('kiwix-resources') res_compiler = find_program('kiwix-compile-resources') install_data(res_compiler.path(), install_dir:get_option('bindir')) diff --git a/static/meson.build b/static/meson.build index 6a8ce1773..965f2c11b 100644 --- a/static/meson.build +++ b/static/meson.build @@ -1,18 +1,27 @@ -resource_files = run_command(find_program('python3'), - '-c', - 'import sys; f=open(sys.argv[1]); print(f.read())', +resource_files = run_command(res_manager, + '--list-all', files('resources_list.txt') ).stdout().strip().split('\n') -lib_resources = custom_target('resources', +preprocessed_resources = custom_target('preprocessed_resource_files', input: 'resources_list.txt', + output: ['resources_list.txt'], + command:[res_manager, + '--preprocess', + '--outdir', '@OUTDIR@', + '@INPUT@'], + depend_files: resource_files +) + +lib_resources = custom_target('resources', + input: preprocessed_resources, output: ['kiwixlib-resources.cpp', 'kiwixlib-resources.h'], command:[res_compiler, '--cxxfile', '@OUTPUT0@', '--hfile', '@OUTPUT1@', '--source_dir', '@OUTDIR@', '@INPUT@'], - depend_files: resource_files + depends: preprocessed_resources ) i18n_resource_files = run_command(find_program('python3'), diff --git a/static/templates/head_taskbar.html b/static/templates/head_taskbar.html index 1ed843638..e595ac138 100644 --- a/static/templates/head_taskbar.html +++ b/static/templates/head_taskbar.html @@ -1,6 +1,6 @@ - - - - - - + + + + + + diff --git a/static/templates/index.html b/static/templates/index.html index b28f58085..39d5afc9e 100644 --- a/static/templates/index.html +++ b/static/templates/index.html @@ -6,41 +6,41 @@ Welcome to Kiwix Server - - - + + +
diff --git a/static/templates/taskbar_part.html b/static/templates/taskbar_part.html index 42c76ed4b..c056d48d0 100644 --- a/static/templates/taskbar_part.html +++ b/static/templates/taskbar_part.html @@ -9,7 +9,7 @@
- +
{{#withlibrarybutton}} diff --git a/test/server.cpp b/test/server.cpp index d95f15fad..422cc2620 100644 --- a/test/server.cpp +++ b/test/server.cpp @@ -310,16 +310,16 @@ TEST_F(ServerTest, CacheIdsOfStaticResources) const std::vector testData{ { /* url */ "/ROOT/", -R"EXPECTEDRESULT( src="/ROOT/skin/jquery-ui/external/jquery/jquery.js" - src="/ROOT/skin/jquery-ui/jquery-ui.min.js" - href="/ROOT/skin/jquery-ui/jquery-ui.min.css" - href="/ROOT/skin/jquery-ui/jquery-ui.theme.min.css" - href="/ROOT/skin/index.css" - src: url("/ROOT/skin/fonts/Poppins.ttf") format("truetype"); - src: url("/ROOT/skin/fonts/Roboto.ttf") format("truetype"); - - - +R"EXPECTEDRESULT( src="/ROOT/skin/jquery-ui/external/jquery/jquery.js?cacheid=1d85f0f3" + src="/ROOT/skin/jquery-ui/jquery-ui.min.js?cacheid=d927c2ff" + href="/ROOT/skin/jquery-ui/jquery-ui.min.css?cacheid=e1de77b3" + href="/ROOT/skin/jquery-ui/jquery-ui.theme.min.css?cacheid=2a5841f9" + href="/ROOT/skin/index.css?cacheid=1aca980a" + src: url("/ROOT/skin/fonts/Poppins.ttf?cacheid=af705837") format("truetype"); + src: url("/ROOT/skin/fonts/Roboto.ttf?cacheid=84d10248") format("truetype"); + + + )EXPECTEDRESULT" }, { @@ -332,13 +332,13 @@ R"EXPECTEDRESULT( - - - - - - +R"EXPECTEDRESULT( + + + + + + )EXPECTEDRESULT" }, { @@ -346,13 +346,13 @@ R"EXPECTEDRESULT( - - - - - - - + + + + + + + )EXPECTEDRESULT" }, }; @@ -512,12 +512,12 @@ std::string TestContentIn404HtmlResponse::expectedResponse() const )FRAG", R"FRAG( - - - - - - + + + + + + @@ -533,7 +533,7 @@ std::string TestContentIn404HtmlResponse::expectedResponse() const R"FRAG(
- +