mirror of https://github.com/kiwix/libkiwix.git
Preprocessing of template resources
In template resources (found under static/templates), strings of the form "PATH/TO/STATIC/RESOURCE?KIWIXCACHEID" are expanded into "PATH/TO/STATIC/RESOURCE?cacheid=CACHEIDVAL" where CACHEIDVAL is a 8-digit hexadecimal hash digest of the file at static/PATH/TO/STATIC/RESOURCE.
This commit is contained in:
Veloman Yunkan
parent
acdc1dfb27
commit
fc85215ea0
|
|
@ -0,0 +1,116 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
'''
|
||||
Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 3 of the License, or any
|
||||
later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful, but
|
||||
WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
||||
02110-1301, USA.
|
||||
'''
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import os.path
|
||||
import re
|
||||
|
||||
def read_resource_file(resource_file_path):
|
||||
with open(resource_file_path, 'r') as f:
|
||||
return [line.strip() for line in f]
|
||||
|
||||
def list_resources(resource_file_path):
|
||||
for resource_path in read_resource_file(resource_file_path):
|
||||
print(resource_path)
|
||||
|
||||
def get_resource_revision(base_dir, resource_path):
|
||||
with open(os.path.join(base_dir, resource_path), 'rb') as f:
|
||||
return hashlib.sha1(f.read()).hexdigest()[:8]
|
||||
|
||||
resource_revisions = {}
|
||||
|
||||
def fill_resource_revisions(resource_file_path):
|
||||
base_dir = os.path.dirname(os.path.realpath(resource_file_path))
|
||||
for resource in read_resource_file(resource_file_path):
|
||||
resource_revisions[resource] = get_resource_revision(base_dir, resource)
|
||||
|
||||
RESOURCE_WITH_CACHEID_URL_PATTERN=r'"([^"?]+)\?KIWIXCACHEID([^"]*)"'
|
||||
|
||||
def set_cacheid(resource_matchobj):
|
||||
path = resource_matchobj.group(1)
|
||||
resource = path
|
||||
root_prefix = '{{root}}/'
|
||||
if resource.startswith(root_prefix):
|
||||
resource = resource[len(root_prefix):]
|
||||
extra_query = resource_matchobj.group(2)
|
||||
cacheid = 'cacheid=' + resource_revisions[resource]
|
||||
return f'"{path}?{cacheid}{extra_query}"'
|
||||
|
||||
def preprocess_line(line):
|
||||
if 'KIWIXCACHEID' in line:
|
||||
line = re.sub(RESOURCE_WITH_CACHEID_URL_PATTERN, set_cacheid, line)
|
||||
assert not 'KIWIXCACHEID' in line
|
||||
return line
|
||||
|
||||
def preprocess_template(srcpath, dstpath):
|
||||
preprocessed_lines = []
|
||||
with open(srcpath, 'r') as source:
|
||||
for line in source:
|
||||
preprocessed_lines.append(preprocess_line(line))
|
||||
|
||||
with open(dstpath, 'w') as target:
|
||||
print("".join(preprocessed_lines), end='', file=target)
|
||||
|
||||
def symlink_resource(src, resource_path):
|
||||
if os.path.exists(resource_path):
|
||||
if os.path.islink(resource_path) and os.readlink(resource_path) == src:
|
||||
return
|
||||
os.remove(resource_path)
|
||||
os.symlink(src, resource_path)
|
||||
|
||||
def preprocess_resource(srcdir, resource_path, outdir):
|
||||
resource_dir = os.path.dirname(resource_path)
|
||||
if resource_dir != '':
|
||||
os.makedirs(os.path.join(outdir, resource_dir), exist_ok=True)
|
||||
srcpath = os.path.join(srcdir, resource_path)
|
||||
outpath = os.path.join(outdir, resource_path)
|
||||
if resource_path.startswith('templates/'):
|
||||
preprocess_template(srcpath, outpath)
|
||||
else:
|
||||
symlink_resource(srcpath, outpath)
|
||||
|
||||
def copy_file(src_path, dst_path):
|
||||
with open(src_path, 'rb') as src:
|
||||
with open(dst_path, 'wb') as dst:
|
||||
dst.write(src.read())
|
||||
|
||||
def preprocess_resources(resource_file_path, outdir):
|
||||
base_dir = os.path.dirname(os.path.realpath(resource_file_path))
|
||||
resource_filename = os.path.basename(resource_file_path)
|
||||
for resource in read_resource_file(resource_file_path):
|
||||
preprocess_resource(base_dir, resource, outdir)
|
||||
copy_file(resource_file_path, os.path.join(outdir, resource_filename))
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
commands = parser.add_mutually_exclusive_group()
|
||||
commands.add_argument('--list-all', action='store_true')
|
||||
commands.add_argument('--preprocess', action='store_true')
|
||||
parser.add_argument('--outdir')
|
||||
parser.add_argument('resource_file')
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.list_all:
|
||||
list_resources(args.resource_file)
|
||||
elif args.preprocess:
|
||||
fill_resource_revisions(args.resource_file)
|
||||
preprocess_resources(args.resource_file, args.outdir)
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
|
||||
res_manager = find_program('kiwix-resources')
|
||||
res_compiler = find_program('kiwix-compile-resources')
|
||||
|
||||
install_data(res_compiler.path(), install_dir:get_option('bindir'))
|
||||
|
|
|
|||
|
|
@ -1,18 +1,27 @@
|
|||
resource_files = run_command(find_program('python3'),
|
||||
'-c',
|
||||
'import sys; f=open(sys.argv[1]); print(f.read())',
|
||||
resource_files = run_command(res_manager,
|
||||
'--list-all',
|
||||
files('resources_list.txt')
|
||||
).stdout().strip().split('\n')
|
||||
|
||||
lib_resources = custom_target('resources',
|
||||
preprocessed_resources = custom_target('preprocessed_resource_files',
|
||||
input: 'resources_list.txt',
|
||||
output: ['resources_list.txt'],
|
||||
command:[res_manager,
|
||||
'--preprocess',
|
||||
'--outdir', '@OUTDIR@',
|
||||
'@INPUT@'],
|
||||
depend_files: resource_files
|
||||
)
|
||||
|
||||
lib_resources = custom_target('resources',
|
||||
input: preprocessed_resources,
|
||||
output: ['kiwixlib-resources.cpp', 'kiwixlib-resources.h'],
|
||||
command:[res_compiler,
|
||||
'--cxxfile', '@OUTPUT0@',
|
||||
'--hfile', '@OUTPUT1@',
|
||||
'--source_dir', '@OUTDIR@',
|
||||
'@INPUT@'],
|
||||
depend_files: resource_files
|
||||
depends: preprocessed_resources
|
||||
)
|
||||
|
||||
i18n_resource_files = run_command(find_program('python3'),
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
<link type="text/css" href="{{root}}/skin/jquery-ui/jquery-ui.min.css" rel="Stylesheet" />
|
||||
<link type="text/css" href="{{root}}/skin/jquery-ui/jquery-ui.theme.min.css" rel="Stylesheet" />
|
||||
<link type="text/css" href="{{root}}/skin/taskbar.css" rel="Stylesheet" />
|
||||
<script type="text/javascript" src="{{root}}/skin/jquery-ui/external/jquery/jquery.js" defer></script>
|
||||
<script type="text/javascript" src="{{root}}/skin/jquery-ui/jquery-ui.min.js" defer></script>
|
||||
<script type="text/javascript" src="{{root}}/skin/taskbar.js" defer></script>
|
||||
<link type="text/css" href="{{root}}/skin/jquery-ui/jquery-ui.min.css?KIWIXCACHEID" rel="Stylesheet" />
|
||||
<link type="text/css" href="{{root}}/skin/jquery-ui/jquery-ui.theme.min.css?KIWIXCACHEID" rel="Stylesheet" />
|
||||
<link type="text/css" href="{{root}}/skin/taskbar.css?KIWIXCACHEID" rel="Stylesheet" />
|
||||
<script type="text/javascript" src="{{root}}/skin/jquery-ui/external/jquery/jquery.js?KIWIXCACHEID" defer></script>
|
||||
<script type="text/javascript" src="{{root}}/skin/jquery-ui/jquery-ui.min.js?KIWIXCACHEID" defer></script>
|
||||
<script type="text/javascript" src="{{root}}/skin/taskbar.js?KIWIXCACHEID" defer></script>
|
||||
|
|
|
|||
|
|
@ -6,41 +6,41 @@
|
|||
<title>Welcome to Kiwix Server</title>
|
||||
<script
|
||||
type="text/javascript"
|
||||
src="{{root}}/skin/jquery-ui/external/jquery/jquery.js"
|
||||
src="{{root}}/skin/jquery-ui/external/jquery/jquery.js?KIWIXCACHEID"
|
||||
></script>
|
||||
<script
|
||||
type="text/javascript"
|
||||
src="{{root}}/skin/jquery-ui/jquery-ui.min.js"
|
||||
src="{{root}}/skin/jquery-ui/jquery-ui.min.js?KIWIXCACHEID"
|
||||
></script>
|
||||
<link
|
||||
type="text/css"
|
||||
href="{{root}}/skin/jquery-ui/jquery-ui.min.css"
|
||||
href="{{root}}/skin/jquery-ui/jquery-ui.min.css?KIWIXCACHEID"
|
||||
rel="Stylesheet"
|
||||
/>
|
||||
<link
|
||||
type="text/css"
|
||||
href="{{root}}/skin/jquery-ui/jquery-ui.theme.min.css"
|
||||
href="{{root}}/skin/jquery-ui/jquery-ui.theme.min.css?KIWIXCACHEID"
|
||||
rel="Stylesheet"
|
||||
/>
|
||||
<link
|
||||
type="text/css"
|
||||
href="{{root}}/skin/index.css"
|
||||
href="{{root}}/skin/index.css?KIWIXCACHEID"
|
||||
rel="Stylesheet"
|
||||
/>
|
||||
<style>
|
||||
@font-face {
|
||||
font-family: "poppins";
|
||||
src: url("{{root}}/skin/fonts/Poppins.ttf") format("truetype");
|
||||
src: url("{{root}}/skin/fonts/Poppins.ttf?KIWIXCACHEID") format("truetype");
|
||||
}
|
||||
|
||||
@font-face {
|
||||
font-family: "roboto";
|
||||
src: url("{{root}}/skin/fonts/Roboto.ttf") format("truetype");
|
||||
src: url("{{root}}/skin/fonts/Roboto.ttf?KIWIXCACHEID") format("truetype");
|
||||
}
|
||||
</style>
|
||||
<script src="{{root}}/skin/isotope.pkgd.min.js" defer></script>
|
||||
<script src="{{root}}/skin/iso6391To3.js"></script>
|
||||
<script type="text/javascript" src="{{root}}/skin/index.js" defer></script>
|
||||
<script src="{{root}}/skin/isotope.pkgd.min.js?KIWIXCACHEID" defer></script>
|
||||
<script src="{{root}}/skin/iso6391To3.js?KIWIXCACHEID"></script>
|
||||
<script type="text/javascript" src="{{root}}/skin/index.js?KIWIXCACHEID" defer></script>
|
||||
</head>
|
||||
<body>
|
||||
<div class='kiwixNav'>
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@
|
|||
</form>
|
||||
</div>
|
||||
<input type="checkbox" id="kiwix_button_show_toggle">
|
||||
<label for="kiwix_button_show_toggle"><img src="{{root}}/skin/caret.png" alt=""></label>
|
||||
<label for="kiwix_button_show_toggle"><img src="{{root}}/skin/caret.png?KIWIXCACHEID" alt=""></label>
|
||||
<div class="kiwix_button_cont">
|
||||
{{#withlibrarybutton}}
|
||||
<a id="kiwix_serve_taskbar_library_button" title="{{{LIBRARY_BUTTON_TEXT}}}" aria-label="{{{LIBRARY_BUTTON_TEXT}}}" href="{{root}}/"><button>🏠</button></a>
|
||||
|
|
|
|||
|
|
@ -310,16 +310,16 @@ TEST_F(ServerTest, CacheIdsOfStaticResources)
|
|||
const std::vector<UrlAndExpectedResult> testData{
|
||||
{
|
||||
/* url */ "/ROOT/",
|
||||
R"EXPECTEDRESULT( src="/ROOT/skin/jquery-ui/external/jquery/jquery.js"
|
||||
src="/ROOT/skin/jquery-ui/jquery-ui.min.js"
|
||||
href="/ROOT/skin/jquery-ui/jquery-ui.min.css"
|
||||
href="/ROOT/skin/jquery-ui/jquery-ui.theme.min.css"
|
||||
href="/ROOT/skin/index.css"
|
||||
src: url("/ROOT/skin/fonts/Poppins.ttf") format("truetype");
|
||||
src: url("/ROOT/skin/fonts/Roboto.ttf") format("truetype");
|
||||
<script src="/ROOT/skin/isotope.pkgd.min.js" defer></script>
|
||||
<script src="/ROOT/skin/iso6391To3.js"></script>
|
||||
<script type="text/javascript" src="/ROOT/skin/index.js" defer></script>
|
||||
R"EXPECTEDRESULT( src="/ROOT/skin/jquery-ui/external/jquery/jquery.js?cacheid=1d85f0f3"
|
||||
src="/ROOT/skin/jquery-ui/jquery-ui.min.js?cacheid=d927c2ff"
|
||||
href="/ROOT/skin/jquery-ui/jquery-ui.min.css?cacheid=e1de77b3"
|
||||
href="/ROOT/skin/jquery-ui/jquery-ui.theme.min.css?cacheid=2a5841f9"
|
||||
href="/ROOT/skin/index.css?cacheid=1aca980a"
|
||||
src: url("/ROOT/skin/fonts/Poppins.ttf?cacheid=af705837") format("truetype");
|
||||
src: url("/ROOT/skin/fonts/Roboto.ttf?cacheid=84d10248") format("truetype");
|
||||
<script src="/ROOT/skin/isotope.pkgd.min.js?cacheid=2e48d392" defer></script>
|
||||
<script src="/ROOT/skin/iso6391To3.js?cacheid=ecde2bb3"></script>
|
||||
<script type="text/javascript" src="/ROOT/skin/index.js?cacheid=ea9ce83c" defer></script>
|
||||
)EXPECTEDRESULT"
|
||||
},
|
||||
{
|
||||
|
|
@ -332,13 +332,13 @@ R"EXPECTEDRESULT( <img src="../skin/download.png"
|
|||
},
|
||||
{
|
||||
/* url */ "/ROOT/zimfile/A/index",
|
||||
R"EXPECTEDRESULT(<link type="root" href="/ROOT"><link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.min.css" rel="Stylesheet" />
|
||||
<link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.theme.min.css" rel="Stylesheet" />
|
||||
<link type="text/css" href="/ROOT/skin/taskbar.css" rel="Stylesheet" />
|
||||
<script type="text/javascript" src="/ROOT/skin/jquery-ui/external/jquery/jquery.js" defer></script>
|
||||
<script type="text/javascript" src="/ROOT/skin/jquery-ui/jquery-ui.min.js" defer></script>
|
||||
<script type="text/javascript" src="/ROOT/skin/taskbar.js" defer></script>
|
||||
<label for="kiwix_button_show_toggle"><img src="/ROOT/skin/caret.png" alt=""></label>
|
||||
R"EXPECTEDRESULT(<link type="root" href="/ROOT"><link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.min.css?cacheid=e1de77b3" rel="Stylesheet" />
|
||||
<link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.theme.min.css?cacheid=2a5841f9" rel="Stylesheet" />
|
||||
<link type="text/css" href="/ROOT/skin/taskbar.css?cacheid=49365e9c" rel="Stylesheet" />
|
||||
<script type="text/javascript" src="/ROOT/skin/jquery-ui/external/jquery/jquery.js?cacheid=1d85f0f3" defer></script>
|
||||
<script type="text/javascript" src="/ROOT/skin/jquery-ui/jquery-ui.min.js?cacheid=d927c2ff" defer></script>
|
||||
<script type="text/javascript" src="/ROOT/skin/taskbar.js?cacheid=5982280c" defer></script>
|
||||
<label for="kiwix_button_show_toggle"><img src="/ROOT/skin/caret.png?cacheid=22b942b4" alt=""></label>
|
||||
)EXPECTEDRESULT"
|
||||
},
|
||||
{
|
||||
|
|
@ -346,13 +346,13 @@ R"EXPECTEDRESULT(<link type="root" href="/ROOT"><link type="text/css" href="/ROO
|
|||
// a page rendered from static/templates/no_search_result_html
|
||||
/* url */ "/ROOT/search?content=poor&pattern=whatever",
|
||||
R"EXPECTEDRESULT( <link type="text/css" href="/ROOT/skin/search_results.css" rel="Stylesheet" />
|
||||
<link type="root" href="/ROOT"><link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.min.css" rel="Stylesheet" />
|
||||
<link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.theme.min.css" rel="Stylesheet" />
|
||||
<link type="text/css" href="/ROOT/skin/taskbar.css" rel="Stylesheet" />
|
||||
<script type="text/javascript" src="/ROOT/skin/jquery-ui/external/jquery/jquery.js" defer></script>
|
||||
<script type="text/javascript" src="/ROOT/skin/jquery-ui/jquery-ui.min.js" defer></script>
|
||||
<script type="text/javascript" src="/ROOT/skin/taskbar.js" defer></script>
|
||||
<label for="kiwix_button_show_toggle"><img src="/ROOT/skin/caret.png" alt=""></label>
|
||||
<link type="root" href="/ROOT"><link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.min.css?cacheid=e1de77b3" rel="Stylesheet" />
|
||||
<link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.theme.min.css?cacheid=2a5841f9" rel="Stylesheet" />
|
||||
<link type="text/css" href="/ROOT/skin/taskbar.css?cacheid=49365e9c" rel="Stylesheet" />
|
||||
<script type="text/javascript" src="/ROOT/skin/jquery-ui/external/jquery/jquery.js?cacheid=1d85f0f3" defer></script>
|
||||
<script type="text/javascript" src="/ROOT/skin/jquery-ui/jquery-ui.min.js?cacheid=d927c2ff" defer></script>
|
||||
<script type="text/javascript" src="/ROOT/skin/taskbar.js?cacheid=5982280c" defer></script>
|
||||
<label for="kiwix_button_show_toggle"><img src="/ROOT/skin/caret.png?cacheid=22b942b4" alt=""></label>
|
||||
)EXPECTEDRESULT"
|
||||
},
|
||||
};
|
||||
|
|
@ -512,12 +512,12 @@ std::string TestContentIn404HtmlResponse::expectedResponse() const
|
|||
)FRAG",
|
||||
|
||||
R"FRAG(
|
||||
<link type="root" href="/ROOT"><link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.min.css" rel="Stylesheet" />
|
||||
<link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.theme.min.css" rel="Stylesheet" />
|
||||
<link type="text/css" href="/ROOT/skin/taskbar.css" rel="Stylesheet" />
|
||||
<script type="text/javascript" src="/ROOT/skin/jquery-ui/external/jquery/jquery.js" defer></script>
|
||||
<script type="text/javascript" src="/ROOT/skin/jquery-ui/jquery-ui.min.js" defer></script>
|
||||
<script type="text/javascript" src="/ROOT/skin/taskbar.js" defer></script>
|
||||
<link type="root" href="/ROOT"><link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.min.css?cacheid=e1de77b3" rel="Stylesheet" />
|
||||
<link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.theme.min.css?cacheid=2a5841f9" rel="Stylesheet" />
|
||||
<link type="text/css" href="/ROOT/skin/taskbar.css?cacheid=49365e9c" rel="Stylesheet" />
|
||||
<script type="text/javascript" src="/ROOT/skin/jquery-ui/external/jquery/jquery.js?cacheid=1d85f0f3" defer></script>
|
||||
<script type="text/javascript" src="/ROOT/skin/jquery-ui/jquery-ui.min.js?cacheid=d927c2ff" defer></script>
|
||||
<script type="text/javascript" src="/ROOT/skin/taskbar.js?cacheid=5982280c" defer></script>
|
||||
</head>
|
||||
<body><span class="kiwix">
|
||||
<span id="kiwixtoolbar" class="ui-widget-header">
|
||||
|
|
@ -533,7 +533,7 @@ std::string TestContentIn404HtmlResponse::expectedResponse() const
|
|||
R"FRAG( </form>
|
||||
</div>
|
||||
<input type="checkbox" id="kiwix_button_show_toggle">
|
||||
<label for="kiwix_button_show_toggle"><img src="/ROOT/skin/caret.png" alt=""></label>
|
||||
<label for="kiwix_button_show_toggle"><img src="/ROOT/skin/caret.png?cacheid=22b942b4" alt=""></label>
|
||||
<div class="kiwix_button_cont">
|
||||
<a id="kiwix_serve_taskbar_library_button" title=")FRAG",
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue