smallsolar
/
libkiwix
mirror of https://github.com/kiwix/libkiwix.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Preprocessing of template resources 

In template resources (found under static/templates), strings of the
form "PATH/TO/STATIC/RESOURCE?KIWIXCACHEID" are expanded into
"PATH/TO/STATIC/RESOURCE?cacheid=CACHEIDVAL" where CACHEIDVAL is a
8-digit hexadecimal hash digest of the file at
static/PATH/TO/STATIC/RESOURCE.
This commit is contained in:
Veloman Yunkan 2022-04-06 19:47:37 +04:00
parent acdc1dfb27
commit fc85215ea0
7 changed files with 179 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

116
scripts/kiwix-resources Executable file
View File

@ -0,0 +1,116 @@
#!/usr/bin/env python3
'''
Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3 of the License, or any
later version.
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
'''
import argparse
import hashlib
import os.path
import re
def read_resource_file(resource_file_path):
with open(resource_file_path, 'r') as f:
return [line.strip() for line in f]
def list_resources(resource_file_path):
for resource_path in read_resource_file(resource_file_path):
print(resource_path)
def get_resource_revision(base_dir, resource_path):
with open(os.path.join(base_dir, resource_path), 'rb') as f:
return hashlib.sha1(f.read()).hexdigest()[:8]
resource_revisions = {}
def fill_resource_revisions(resource_file_path):
base_dir = os.path.dirname(os.path.realpath(resource_file_path))
for resource in read_resource_file(resource_file_path):
resource_revisions[resource] = get_resource_revision(base_dir, resource)
RESOURCE_WITH_CACHEID_URL_PATTERN=r'"([^"?]+)\?KIWIXCACHEID([^"]*)"'
def set_cacheid(resource_matchobj):
path = resource_matchobj.group(1)
resource = path
root_prefix = '{{root}}/'
if resource.startswith(root_prefix):
resource = resource[len(root_prefix):]
extra_query = resource_matchobj.group(2)
cacheid = 'cacheid=' + resource_revisions[resource]
return f'"{path}?{cacheid}{extra_query}"'
def preprocess_line(line):
if 'KIWIXCACHEID' in line:
line = re.sub(RESOURCE_WITH_CACHEID_URL_PATTERN, set_cacheid, line)
assert not 'KIWIXCACHEID' in line
return line
def preprocess_template(srcpath, dstpath):
preprocessed_lines = []
with open(srcpath, 'r') as source:
for line in source:
preprocessed_lines.append(preprocess_line(line))
with open(dstpath, 'w') as target:
print("".join(preprocessed_lines), end='', file=target)
def symlink_resource(src, resource_path):
if os.path.exists(resource_path):
if os.path.islink(resource_path) and os.readlink(resource_path) == src:
return
os.remove(resource_path)
os.symlink(src, resource_path)
def preprocess_resource(srcdir, resource_path, outdir):
resource_dir = os.path.dirname(resource_path)
if resource_dir != '':
os.makedirs(os.path.join(outdir, resource_dir), exist_ok=True)
srcpath = os.path.join(srcdir, resource_path)
outpath = os.path.join(outdir, resource_path)
if resource_path.startswith('templates/'):
preprocess_template(srcpath, outpath)
else:
symlink_resource(srcpath, outpath)
def copy_file(src_path, dst_path):
with open(src_path, 'rb') as src:
with open(dst_path, 'wb') as dst:
dst.write(src.read())
def preprocess_resources(resource_file_path, outdir):
base_dir = os.path.dirname(os.path.realpath(resource_file_path))
resource_filename = os.path.basename(resource_file_path)
for resource in read_resource_file(resource_file_path):
preprocess_resource(base_dir, resource, outdir)
copy_file(resource_file_path, os.path.join(outdir, resource_filename))
if __name__ == "__main__":
parser = argparse.ArgumentParser()
commands = parser.add_mutually_exclusive_group()
commands.add_argument('--list-all', action='store_true')
commands.add_argument('--preprocess', action='store_true')
parser.add_argument('--outdir')
parser.add_argument('resource_file')
args = parser.parse_args()
if args.list_all:
list_resources(args.resource_file)
elif args.preprocess:
fill_resource_revisions(args.resource_file)
preprocess_resources(args.resource_file, args.outdir)

1
scripts/meson.build
View File

@ -1,4 +1,5 @@
res_manager = find_program('kiwix-resources')
res_compiler = find_program('kiwix-compile-resources') res_compiler = find_program('kiwix-compile-resources')
install_data(res_compiler.path(), install_dir:get_option('bindir')) install_data(res_compiler.path(), install_dir:get_option('bindir'))

19
static/meson.build
View File

@ -1,18 +1,27 @@
resource_files = run_command(find_program('python3'), resource_files = run_command(res_manager,
'-c', '--list-all',
'import sys; f=open(sys.argv[1]); print(f.read())',
files('resources_list.txt') files('resources_list.txt')
).stdout().strip().split('\n') ).stdout().strip().split('\n')
lib_resources = custom_target('resources', preprocessed_resources = custom_target('preprocessed_resource_files',
input: 'resources_list.txt', input: 'resources_list.txt',
output: ['resources_list.txt'],
command:[res_manager,
'--preprocess',
'--outdir', '@OUTDIR@',
'@INPUT@'],
depend_files: resource_files
)
lib_resources = custom_target('resources',
input: preprocessed_resources,
output: ['kiwixlib-resources.cpp', 'kiwixlib-resources.h'], output: ['kiwixlib-resources.cpp', 'kiwixlib-resources.h'],
command:[res_compiler, command:[res_compiler,
'--cxxfile', '@OUTPUT0@', '--cxxfile', '@OUTPUT0@',
'--hfile', '@OUTPUT1@', '--hfile', '@OUTPUT1@',
'--source_dir', '@OUTDIR@', '--source_dir', '@OUTDIR@',
'@INPUT@'], '@INPUT@'],
depend_files: resource_files depends: preprocessed_resources
) )
i18n_resource_files = run_command(find_program('python3'), i18n_resource_files = run_command(find_program('python3'),

12
static/templates/head_taskbar.html
View File

@ -1,6 +1,6 @@
<link type="text/css" href="{{root}}/skin/jquery-ui/jquery-ui.min.css" rel="Stylesheet" /> <link type="text/css" href="{{root}}/skin/jquery-ui/jquery-ui.min.css?KIWIXCACHEID" rel="Stylesheet" />
<link type="text/css" href="{{root}}/skin/jquery-ui/jquery-ui.theme.min.css" rel="Stylesheet" /> <link type="text/css" href="{{root}}/skin/jquery-ui/jquery-ui.theme.min.css?KIWIXCACHEID" rel="Stylesheet" />
<link type="text/css" href="{{root}}/skin/taskbar.css" rel="Stylesheet" /> <link type="text/css" href="{{root}}/skin/taskbar.css?KIWIXCACHEID" rel="Stylesheet" />
<script type="text/javascript" src="{{root}}/skin/jquery-ui/external/jquery/jquery.js" defer></script> <script type="text/javascript" src="{{root}}/skin/jquery-ui/external/jquery/jquery.js?KIWIXCACHEID" defer></script>
<script type="text/javascript" src="{{root}}/skin/jquery-ui/jquery-ui.min.js" defer></script> <script type="text/javascript" src="{{root}}/skin/jquery-ui/jquery-ui.min.js?KIWIXCACHEID" defer></script>
<script type="text/javascript" src="{{root}}/skin/taskbar.js" defer></script> <script type="text/javascript" src="{{root}}/skin/taskbar.js?KIWIXCACHEID" defer></script>

20
static/templates/index.html
View File

@ -6,41 +6,41 @@
<title>Welcome to Kiwix Server</title> <title>Welcome to Kiwix Server</title>
<script <script
type="text/javascript" type="text/javascript"
src="{{root}}/skin/jquery-ui/external/jquery/jquery.js" src="{{root}}/skin/jquery-ui/external/jquery/jquery.js?KIWIXCACHEID"
></script> ></script>
<script <script
type="text/javascript" type="text/javascript"
src="{{root}}/skin/jquery-ui/jquery-ui.min.js" src="{{root}}/skin/jquery-ui/jquery-ui.min.js?KIWIXCACHEID"
></script> ></script>
<link <link
type="text/css" type="text/css"
href="{{root}}/skin/jquery-ui/jquery-ui.min.css" href="{{root}}/skin/jquery-ui/jquery-ui.min.css?KIWIXCACHEID"
rel="Stylesheet" rel="Stylesheet"
/> />
<link <link
type="text/css" type="text/css"
href="{{root}}/skin/jquery-ui/jquery-ui.theme.min.css" href="{{root}}/skin/jquery-ui/jquery-ui.theme.min.css?KIWIXCACHEID"
rel="Stylesheet" rel="Stylesheet"
/> />
<link <link
type="text/css" type="text/css"
href="{{root}}/skin/index.css" href="{{root}}/skin/index.css?KIWIXCACHEID"
rel="Stylesheet" rel="Stylesheet"
/> />
<style> <style>
@font-face { @font-face {
font-family: "poppins"; font-family: "poppins";
src: url("{{root}}/skin/fonts/Poppins.ttf") format("truetype"); src: url("{{root}}/skin/fonts/Poppins.ttf?KIWIXCACHEID") format("truetype");
} }
@font-face { @font-face {
font-family: "roboto"; font-family: "roboto";
src: url("{{root}}/skin/fonts/Roboto.ttf") format("truetype"); src: url("{{root}}/skin/fonts/Roboto.ttf?KIWIXCACHEID") format("truetype");
} }
</style> </style>
<script src="{{root}}/skin/isotope.pkgd.min.js" defer></script> <script src="{{root}}/skin/isotope.pkgd.min.js?KIWIXCACHEID" defer></script>
<script src="{{root}}/skin/iso6391To3.js"></script> <script src="{{root}}/skin/iso6391To3.js?KIWIXCACHEID"></script>
<script type="text/javascript" src="{{root}}/skin/index.js" defer></script> <script type="text/javascript" src="{{root}}/skin/index.js?KIWIXCACHEID" defer></script>
</head> </head>
<body> <body>
<div class='kiwixNav'> <div class='kiwixNav'>

2
static/templates/taskbar_part.html
View File

@ -9,7 +9,7 @@
</form> </form>
</div> </div>
<input type="checkbox" id="kiwix_button_show_toggle"> <input type="checkbox" id="kiwix_button_show_toggle">
<label for="kiwix_button_show_toggle"><img src="{{root}}/skin/caret.png" alt=""></label> <label for="kiwix_button_show_toggle"><img src="{{root}}/skin/caret.png?KIWIXCACHEID" alt=""></label>
<div class="kiwix_button_cont"> <div class="kiwix_button_cont">
{{#withlibrarybutton}} {{#withlibrarybutton}}
<a id="kiwix_serve_taskbar_library_button" title="{{{LIBRARY_BUTTON_TEXT}}}" aria-label="{{{LIBRARY_BUTTON_TEXT}}}" href="{{root}}/"><button>&#x1f3e0;</button></a> <a id="kiwix_serve_taskbar_library_button" title="{{{LIBRARY_BUTTON_TEXT}}}" aria-label="{{{LIBRARY_BUTTON_TEXT}}}" href="{{root}}/"><button>&#x1f3e0;</button></a>

62
test/server.cpp
View File

@ -310,16 +310,16 @@ TEST_F(ServerTest, CacheIdsOfStaticResources)
const std::vector<UrlAndExpectedResult> testData{ const std::vector<UrlAndExpectedResult> testData{
{ {
/* url */ "/ROOT/", /* url */ "/ROOT/",
R"EXPECTEDRESULT( src="/ROOT/skin/jquery-ui/external/jquery/jquery.js" R"EXPECTEDRESULT( src="/ROOT/skin/jquery-ui/external/jquery/jquery.js?cacheid=1d85f0f3"
src="/ROOT/skin/jquery-ui/jquery-ui.min.js" src="/ROOT/skin/jquery-ui/jquery-ui.min.js?cacheid=d927c2ff"
href="/ROOT/skin/jquery-ui/jquery-ui.min.css" href="/ROOT/skin/jquery-ui/jquery-ui.min.css?cacheid=e1de77b3"
href="/ROOT/skin/jquery-ui/jquery-ui.theme.min.css" href="/ROOT/skin/jquery-ui/jquery-ui.theme.min.css?cacheid=2a5841f9"
href="/ROOT/skin/index.css" href="/ROOT/skin/index.css?cacheid=1aca980a"
src: url("/ROOT/skin/fonts/Poppins.ttf") format("truetype"); src: url("/ROOT/skin/fonts/Poppins.ttf?cacheid=af705837") format("truetype");
src: url("/ROOT/skin/fonts/Roboto.ttf") format("truetype"); src: url("/ROOT/skin/fonts/Roboto.ttf?cacheid=84d10248") format("truetype");
<script src="/ROOT/skin/isotope.pkgd.min.js" defer></script> <script src="/ROOT/skin/isotope.pkgd.min.js?cacheid=2e48d392" defer></script>
<script src="/ROOT/skin/iso6391To3.js"></script> <script src="/ROOT/skin/iso6391To3.js?cacheid=ecde2bb3"></script>
<script type="text/javascript" src="/ROOT/skin/index.js" defer></script> <script type="text/javascript" src="/ROOT/skin/index.js?cacheid=ea9ce83c" defer></script>
)EXPECTEDRESULT" )EXPECTEDRESULT"
}, },
{ {
@ -332,13 +332,13 @@ R"EXPECTEDRESULT( <img src="../skin/download.png"
}, },
{ {
/* url */ "/ROOT/zimfile/A/index", /* url */ "/ROOT/zimfile/A/index",
R"EXPECTEDRESULT(<link type="root" href="/ROOT"><link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.min.css" rel="Stylesheet" /> R"EXPECTEDRESULT(<link type="root" href="/ROOT"><link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.min.css?cacheid=e1de77b3" rel="Stylesheet" />
<link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.theme.min.css" rel="Stylesheet" /> <link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.theme.min.css?cacheid=2a5841f9" rel="Stylesheet" />
<link type="text/css" href="/ROOT/skin/taskbar.css" rel="Stylesheet" /> <link type="text/css" href="/ROOT/skin/taskbar.css?cacheid=49365e9c" rel="Stylesheet" />
<script type="text/javascript" src="/ROOT/skin/jquery-ui/external/jquery/jquery.js" defer></script> <script type="text/javascript" src="/ROOT/skin/jquery-ui/external/jquery/jquery.js?cacheid=1d85f0f3" defer></script>
<script type="text/javascript" src="/ROOT/skin/jquery-ui/jquery-ui.min.js" defer></script> <script type="text/javascript" src="/ROOT/skin/jquery-ui/jquery-ui.min.js?cacheid=d927c2ff" defer></script>
<script type="text/javascript" src="/ROOT/skin/taskbar.js" defer></script> <script type="text/javascript" src="/ROOT/skin/taskbar.js?cacheid=5982280c" defer></script>
<label for="kiwix_button_show_toggle"><img src="/ROOT/skin/caret.png" alt=""></label> <label for="kiwix_button_show_toggle"><img src="/ROOT/skin/caret.png?cacheid=22b942b4" alt=""></label>
)EXPECTEDRESULT" )EXPECTEDRESULT"
}, },
{ {
@ -346,13 +346,13 @@ R"EXPECTEDRESULT(<link type="root" href="/ROOT"><link type="text/css" href="/ROO
// a page rendered from static/templates/no_search_result_html // a page rendered from static/templates/no_search_result_html
/* url */ "/ROOT/search?content=poor&pattern=whatever", /* url */ "/ROOT/search?content=poor&pattern=whatever",
R"EXPECTEDRESULT( <link type="text/css" href="/ROOT/skin/search_results.css" rel="Stylesheet" /> R"EXPECTEDRESULT( <link type="text/css" href="/ROOT/skin/search_results.css" rel="Stylesheet" />
<link type="root" href="/ROOT"><link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.min.css" rel="Stylesheet" /> <link type="root" href="/ROOT"><link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.min.css?cacheid=e1de77b3" rel="Stylesheet" />
<link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.theme.min.css" rel="Stylesheet" /> <link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.theme.min.css?cacheid=2a5841f9" rel="Stylesheet" />
<link type="text/css" href="/ROOT/skin/taskbar.css" rel="Stylesheet" /> <link type="text/css" href="/ROOT/skin/taskbar.css?cacheid=49365e9c" rel="Stylesheet" />
<script type="text/javascript" src="/ROOT/skin/jquery-ui/external/jquery/jquery.js" defer></script> <script type="text/javascript" src="/ROOT/skin/jquery-ui/external/jquery/jquery.js?cacheid=1d85f0f3" defer></script>
<script type="text/javascript" src="/ROOT/skin/jquery-ui/jquery-ui.min.js" defer></script> <script type="text/javascript" src="/ROOT/skin/jquery-ui/jquery-ui.min.js?cacheid=d927c2ff" defer></script>
<script type="text/javascript" src="/ROOT/skin/taskbar.js" defer></script> <script type="text/javascript" src="/ROOT/skin/taskbar.js?cacheid=5982280c" defer></script>
<label for="kiwix_button_show_toggle"><img src="/ROOT/skin/caret.png" alt=""></label> <label for="kiwix_button_show_toggle"><img src="/ROOT/skin/caret.png?cacheid=22b942b4" alt=""></label>
)EXPECTEDRESULT" )EXPECTEDRESULT"
}, },
}; };
@ -512,12 +512,12 @@ std::string TestContentIn404HtmlResponse::expectedResponse() const
)FRAG", )FRAG",
R"FRAG( R"FRAG(
<link type="root" href="/ROOT"><link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.min.css" rel="Stylesheet" /> <link type="root" href="/ROOT"><link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.min.css?cacheid=e1de77b3" rel="Stylesheet" />
<link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.theme.min.css" rel="Stylesheet" /> <link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.theme.min.css?cacheid=2a5841f9" rel="Stylesheet" />
<link type="text/css" href="/ROOT/skin/taskbar.css" rel="Stylesheet" /> <link type="text/css" href="/ROOT/skin/taskbar.css?cacheid=49365e9c" rel="Stylesheet" />
<script type="text/javascript" src="/ROOT/skin/jquery-ui/external/jquery/jquery.js" defer></script> <script type="text/javascript" src="/ROOT/skin/jquery-ui/external/jquery/jquery.js?cacheid=1d85f0f3" defer></script>
<script type="text/javascript" src="/ROOT/skin/jquery-ui/jquery-ui.min.js" defer></script> <script type="text/javascript" src="/ROOT/skin/jquery-ui/jquery-ui.min.js?cacheid=d927c2ff" defer></script>
<script type="text/javascript" src="/ROOT/skin/taskbar.js" defer></script> <script type="text/javascript" src="/ROOT/skin/taskbar.js?cacheid=5982280c" defer></script>
</head> </head>
<body><span class="kiwix"> <body><span class="kiwix">
<span id="kiwixtoolbar" class="ui-widget-header"> <span id="kiwixtoolbar" class="ui-widget-header">
@ -533,7 +533,7 @@ std::string TestContentIn404HtmlResponse::expectedResponse() const
R"FRAG( </form> R"FRAG( </form>
</div> </div>
<input type="checkbox" id="kiwix_button_show_toggle"> <input type="checkbox" id="kiwix_button_show_toggle">
<label for="kiwix_button_show_toggle"><img src="/ROOT/skin/caret.png" alt=""></label> <label for="kiwix_button_show_toggle"><img src="/ROOT/skin/caret.png?cacheid=22b942b4" alt=""></label>
<div class="kiwix_button_cont"> <div class="kiwix_button_cont">
<a id="kiwix_serve_taskbar_library_button" title=")FRAG", <a id="kiwix_serve_taskbar_library_button" title=")FRAG",