Merge pull request #712 from kiwix/static_resource_versioning

Static resource versioning
This commit is contained in:
Kelson 2022-05-02 23:49:55 +02:00 committed by GitHub
commit 26eccb5a5f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 264 additions and 35 deletions

127
scripts/kiwix-resources Executable file
View File

@ -0,0 +1,127 @@
#!/usr/bin/env python3
'''
Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3 of the License, or any
later version.
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
02110-1301, USA.
'''
import argparse
import hashlib
import os.path
import re
def read_resource_file(resource_file_path):
with open(resource_file_path, 'r') as f:
return [line.strip() for line in f]
def list_resources(resource_file_path):
for resource_path in read_resource_file(resource_file_path):
print(resource_path)
def compute_resource_revision(resource_path):
with open(os.path.join(OUT_DIR, resource_path), 'rb') as f:
return hashlib.sha1(f.read()).hexdigest()[:8]
resource_revisions = {}
def get_resource_revision(res):
if not res in resource_revisions:
preprocess_resource(res)
resource_revisions[res] = compute_resource_revision(res)
return resource_revisions[res]
RESOURCE_WITH_CACHEID_URL_PATTERN=r'(?P<pre>.*/(?P<resource>skin/[^"?]+)\?)KIWIXCACHEID(?P<post>[^"]*)'
def set_cacheid(resource_matchobj):
pre = resource_matchobj.group('pre')
resource = resource_matchobj.group('resource')
post = resource_matchobj.group('post')
cacheid = 'cacheid=' + get_resource_revision(resource)
return pre + cacheid + post
def preprocess_text(s):
if 'KIWIXCACHEID' in s:
s = re.sub(RESOURCE_WITH_CACHEID_URL_PATTERN, set_cacheid, s)
assert not 'KIWIXCACHEID' in s
return s
def get_preprocessed_resource(srcpath):
"""Get the transformed content of a resource
If the resource at srcpath is modified by preprocessing then this function
returns the transformed content of the resource. Otherwise it returns None.
"""
try:
with open(srcpath, 'r') as resource_file:
content = resource_file.read()
preprocessed_content = preprocess_text(content)
return preprocessed_content if preprocessed_content != content else None
except UnicodeDecodeError:
# It was a binary resource
return None
def symlink_resource(src, resource_path):
if os.path.exists(resource_path):
if os.path.islink(resource_path) and os.readlink(resource_path) == src:
return
os.remove(resource_path)
os.symlink(src, resource_path)
def preprocess_resource(resource_path):
print('Preprocessing', resource_path, '...')
resource_dir = os.path.dirname(resource_path)
if resource_dir != '':
os.makedirs(os.path.join(OUT_DIR, resource_dir), exist_ok=True)
srcpath = os.path.join(BASE_DIR, resource_path)
outpath = os.path.join(OUT_DIR, resource_path)
if os.path.exists(outpath):
os.remove(outpath)
preprocessed_content = get_preprocessed_resource(srcpath)
if preprocessed_content is None:
symlink_resource(srcpath, outpath)
else:
with open(outpath, 'w') as target:
print(preprocessed_content, end='', file=target)
def copy_file(src_path, dst_path):
with open(src_path, 'rb') as src:
with open(dst_path, 'wb') as dst:
dst.write(src.read())
def preprocess_resources(resource_file_path):
resource_filename = os.path.basename(resource_file_path)
for resource in read_resource_file(resource_file_path):
preprocess_resource(resource)
copy_file(resource_file_path, os.path.join(OUT_DIR, resource_filename))
if __name__ == "__main__":
parser = argparse.ArgumentParser()
commands = parser.add_mutually_exclusive_group()
commands.add_argument('--list-all', action='store_true')
commands.add_argument('--preprocess', action='store_true')
parser.add_argument('--outdir')
parser.add_argument('resource_file')
args = parser.parse_args()
BASE_DIR = os.path.dirname(os.path.realpath(args.resource_file))
OUT_DIR = args.outdir
if args.list_all:
list_resources(args.resource_file)
elif args.preprocess:
preprocess_resources(args.resource_file)

View File

@ -1,4 +1,5 @@
res_manager = find_program('kiwix-resources')
res_compiler = find_program('kiwix-compile-resources')
install_data(res_compiler.path(), install_dir:get_option('bindir'))

View File

@ -442,6 +442,16 @@ SuggestionsList_t getSuggestions(SuggestionSearcherCache& cache, const zim::Arch
namespace
{
std::string renderUrl(const std::string& root, const std::string& urlTemplate)
{
MustacheData data;
data.set("root", root);
auto url = kainjow::mustache::mustache(urlTemplate).render(data);
if ( url.back() == '\n' )
url.pop_back();
return url;
}
std::string makeFulltextSearchSuggestion(const std::string& lang, const std::string& queryString)
{
return i18n::expandParameterizedString(lang, "suggest-full-text-search",
@ -622,10 +632,11 @@ std::unique_ptr<Response> InternalServer::handle_search(const RequestContext& re
} catch(std::runtime_error& e) {
// Searcher->search will throw a runtime error if there is no valid xapian database to do the search.
// (in case of zim file not containing a index)
const auto cssUrl = renderUrl(m_root, RESOURCE::templates::url_of_search_results_css);
return HTTPErrorHtmlResponse(*this, request, MHD_HTTP_NOT_FOUND,
"fulltext-search-unavailable",
"404-page-heading",
m_root + "/skin/search_results.css")
cssUrl)
+ nonParameterizedMessage("no-search-results")
+ TaskbarInfo(searchInfo.bookName, archive.get());
}

View File

@ -1,18 +1,27 @@
resource_files = run_command(find_program('python3'),
'-c',
'import sys; f=open(sys.argv[1]); print(f.read())',
resource_files = run_command(res_manager,
'--list-all',
files('resources_list.txt')
).stdout().strip().split('\n')
lib_resources = custom_target('resources',
preprocessed_resources = custom_target('preprocessed_resource_files',
input: 'resources_list.txt',
output: ['resources_list.txt'],
command:[res_manager,
'--preprocess',
'--outdir', '@OUTDIR@',
'@INPUT@'],
depend_files: resource_files
)
lib_resources = custom_target('resources',
input: preprocessed_resources,
output: ['kiwixlib-resources.cpp', 'kiwixlib-resources.h'],
command:[res_compiler,
'--cxxfile', '@OUTPUT0@',
'--hfile', '@OUTPUT1@',
'--source_dir', '@OUTDIR@',
'@INPUT@'],
depend_files: resource_files
depends: preprocessed_resources
)
i18n_resource_files = run_command(find_program('python3'),

View File

@ -47,5 +47,6 @@ templates/catalog_v2_entries.xml
templates/catalog_v2_entry.xml
templates/catalog_v2_categories.xml
templates/catalog_v2_languages.xml
templates/url_of_search_results_css
opensearchdescription.xml
catalog_v2_searchdescription.xml

View File

@ -171,25 +171,25 @@
<div class="modal-content">
<div class="modal-regular-download">
<a href="${downloadLink}" download>
<img src="../skin/download.png" alt="direct download" />
<img src="../skin/download.png?KIWIXCACHEID" alt="direct download" />
<div>Direct</div>
</a>
</div>
<div class="modal-regular-download">
<a href="${downloadLink}.sha256" download>
<img src="../skin/hash.png" alt="download hash" />
<img src="../skin/hash.png?KIWIXCACHEID" alt="download hash" />
<div>Sha256 hash</div>
</a>
</div>
<div class="modal-regular-download">
<a href="${downloadLink}.magnet" target="_blank">
<img src="../skin/magnet.png" alt="download magnet" />
<img src="../skin/magnet.png?KIWIXCACHEID" alt="download magnet" />
<div>Magnet link</div>
</a>
</div>
<div class="modal-regular-download">
<a href="${downloadLink}.torrent" download>
<img src="../skin/bittorrent.png" alt="download torrent" />
<img src="../skin/bittorrent.png?KIWIXCACHEID" alt="download torrent" />
<div>Torrent file</div>
</a>
</div>

View File

@ -1,6 +1,6 @@
<link type="text/css" href="{{root}}/skin/jquery-ui/jquery-ui.min.css" rel="Stylesheet" />
<link type="text/css" href="{{root}}/skin/jquery-ui/jquery-ui.theme.min.css" rel="Stylesheet" />
<link type="text/css" href="{{root}}/skin/taskbar.css" rel="Stylesheet" />
<script type="text/javascript" src="{{root}}/skin/jquery-ui/external/jquery/jquery.js" defer></script>
<script type="text/javascript" src="{{root}}/skin/jquery-ui/jquery-ui.min.js" defer></script>
<script type="text/javascript" src="{{root}}/skin/taskbar.js" defer></script>
<link type="text/css" href="{{root}}/skin/jquery-ui/jquery-ui.min.css?KIWIXCACHEID" rel="Stylesheet" />
<link type="text/css" href="{{root}}/skin/jquery-ui/jquery-ui.theme.min.css?KIWIXCACHEID" rel="Stylesheet" />
<link type="text/css" href="{{root}}/skin/taskbar.css?KIWIXCACHEID" rel="Stylesheet" />
<script type="text/javascript" src="{{root}}/skin/jquery-ui/external/jquery/jquery.js?KIWIXCACHEID" defer></script>
<script type="text/javascript" src="{{root}}/skin/jquery-ui/jquery-ui.min.js?KIWIXCACHEID" defer></script>
<script type="text/javascript" src="{{root}}/skin/taskbar.js?KIWIXCACHEID" defer></script>

View File

@ -6,41 +6,41 @@
<title>Welcome to Kiwix Server</title>
<script
type="text/javascript"
src="{{root}}/skin/jquery-ui/external/jquery/jquery.js"
src="{{root}}/skin/jquery-ui/external/jquery/jquery.js?KIWIXCACHEID"
></script>
<script
type="text/javascript"
src="{{root}}/skin/jquery-ui/jquery-ui.min.js"
src="{{root}}/skin/jquery-ui/jquery-ui.min.js?KIWIXCACHEID"
></script>
<link
type="text/css"
href="{{root}}/skin/jquery-ui/jquery-ui.min.css"
href="{{root}}/skin/jquery-ui/jquery-ui.min.css?KIWIXCACHEID"
rel="Stylesheet"
/>
<link
type="text/css"
href="{{root}}/skin/jquery-ui/jquery-ui.theme.min.css"
href="{{root}}/skin/jquery-ui/jquery-ui.theme.min.css?KIWIXCACHEID"
rel="Stylesheet"
/>
<link
type="text/css"
href="{{root}}/skin/index.css"
href="{{root}}/skin/index.css?KIWIXCACHEID"
rel="Stylesheet"
/>
<style>
@font-face {
font-family: "poppins";
src: url("{{root}}/skin/fonts/Poppins.ttf") format("truetype");
src: url("{{root}}/skin/fonts/Poppins.ttf?KIWIXCACHEID") format("truetype");
}
@font-face {
font-family: "roboto";
src: url("{{root}}/skin/fonts/Roboto.ttf") format("truetype");
src: url("{{root}}/skin/fonts/Roboto.ttf?KIWIXCACHEID") format("truetype");
}
</style>
<script src="{{root}}/skin/isotope.pkgd.min.js" defer></script>
<script src="{{root}}/skin/iso6391To3.js"></script>
<script type="text/javascript" src="{{root}}/skin/index.js" defer></script>
<script src="{{root}}/skin/isotope.pkgd.min.js?KIWIXCACHEID" defer></script>
<script src="{{root}}/skin/iso6391To3.js?KIWIXCACHEID"></script>
<script type="text/javascript" src="{{root}}/skin/index.js?KIWIXCACHEID" defer></script>
</head>
<body>
<div class='kiwixNav'>

View File

@ -9,7 +9,7 @@
</form>
</div>
<input type="checkbox" id="kiwix_button_show_toggle">
<label for="kiwix_button_show_toggle"><img src="{{root}}/skin/caret.png" alt=""></label>
<label for="kiwix_button_show_toggle"><img src="{{root}}/skin/caret.png?KIWIXCACHEID" alt=""></label>
<div class="kiwix_button_cont">
{{#withlibrarybutton}}
<a id="kiwix_serve_taskbar_library_button" title="{{{LIBRARY_BUTTON_TEXT}}}" aria-label="{{{LIBRARY_BUTTON_TEXT}}}" href="{{root}}/"><button>&#x1f3e0;</button></a>

View File

@ -0,0 +1 @@
{{root}}/skin/search_results.css?KIWIXCACHEID

View File

@ -288,6 +288,85 @@ TEST_F(ServerTest, UncompressibleContentIsNotCompressed)
}
}
// Selects from text only the lines containing the specified (fixed string)
// pattern
std::string fgrep(const std::string& pattern, const std::string& text)
{
std::istringstream iss(text);
std::string line;
std::string result;
while ( getline(iss, line) ) {
if ( line.find(pattern) != std::string::npos ) {
result += line + "\n";
}
}
return result;
}
TEST_F(ServerTest, CacheIdsOfStaticResources)
{
typedef std::pair<std::string, std::string> UrlAndExpectedResult;
const std::vector<UrlAndExpectedResult> testData{
{
/* url */ "/ROOT/",
R"EXPECTEDRESULT( src="/ROOT/skin/jquery-ui/external/jquery/jquery.js?cacheid=1d85f0f3"
src="/ROOT/skin/jquery-ui/jquery-ui.min.js?cacheid=d927c2ff"
href="/ROOT/skin/jquery-ui/jquery-ui.min.css?cacheid=e1de77b3"
href="/ROOT/skin/jquery-ui/jquery-ui.theme.min.css?cacheid=2a5841f9"
href="/ROOT/skin/index.css?cacheid=1aca980a"
src: url("/ROOT/skin/fonts/Poppins.ttf?cacheid=af705837") format("truetype");
src: url("/ROOT/skin/fonts/Roboto.ttf?cacheid=84d10248") format("truetype");
<script src="/ROOT/skin/isotope.pkgd.min.js?cacheid=2e48d392" defer></script>
<script src="/ROOT/skin/iso6391To3.js?cacheid=ecde2bb3"></script>
<script type="text/javascript" src="/ROOT/skin/index.js?cacheid=0951f06f" defer></script>
)EXPECTEDRESULT"
},
{
/* url */ "/ROOT/skin/index.js",
R"EXPECTEDRESULT( <img src="../skin/download.png?cacheid=a39aa502" alt="direct download" />
<img src="../skin/hash.png?cacheid=f836e872" alt="download hash" />
<img src="../skin/magnet.png?cacheid=73b6bddf" alt="download magnet" />
<img src="../skin/bittorrent.png?cacheid=4f5c6882" alt="download torrent" />
)EXPECTEDRESULT"
},
{
/* url */ "/ROOT/zimfile/A/index",
R"EXPECTEDRESULT(<link type="root" href="/ROOT"><link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.min.css?cacheid=e1de77b3" rel="Stylesheet" />
<link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.theme.min.css?cacheid=2a5841f9" rel="Stylesheet" />
<link type="text/css" href="/ROOT/skin/taskbar.css?cacheid=49365e9c" rel="Stylesheet" />
<script type="text/javascript" src="/ROOT/skin/jquery-ui/external/jquery/jquery.js?cacheid=1d85f0f3" defer></script>
<script type="text/javascript" src="/ROOT/skin/jquery-ui/jquery-ui.min.js?cacheid=d927c2ff" defer></script>
<script type="text/javascript" src="/ROOT/skin/taskbar.js?cacheid=5982280c" defer></script>
<label for="kiwix_button_show_toggle"><img src="/ROOT/skin/caret.png?cacheid=22b942b4" alt=""></label>
)EXPECTEDRESULT"
},
{
// Searching in a ZIM file without a full-text index returns
// a page rendered from static/templates/no_search_result_html
/* url */ "/ROOT/search?content=poor&pattern=whatever",
R"EXPECTEDRESULT( <link type="text/css" href="/ROOT/skin/search_results.css?cacheid=76d39c84" rel="Stylesheet" />
<link type="root" href="/ROOT"><link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.min.css?cacheid=e1de77b3" rel="Stylesheet" />
<link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.theme.min.css?cacheid=2a5841f9" rel="Stylesheet" />
<link type="text/css" href="/ROOT/skin/taskbar.css?cacheid=49365e9c" rel="Stylesheet" />
<script type="text/javascript" src="/ROOT/skin/jquery-ui/external/jquery/jquery.js?cacheid=1d85f0f3" defer></script>
<script type="text/javascript" src="/ROOT/skin/jquery-ui/jquery-ui.min.js?cacheid=d927c2ff" defer></script>
<script type="text/javascript" src="/ROOT/skin/taskbar.js?cacheid=5982280c" defer></script>
<label for="kiwix_button_show_toggle"><img src="/ROOT/skin/caret.png?cacheid=22b942b4" alt=""></label>
)EXPECTEDRESULT"
},
};
for ( const auto& urlAndExpectedResult : testData ) {
const std::string url = urlAndExpectedResult.first;
const std::string expectedResult = urlAndExpectedResult.second;
const TestContext ctx{ {"url", url} };
const auto r = zfs1_->GET(url.c_str());
EXPECT_EQ(r->body.find("KIWIXCACHEID"), std::string::npos) << ctx;
EXPECT_EQ(fgrep("/skin/", r->body), expectedResult) << ctx;
}
}
const char* urls400[] = {
"/ROOT/search",
"/ROOT/search?content=zimfile",
@ -433,12 +512,12 @@ std::string TestContentIn404HtmlResponse::expectedResponse() const
)FRAG",
R"FRAG(
<link type="root" href="/ROOT"><link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.min.css" rel="Stylesheet" />
<link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.theme.min.css" rel="Stylesheet" />
<link type="text/css" href="/ROOT/skin/taskbar.css" rel="Stylesheet" />
<script type="text/javascript" src="/ROOT/skin/jquery-ui/external/jquery/jquery.js" defer></script>
<script type="text/javascript" src="/ROOT/skin/jquery-ui/jquery-ui.min.js" defer></script>
<script type="text/javascript" src="/ROOT/skin/taskbar.js" defer></script>
<link type="root" href="/ROOT"><link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.min.css?cacheid=e1de77b3" rel="Stylesheet" />
<link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.theme.min.css?cacheid=2a5841f9" rel="Stylesheet" />
<link type="text/css" href="/ROOT/skin/taskbar.css?cacheid=49365e9c" rel="Stylesheet" />
<script type="text/javascript" src="/ROOT/skin/jquery-ui/external/jquery/jquery.js?cacheid=1d85f0f3" defer></script>
<script type="text/javascript" src="/ROOT/skin/jquery-ui/jquery-ui.min.js?cacheid=d927c2ff" defer></script>
<script type="text/javascript" src="/ROOT/skin/taskbar.js?cacheid=5982280c" defer></script>
</head>
<body><span class="kiwix">
<span id="kiwixtoolbar" class="ui-widget-header">
@ -454,7 +533,7 @@ std::string TestContentIn404HtmlResponse::expectedResponse() const
R"FRAG( </form>
</div>
<input type="checkbox" id="kiwix_button_show_toggle">
<label for="kiwix_button_show_toggle"><img src="/ROOT/skin/caret.png" alt=""></label>
<label for="kiwix_button_show_toggle"><img src="/ROOT/skin/caret.png?cacheid=22b942b4" alt=""></label>
<div class="kiwix_button_cont">
<a id="kiwix_serve_taskbar_library_button" title=")FRAG",
@ -768,7 +847,7 @@ TEST_F(ServerTest, 404WithBodyTesting)
{ /* url */ "/ROOT/search?content=poor&pattern=whatever",
expected_page_title=="Fulltext search unavailable" &&
expected_css_url=="/ROOT/skin/search_results.css" &&
expected_css_url=="/ROOT/skin/search_results.css?cacheid=76d39c84" &&
book_name=="poor" &&
book_title=="poor" &&
expected_body==R"(