Merge pull request #712 from kiwix/static_resource_versioning

Static resource versioning

scripts/kiwix-resources

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+
+'''
+Copyright 2022 Veloman Yunkan <veloman.yunkan@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 3 of the License, or any
+later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+02110-1301, USA.
+'''
+
+import argparse
+import hashlib
+import os.path
+import re
+
+def read_resource_file(resource_file_path):
+    with open(resource_file_path, 'r') as f:
+        return [line.strip() for line in f]
+
+def list_resources(resource_file_path):
+    for resource_path in read_resource_file(resource_file_path):
+        print(resource_path)
+
+def compute_resource_revision(resource_path):
+    with open(os.path.join(OUT_DIR, resource_path), 'rb') as f:
+        return hashlib.sha1(f.read()).hexdigest()[:8]
+
+resource_revisions = {}
+
+def get_resource_revision(res):
+    if not res in resource_revisions:
+        preprocess_resource(res)
+        resource_revisions[res] = compute_resource_revision(res)
+    return resource_revisions[res]
+
+RESOURCE_WITH_CACHEID_URL_PATTERN=r'(?P<pre>.*/(?P<resource>skin/[^"?]+)\?)KIWIXCACHEID(?P<post>[^"]*)'
+
+def set_cacheid(resource_matchobj):
+    pre = resource_matchobj.group('pre')
+    resource = resource_matchobj.group('resource')
+    post = resource_matchobj.group('post')
+    cacheid = 'cacheid=' + get_resource_revision(resource)
+    return pre + cacheid + post
+
+def preprocess_text(s):
+    if 'KIWIXCACHEID' in s:
+        s = re.sub(RESOURCE_WITH_CACHEID_URL_PATTERN, set_cacheid, s)
+        assert not 'KIWIXCACHEID' in s
+    return s
+
+def get_preprocessed_resource(srcpath):
+    """Get the transformed content of a resource
+
+    If the resource at srcpath is modified by preprocessing then this function
+    returns the transformed content of the resource. Otherwise it returns None.
+    """
+    try:
+        with open(srcpath, 'r') as resource_file:
+            content = resource_file.read()
+            preprocessed_content = preprocess_text(content)
+        return preprocessed_content if preprocessed_content != content else None
+    except UnicodeDecodeError:
+        # It was a binary resource
+        return None
+
+
+def symlink_resource(src, resource_path):
+    if os.path.exists(resource_path):
+        if os.path.islink(resource_path) and os.readlink(resource_path) == src:
+            return
+        os.remove(resource_path)
+    os.symlink(src, resource_path)
+
+def preprocess_resource(resource_path):
+    print('Preprocessing', resource_path, '...')
+    resource_dir = os.path.dirname(resource_path)
+    if resource_dir != '':
+        os.makedirs(os.path.join(OUT_DIR, resource_dir), exist_ok=True)
+    srcpath = os.path.join(BASE_DIR, resource_path)
+    outpath = os.path.join(OUT_DIR, resource_path)
+    if os.path.exists(outpath):
+        os.remove(outpath)
+    preprocessed_content = get_preprocessed_resource(srcpath)
+    if preprocessed_content is None:
+        symlink_resource(srcpath, outpath)
+    else:
+        with open(outpath, 'w') as target:
+            print(preprocessed_content, end='', file=target)
+
+
+def copy_file(src_path, dst_path):
+    with open(src_path, 'rb') as src:
+        with open(dst_path, 'wb') as dst:
+            dst.write(src.read())
+
+def preprocess_resources(resource_file_path):
+    resource_filename = os.path.basename(resource_file_path)
+    for resource in read_resource_file(resource_file_path):
+        preprocess_resource(resource)
+    copy_file(resource_file_path, os.path.join(OUT_DIR, resource_filename))
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    commands = parser.add_mutually_exclusive_group()
+    commands.add_argument('--list-all', action='store_true')
+    commands.add_argument('--preprocess', action='store_true')
+    parser.add_argument('--outdir')
+    parser.add_argument('resource_file')
+    args = parser.parse_args()
+    BASE_DIR = os.path.dirname(os.path.realpath(args.resource_file))
+    OUT_DIR = args.outdir
+
+    if args.list_all:
+        list_resources(args.resource_file)
+    elif args.preprocess:
+        preprocess_resources(args.resource_file)

scripts/meson.build

@@ -1,4 +1,5 @@
 
+res_manager = find_program('kiwix-resources')
 res_compiler = find_program('kiwix-compile-resources')
 
 install_data(res_compiler.path(), install_dir:get_option('bindir'))

src/server/internalServer.cpp

@@ -442,6 +442,16 @@ SuggestionsList_t getSuggestions(SuggestionSearcherCache& cache, const zim::Arch
 namespace
 {
 
+std::string renderUrl(const std::string& root, const std::string& urlTemplate)
+{
+  MustacheData data;
+  data.set("root", root);
+  auto url = kainjow::mustache::mustache(urlTemplate).render(data);
+  if ( url.back() == '\n' )
+    url.pop_back();
+  return url;
+}
+
 std::string makeFulltextSearchSuggestion(const std::string& lang, const std::string& queryString)
 {
   return i18n::expandParameterizedString(lang, "suggest-full-text-search",
@@ -622,10 +632,11 @@ std::unique_ptr<Response> InternalServer::handle_search(const RequestContext& re
     } catch(std::runtime_error& e) {
       // Searcher->search will throw a runtime error if there is no valid xapian database to do the search.
       // (in case of zim file not containing a index)
+      const auto cssUrl = renderUrl(m_root, RESOURCE::templates::url_of_search_results_css);
       return HTTPErrorHtmlResponse(*this, request, MHD_HTTP_NOT_FOUND,
                                    "fulltext-search-unavailable",
                                    "404-page-heading",
-                                   m_root + "/skin/search_results.css")
+                                   cssUrl)
            + nonParameterizedMessage("no-search-results")
            + TaskbarInfo(searchInfo.bookName, archive.get());
     }

static/meson.build

@@ -1,18 +1,27 @@
-resource_files = run_command(find_program('python3'),
+resource_files = run_command(res_manager,
-                    '-c',
+                    '--list-all',
-                    'import sys; f=open(sys.argv[1]); print(f.read())',
                     files('resources_list.txt')
                  ).stdout().strip().split('\n')
 
-lib_resources = custom_target('resources',
+preprocessed_resources = custom_target('preprocessed_resource_files',
     input: 'resources_list.txt',
+    output: ['resources_list.txt'],
+    command:[res_manager,
+             '--preprocess',
+             '--outdir', '@OUTDIR@',
+             '@INPUT@'],
+    depend_files: resource_files
+)
+
+lib_resources = custom_target('resources',
+    input: preprocessed_resources,
     output: ['kiwixlib-resources.cpp', 'kiwixlib-resources.h'],
     command:[res_compiler,
              '--cxxfile', '@OUTPUT0@',
              '--hfile', '@OUTPUT1@',
              '--source_dir', '@OUTDIR@',
              '@INPUT@'],
-    depend_files: resource_files
+    depends: preprocessed_resources
 )
 
 i18n_resource_files = run_command(find_program('python3'),

static/resources_list.txt

@@ -47,5 +47,6 @@ templates/catalog_v2_entries.xml
 templates/catalog_v2_entry.xml
 templates/catalog_v2_categories.xml
 templates/catalog_v2_languages.xml
+templates/url_of_search_results_css
 opensearchdescription.xml
 catalog_v2_searchdescription.xml

static/skin/index.js

@@ -171,25 +171,25 @@
                     <div class="modal-content">
                         <div class="modal-regular-download">
                             <a href="${downloadLink}" download>
-                                <img src="../skin/download.png" alt="direct download" />
+                                <img src="../skin/download.png?KIWIXCACHEID" alt="direct download" />
                                 <div>Direct</div>
                             </a>
                         </div>
                         <div class="modal-regular-download">
                             <a href="${downloadLink}.sha256" download>
-                                <img src="../skin/hash.png" alt="download hash" />
+                                <img src="../skin/hash.png?KIWIXCACHEID" alt="download hash" />
                                 <div>Sha256 hash</div>
                             </a>
                         </div>
                         <div class="modal-regular-download">
                             <a href="${downloadLink}.magnet" target="_blank">
-                                <img src="../skin/magnet.png" alt="download magnet" />
+                                <img src="../skin/magnet.png?KIWIXCACHEID" alt="download magnet" />
                                 <div>Magnet link</div>
                             </a>
                         </div>
                         <div class="modal-regular-download">
                             <a href="${downloadLink}.torrent" download>
-                                <img src="../skin/bittorrent.png" alt="download torrent" />
+                                <img src="../skin/bittorrent.png?KIWIXCACHEID" alt="download torrent" />
                                 <div>Torrent file</div>
                             </a>
                         </div>

static/templates/head_taskbar.html

@@ -1,6 +1,6 @@
-<link type="text/css" href="{{root}}/skin/jquery-ui/jquery-ui.min.css" rel="Stylesheet" />
+<link type="text/css" href="{{root}}/skin/jquery-ui/jquery-ui.min.css?KIWIXCACHEID" rel="Stylesheet" />
-<link type="text/css" href="{{root}}/skin/jquery-ui/jquery-ui.theme.min.css" rel="Stylesheet" />
+<link type="text/css" href="{{root}}/skin/jquery-ui/jquery-ui.theme.min.css?KIWIXCACHEID" rel="Stylesheet" />
-<link type="text/css" href="{{root}}/skin/taskbar.css" rel="Stylesheet" />
+<link type="text/css" href="{{root}}/skin/taskbar.css?KIWIXCACHEID" rel="Stylesheet" />
-<script type="text/javascript" src="{{root}}/skin/jquery-ui/external/jquery/jquery.js" defer></script>
+<script type="text/javascript" src="{{root}}/skin/jquery-ui/external/jquery/jquery.js?KIWIXCACHEID" defer></script>
-<script type="text/javascript" src="{{root}}/skin/jquery-ui/jquery-ui.min.js" defer></script>
+<script type="text/javascript" src="{{root}}/skin/jquery-ui/jquery-ui.min.js?KIWIXCACHEID" defer></script>
-<script type="text/javascript" src="{{root}}/skin/taskbar.js" defer></script>
+<script type="text/javascript" src="{{root}}/skin/taskbar.js?KIWIXCACHEID" defer></script>

static/templates/index.html

@@ -6,41 +6,41 @@
     <title>Welcome to Kiwix Server</title>
     <script
       type="text/javascript"
-      src="{{root}}/skin/jquery-ui/external/jquery/jquery.js"
+      src="{{root}}/skin/jquery-ui/external/jquery/jquery.js?KIWIXCACHEID"
     ></script>
     <script
       type="text/javascript"
-      src="{{root}}/skin/jquery-ui/jquery-ui.min.js"
+      src="{{root}}/skin/jquery-ui/jquery-ui.min.js?KIWIXCACHEID"
     ></script>
     <link
       type="text/css"
-      href="{{root}}/skin/jquery-ui/jquery-ui.min.css"
+      href="{{root}}/skin/jquery-ui/jquery-ui.min.css?KIWIXCACHEID"
       rel="Stylesheet"
     />
     <link
       type="text/css"
-      href="{{root}}/skin/jquery-ui/jquery-ui.theme.min.css"
+      href="{{root}}/skin/jquery-ui/jquery-ui.theme.min.css?KIWIXCACHEID"
       rel="Stylesheet"
     />
     <link
       type="text/css"
-      href="{{root}}/skin/index.css"
+      href="{{root}}/skin/index.css?KIWIXCACHEID"
       rel="Stylesheet"
     />
     <style>
       @font-face {
         font-family: "poppins";
-        src: url("{{root}}/skin/fonts/Poppins.ttf") format("truetype");
+        src: url("{{root}}/skin/fonts/Poppins.ttf?KIWIXCACHEID") format("truetype");
       }
 
       @font-face {
           font-family: "roboto";
-          src: url("{{root}}/skin/fonts/Roboto.ttf") format("truetype");
+          src: url("{{root}}/skin/fonts/Roboto.ttf?KIWIXCACHEID") format("truetype");
       }
     </style>
-    <script src="{{root}}/skin/isotope.pkgd.min.js" defer></script>
+    <script src="{{root}}/skin/isotope.pkgd.min.js?KIWIXCACHEID" defer></script>
-    <script src="{{root}}/skin/iso6391To3.js"></script>
+    <script src="{{root}}/skin/iso6391To3.js?KIWIXCACHEID"></script>
-    <script type="text/javascript" src="{{root}}/skin/index.js" defer></script>
+    <script type="text/javascript" src="{{root}}/skin/index.js?KIWIXCACHEID" defer></script>
   </head>
   <body>
     <div class='kiwixNav'>

static/templates/taskbar_part.html

@@ -9,7 +9,7 @@
         </form>
       </div>
         <input type="checkbox" id="kiwix_button_show_toggle">
-        <label for="kiwix_button_show_toggle"><img src="{{root}}/skin/caret.png" alt=""></label>
+        <label for="kiwix_button_show_toggle"><img src="{{root}}/skin/caret.png?KIWIXCACHEID" alt=""></label>
         <div class="kiwix_button_cont">
           {{#withlibrarybutton}}
             <a id="kiwix_serve_taskbar_library_button" title="{{{LIBRARY_BUTTON_TEXT}}}" aria-label="{{{LIBRARY_BUTTON_TEXT}}}" href="{{root}}/"><button>&#x1f3e0;</button></a>

static/templates/url_of_search_results_css

@@ -0,0 +1 @@
+{{root}}/skin/search_results.css?KIWIXCACHEID

test/server.cpp

@@ -288,6 +288,85 @@ TEST_F(ServerTest, UncompressibleContentIsNotCompressed)
   }
 }
 
+
+// Selects from text only the lines containing the specified (fixed string)
+// pattern
+std::string fgrep(const std::string& pattern, const std::string& text)
+{
+  std::istringstream iss(text);
+  std::string line;
+  std::string result;
+  while ( getline(iss, line) ) {
+    if ( line.find(pattern) != std::string::npos ) {
+      result += line + "\n";
+    }
+  }
+  return result;
+}
+
+TEST_F(ServerTest, CacheIdsOfStaticResources)
+{
+  typedef std::pair<std::string, std::string> UrlAndExpectedResult;
+  const std::vector<UrlAndExpectedResult> testData{
+    {
+      /* url */ "/ROOT/",
+R"EXPECTEDRESULT(      src="/ROOT/skin/jquery-ui/external/jquery/jquery.js?cacheid=1d85f0f3"
+      src="/ROOT/skin/jquery-ui/jquery-ui.min.js?cacheid=d927c2ff"
+      href="/ROOT/skin/jquery-ui/jquery-ui.min.css?cacheid=e1de77b3"
+      href="/ROOT/skin/jquery-ui/jquery-ui.theme.min.css?cacheid=2a5841f9"
+      href="/ROOT/skin/index.css?cacheid=1aca980a"
+        src: url("/ROOT/skin/fonts/Poppins.ttf?cacheid=af705837") format("truetype");
+          src: url("/ROOT/skin/fonts/Roboto.ttf?cacheid=84d10248") format("truetype");
+    <script src="/ROOT/skin/isotope.pkgd.min.js?cacheid=2e48d392" defer></script>
+    <script src="/ROOT/skin/iso6391To3.js?cacheid=ecde2bb3"></script>
+    <script type="text/javascript" src="/ROOT/skin/index.js?cacheid=0951f06f" defer></script>
+)EXPECTEDRESULT"
+    },
+    {
+      /* url */ "/ROOT/skin/index.js",
+R"EXPECTEDRESULT(                                <img src="../skin/download.png?cacheid=a39aa502" alt="direct download" />
+                                <img src="../skin/hash.png?cacheid=f836e872" alt="download hash" />
+                                <img src="../skin/magnet.png?cacheid=73b6bddf" alt="download magnet" />
+                                <img src="../skin/bittorrent.png?cacheid=4f5c6882" alt="download torrent" />
+)EXPECTEDRESULT"
+    },
+    {
+      /* url */ "/ROOT/zimfile/A/index",
+R"EXPECTEDRESULT(<link type="root" href="/ROOT"><link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.min.css?cacheid=e1de77b3" rel="Stylesheet" />
+<link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.theme.min.css?cacheid=2a5841f9" rel="Stylesheet" />
+<link type="text/css" href="/ROOT/skin/taskbar.css?cacheid=49365e9c" rel="Stylesheet" />
+<script type="text/javascript" src="/ROOT/skin/jquery-ui/external/jquery/jquery.js?cacheid=1d85f0f3" defer></script>
+<script type="text/javascript" src="/ROOT/skin/jquery-ui/jquery-ui.min.js?cacheid=d927c2ff" defer></script>
+<script type="text/javascript" src="/ROOT/skin/taskbar.js?cacheid=5982280c" defer></script>
+        <label for="kiwix_button_show_toggle"><img src="/ROOT/skin/caret.png?cacheid=22b942b4" alt=""></label>
+)EXPECTEDRESULT"
+    },
+    {
+      // Searching in a ZIM file without a full-text index returns
+      // a page rendered from static/templates/no_search_result_html
+      /* url */ "/ROOT/search?content=poor&pattern=whatever",
+R"EXPECTEDRESULT(    <link type="text/css" href="/ROOT/skin/search_results.css?cacheid=76d39c84" rel="Stylesheet" />
+  <link type="root" href="/ROOT"><link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.min.css?cacheid=e1de77b3" rel="Stylesheet" />
+<link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.theme.min.css?cacheid=2a5841f9" rel="Stylesheet" />
+<link type="text/css" href="/ROOT/skin/taskbar.css?cacheid=49365e9c" rel="Stylesheet" />
+<script type="text/javascript" src="/ROOT/skin/jquery-ui/external/jquery/jquery.js?cacheid=1d85f0f3" defer></script>
+<script type="text/javascript" src="/ROOT/skin/jquery-ui/jquery-ui.min.js?cacheid=d927c2ff" defer></script>
+<script type="text/javascript" src="/ROOT/skin/taskbar.js?cacheid=5982280c" defer></script>
+        <label for="kiwix_button_show_toggle"><img src="/ROOT/skin/caret.png?cacheid=22b942b4" alt=""></label>
+)EXPECTEDRESULT"
+    },
+  };
+
+  for ( const auto& urlAndExpectedResult : testData ) {
+    const std::string url = urlAndExpectedResult.first;
+    const std::string expectedResult = urlAndExpectedResult.second;
+    const TestContext ctx{ {"url", url} };
+    const auto r = zfs1_->GET(url.c_str());
+    EXPECT_EQ(r->body.find("KIWIXCACHEID"), std::string::npos) << ctx;
+    EXPECT_EQ(fgrep("/skin/", r->body), expectedResult) << ctx;
+  }
+}
+
 const char* urls400[] = {
   "/ROOT/search",
   "/ROOT/search?content=zimfile",
@@ -433,12 +512,12 @@ std::string TestContentIn404HtmlResponse::expectedResponse() const
 )FRAG",
 
     R"FRAG(
-  <link type="root" href="/ROOT"><link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.min.css" rel="Stylesheet" />
+  <link type="root" href="/ROOT"><link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.min.css?cacheid=e1de77b3" rel="Stylesheet" />
-<link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.theme.min.css" rel="Stylesheet" />
+<link type="text/css" href="/ROOT/skin/jquery-ui/jquery-ui.theme.min.css?cacheid=2a5841f9" rel="Stylesheet" />
-<link type="text/css" href="/ROOT/skin/taskbar.css" rel="Stylesheet" />
+<link type="text/css" href="/ROOT/skin/taskbar.css?cacheid=49365e9c" rel="Stylesheet" />
-<script type="text/javascript" src="/ROOT/skin/jquery-ui/external/jquery/jquery.js" defer></script>
+<script type="text/javascript" src="/ROOT/skin/jquery-ui/external/jquery/jquery.js?cacheid=1d85f0f3" defer></script>
-<script type="text/javascript" src="/ROOT/skin/jquery-ui/jquery-ui.min.js" defer></script>
+<script type="text/javascript" src="/ROOT/skin/jquery-ui/jquery-ui.min.js?cacheid=d927c2ff" defer></script>
-<script type="text/javascript" src="/ROOT/skin/taskbar.js" defer></script>
+<script type="text/javascript" src="/ROOT/skin/taskbar.js?cacheid=5982280c" defer></script>
 </head>
   <body><span class="kiwix">
   <span id="kiwixtoolbar" class="ui-widget-header">
@@ -454,7 +533,7 @@ std::string TestContentIn404HtmlResponse::expectedResponse() const
   R"FRAG(        </form>
       </div>
         <input type="checkbox" id="kiwix_button_show_toggle">
-        <label for="kiwix_button_show_toggle"><img src="/ROOT/skin/caret.png" alt=""></label>
+        <label for="kiwix_button_show_toggle"><img src="/ROOT/skin/caret.png?cacheid=22b942b4" alt=""></label>
         <div class="kiwix_button_cont">
             <a id="kiwix_serve_taskbar_library_button" title=")FRAG",
 
@@ -768,7 +847,7 @@ TEST_F(ServerTest, 404WithBodyTesting)
 
     { /* url */ "/ROOT/search?content=poor&pattern=whatever",
       expected_page_title=="Fulltext search unavailable" &&
-      expected_css_url=="/ROOT/skin/search_results.css" &&
+      expected_css_url=="/ROOT/skin/search_results.css?cacheid=76d39c84" &&
       book_name=="poor" &&
       book_title=="poor" &&
       expected_body==R"(