Add a new script to compile resources.

- No more dependency to reswrap binary (everything is done in python)
- Resource strings can be directly accessed.
  As side effect, it add a check at compilation if the resource is
  declared and compiled in the binary.
- The resource content can be overwritten at runtime with a env variable.

There is also few clean in the static as some files shoul be in the tools
directory.

The compile_resource script is install to let other project use it.

include/ctpp2/CTPP2VMStringLoader.hpp

@@ -34,7 +34,6 @@
 #include <stdio.h>
 #include <stdlib.h>
 
-// #include <common/resourceTools.h>
 #include <iostream>
 #include <string>
 

include/indexer.h

@@ -31,7 +31,6 @@
 #include <pthread.h>
 #include "common/stringTools.h"
 #include "common/otherTools.h"
-#include "common/resourceTools.h"
 #include <zim/file.h>
 #include <zim/article.h>
 #include <zim/fileiterator.h>

include/meson.build

@@ -18,7 +18,6 @@ install_headers(
   'common/otherTools.h',
   'common/pathTools.h',
   'common/regexTools.h',
-  'common/resourceTools.h',
   'common/stringTools.h',
   'common/tree.h',
   subdir:'kiwix/common'

include/searcher.h

@@ -28,7 +28,6 @@
 #include <locale>
 #include <cctype>
 #include <vector>
-#include "common/resourceTools.h"
 #include "common/pathTools.h"
 #include "common/stringTools.h"
 #include <unicode/putil.h>

meson.build

@@ -49,6 +49,8 @@ all_deps = [thread_dep, libicu_dep, libzim_dep, ctpp2_dep, xapian_dep, pugixml_d
 inc = include_directories('include')
 
 subdir('include')
+subdir('scripts')
+subdir('static')
 subdir('src')
 
 pkg_mod = import('pkgconfig')

scripts/compile_resources.py

@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+
+import argparse
+import os.path
+import re
+
+def full_identifier(filename):
+    parts = os.path.normpath(filename).split(os.sep)
+    parts = [to_identifier(part) for part in parts]
+    print(filename, parts)
+    return parts
+
+def to_identifier(name):
+    ident = re.sub(r'[^0-9a-zA-Z]', '_', name)
+    if ident[0].isnumeric():
+        return "_"+ident
+    return ident
+
+resource_impl_template = """
+static const unsigned char {data_identifier}[] = {{
+    {resource_content}
+}};
+
+namespace RESOURCE {{
+{namespaces_open}
+const std::string {identifier} = init_resource("{env_identifier}", {data_identifier}, {resource_len});
+{namespaces_close}
+}}
+"""
+
+resource_getter_template = """
+    if (name == "{common_name}")
+        return RESOURCE::{identifier};
+"""
+
+resource_decl_template = """{namespaces_open}
+extern const std::string {identifier};
+{namespaces_close}"""
+
+class Resource:
+    def __init__(self, base_dir, filename):
+        filename = filename.strip()
+        self.filename = filename
+        self.identifier = full_identifier(filename)
+        with open(os.path.join(base_dir, filename), 'rb') as f:
+            self.data = f.read()
+
+    def dump_impl(self):
+        nb_row = len(self.data)//16 + (1 if len(self.data) % 16 else 0)
+        sliced = (self.data[i*16:(i+1)*16] for i in range(nb_row))
+
+        return resource_impl_template.format(
+            data_identifier="_".join([""]+self.identifier),
+            resource_content=",\n    ".join(", ".join("{:#04x}".format(i) for i in r) for r in sliced),
+            resource_len=len(self.data),
+            namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]), 
+            namespaces_close=" ".join(["}"]*(len(self.identifier)-1)),
+            identifier=self.identifier[-1],
+            env_identifier="RES_"+"_".join(self.identifier)+"_PATH"
+        )
+    
+    def dump_getter(self):
+        return resource_getter_template.format(
+            common_name=self.filename,
+            identifier="::".join(self.identifier)
+        )
+
+    def dump_decl(self):
+        return resource_decl_template.format(
+            namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]), 
+            namespaces_close=" ".join(["}"]*(len(self.identifier)-1)),
+            identifier=self.identifier[-1]
+        )
+    
+
+
+master_c_template = """//This file is automaically generated. Do not modify it.
+
+#include <stdlib.h>
+#include <fstream>
+#include <exception>
+#include "{basename}"
+
+class ResourceNotFound : public std::runtime_error {{
+  public:
+    ResourceNotFound(const std::string& what_arg):
+      std::runtime_error(what_arg)
+    {{ }};
+}};
+
+static std::string init_resource(const char* name, const unsigned char* content, int len)
+{{
+    char * resPath = getenv(name);
+    if (NULL == resPath)
+        return std::string(reinterpret_cast<const char*>(content), len);
+    
+    std::ifstream ifs(resPath);
+    if (!ifs.good())
+        return std::string(reinterpret_cast<const char*>(content), len);
+    return std::string( (std::istreambuf_iterator<char>(ifs)),
+                        (std::istreambuf_iterator<char>()   ));
+}}
+
+const std::string& getResource(const std::string& name) {{
+{RESOURCES_GETTER}
+    throw ResourceNotFound("Resource not found.");
+}}
+
+{RESOURCES}
+
+"""
+
+def gen_c_file(resources, basename):
+    return master_c_template.format(
+       RESOURCES="\n\n".join(r.dump_impl() for r in resources),
+       RESOURCES_GETTER="\n\n".join(r.dump_getter() for r in resources),
+       basename=basename
+    )
+ 
+
+
+master_h_template = """//This file is automaically generated. Do not modify it.
+#ifndef KIWIX_{BASENAME}
+#define KIWIX_{BASENAME}
+
+#include <string>
+
+namespace RESOURCE {{
+    {RESOURCES}
+}};
+
+const std::string& getResource(const std::string& name);
+
+#endif // KIWIX_{BASENAME}
+
+"""
+
+def gen_h_file(resources, basename):
+    return master_h_template.format(
+       RESOURCES="\n    ".join(r.dump_decl() for r in resources),
+       BASENAME=basename.upper()
+    )
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--cxxfile',
+                        help='The Cpp file name to generate')
+    parser.add_argument('--hfile',
+                        help='The h file name to generate')
+    parser.add_argument('resource_file',
+                        help='The list of resources to compile.')
+    args = parser.parse_args()
+
+    base_dir = os.path.dirname(os.path.realpath(args.resource_file))
+    with open(args.resource_file, 'r') as f:
+        resources = [Resource(base_dir, filename) for filename in f.readlines()]
+
+    h_identifier = to_identifier(os.path.basename(args.hfile))
+    with open(args.hfile, 'w') as f:
+        f.write(gen_h_file(resources, h_identifier))
+
+    with open(args.cxxfile, 'w') as f:
+        f.write(gen_c_file(resources, os.path.basename(args.hfile)))
+

scripts/meson.build

@@ -0,0 +1,4 @@
+
+res_compiler = find_program('compile_resources.py')
+
+install_data(res_compiler.path(), install_dir:get_option('bindir'))

src/common/resourceTools.cpp

@@ -1,10 +0,0 @@
-#include <common/resourceTools.h>
-#include <iostream>
-
-std::string getResourceAsString(const std::string &name) {
-  std::map<std::string, std::pair<const unsigned char*, unsigned int> >::iterator it = resourceMap.find(name);
-  if (it != resourceMap.end()) {
-    return std::string((const char*)resourceMap[name].first, resourceMap[name].second);
-  }
-  return "";
-}

src/indexer.cpp

@@ -18,6 +18,7 @@
  */
 
 #include "indexer.h"
+#include "kiwixlib-resources.h"
 
 namespace kiwix {
 
@@ -64,7 +65,7 @@ namespace kiwix {
   /* Read the stopwords */
   void Indexer::readStopWords(const string languageCode) {
     std::string stopWord;
-    std::istringstream file(getResourceAsString("stopwords/" + languageCode));
+    std::istringstream file(getResource("stopwords/" + languageCode));
 
     this->stopWords.clear();
 

src/meson.build

@@ -8,13 +8,13 @@ kiwix_sources = [
   'common/pathTools.cpp',
   'common/regexTools.cpp',
   'common/stringTools.cpp',
-  'common/resourceTools.cpp',
   'common/networkTools.cpp',
   'common/otherTools.cpp',
   'ctpp2/CTPP2VMStringLoader.cpp',
   'xapian/htmlparse.cc',
   'xapian/myhtmlparse.cc'
 ]
+kiwix_sources += lib_resources
 
 if xapian_dep.found()
   kiwix_sources += ['xapianIndexer.cpp', 'xapianSearcher.cpp']

src/searcher.cpp

@@ -18,6 +18,7 @@
  */
 
 #include "searcher.h"
+#include "kiwixlib-resources.h"
 
 
 namespace kiwix {
@@ -32,7 +33,7 @@ namespace kiwix {
     resultStart(0),
     resultEnd(0)
   {
-    template_ct2 = getResourceAsString("results.ct2");
+    template_ct2 = RESOURCE::results_ct2;
     loadICUExternalTables();
   }
   

static/buildResourceCode.sh

@@ -1,43 +0,0 @@
-#!/bin/sh
-SCRIPT_DIR=$(dirname $0)
-RESOURCE_FILE=$SCRIPT_DIR/../src/common/resourceTools.h
-MAP="static std::map<std::string, std::pair<const unsigned char*, unsigned int> > createResourceMap() { \n"
-MAP=$MAP"\tstd::map<std::string, std::pair<const unsigned char*, unsigned int> > m; \n"
-
-# Delete old version of the file
-rm -f "$RESOURCE_FILE"
-
-# Create header of resourceTools.h
-cat << EOF > "$RESOURCE_FILE"
-#ifndef KIWIX_RESOURCETOOLS_H
-#define KIWIX_RESOURCETOOLS_H
-
-#include <string>
-#include <string.h>
-#include <map>
-
-std::string getResourceAsString(const std::string &name);
-
-EOF
-
-# Goes through all files in /static/
-for FILE in `find . -type f | sed 's/\.\///' | grep -v .svn | grep -v Makefile | grep -v .sh | grep -v "~" | grep -v "#"`
-do
-    FILE_ID=`echo "$FILE" | sed "s/\//_/g" | sed "s/\./_/g" | sed "s/\-/_/g"`
-    echo "Inserting $FILE... [$FILE_ID]"
-    reswrap -s -x -oa $RESOURCE_FILE -r $FILE_ID $FILE
-    MAP=$MAP"\tm[\""$FILE"\"] = std::pair <const unsigned char*, unsigned int>("$FILE_ID", sizeof "$FILE_ID"); \n";
-done;
-MAP=$MAP"\treturn m; \n";
-MAP=$MAP"} \n\n"
-MAP=$MAP"static std::map<std::string, std::pair<const unsigned char*, unsigned int> > resourceMap = createResourceMap(); \n\n"
-
-# Create the map table
-# map<int, int> m = map_list_of (1,2) (3,4) (5,6) (7,8);
-echo $MAP >> "$RESOURCE_FILE"
-
-# Create the footer
-cat << EOF >> "$RESOURCE_FILE"
-#endif
-
-EOF

static/meson.build

@@ -0,0 +1,7 @@
+
+
+lib_resources = custom_target('resources',
+  input: 'resources_list.txt',
+  output: ['kiwixlib-resources.cpp', 'kiwixlib-resources.h'],
+  command:[res_compiler, '--cxxfile', '@OUTPUT0@', '--hfile', '@OUTPUT1@', '@INPUT@']
+)

static/resources_list.txt

@@ -0,0 +1,4 @@
+results.ct2
+stopwords/en
+stopwords/he
+stopwords/fra

static/results.tmpl

@@ -1,127 +0,0 @@
-<!DOCTYPE html>
-<html xmlns="http://www.w3.org/1999/xhtml">
-  <head>
-    <meta content="text/html; charset=utf-8" http-equiv="content-type" />
-    <style type="text/css">
-      body{
-      color: #00000;
-      font: small/normal Arial,Helvetica,Sans-Serif;
-      margin-top: 0.5em;
-      font-size: 90%;
-      }
-
-      a{
-      color: #04c;
-      }
-
-      a:visited {
-      color: #639
-      }
-
-      a:hover {
-      text-decoration: underline
-      }
-
-      .header {
-      font-size: 120%;
-      }
-
-      ul {
-      margin:0;
-      padding:0
-      }
-
-      .results {
-      font-size: 110%;
-      }
-
-      .results li {
-      list-style-type:none;
-      margin-top: 0.5em;
-      }
-
-      .results a {
-      font-size: 110%;
-      text-decoration: underline
-      }
-
-      cite {
-      font-style:normal;
-      word-wrap:break-word;
-      display: block;
-      font-size: 100%;
-      }
-
-      .informations {
-      color: #388222;
-      font-size: 100%;
-      }
-
-      .footer {
-      padding: 0;
-      margin-top: 1em;
-      width: 100%;
-      float: left
-      }
-
-      .footer a, .footer span {
-      display: block;
-      padding: .3em .7em;
-      margin: 0 .38em 0 0;
-      text-align:center;
-      text-decoration: none;
-      }
-
-      .footer a:hover {
-      background: #ededed;
-      }
-
-      .footer ul, .footer li {
-      list-style:none;
-      margin: 0;
-      padding: 0;
-      }
-
-      .footer li {
-      float: left;
-      }
-
-      .selected {
-      background: #ededed;
-      }
-
-    </style>
-    <title>Search: <TMPL_var searchPattern></title>
-</head>
-<body bgcolor="white">
-    <div class="header">
-        <TMPL_if results>Results <b><TMPL_var resultStart>-<TMPL_var resultEnd></b> of <b><TMPL_var count></b> for <b><TMPL_var searchPattern></b><TMPL_else>No result were found for <b><TMPL_var searchPattern></b></TMPL_if>
-    </div>
-
-    <div class="results">
-        <ul>
-        <TMPL_foreach results as result>
-            <li><a href="<TMPL_var protocolPrefix><TMPL_var contentId>/<TMPL_var result.url>"><TMPL_var result.title></a>
-            <cite><TMPL_if result.snippet><TMPL_var result.snippet>...</TMPL_if></cite>
-            <TMPL_if wordCount><div class="informations"><TMPL_var wordCount> words</div></TMPL_if>
-            </li>
-        </TMPL_foreach>
-        </ul>
-    </div>
-
-    <div class="footer">
-        <ul>
-            <TMPL_if (resultLastPageStart>0)>
-            <li><a href="<TMPL_var searchProtocolPrefix>content=<TMPL_var contentId>&pattern=<TMPL_var searchPatternEncoded>&start=0&end=<TMPL_var resultRange>">◀</a></li>
-            </TMPL_if>
-            <TMPL_foreach pages as page>
-            <li><a <TMPL_if page.selected>class="selected"</TMPL_if> href="<TMPL_var searchProtocolPrefix>content=<TMPL_var contentId>&pattern=<TMPL_var searchPatternEncoded>&start=<TMPL_var page.start>&end=<TMPL_var page.end>"><TMPL_var page.label></a></li>
-            </TMPL_foreach>
-            <TMPL_if (resultLastPageStart>0)>
-            <li><a href="<TMPL_var searchProtocolPrefix>content=<TMPL_var contentId>&pattern=<TMPL_var searchPatternEncoded>&start=<TMPL_var resultLastPageStart>&end=<TMPL_var (resultLastPageStart+resultRange)>">▶</a></li>
-            </TMPL_if>
-        </ul>
-     </div>
-
-</body>
-</html>

static/server/home.html.tmpl

@@ -1,22 +0,0 @@
-<!DOCTYPE html>
-<html xmlns="http://www.w3.org/1999/xhtml">
-<head>
-  <meta charset="UTF-8" />
-  <title>Welcome to Kiwix Server</title>
-  <script type="text/javascript" src="/skin/jquery-ui/external/jquery/jquery.js"></script>
-  <script type="text/javascript" src="/skin/jquery-ui/jquery-ui.min.js"></script>
-  <link type="text/css" href="/skin/jquery-ui/jquery-ui.min.css" rel="Stylesheet" />
-  <link type="text/css" href="/skin/jquery-ui/jquery-ui.theme.min.css" rel="Stylesheet" />
-<script>
-$(function() {
-$( "#accordion" ).accordion();
-});
-</script>
-</head>
-<body class="kiwix">
-
-<div id="accordion" class="kiwix">
-  __BOOKS__
-</div>
-</body>
-</html>

static/server/include.html.part

@@ -1,24 +0,0 @@
-<script type="text/javascript" src="/skin/jquery-ui/external/jquery/jquery.js"></script>
-<script type="text/javascript" src="/skin/jquery-ui/jquery-ui.min.js"></script>
-<script>
-  var jk = jQuery.noConflict();
-  jk(function() {
-  jk( "#kiwixsearchbox" ).autocomplete({
-
-  source: "/suggest?content=__CONTENT__",
-  dataType: "json",
-  cache: false,
-
-  select: function(event, ui) {
-  jk( "#kiwixsearchbox" ).val(ui.item.value);
-  jk( "#kiwixsearchform" ).submit();
-  },
-
-  });
-  });
-
-  /* cybook hack */
-  if (navigator.userAgent.indexOf("bookeen/cybook") != -1) {
-  	jk("html").addClass("cybook");
-  }
-</script>

static/server/taskbar.css

@@ -1,50 +0,0 @@
-#kiwixtoolbar {
-    position: fixed;
-    padding: .5em;
-    left:0;
-    right:0;
-    top: 0;
-    z-index:100;
-    max-height: 2.8em;
-}
-
-#kiwixsearchbox {
-    width: 20em;
-}
-
-.kiwixsearch {
-	float: right;
-}s
-
-/* Try to fix buggy stuff in jquery-ui autocomplete */
-#ui-id-1, .ui-autocomplete {
-    background: white !important;
-    border: solid 1px grey !important;
-}
-
-li.ui-state-focus {
-    font-weight: bold;
-}
-
-/* Specific CSS for Bookeen Cybook device (800x600 B&W ereader) */
-.cybook #kiwixtoolbar button, .cybook #kiwixtoolbar input {
-	font-size: 1.5em;
-}
-.cybook #kiwixsearchbox {
-	width: 7em;
-}
-.cybook a {
-	text-decoration: underline;
-}
-
-@media only screen and (min--moz-device-pixel-ratio: 1.5),
-only screen and (-o-min-device-pixel-ratio: 1.5/1),
-only screen and (-webkit-min-device-pixel-ratio: 1.5),
-only screen and (min-device-pixel-ratio: 1.5) {
-    #kiwixtoolbar button, #kiwixtoolbar input {
-    	font-size: 2em;
-    }
-    #kiwixsearchbox {
-		width: 7em;
-	}
-}

static/server/taskbar.html.part

@@ -1,15 +0,0 @@
-<link type="text/css" href="/skin/jquery-ui/jquery-ui.min.css" rel="Stylesheet" />
-<link type="text/css" href="/skin/jquery-ui/jquery-ui.theme.min.css" rel="Stylesheet" />
-<span class="kiwix">
-  <span id="kiwixtoolbar" class="ui-widget-header">
-  	<a href="/"><button>Library</button></a>
-  	<a href="/__CONTENT__/"><button>Home</button></a>
-  	<a href="/random?content=__CONTENT__"><button>Random</button></a>
-  	<form class="kiwixsearch" method="GET" action="/search" id="kiwixsearchform">
-  	<input type="hidden" name="content" value="__CONTENT__" />
-  	<input autocomplete="off" class="ui-autocomplete-input" id="kiwixsearchbox" name="pattern" type="text">
-  	<input type="submit" value="Search">
-  	</form>
-  </span>
-</span>
-<div style="display: block; height: 3em;"></div>