mirror of https://github.com/kiwix/libkiwix.git
Merge pull request #833 from kiwix/http_caching
This commit is contained in:
Matthieu Gautier
GitHub
commit
0e20f50443
|
|
@ -332,8 +332,8 @@ class Library
|
|||
/**
|
||||
* Return the current revision of the library.
|
||||
*
|
||||
* The revision of the library is updated (incremented by one) only by
|
||||
* the addBook() operation.
|
||||
* The revision of the library is updated (incremented by one) by
|
||||
* the addBook() and removeBookById() operations.
|
||||
*
|
||||
* @return Current revision of the library.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -52,15 +52,21 @@ resource_getter_template = """
|
|||
return RESOURCE::{identifier};
|
||||
"""
|
||||
|
||||
resource_cacheid_getter_template = """
|
||||
if (name == "{common_name}")
|
||||
return "{cacheid}";
|
||||
"""
|
||||
|
||||
resource_decl_template = """{namespaces_open}
|
||||
extern const std::string {identifier};
|
||||
{namespaces_close}"""
|
||||
|
||||
class Resource:
|
||||
def __init__(self, base_dirs, filename):
|
||||
filename = filename.strip()
|
||||
def __init__(self, base_dirs, filename, cacheid=None):
|
||||
filename = filename
|
||||
self.filename = filename
|
||||
self.identifier = full_identifier(filename)
|
||||
self.cacheid = cacheid
|
||||
found = False
|
||||
for base_dir in base_dirs:
|
||||
try:
|
||||
|
|
@ -71,7 +77,7 @@ class Resource:
|
|||
except FileNotFoundError:
|
||||
continue
|
||||
if not found:
|
||||
raise Exception("Impossible to found {}".format(filename))
|
||||
raise Exception("Resource not found: {}".format(filename))
|
||||
|
||||
def dump_impl(self):
|
||||
nb_row = len(self.data)//16 + (1 if len(self.data) % 16 else 0)
|
||||
|
|
@ -93,6 +99,12 @@ class Resource:
|
|||
identifier="::".join(self.identifier)
|
||||
)
|
||||
|
||||
def dump_cacheid_getter(self):
|
||||
return resource_cacheid_getter_template.format(
|
||||
common_name=self.filename,
|
||||
cacheid=self.cacheid
|
||||
)
|
||||
|
||||
def dump_decl(self):
|
||||
return resource_decl_template.format(
|
||||
namespaces_open=" ".join("namespace {} {{".format(id) for id in self.identifier[:-1]),
|
||||
|
|
@ -123,7 +135,12 @@ static std::string init_resource(const char* name, const unsigned char* content,
|
|||
|
||||
const std::string& getResource_{basename}(const std::string& name) {{
|
||||
{RESOURCES_GETTER}
|
||||
throw ResourceNotFound("Resource not found.");
|
||||
throw ResourceNotFound("Resource not found: " + name);
|
||||
}}
|
||||
|
||||
const char* getResourceCacheId_{basename}(const std::string& name) {{
|
||||
{RESOURCE_CACHEID_GETTER}
|
||||
return nullptr;
|
||||
}}
|
||||
|
||||
{RESOURCES}
|
||||
|
|
@ -134,6 +151,7 @@ def gen_c_file(resources, basename):
|
|||
return master_c_template.format(
|
||||
RESOURCES="\n\n".join(r.dump_impl() for r in resources),
|
||||
RESOURCES_GETTER="\n\n".join(r.dump_getter() for r in resources),
|
||||
RESOURCE_CACHEID_GETTER="\n\n".join(r.dump_cacheid_getter() for r in resources if r.cacheid is not None),
|
||||
include_file=basename,
|
||||
basename=to_identifier(basename)
|
||||
)
|
||||
|
|
@ -159,8 +177,10 @@ class ResourceNotFound : public std::runtime_error {{
|
|||
}};
|
||||
|
||||
const std::string& getResource_{basename}(const std::string& name);
|
||||
const char* getResourceCacheId_{basename}(const std::string& name);
|
||||
|
||||
#define getResource(a) (getResource_{basename}(a))
|
||||
#define getResourceCacheId(a) (getResourceCacheId_{basename}(a))
|
||||
|
||||
#endif // KIWIX_{BASENAME}
|
||||
|
||||
|
|
@ -189,8 +209,8 @@ if __name__ == "__main__":
|
|||
base_dir = os.path.dirname(os.path.realpath(args.resource_file))
|
||||
source_dir = args.source_dir or []
|
||||
with open(args.resource_file, 'r') as f:
|
||||
resources = [Resource([base_dir]+source_dir, filename)
|
||||
for filename in f.readlines()]
|
||||
resources = [Resource([base_dir]+source_dir, *line.strip().split())
|
||||
for line in f.readlines()]
|
||||
|
||||
h_identifier = to_identifier(os.path.basename(args.hfile))
|
||||
with open(args.hfile, 'w') as f:
|
||||
|
|
|
|||
|
|
@ -99,16 +99,24 @@ def preprocess_resource(resource_path):
|
|||
print(preprocessed_content, end='', file=target)
|
||||
|
||||
|
||||
def copy_file(src_path, dst_path):
|
||||
with open(src_path, 'rb') as src:
|
||||
with open(dst_path, 'wb') as dst:
|
||||
dst.write(src.read())
|
||||
def copy_resource_list_file(src_path, dst_path):
|
||||
with open(src_path, 'r') as src:
|
||||
with open(dst_path, 'w') as dst:
|
||||
for line in src:
|
||||
res = line.strip()
|
||||
if line.startswith("skin/") and res in resource_revisions:
|
||||
dst.write(res + " " + resource_revisions[res] + "\n")
|
||||
else:
|
||||
dst.write(line)
|
||||
|
||||
def preprocess_resources(resource_file_path):
|
||||
resource_filename = os.path.basename(resource_file_path)
|
||||
for resource in read_resource_file(resource_file_path):
|
||||
if resource.startswith('skin/'):
|
||||
get_resource_revision(resource)
|
||||
else:
|
||||
preprocess_resource(resource)
|
||||
copy_file(resource_file_path, os.path.join(OUT_DIR, resource_filename))
|
||||
copy_resource_list_file(resource_file_path, os.path.join(OUT_DIR, resource_filename))
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
|
|
|
|||
|
|
@ -221,7 +221,11 @@ bool Library::removeBookById(const std::string& id)
|
|||
// Having a too big cache is not a problem here (or it would have been before)
|
||||
// (And setMaxSize doesn't actually reduce the cache size, extra cached items
|
||||
// will be removed in put or getOrPut).
|
||||
return mp_impl->m_books.erase(id) == 1;
|
||||
const bool bookWasRemoved = mp_impl->m_books.erase(id) == 1;
|
||||
if ( bookWasRemoved ) {
|
||||
++mp_impl->m_revision;
|
||||
}
|
||||
return bookWasRemoved;
|
||||
}
|
||||
|
||||
Library::Revision Library::getRevision() const
|
||||
|
|
|
|||
|
|
@ -37,11 +37,11 @@ namespace {
|
|||
// into the ETag for ETag::Option opt.
|
||||
// IMPORTANT: The characters in all_options must come in sorted order (so that
|
||||
// IMPORTANT: isValidOptionsString() works correctly).
|
||||
const char all_options[] = "cz";
|
||||
const char all_options[] = "Zz";
|
||||
|
||||
static_assert(ETag::OPTION_COUNT == sizeof(all_options) - 1, "");
|
||||
|
||||
bool isValidServerId(const std::string& s)
|
||||
bool isValidETagBody(const std::string& s)
|
||||
{
|
||||
return !s.empty() && s.find_first_of("\"/") == std::string::npos;
|
||||
}
|
||||
|
|
@ -83,17 +83,17 @@ bool ETag::get_option(Option opt) const
|
|||
|
||||
std::string ETag::get_etag() const
|
||||
{
|
||||
if ( m_serverId.empty() )
|
||||
if ( m_body.empty() )
|
||||
return std::string();
|
||||
|
||||
return "\"" + m_serverId + "/" + m_options + "\"";
|
||||
return "\"" + m_body + "/" + m_options + "\"";
|
||||
}
|
||||
|
||||
ETag::ETag(const std::string& serverId, const std::string& options)
|
||||
ETag::ETag(const std::string& body, const std::string& options)
|
||||
{
|
||||
if ( isValidServerId(serverId) && isValidOptionsString(options) )
|
||||
if ( isValidETagBody(body) && isValidOptionsString(options) )
|
||||
{
|
||||
m_serverId = serverId;
|
||||
m_body = body;
|
||||
m_options = options;
|
||||
}
|
||||
}
|
||||
|
|
@ -115,7 +115,7 @@ ETag ETag::parse(std::string s)
|
|||
return ETag(s.substr(0, i), s.substr(i+1));
|
||||
}
|
||||
|
||||
ETag ETag::match(const std::string& etags, const std::string& server_id)
|
||||
ETag ETag::match(const std::string& etags, const std::string& body)
|
||||
{
|
||||
std::istringstream ss(etags);
|
||||
std::string etag_str;
|
||||
|
|
@ -125,7 +125,7 @@ ETag ETag::match(const std::string& etags, const std::string& server_id)
|
|||
etag_str.pop_back();
|
||||
|
||||
const ETag etag = parse(etag_str);
|
||||
if ( etag && etag.m_serverId == server_id )
|
||||
if ( etag && etag.m_body == body )
|
||||
return etag;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -28,10 +28,11 @@ namespace kiwix {
|
|||
// The ETag string used by Kiwix server (more precisely, its value inside the
|
||||
// double quotes) consists of two parts:
|
||||
//
|
||||
// 1. ServerId - The string obtained on server start up
|
||||
// 1. Body - A string uniquely identifying the object or state from which
|
||||
// the resource has been obtained.
|
||||
//
|
||||
// 2. Options - Zero or more characters encoding the values of some of the
|
||||
// headers of the response
|
||||
// 2. Options - Zero or more characters encoding the type of the ETag and/or
|
||||
// the values of some of the headers of the response
|
||||
//
|
||||
// The two parts are separated with a slash (/) symbol (which is always present,
|
||||
// even when the the options part is empty). Neither portion of a Kiwix ETag
|
||||
|
|
@ -40,7 +41,7 @@ namespace kiwix {
|
|||
//
|
||||
// "abcdefghijklmn/"
|
||||
// "1234567890/z"
|
||||
// "1234567890/cz"
|
||||
// "6f1d19d0-633f-087b-fb55-7ac324ff9baf/Zz"
|
||||
//
|
||||
// The options part of the Kiwix ETag allows to correctly set the required
|
||||
// headers when responding to a conditional If-None-Match request with a 304
|
||||
|
|
@ -51,7 +52,7 @@ class ETag
|
|||
{
|
||||
public: // types
|
||||
enum Option {
|
||||
CACHEABLE_ENTITY,
|
||||
ZIM_CONTENT,
|
||||
COMPRESSED_CONTENT,
|
||||
OPTION_COUNT
|
||||
};
|
||||
|
|
@ -59,10 +60,10 @@ class ETag
|
|||
public: // functions
|
||||
ETag() {}
|
||||
|
||||
void set_server_id(const std::string& id) { m_serverId = id; }
|
||||
void set_body(const std::string& s) { m_body = s; }
|
||||
void set_option(Option opt);
|
||||
|
||||
explicit operator bool() const { return !m_serverId.empty(); }
|
||||
explicit operator bool() const { return !m_body.empty(); }
|
||||
|
||||
bool get_option(Option opt) const;
|
||||
std::string get_etag() const;
|
||||
|
|
@ -76,7 +77,7 @@ class ETag
|
|||
static ETag parse(std::string s);
|
||||
|
||||
private: // data
|
||||
std::string m_serverId;
|
||||
std::string m_body;
|
||||
std::string m_options;
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -218,6 +218,24 @@ struct CustomizedResourceData
|
|||
std::string resourceFilePath;
|
||||
};
|
||||
|
||||
bool responseMustBeETaggedWithLibraryId(const Response& response, const RequestContext& request)
|
||||
{
|
||||
return response.getReturnCode() == MHD_HTTP_OK
|
||||
&& response.get_kind() == Response::DYNAMIC_CONTENT
|
||||
&& request.get_url() != "/random";
|
||||
}
|
||||
|
||||
ETag
|
||||
get_matching_if_none_match_etag(const RequestContext& r, const std::string& etagBody)
|
||||
{
|
||||
try {
|
||||
const std::string etag_list = r.get_header(MHD_HTTP_HEADER_IF_NONE_MATCH);
|
||||
return ETag::match(etag_list, etagBody);
|
||||
} catch (const std::out_of_range&) {
|
||||
return ETag();
|
||||
}
|
||||
}
|
||||
|
||||
} // unnamed namespace
|
||||
|
||||
std::pair<std::string, Library::BookIdSet> InternalServer::selectBooks(const RequestContext& request) const
|
||||
|
|
@ -443,7 +461,6 @@ bool InternalServer::start() {
|
|||
}
|
||||
auto server_start_time = std::chrono::system_clock::now().time_since_epoch();
|
||||
m_server_id = kiwix::to_string(server_start_time.count());
|
||||
m_library_id = m_server_id;
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
@ -511,8 +528,9 @@ MHD_Result InternalServer::handlerCallback(struct MHD_Connection* connection,
|
|||
}
|
||||
}
|
||||
|
||||
if (response->getReturnCode() == MHD_HTTP_OK && !etag_not_needed(request))
|
||||
response->set_server_id(m_server_id);
|
||||
if ( responseMustBeETaggedWithLibraryId(*response, request) ) {
|
||||
response->set_etag_body(getLibraryId());
|
||||
}
|
||||
|
||||
auto ret = response->send(request, connection);
|
||||
auto end_time = std::chrono::steady_clock::now();
|
||||
|
|
@ -534,6 +552,11 @@ bool isEndpointUrl(const std::string& url, const std::string& endpoint)
|
|||
|
||||
} // unnamed namespace
|
||||
|
||||
std::string InternalServer::getLibraryId() const
|
||||
{
|
||||
return m_server_id + "." + kiwix::to_string(mp_library->getRevision());
|
||||
}
|
||||
|
||||
std::unique_ptr<Response> InternalServer::handle_request(const RequestContext& request)
|
||||
{
|
||||
try {
|
||||
|
|
@ -542,7 +565,7 @@ std::unique_ptr<Response> InternalServer::handle_request(const RequestContext& r
|
|||
+ urlNotFoundMsg;
|
||||
}
|
||||
|
||||
const ETag etag = get_matching_if_none_match_etag(request);
|
||||
const ETag etag = get_matching_if_none_match_etag(request, getLibraryId());
|
||||
if ( etag )
|
||||
return Response::build_304(*this, etag);
|
||||
|
||||
|
|
@ -603,27 +626,6 @@ MustacheData InternalServer::get_default_data() const
|
|||
return data;
|
||||
}
|
||||
|
||||
bool InternalServer::etag_not_needed(const RequestContext& request) const
|
||||
{
|
||||
const std::string url = request.get_url();
|
||||
return kiwix::startsWith(url, "/catalog")
|
||||
|| url == "/search"
|
||||
|| url == "/suggest"
|
||||
|| url == "/random"
|
||||
|| url == "/catch/external";
|
||||
}
|
||||
|
||||
ETag
|
||||
InternalServer::get_matching_if_none_match_etag(const RequestContext& r) const
|
||||
{
|
||||
try {
|
||||
const std::string etag_list = r.get_header(MHD_HTTP_HEADER_IF_NONE_MATCH);
|
||||
return ETag::match(etag_list, m_server_id);
|
||||
} catch (const std::out_of_range&) {
|
||||
return ETag();
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_ptr<Response> InternalServer::build_homepage(const RequestContext& request)
|
||||
{
|
||||
return ContentResponse::build(*this, m_indexTemplateString, get_default_data(), "text/html; charset=utf-8");
|
||||
|
|
@ -746,6 +748,25 @@ std::unique_ptr<Response> InternalServer::handle_viewer_settings(const RequestCo
|
|||
return ContentResponse::build(*this, RESOURCE::templates::viewer_settings_js, data, "application/javascript; charset=utf-8");
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
Response::Kind staticResourceAccessType(const RequestContext& req, const char* expectedCacheid)
|
||||
{
|
||||
if ( expectedCacheid == nullptr )
|
||||
return Response::DYNAMIC_CONTENT;
|
||||
|
||||
try {
|
||||
if ( expectedCacheid != req.get_argument("cacheid") )
|
||||
throw ResourceNotFound("Wrong cacheid");
|
||||
return Response::STATIC_RESOURCE;
|
||||
} catch( const std::out_of_range& ) {
|
||||
return Response::DYNAMIC_CONTENT;
|
||||
}
|
||||
}
|
||||
|
||||
} // unnamed namespace
|
||||
|
||||
std::unique_ptr<Response> InternalServer::handle_skin(const RequestContext& request)
|
||||
{
|
||||
if (m_verbose.load()) {
|
||||
|
|
@ -756,12 +777,16 @@ std::unique_ptr<Response> InternalServer::handle_skin(const RequestContext& requ
|
|||
auto resourceName = isRequestForViewer
|
||||
? "viewer.html"
|
||||
: request.get_url().substr(1);
|
||||
|
||||
const char* const resourceCacheId = getResourceCacheId(resourceName);
|
||||
|
||||
try {
|
||||
const auto accessType = staticResourceAccessType(request, resourceCacheId);
|
||||
auto response = ContentResponse::build(
|
||||
*this,
|
||||
getResource(resourceName),
|
||||
getMimeTypeForFile(resourceName));
|
||||
response->set_cacheable();
|
||||
response->set_kind(accessType);
|
||||
return std::move(response);
|
||||
} catch (const ResourceNotFound& e) {
|
||||
return HTTP404Response(*this, request)
|
||||
|
|
@ -969,7 +994,7 @@ std::unique_ptr<Response> InternalServer::handle_catalog(const RequestContext& r
|
|||
zim::Uuid uuid;
|
||||
kiwix::OPDSDumper opdsDumper(mp_library);
|
||||
opdsDumper.setRootLocation(m_root);
|
||||
opdsDumper.setLibraryId(m_library_id);
|
||||
opdsDumper.setLibraryId(getLibraryId());
|
||||
std::vector<std::string> bookIdsToDump;
|
||||
if (url == "root.xml") {
|
||||
uuid = zim::Uuid::generate(host);
|
||||
|
|
@ -1052,6 +1077,11 @@ std::unique_ptr<Response> InternalServer::handle_content(const RequestContext& r
|
|||
+ suggestSearchMsg(searchURL, kiwix::urlDecode(pattern));
|
||||
}
|
||||
|
||||
const std::string archiveUuid(archive->getUuid());
|
||||
const ETag etag = get_matching_if_none_match_etag(request, archiveUuid);
|
||||
if ( etag )
|
||||
return Response::build_304(*this, etag);
|
||||
|
||||
auto urlStr = url.substr(prefixLength + bookName.size());
|
||||
if (urlStr[0] == '/') {
|
||||
urlStr = urlStr.substr(1);
|
||||
|
|
@ -1070,6 +1100,7 @@ std::unique_ptr<Response> InternalServer::handle_content(const RequestContext& r
|
|||
return build_redirect(bookName, getFinalItem(*archive, entry));
|
||||
}
|
||||
auto response = ItemResponse::build(*this, request, entry.getItem());
|
||||
response->set_etag_body(archiveUuid);
|
||||
|
||||
if (m_verbose.load()) {
|
||||
printf("Found %s\n", entry.getPath().c_str());
|
||||
|
|
@ -1123,6 +1154,11 @@ std::unique_ptr<Response> InternalServer::handle_raw(const RequestContext& reque
|
|||
+ noSuchBookErrorMsg(bookName);
|
||||
}
|
||||
|
||||
const std::string archiveUuid(archive->getUuid());
|
||||
const ETag etag = get_matching_if_none_match_etag(request, archiveUuid);
|
||||
if ( etag )
|
||||
return Response::build_304(*this, etag);
|
||||
|
||||
// Remove the beggining of the path:
|
||||
// /raw/<bookName>/<kind>/foo
|
||||
// ^^^^^ ^ ^
|
||||
|
|
@ -1132,13 +1168,17 @@ std::unique_ptr<Response> InternalServer::handle_raw(const RequestContext& reque
|
|||
try {
|
||||
if (kind == "meta") {
|
||||
auto item = archive->getMetadataItem(itemPath);
|
||||
return ItemResponse::build(*this, request, item);
|
||||
auto response = ItemResponse::build(*this, request, item);
|
||||
response->set_etag_body(archiveUuid);
|
||||
return response;
|
||||
} else {
|
||||
auto entry = archive->getEntryByPath(itemPath);
|
||||
if (entry.isRedirect()) {
|
||||
return build_redirect(bookName, entry.getItem(true));
|
||||
}
|
||||
return ItemResponse::build(*this, request, entry.getItem());
|
||||
auto response = ItemResponse::build(*this, request, entry.getItem());
|
||||
response->set_etag_body(archiveUuid);
|
||||
return response;
|
||||
}
|
||||
} catch (zim::EntryNotFound& e ) {
|
||||
if (m_verbose.load()) {
|
||||
|
|
|
|||
|
|
@ -147,13 +147,13 @@ class InternalServer {
|
|||
|
||||
MustacheData get_default_data() const;
|
||||
|
||||
bool etag_not_needed(const RequestContext& r) const;
|
||||
ETag get_matching_if_none_match_etag(const RequestContext& request) const;
|
||||
std::pair<std::string, Library::BookIdSet> selectBooks(const RequestContext& r) const;
|
||||
SearchInfo getSearchInfo(const RequestContext& r) const;
|
||||
|
||||
bool isLocallyCustomizedResource(const std::string& url) const;
|
||||
|
||||
std::string getLibraryId() const;
|
||||
|
||||
private: // types
|
||||
class LockableSuggestionSearcher;
|
||||
typedef ConcurrentCache<SearchInfo, std::shared_ptr<zim::Search>> SearchCache;
|
||||
|
|
@ -180,7 +180,6 @@ class InternalServer {
|
|||
SuggestionSearcherCache suggestionSearcherCache;
|
||||
|
||||
std::string m_server_id;
|
||||
std::string m_library_id;
|
||||
|
||||
class CustomizedResources;
|
||||
std::unique_ptr<CustomizedResources> m_customizedResources;
|
||||
|
|
|
|||
|
|
@ -77,17 +77,18 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2(const RequestContext
|
|||
|
||||
std::unique_ptr<Response> InternalServer::handle_catalog_v2_root(const RequestContext& request)
|
||||
{
|
||||
const std::string libraryId = getLibraryId();
|
||||
return ContentResponse::build(
|
||||
*this,
|
||||
RESOURCE::templates::catalog_v2_root_xml,
|
||||
kainjow::mustache::object{
|
||||
{"date", gen_date_str()},
|
||||
{"endpoint_root", m_root + "/catalog/v2"},
|
||||
{"feed_id", gen_uuid(m_library_id)},
|
||||
{"all_entries_feed_id", gen_uuid(m_library_id + "/entries")},
|
||||
{"partial_entries_feed_id", gen_uuid(m_library_id + "/partial_entries")},
|
||||
{"category_list_feed_id", gen_uuid(m_library_id + "/categories")},
|
||||
{"language_list_feed_id", gen_uuid(m_library_id + "/languages")}
|
||||
{"feed_id", gen_uuid(libraryId)},
|
||||
{"all_entries_feed_id", gen_uuid(libraryId + "/entries")},
|
||||
{"partial_entries_feed_id", gen_uuid(libraryId + "/partial_entries")},
|
||||
{"category_list_feed_id", gen_uuid(libraryId + "/categories")},
|
||||
{"language_list_feed_id", gen_uuid(libraryId + "/languages")}
|
||||
},
|
||||
"application/atom+xml;profile=opds-catalog;kind=navigation"
|
||||
);
|
||||
|
|
@ -97,7 +98,7 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_entries(const Reques
|
|||
{
|
||||
OPDSDumper opdsDumper(mp_library);
|
||||
opdsDumper.setRootLocation(m_root);
|
||||
opdsDumper.setLibraryId(m_library_id);
|
||||
opdsDumper.setLibraryId(getLibraryId());
|
||||
const auto bookIds = search_catalog(request, opdsDumper);
|
||||
const auto opdsFeed = opdsDumper.dumpOPDSFeedV2(bookIds, request.get_query(), partial);
|
||||
return ContentResponse::build(
|
||||
|
|
@ -118,7 +119,7 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_complete_entry(const
|
|||
|
||||
OPDSDumper opdsDumper(mp_library);
|
||||
opdsDumper.setRootLocation(m_root);
|
||||
opdsDumper.setLibraryId(m_library_id);
|
||||
opdsDumper.setLibraryId(getLibraryId());
|
||||
const auto opdsFeed = opdsDumper.dumpOPDSCompleteEntry(entryId);
|
||||
return ContentResponse::build(
|
||||
*this,
|
||||
|
|
@ -131,7 +132,7 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_categories(const Req
|
|||
{
|
||||
OPDSDumper opdsDumper(mp_library);
|
||||
opdsDumper.setRootLocation(m_root);
|
||||
opdsDumper.setLibraryId(m_library_id);
|
||||
opdsDumper.setLibraryId(getLibraryId());
|
||||
return ContentResponse::build(
|
||||
*this,
|
||||
opdsDumper.categoriesOPDSFeed(),
|
||||
|
|
@ -143,7 +144,7 @@ std::unique_ptr<Response> InternalServer::handle_catalog_v2_languages(const Requ
|
|||
{
|
||||
OPDSDumper opdsDumper(mp_library);
|
||||
opdsDumper.setRootLocation(m_root);
|
||||
opdsDumper.setLibraryId(m_library_id);
|
||||
opdsDumper.setLibraryId(getLibraryId());
|
||||
return ContentResponse::build(
|
||||
*this,
|
||||
opdsDumper.languagesOPDSFeed(),
|
||||
|
|
|
|||
|
|
@ -102,6 +102,14 @@ bool compress(std::string &content) {
|
|||
}
|
||||
|
||||
|
||||
const char* getCacheControlHeader(Response::Kind k)
|
||||
{
|
||||
switch(k) {
|
||||
case Response::STATIC_RESOURCE: return "max-age=31536000, immutable";
|
||||
case Response::ZIM_CONTENT: return "max-age=3600, must-revalidate";
|
||||
default: return "max-age=0, must-revalidate";
|
||||
}
|
||||
}
|
||||
|
||||
} // unnamed namespace
|
||||
|
||||
|
|
@ -112,6 +120,13 @@ Response::Response(bool verbose)
|
|||
add_header(MHD_HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN, "*");
|
||||
}
|
||||
|
||||
void Response::set_kind(Kind k)
|
||||
{
|
||||
m_kind = k;
|
||||
if ( k == ZIM_CONTENT )
|
||||
m_etag.set_option(ETag::ZIM_CONTENT);
|
||||
}
|
||||
|
||||
std::unique_ptr<Response> Response::build(const InternalServer& server)
|
||||
{
|
||||
return std::unique_ptr<Response>(new Response(server.m_verbose.load()));
|
||||
|
|
@ -122,6 +137,9 @@ std::unique_ptr<Response> Response::build_304(const InternalServer& server, cons
|
|||
auto response = Response::build(server);
|
||||
response->set_code(MHD_HTTP_NOT_MODIFIED);
|
||||
response->m_etag = etag;
|
||||
if ( etag.get_option(ETag::ZIM_CONTENT) ) {
|
||||
response->set_kind(Response::ZIM_CONTENT);
|
||||
}
|
||||
if ( etag.get_option(ETag::COMPRESSED_CONTENT) ) {
|
||||
response->add_header(MHD_HTTP_HEADER_VARY, "Accept-Encoding");
|
||||
}
|
||||
|
|
@ -355,7 +373,7 @@ MHD_Result Response::send(const RequestContext& request, MHD_Connection* connect
|
|||
MHD_Response* response = create_mhd_response(request);
|
||||
|
||||
MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL,
|
||||
m_etag.get_option(ETag::CACHEABLE_ENTITY) ? "max-age=2723040, public" : "no-cache, no-store, must-revalidate");
|
||||
getCacheControlHeader(m_kind));
|
||||
const std::string etag = m_etag.get_etag();
|
||||
if ( ! etag.empty() )
|
||||
MHD_add_response_header(response, MHD_HTTP_HEADER_ETAG, etag.c_str());
|
||||
|
|
@ -411,7 +429,7 @@ ItemResponse::ItemResponse(bool verbose, const zim::Item& item, const std::strin
|
|||
m_mimeType(mimetype)
|
||||
{
|
||||
m_byteRange = byterange;
|
||||
set_cacheable();
|
||||
set_kind(Response::ZIM_CONTENT);
|
||||
add_header(MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType);
|
||||
}
|
||||
|
||||
|
|
@ -423,14 +441,14 @@ std::unique_ptr<Response> ItemResponse::build(const InternalServer& server, cons
|
|||
if (noRange && is_compressible_mime_type(mimetype)) {
|
||||
// Return a contentResponse
|
||||
auto response = ContentResponse::build(server, item.getData(), mimetype);
|
||||
response->set_cacheable();
|
||||
response->set_kind(Response::ZIM_CONTENT);
|
||||
response->m_byteRange = byteRange;
|
||||
return std::move(response);
|
||||
}
|
||||
|
||||
if (byteRange.kind() == ByteRange::RESOLVED_UNSATISFIABLE) {
|
||||
auto response = Response::build_416(server, item.getSize());
|
||||
response->set_cacheable();
|
||||
response->set_kind(Response::ZIM_CONTENT);
|
||||
return response;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -45,6 +45,14 @@ class InternalServer;
|
|||
class RequestContext;
|
||||
|
||||
class Response {
|
||||
public:
|
||||
enum Kind
|
||||
{
|
||||
STATIC_RESOURCE,
|
||||
ZIM_CONTENT,
|
||||
DYNAMIC_CONTENT
|
||||
};
|
||||
|
||||
public:
|
||||
Response(bool verbose);
|
||||
virtual ~Response() = default;
|
||||
|
|
@ -57,8 +65,9 @@ class Response {
|
|||
MHD_Result send(const RequestContext& request, MHD_Connection* connection);
|
||||
|
||||
void set_code(int code) { m_returnCode = code; }
|
||||
void set_cacheable() { m_etag.set_option(ETag::CACHEABLE_ENTITY); }
|
||||
void set_server_id(const std::string& id) { m_etag.set_server_id(id); }
|
||||
void set_kind(Kind k);
|
||||
Kind get_kind() const { return m_kind; }
|
||||
void set_etag_body(const std::string& id) { m_etag.set_body(id); }
|
||||
void add_header(const std::string& name, const std::string& value) { m_customHeaders[name] = value; }
|
||||
|
||||
int getReturnCode() const { return m_returnCode; }
|
||||
|
|
@ -68,6 +77,7 @@ class Response {
|
|||
MHD_Response* create_error_response(const RequestContext& request) const;
|
||||
|
||||
protected: // data
|
||||
Kind m_kind = DYNAMIC_CONTENT;
|
||||
bool m_verbose;
|
||||
int m_returnCode;
|
||||
ByteRange m_byteRange;
|
||||
|
|
|
|||
|
|
@ -36,7 +36,6 @@ opensearchdescription.xml
|
|||
ft_opensearchdescription.xml
|
||||
catalog_v2_searchdescription.xml
|
||||
skin/css/autoComplete.css
|
||||
skin/css/images/search.svg
|
||||
skin/favicon/android-chrome-192x192.png
|
||||
skin/favicon/android-chrome-512x512.png
|
||||
skin/favicon/apple-touch-icon.png
|
||||
|
|
|
|||
|
|
@ -1,8 +0,0 @@
|
|||
<svg xmlns="http://www.w3.org/2000/svg" focusable="false" x="0px" y="0px" width="30" height="30" viewBox="0 0 171 171" style=" fill:#000000;">
|
||||
<g fill="none" fill-rule="nonzero" stroke="none" stroke-width="1" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="10" stroke-dasharray="" stroke-dashoffset="0" font-family="none" font-weight="none" font-size="none" text-anchor="none" style="mix-blend-mode: normal">
|
||||
<path d="M0,171.99609v-171.99609h171.99609v171.99609z" fill="none"></path>
|
||||
<g fill="#ff7a7a">
|
||||
<path d="M74.1,17.1c-31.41272,0 -57,25.58728 -57,57c0,31.41272 25.58728,57 57,57c13.6601,0 26.20509,-4.85078 36.03692,-12.90293l34.03301,34.03301c1.42965,1.48907 3.55262,2.08891 5.55014,1.56818c1.99752,-0.52073 3.55746,-2.08067 4.07819,-4.07819c0.52073,-1.99752 -0.0791,-4.12049 -1.56818,-5.55014l-34.03301,-34.03301c8.05215,-9.83182 12.90293,-22.37682 12.90293,-36.03692c0,-31.41272 -25.58728,-57 -57,-57zM74.1,28.5c25.2517,0 45.6,20.3483 45.6,45.6c0,25.2517 -20.3483,45.6 -45.6,45.6c-25.2517,0 -45.6,-20.3483 -45.6,-45.6c0,-25.2517 20.3483,-45.6 45.6,-45.6z"></path>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 1.1 KiB |
|
|
@ -105,7 +105,7 @@ body {
|
|||
border-radius: 10px;
|
||||
border: solid 1px #b5b2b2;
|
||||
padding: 10px;
|
||||
background-image: url('./search-icon.svg');
|
||||
background-image: url('../skin/search-icon.svg?KIWIXCACHEID');
|
||||
background-repeat: no-repeat;
|
||||
background-position: right center;
|
||||
background-origin: content-box;
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
<link rel="apple-touch-icon" sizes="180x180" href="{{root}}/skin/favicon/apple-touch-icon.png?KIWIXCACHEID">
|
||||
<link rel="icon" type="image/png" sizes="32x32" href="{{root}}/skin/favicon/favicon-32x32.png?KIWIXCACHEID">
|
||||
<link rel="icon" type="image/png" sizes="16x16" href="{{root}}/skin/favicon/favicon-16x16.png?KIWIXCACHEID">
|
||||
<link rel="manifest" href="{{root}}/skin/favicon/site.webmanifest">
|
||||
<link rel="manifest" href="{{root}}/skin/favicon/site.webmanifest?KIWIXCACHEID">
|
||||
<link rel="mask-icon" href="{{root}}/skin/favicon/safari-pinned-tab.svg?KIWIXCACHEID" color="#5bbad5">
|
||||
<link rel="shortcut icon" href="{{root}}/skin/favicon/favicon.ico?KIWIXCACHEID">
|
||||
<meta name="msapplication-TileColor" content="#da532c">
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@
|
|||
}
|
||||
|
||||
const root = getRootLocation();
|
||||
const blankPageUrl = `${root}/skin/blank.html`;
|
||||
const blankPageUrl = root + "/skin/blank.html?KIWIXCACHEID";
|
||||
|
||||
if ( location.hash == '' ) {
|
||||
location.href = root + '/';
|
||||
|
|
@ -58,7 +58,7 @@
|
|||
<iframe id="content_iframe"
|
||||
referrerpolicy="same-origin"
|
||||
onload="on_content_load()"
|
||||
src="skin/blank.html" title="ZIM content" width="100%"
|
||||
src="./skin/blank.html?KIWIXCACHEID" title="ZIM content" width="100%"
|
||||
style="border:0px">
|
||||
</iframe>
|
||||
|
||||
|
|
|
|||
|
|
@ -801,8 +801,14 @@ TEST_F(LibraryTest, removeBooksNotUpdatedSince)
|
|||
lib.addBook(lib.getBookByIdThreadSafe(id));
|
||||
}
|
||||
|
||||
EXPECT_GT(lib.getRevision(), rev);
|
||||
|
||||
const uint64_t rev2 = lib.getRevision();
|
||||
|
||||
EXPECT_EQ(9u, lib.removeBooksNotUpdatedSince(rev));
|
||||
|
||||
EXPECT_GT(lib.getRevision(), rev2);
|
||||
|
||||
EXPECT_FILTER_RESULTS(kiwix::Filter(),
|
||||
"Islam Stack Exchange",
|
||||
"Movies & TV Stack Exchange",
|
||||
|
|
|
|||
232
test/server.cpp
232
test/server.cpp
|
|
@ -23,13 +23,19 @@ T1 concat(T1 a, const T2& b)
|
|||
return a;
|
||||
}
|
||||
|
||||
const bool WITH_ETAG = true;
|
||||
const bool NO_ETAG = false;
|
||||
enum ResourceKind
|
||||
{
|
||||
ZIM_CONTENT,
|
||||
STATIC_CONTENT,
|
||||
DYNAMIC_CONTENT,
|
||||
};
|
||||
|
||||
struct Resource
|
||||
{
|
||||
bool etag_expected;
|
||||
ResourceKind kind;
|
||||
const char* url;
|
||||
|
||||
bool etag_expected() const { return kind != STATIC_CONTENT; }
|
||||
};
|
||||
|
||||
std::ostream& operator<<(std::ostream& out, const Resource& r)
|
||||
|
|
@ -41,55 +47,127 @@ std::ostream& operator<<(std::ostream& out, const Resource& r)
|
|||
typedef std::vector<Resource> ResourceCollection;
|
||||
|
||||
const ResourceCollection resources200Compressible{
|
||||
{ WITH_ETAG, "/ROOT/" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/" },
|
||||
|
||||
{ WITH_ETAG, "/ROOT/skin/autoComplete.min.js" },
|
||||
{ WITH_ETAG, "/ROOT/skin/css/autoComplete.css" },
|
||||
{ WITH_ETAG, "/ROOT/skin/taskbar.css" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/viewer" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/viewer?cacheid=whatever" },
|
||||
|
||||
{ NO_ETAG, "/ROOT/catalog/search" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/skin/autoComplete.min.js" },
|
||||
{ STATIC_CONTENT, "/ROOT/skin/autoComplete.min.js?cacheid=1191aaaf" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/skin/css/autoComplete.css" },
|
||||
{ STATIC_CONTENT, "/ROOT/skin/css/autoComplete.css?cacheid=08951e06" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/skin/favicon/favicon.ico" },
|
||||
{ STATIC_CONTENT, "/ROOT/skin/favicon/favicon.ico?cacheid=fba03a27" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/skin/index.css" },
|
||||
{ STATIC_CONTENT, "/ROOT/skin/index.css?cacheid=0f9ba34e" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/skin/index.js" },
|
||||
{ STATIC_CONTENT, "/ROOT/skin/index.js?cacheid=2f5a81ac" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/skin/iso6391To3.js" },
|
||||
{ STATIC_CONTENT, "/ROOT/skin/iso6391To3.js?cacheid=ecde2bb3" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/skin/isotope.pkgd.min.js" },
|
||||
{ STATIC_CONTENT, "/ROOT/skin/isotope.pkgd.min.js?cacheid=2e48d392" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/skin/taskbar.css" },
|
||||
{ STATIC_CONTENT, "/ROOT/skin/taskbar.css?cacheid=216d6b5d" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/skin/viewer.js" },
|
||||
{ STATIC_CONTENT, "/ROOT/skin/viewer.js?cacheid=51e745c2" },
|
||||
|
||||
{ NO_ETAG, "/ROOT/search?content=zimfile&pattern=a" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/catalog/search" },
|
||||
|
||||
{ NO_ETAG, "/ROOT/suggest?content=zimfile&term=ray" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/catalog/v2/root.xml" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/catalog/v2/languages" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/catalog/v2/entries" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/catalog/v2/partial_entries" },
|
||||
|
||||
{ WITH_ETAG, "/ROOT/content/zimfile/A/index" },
|
||||
{ WITH_ETAG, "/ROOT/content/zimfile/A/Ray_Charles" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/search?content=zimfile&pattern=a" },
|
||||
|
||||
{ WITH_ETAG, "/ROOT/raw/zimfile/content/A/index" },
|
||||
{ WITH_ETAG, "/ROOT/raw/zimfile/content/A/Ray_Charles" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/suggest?content=zimfile&term=ray" },
|
||||
|
||||
{ ZIM_CONTENT, "/ROOT/content/zimfile/A/index" },
|
||||
{ ZIM_CONTENT, "/ROOT/content/zimfile/A/Ray_Charles" },
|
||||
|
||||
{ ZIM_CONTENT, "/ROOT/raw/zimfile/content/A/index" },
|
||||
{ ZIM_CONTENT, "/ROOT/raw/zimfile/content/A/Ray_Charles" },
|
||||
};
|
||||
|
||||
const ResourceCollection resources200Uncompressible{
|
||||
{ WITH_ETAG, "/ROOT/skin/caret.png" },
|
||||
{ WITH_ETAG, "/ROOT/skin/css/images/search.svg" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/skin/bittorrent.png" },
|
||||
{ STATIC_CONTENT, "/ROOT/skin/bittorrent.png?cacheid=4f5c6882" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/skin/blank.html" },
|
||||
{ STATIC_CONTENT, "/ROOT/skin/blank.html?cacheid=6b1fa032" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/skin/caret.png" },
|
||||
{ STATIC_CONTENT, "/ROOT/skin/caret.png?cacheid=22b942b4" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/skin/download.png" },
|
||||
{ STATIC_CONTENT, "/ROOT/skin/download.png?cacheid=a39aa502" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/skin/favicon/android-chrome-192x192.png" },
|
||||
{ STATIC_CONTENT, "/ROOT/skin/favicon/android-chrome-192x192.png?cacheid=bfac158b" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/skin/favicon/android-chrome-512x512.png" },
|
||||
{ STATIC_CONTENT, "/ROOT/skin/favicon/android-chrome-512x512.png?cacheid=380c3653" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/skin/favicon/apple-touch-icon.png" },
|
||||
{ STATIC_CONTENT, "/ROOT/skin/favicon/apple-touch-icon.png?cacheid=f86f8df3" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/skin/favicon/browserconfig.xml" },
|
||||
{ STATIC_CONTENT, "/ROOT/skin/favicon/browserconfig.xml?cacheid=f29a7c4a" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/skin/favicon/favicon-16x16.png" },
|
||||
{ STATIC_CONTENT, "/ROOT/skin/favicon/favicon-16x16.png?cacheid=a986fedc" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/skin/favicon/favicon-32x32.png" },
|
||||
{ STATIC_CONTENT, "/ROOT/skin/favicon/favicon-32x32.png?cacheid=79ded625" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-144x144.png" },
|
||||
{ STATIC_CONTENT, "/ROOT/skin/favicon/mstile-144x144.png?cacheid=c25a7641" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-150x150.png" },
|
||||
{ STATIC_CONTENT, "/ROOT/skin/favicon/mstile-150x150.png?cacheid=6fa6f467" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-310x150.png" },
|
||||
{ STATIC_CONTENT, "/ROOT/skin/favicon/mstile-310x150.png?cacheid=e0ed9032" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-310x310.png" },
|
||||
{ STATIC_CONTENT, "/ROOT/skin/favicon/mstile-310x310.png?cacheid=26b20530" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/skin/favicon/mstile-70x70.png" },
|
||||
{ STATIC_CONTENT, "/ROOT/skin/favicon/mstile-70x70.png?cacheid=64ffd9dc" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/skin/favicon/safari-pinned-tab.svg" },
|
||||
{ STATIC_CONTENT, "/ROOT/skin/favicon/safari-pinned-tab.svg?cacheid=8d487e95" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/skin/favicon/site.webmanifest" },
|
||||
{ STATIC_CONTENT, "/ROOT/skin/favicon/site.webmanifest?cacheid=bc396efb" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/skin/fonts/Poppins.ttf" },
|
||||
{ STATIC_CONTENT, "/ROOT/skin/fonts/Poppins.ttf?cacheid=af705837" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/skin/fonts/Roboto.ttf" },
|
||||
{ STATIC_CONTENT, "/ROOT/skin/fonts/Roboto.ttf?cacheid=84d10248" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/skin/hash.png" },
|
||||
{ STATIC_CONTENT, "/ROOT/skin/hash.png?cacheid=f836e872" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/skin/magnet.png" },
|
||||
{ STATIC_CONTENT, "/ROOT/skin/magnet.png?cacheid=73b6bddf" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/skin/search-icon.svg" },
|
||||
{ STATIC_CONTENT, "/ROOT/skin/search-icon.svg?cacheid=b10ae7ed" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/skin/search_results.css" },
|
||||
{ STATIC_CONTENT, "/ROOT/skin/search_results.css?cacheid=76d39c84" },
|
||||
|
||||
{ WITH_ETAG, "/ROOT/raw/zimfile/meta/Title" },
|
||||
{ WITH_ETAG, "/ROOT/raw/zimfile/meta/Description" },
|
||||
{ WITH_ETAG, "/ROOT/raw/zimfile/meta/Language" },
|
||||
{ WITH_ETAG, "/ROOT/raw/zimfile/meta/Name" },
|
||||
{ WITH_ETAG, "/ROOT/raw/zimfile/meta/Tags" },
|
||||
{ WITH_ETAG, "/ROOT/raw/zimfile/meta/Date" },
|
||||
{ WITH_ETAG, "/ROOT/raw/zimfile/meta/Creator" },
|
||||
{ WITH_ETAG, "/ROOT/raw/zimfile/meta/Publisher" },
|
||||
{ ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Title" },
|
||||
{ ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Description" },
|
||||
{ ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Language" },
|
||||
{ ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Name" },
|
||||
{ ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Tags" },
|
||||
{ ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Date" },
|
||||
{ ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Creator" },
|
||||
{ ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Publisher" },
|
||||
|
||||
{ NO_ETAG, "/ROOT/catalog/v2/illustration/6f1d19d0-633f-087b-fb55-7ac324ff9baf?size=48" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/catalog/root.xml" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/catalog/searchdescription.xml" },
|
||||
|
||||
{ NO_ETAG, "/ROOT/catch/external?source=www.example.com" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/catalog/v2/categories" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/catalog/v2/searchdescription.xml" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/catalog/v2/illustration/6f1d19d0-633f-087b-fb55-7ac324ff9baf?size=48" },
|
||||
|
||||
{ WITH_ETAG, "/ROOT/content/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/catch/external?source=www.example.com" },
|
||||
|
||||
{ WITH_ETAG, "/ROOT/content/corner_cases/A/empty.html" },
|
||||
{ WITH_ETAG, "/ROOT/content/corner_cases/-/empty.css" },
|
||||
{ WITH_ETAG, "/ROOT/content/corner_cases/-/empty.js" },
|
||||
{ ZIM_CONTENT, "/ROOT/content/zimfile/I/m/Ray_Charles_classic_piano_pose.jpg" },
|
||||
|
||||
{ ZIM_CONTENT, "/ROOT/content/corner_cases/A/empty.html" },
|
||||
{ ZIM_CONTENT, "/ROOT/content/corner_cases/-/empty.css" },
|
||||
{ ZIM_CONTENT, "/ROOT/content/corner_cases/-/empty.js" },
|
||||
|
||||
|
||||
// The following url's responses are too small to be compressed
|
||||
{ NO_ETAG, "/ROOT/catalog/root.xml" },
|
||||
{ NO_ETAG, "/ROOT/catalog/searchdescription.xml" },
|
||||
{ NO_ETAG, "/ROOT/suggest?content=zimfile" },
|
||||
{ WITH_ETAG, "/ROOT/raw/zimfile/meta/Creator" },
|
||||
{ WITH_ETAG, "/ROOT/raw/zimfile/meta/Title" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/catalog/root.xml" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/catalog/searchdescription.xml" },
|
||||
{ DYNAMIC_CONTENT, "/ROOT/suggest?content=zimfile" },
|
||||
{ ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Creator" },
|
||||
{ ZIM_CONTENT, "/ROOT/raw/zimfile/meta/Title" },
|
||||
};
|
||||
|
||||
ResourceCollection all200Resources()
|
||||
|
|
@ -172,11 +250,11 @@ TEST_F(ServerTest, CacheIdsOfStaticResources)
|
|||
const std::vector<UrlAndExpectedResult> testData{
|
||||
{
|
||||
/* url */ "/ROOT/",
|
||||
R"EXPECTEDRESULT( href="/ROOT/skin/index.css?cacheid=3b470cee"
|
||||
R"EXPECTEDRESULT( href="/ROOT/skin/index.css?cacheid=0f9ba34e"
|
||||
<link rel="apple-touch-icon" sizes="180x180" href="/ROOT/skin/favicon/apple-touch-icon.png?cacheid=f86f8df3">
|
||||
<link rel="icon" type="image/png" sizes="32x32" href="/ROOT/skin/favicon/favicon-32x32.png?cacheid=79ded625">
|
||||
<link rel="icon" type="image/png" sizes="16x16" href="/ROOT/skin/favicon/favicon-16x16.png?cacheid=a986fedc">
|
||||
<link rel="manifest" href="/ROOT/skin/favicon/site.webmanifest">
|
||||
<link rel="manifest" href="/ROOT/skin/favicon/site.webmanifest?cacheid=bc396efb">
|
||||
<link rel="mask-icon" href="/ROOT/skin/favicon/safari-pinned-tab.svg?cacheid=8d487e95" color="#5bbad5">
|
||||
<link rel="shortcut icon" href="/ROOT/skin/favicon/favicon.ico?cacheid=fba03a27">
|
||||
<meta name="msapplication-config" content="/ROOT/skin/favicon/browserconfig.xml?cacheid=f29a7c4a">
|
||||
|
|
@ -185,6 +263,11 @@ R"EXPECTEDRESULT( href="/ROOT/skin/index.css?cacheid=3b470cee"
|
|||
<script src="/ROOT/skin/isotope.pkgd.min.js?cacheid=2e48d392" defer></script>
|
||||
<script src="/ROOT/skin/iso6391To3.js?cacheid=ecde2bb3"></script>
|
||||
<script type="text/javascript" src="/ROOT/skin/index.js?cacheid=2f5a81ac" defer></script>
|
||||
)EXPECTEDRESULT"
|
||||
},
|
||||
{
|
||||
/* url */ "/ROOT/skin/index.css",
|
||||
R"EXPECTEDRESULT( background-image: url('../skin/search-icon.svg?cacheid=b10ae7ed');
|
||||
)EXPECTEDRESULT"
|
||||
},
|
||||
{
|
||||
|
|
@ -201,8 +284,9 @@ R"EXPECTEDRESULT( <link type="text/css" href="./skin/taskbar.css?cacheid=216d
|
|||
<link type="text/css" href="./skin/css/autoComplete.css?cacheid=08951e06" rel="Stylesheet" />
|
||||
<script type="text/javascript" src="./skin/viewer.js?cacheid=51e745c2" defer></script>
|
||||
<script type="text/javascript" src="./skin/autoComplete.min.js?cacheid=1191aaaf"></script>
|
||||
const blankPageUrl = `${root}/skin/blank.html`;
|
||||
const blankPageUrl = root + "/skin/blank.html?cacheid=6b1fa032";
|
||||
<label for="kiwix_button_show_toggle"><img src="./skin/caret.png?cacheid=22b942b4" alt=""></label>
|
||||
src="./skin/blank.html?cacheid=6b1fa032" title="ZIM content" width="100%"
|
||||
)EXPECTEDRESULT"
|
||||
},
|
||||
{
|
||||
|
|
@ -252,6 +336,7 @@ const char* urls404[] = {
|
|||
"/",
|
||||
"/zimfile",
|
||||
"/ROOT/skin/non-existent-skin-resource",
|
||||
"/ROOT/skin/autoComplete.min.js?cacheid=wrongcacheid",
|
||||
"/ROOT/catalog",
|
||||
"/ROOT/catalog/",
|
||||
"/ROOT/catalog/non-existent-item",
|
||||
|
|
@ -310,6 +395,11 @@ std::string getHeaderValue(const Headers& headers, const std::string& name)
|
|||
return er.first->second;
|
||||
}
|
||||
|
||||
std::string getCacheControlHeader(const httplib::Response& r)
|
||||
{
|
||||
return getHeaderValue(r.headers, "Cache-Control");
|
||||
}
|
||||
|
||||
TEST_F(CustomizedServerTest, NewResourcesCanBeAdded)
|
||||
{
|
||||
// ServerTest.404 verifies that "/ROOT/non-existent-item" doesn't exist
|
||||
|
|
@ -952,6 +1042,8 @@ TEST_F(ServerTest, RandomPageRedirectsToAnExistingArticle)
|
|||
ASSERT_EQ(302, g->status);
|
||||
ASSERT_TRUE(g->has_header("Location"));
|
||||
ASSERT_TRUE(kiwix::startsWith(g->get_header_value("Location"), "/ROOT/content/zimfile/A/"));
|
||||
ASSERT_EQ(getCacheControlHeader(*g), "max-age=0, must-revalidate");
|
||||
ASSERT_FALSE(g->has_header("ETag"));
|
||||
}
|
||||
|
||||
TEST_F(ServerTest, NonEndpointUrlsAreRedirectedToContentUrls)
|
||||
|
|
@ -995,6 +1087,8 @@ TEST_F(ServerTest, NonEndpointUrlsAreRedirectedToContentUrls)
|
|||
ASSERT_EQ(302, g->status) << ctx;
|
||||
ASSERT_TRUE(g->has_header("Location")) << ctx;
|
||||
ASSERT_EQ("/ROOT/content" + p, g->get_header_value("Location")) << ctx;
|
||||
ASSERT_EQ(getCacheControlHeader(*g), "max-age=0, must-revalidate");
|
||||
ASSERT_FALSE(g->has_header("ETag"));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1059,12 +1153,45 @@ TEST_F(ServerTest, HeadersAreTheSameInResponsesToHeadAndGetRequests)
|
|||
}
|
||||
}
|
||||
|
||||
TEST_F(ServerTest, CacheControlOfZimContent)
|
||||
{
|
||||
for ( const Resource& res : all200Resources() ) {
|
||||
if ( res.kind == ZIM_CONTENT ) {
|
||||
const auto g = zfs1_->GET(res.url);
|
||||
EXPECT_EQ(getCacheControlHeader(*g), "max-age=3600, must-revalidate") << res;
|
||||
EXPECT_TRUE(g->has_header("ETag")) << res;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ServerTest, CacheControlOfStaticContent)
|
||||
{
|
||||
for ( const Resource& res : all200Resources() ) {
|
||||
if ( res.kind == STATIC_CONTENT ) {
|
||||
const auto g = zfs1_->GET(res.url);
|
||||
EXPECT_EQ(getCacheControlHeader(*g), "max-age=31536000, immutable") << res;
|
||||
EXPECT_FALSE(g->has_header("ETag")) << res;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ServerTest, CacheControlOfDynamicContent)
|
||||
{
|
||||
for ( const Resource& res : all200Resources() ) {
|
||||
if ( res.kind == DYNAMIC_CONTENT ) {
|
||||
const auto g = zfs1_->GET(res.url);
|
||||
EXPECT_EQ(getCacheControlHeader(*g), "max-age=0, must-revalidate") << res;
|
||||
EXPECT_TRUE(g->has_header("ETag")) << res;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ServerTest, ETagHeaderIsSetAsNeeded)
|
||||
{
|
||||
for ( const Resource& res : all200Resources() ) {
|
||||
const auto responseToGet = zfs1_->GET(res.url);
|
||||
EXPECT_EQ(res.etag_expected, responseToGet->has_header("ETag")) << res;
|
||||
if ( res.etag_expected ) {
|
||||
EXPECT_EQ(res.etag_expected(), responseToGet->has_header("ETag")) << res;
|
||||
if ( res.etag_expected() ) {
|
||||
EXPECT_TRUE(is_valid_etag(responseToGet->get_header_value("ETag")));
|
||||
}
|
||||
}
|
||||
|
|
@ -1088,21 +1215,32 @@ TEST_F(ServerTest, ETagIsTheSameAcrossHeadAndGet)
|
|||
}
|
||||
}
|
||||
|
||||
TEST_F(ServerTest, DifferentServerInstancesProduceDifferentETags)
|
||||
TEST_F(ServerTest, DifferentServerInstancesProduceDifferentETagsForDynamicContent)
|
||||
{
|
||||
ZimFileServer zfs2(SERVER_PORT + 1, ZimFileServer::DEFAULT_OPTIONS, ZIMFILES);
|
||||
for ( const Resource& res : all200Resources() ) {
|
||||
if ( !res.etag_expected ) continue;
|
||||
if ( res.kind != DYNAMIC_CONTENT ) continue;
|
||||
const auto h1 = zfs1_->HEAD(res.url);
|
||||
const auto h2 = zfs2.HEAD(res.url);
|
||||
EXPECT_NE(h1->get_header_value("ETag"), h2->get_header_value("ETag"));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ServerTest, DifferentServerInstancesProduceIdenticalETagsForZimContent)
|
||||
{
|
||||
ZimFileServer zfs2(SERVER_PORT + 1, ZimFileServer::DEFAULT_OPTIONS, ZIMFILES);
|
||||
for ( const Resource& res : all200Resources() ) {
|
||||
if ( res.kind != ZIM_CONTENT ) continue;
|
||||
const auto h1 = zfs1_->HEAD(res.url);
|
||||
const auto h2 = zfs2.HEAD(res.url);
|
||||
EXPECT_EQ(h1->get_header_value("ETag"), h2->get_header_value("ETag"));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(ServerTest, CompressionInfluencesETag)
|
||||
{
|
||||
for ( const Resource& res : resources200Compressible ) {
|
||||
if ( ! res.etag_expected ) continue;
|
||||
if ( ! res.etag_expected() ) continue;
|
||||
const auto g1 = zfs1_->GET(res.url);
|
||||
const auto g2 = zfs1_->GET(res.url, { {"Accept-Encoding", ""} } );
|
||||
const auto g3 = zfs1_->GET(res.url, { {"Accept-Encoding", "gzip"} } );
|
||||
|
|
@ -1115,7 +1253,7 @@ TEST_F(ServerTest, CompressionInfluencesETag)
|
|||
TEST_F(ServerTest, ETagOfUncompressibleContentIsNotAffectedByAcceptEncoding)
|
||||
{
|
||||
for ( const Resource& res : resources200Uncompressible ) {
|
||||
if ( ! res.etag_expected ) continue;
|
||||
if ( ! res.etag_expected() ) continue;
|
||||
const auto g1 = zfs1_->GET(res.url);
|
||||
const auto g2 = zfs1_->GET(res.url, { {"Accept-Encoding", ""} } );
|
||||
const auto g3 = zfs1_->GET(res.url, { {"Accept-Encoding", "gzip"} } );
|
||||
|
|
@ -1160,7 +1298,7 @@ TEST_F(ServerTest, IfNoneMatchRequestsWithMatchingETagResultIn304Responses)
|
|||
const char* const encodings[] = { "", "gzip" };
|
||||
for ( const Resource& res : all200Resources() ) {
|
||||
for ( const char* enc: encodings ) {
|
||||
if ( ! res.etag_expected ) continue;
|
||||
if ( ! res.etag_expected() ) continue;
|
||||
const TestContext ctx{ {"url", res.url}, {"encoding", enc} };
|
||||
|
||||
const auto g = zfs1_->GET(res.url, { {"Accept-Encoding", enc} });
|
||||
|
|
@ -1187,8 +1325,8 @@ TEST_F(ServerTest, IfNoneMatchRequestsWithMismatchingETagResultIn200Responses)
|
|||
const auto etag2 = etag.substr(0, etag.size() - 1) + "x\"";
|
||||
const auto h = zfs1_->HEAD(res.url, { {"If-None-Match", etag2} } );
|
||||
const auto g2 = zfs1_->GET(res.url, { {"If-None-Match", etag2} } );
|
||||
EXPECT_EQ(200, h->status);
|
||||
EXPECT_EQ(200, g2->status);
|
||||
EXPECT_EQ(200, h->status) << res;
|
||||
EXPECT_EQ(200, g2->status) << res;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue