smallsolar
/
libkiwix
mirror of https://github.com/kiwix/libkiwix.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Http integration (#246) 

Http integration
This commit is contained in:
Matthieu Gautier 2019-08-11 11:54:09 +02:00 committed by GitHub
parent 1b8e7849bd 7060afae66
commit 1e94665c07
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
75 changed files with 12941 additions and 294 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/book.h
View File

@ -45,7 +45,7 @@ class Book
void update(const Reader& reader);
void updateFromXml(const pugi::xml_node& node, const std::string& baseDir);
void updateFromOpds(const pugi::xml_node& node, const std::string& urlHost);
std::string getHumanReadableIdFromPath();
std::string getHumanReadableIdFromPath() const;
bool readOnly() const { return m_readOnly; }
const std::string& getId() const { return m_id; }

2
include/entry.h
View File

@ -25,6 +25,8 @@
#include <exception>
#include <string>
#include "common.h"
using namespace std;
namespace kiwix

3
include/library.h
View File

@ -23,6 +23,7 @@
#include <string>
#include <vector>
#include <map>
#include <memory>
#include "book.h"
#include "bookmark.h"
@ -110,6 +111,7 @@ class Filter {
class Library
{
std::map<std::string, kiwix::Book> m_books;
std::map<std::string, std::shared_ptr<Reader>> m_readers;
std::vector<kiwix::Bookmark> m_bookmarks;
public:
@ -145,6 +147,7 @@ class Library
bool removeBookmark(const std::string& zimId, const std::string& url);
Book& getBookById(const std::string& id);
std::shared_ptr<Reader> getReaderById(const std::string& id);
/**
* Remove a book from the library.

6
include/meson.build
View File

@ -10,7 +10,10 @@ headers = [
'reader.h',
'entry.h',
'searcher.h',
'kiwixserve.h'
'search_renderer.h',
'server.h',
'kiwixserve.h',
'name_mapper.h'
]
install_headers(headers, subdir:'kiwix')
@ -22,6 +25,7 @@ install_headers(
'tools/pathTools.h',
'tools/regexTools.h',
'tools/stringTools.h',
'tools/lock.h',
subdir:'kiwix/tools'
)

61
include/name_mapper.h Normal file
View File

@ -0,0 +1,61 @@
/*
* Copyright 2019 Matthieu Gautier <mgautier@kymeria.fr>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#ifndef KIWIX_NAMEMAPPER_H
#define KIWIX_NAMEMAPPER_H
#include <string>
#include <map>
namespace kiwix
{
class Library;
class NameMapper {
public:
virtual ~NameMapper() = default;
virtual std::string getNameForId(const std::string& id) = 0;
virtual std::string getIdForName(const std::string& name) = 0;
};
class IdNameMapper : public NameMapper {
public:
virtual std::string getNameForId(const std::string& id) { return id; };
virtual std::string getIdForName(const std::string& name) { return name; };
};
class HumanReadableNameMapper : public NameMapper {
private:
std::map<std::string, std::string> m_idToName;
std::map<std::string, std::string> m_nameToId;
public:
HumanReadableNameMapper(kiwix::Library& library, bool withAlias);
virtual ~HumanReadableNameMapper() = default;
virtual std::string getNameForId(const std::string& id);
virtual std::string getIdForName(const std::string& name);
};
}
#endif

85
include/search_renderer.h Normal file
View File

@ -0,0 +1,85 @@
/*
* Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#ifndef KIWIX_SEARCH_RENDERER_H
#define KIWIX_SEARCH_RENDERER_H
#include <string>
namespace kiwix
{
class Searcher;
class NameMapper;
/**
* The SearcherRenderer class is used to render a search result to a html page.
*/
class SearchRenderer
{
public:
/**
* The default constructor.
*
* @param humanReadableName The global zim's humanReadableName.
* Used to generate pagination links.
*/
SearchRenderer(Searcher* searcher, NameMapper* mapper);
~SearchRenderer();
void setSearchPattern(const std::string& pattern);
/**
* Set the search content id.
*/
void setSearchContent(const std::string& name);
/**
* Set protocol prefix.
*/
void setProtocolPrefix(const std::string& prefix);
/**
* Set search protocol prefix.
*/
void setSearchProtocolPrefix(const std::string& prefix);
/**
* Generate the html page with the resutls of the search.
*/
std::string getHtml();
protected:
std::string beautifyInteger(const unsigned int number);
Searcher* mp_searcher;
NameMapper* mp_nameMapper;
std::string searchContent;
std::string searchPattern;
std::string protocolPrefix;
std::string searchProtocolPrefix;
unsigned int resultCountPerPage;
unsigned int estimatedResultCount;
unsigned int resultStart;
unsigned int resultEnd;
};
}
#endif

41
include/searcher.h
View File

@ -56,24 +56,14 @@ struct SearcherInternal;
/**
* The Searcher class is reponsible to do different kind of search using the
* fulltext index.
*
* Searcher may (if compiled with ctpp2) be used to
* generate a html page for the search result. This use a template that need a
* humanReaderName. This feature is only used by kiwix-serve and this should be
* move outside of Searcher (and with a better API). If you don't use the html
* rendering (getHtml method), you better should simply ignore the different
* humanReadeableName attributes (or give an empty string).
*/
class Searcher
{
public:
/**
* The default constructor.
*
* @param humanReadableName The global zim's humanReadableName.
* Used to generate pagination links.
*/
Searcher(const string& humanReadableName = "");
Searcher();
~Searcher();
@ -81,11 +71,13 @@ class Searcher
* Add a reader (containing embedded fulltext index) to the search.
*
* @param reader The Reader for the zim containing the fulltext index.
* @param humanReaderName The human readable name of the reader.
* @return true if the reader has been added.
* false if the reader cannot be added (no embedded fulltext index present)
*/
bool add_reader(Reader* reader, const std::string& humanReaderName);
bool add_reader(Reader* reader);
Reader* get_reader(int index);
/**
* Start a search on the zim associated to the Searcher.
@ -151,22 +143,8 @@ class Searcher
*/
unsigned int getEstimatedResultCount();
/**
* Set protocol prefix.
* Only used by getHtml.
*/
bool setProtocolPrefix(const std::string prefix);
/**
* Set search protocol prefix.
* Only used by getHtml.
*/
bool setSearchProtocolPrefix(const std::string prefix);
/**
* Generate the html page with the resutls of the search.
*/
string getHtml();
unsigned int getResultStart() { return resultStart; }
unsigned int getResultEnd() { return resultEnd; }
protected:
std::string beautifyInteger(const unsigned int number);
@ -177,16 +155,11 @@ class Searcher
const bool verbose = false);
std::vector<Reader*> readers;
std::vector<std::string> humanReaderNames;
SearcherInternal* internal;
std::string searchPattern;
std::string protocolPrefix;
std::string searchProtocolPrefix;
unsigned int resultCountPerPage;
unsigned int estimatedResultCount;
unsigned int resultStart;
unsigned int resultEnd;
std::string contentHumanReadableId;
private:
void reset();

75
include/server.h Normal file
View File

@ -0,0 +1,75 @@
/*
* Copyright 2019 Matthieu Gautier <mgautier@kymeria.fr>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#ifndef KIWIX_SERVER_H
#define KIWIX_SERVER_H
#include <string>
#include <memory>
namespace kiwix
{
class Library;
class NameMapper;
class InternalServer;
class Server {
public:
/**
* The default constructor.
*
* @param library The library to serve.
*/
Server(Library* library, NameMapper* nameMapper=nullptr);
virtual ~Server();
/**
* Serve the content.
*/
bool start();
/**
* Stop the daemon.
*/
void stop();
void setRoot(const std::string& root);
void setAddress(const std::string& addr) { m_addr = addr; }
void setPort(int port) { m_port = port; }
void setNbThreads(int threads) { m_nbThreads = threads; }
void setVerbose(bool verbose) { m_verbose = verbose; }
void setTaskbar(bool withTaskbar, bool withLibraryButton)
{ m_withTaskbar = withTaskbar; m_withLibraryButton = withLibraryButton; }
protected:
Library* mp_library;
NameMapper* mp_nameMapper;
std::string m_root = "";
std::string m_addr = "";
int m_port = 80;
int m_nbThreads = 1;
bool m_verbose = false;
bool m_withTaskbar = true;
bool m_withLibraryButton = true;
std::unique_ptr<InternalServer> mp_server;
};
}
#endif

46
include/tools/lock.h Normal file
View File

@ -0,0 +1,46 @@
#ifndef KIWIXLIB_TOOL_LOCK_H
#define KIWIXLIB_TOOL_LOCK_H
#include <pthread.h>
namespace kiwix {
class Lock
{
public:
explicit Lock(pthread_mutex_t* mutex) :
mp_mutex(mutex)
{
pthread_mutex_lock(mp_mutex);
}
~Lock() {
if (mp_mutex != nullptr) {
pthread_mutex_unlock(mp_mutex);
}
}
Lock(Lock && other) :
mp_mutex(other.mp_mutex)
{
other.mp_mutex = nullptr;
}
Lock & operator=(Lock && other)
{
mp_mutex = other.mp_mutex;
other.mp_mutex = nullptr;
return *this;
}
private:
pthread_mutex_t* mp_mutex;
Lock(Lock const &) = delete;
Lock & operator=(Lock const &) = delete;
};
}
#endif //KIWIXLIB_TOOL_LOCK_H

12
include/tools/otherTools.h
View File

@ -20,18 +20,16 @@
#ifndef KIWIX_OTHERTOOLS_H
#define KIWIX_OTHERTOOLS_H
#ifdef _WIN32
#include <windows.h>
#else
#include <unistd.h>
#endif
#include <string>
#include <pugixml.hpp>
namespace pugi {
class xml_node;
}
namespace kiwix
{
void sleep(unsigned int milliseconds);
std::string nodeToString(pugi::xml_node node);
std::string nodeToString(const pugi::xml_node& node);
std::string converta2toa3(const std::string& a2code);
}

54
include/tools/pathTools.h
View File

@ -20,45 +20,27 @@
#ifndef KIWIX_PATHTOOLS_H
#define KIWIX_PATHTOOLS_H
#include <fcntl.h>
#include <limits.h>
#include <stdio.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <fstream>
#include <ios>
#include <iostream>
#include <sstream>
#include <string>
#include <vector>
#ifdef _WIN32
#include <direct.h>
#endif
#include "stringTools.h"
using namespace std;
bool isRelativePath(const string& path);
string computeAbsolutePath(const string path, const string relativePath);
string computeRelativePath(const string path, const string absolutePath);
string removeLastPathElement(const string path,
bool isRelativePath(const std::string& path);
std::string computeAbsolutePath(const std::string& path, const std::string& relativePath);
std::string computeRelativePath(const std::string& path, const std::string& absolutePath);
std::string removeLastPathElement(const std::string& path,
const bool removePreSeparator = false,
const bool removePostSeparator = false);
string appendToDirectory(const string& directoryPath, const string& filename);
std::string appendToDirectory(const std::string& directoryPath, const std::string& filename);
unsigned int getFileSize(const string& path);
string getFileSizeAsString(const string& path);
string getFileContent(const string& path);
bool fileExists(const string& path);
bool makeDirectory(const string& path);
string makeTmpDirectory();
bool copyFile(const string& sourcePath, const string& destPath);
string getLastPathElement(const string& path);
string getExecutablePath();
string getCurrentDirectory();
string getDataDirectory();
bool writeTextFile(const string& path, const string& content);
unsigned int getFileSize(const std::string& path);
std::string getFileSizeAsString(const std::string& path);
std::string getFileContent(const std::string& path);
bool fileExists(const std::string& path);
bool makeDirectory(const std::string& path);
std::string makeTmpDirectory();
bool copyFile(const std::string& sourcePath, const std::string& destPath);
std::string getLastPathElement(const std::string& path);
std::string getExecutablePath();
std::string getCurrentDirectory();
std::string getDataDirectory();
bool writeTextFile(const std::string& path, const std::string& content);
std::string getMimeTypeForFile(const std::string& filename);
#endif

5
include/tools/regexTools.h
View File

@ -20,9 +20,6 @@
#ifndef KIWIX_REGEXTOOLS_H
#define KIWIX_REGEXTOOLS_H
#include <unicode/regex.h>
#include <unicode/ucnv.h>
#include <map>
#include <string>
bool matchRegex(const std::string& content, const std::string& regex);
@ -30,7 +27,7 @@ std::string replaceRegex(const std::string& content,
const std::string& replacement,
const std::string& regex);
std::string appendToFirstOccurence(const std::string& content,
const std::string regex,
const std::string& regex,
const std::string& replacement);
#endif

9
include/tools/stringTools.h
View File

@ -22,14 +22,9 @@
#include <unicode/unistr.h>
#include <fstream>
#include <iomanip>
#include <iostream>
#include <sstream>
#include <string>
#include <vector>
#include "pathTools.h"
#include <sstream>
namespace kiwix
{
@ -75,5 +70,7 @@ T extractFromString(const std::string& str) {
iss >> ret;
return ret;
}
bool startsWith(const std::string& base, const std::string& start);
} //namespace kiwix
#endif

3
meson.build
View File

@ -17,6 +17,7 @@ libicu_dep = dependency('icu-i18n', static:static_deps)
libzim_dep = dependency('libzim', version : '>=5.0.0', static:static_deps)
pugixml_dep = dependency('pugixml', static:static_deps)
libcurl_dep = dependency('libcurl', static:static_deps)
microhttpd_dep = dependency('libmicrohttpd', static:static_deps)
if not compiler.has_header('mustache.hpp')
error('Cannot found header mustache.hpp')
@ -28,7 +29,7 @@ if target_machine.system() == 'windows' and static_deps
extra_cflags += '-DCURL_STATICLIB'
endif
all_deps = [thread_dep, libicu_dep, libzim_dep, pugixml_dep, libcurl_dep]
all_deps = [thread_dep, libicu_dep, libzim_dep, pugixml_dep, libcurl_dep, microhttpd_dep]
inc = include_directories('include')

72
src/android/kiwixlibrary.cpp Normal file
View File

@ -0,0 +1,72 @@
/*
* Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
* Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#include <jni.h>
#include <android/log.h>
#include "org_kiwix_kiwixlib_JNIKiwixLibrary.h"
#include "library.h"
#include "reader.h"
#include "utils.h"
/* Kiwix Reader JNI functions */
JNIEXPORT jlong JNICALL Java_org_kiwix_kiwixlib_JNIKiwixLibrary_getNativeLibrary(
JNIEnv* env, jobject obj)
{
__android_log_print(ANDROID_LOG_INFO, "kiwix", "Attempting to create library");
Lock l;
try {
kiwix::Library* library = new kiwix::Library();
return reinterpret_cast<jlong>(new Handle<kiwix::Library>(library));
} catch (std::exception& e) {
__android_log_print(ANDROID_LOG_WARN, "kiwix", "Error creating ZIM library");
__android_log_print(ANDROID_LOG_WARN, "kiwix", e.what());
return 0;
}
}
JNIEXPORT void JNICALL
Java_org_kiwix_kiwixlib_JNIKiwixLibrary_dispose(JNIEnv* env, jobject obj)
{
Handle<kiwix::Library>::dispose(env, obj);
}
#define LIBRARY (Handle<kiwix::Library>::getHandle(env, obj))
/* Kiwix library functions */
JNIEXPORT jboolean JNICALL
Java_org_kiwix_kiwixlib_JNIKiwixLibrary_addBook(JNIEnv* env, jobject obj, jstring path)
{
std::string cPath = jni2c(path, env);
bool ret;
try {
kiwix::Reader reader(cPath);
kiwix::Book book;
book.update(reader);
ret = LIBRARY->addBook(book);
} catch (std::exception& e) {
__android_log_print(ANDROID_LOG_ERROR, "kiwix", "Unable to get ZIM main page");
__android_log_print(ANDROID_LOG_ERROR, "kiwix", e.what());
ret = false;
}
return ret;
}

2
src/android/kiwixsearcher.cpp
View File

@ -52,7 +52,7 @@ JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIKiwixSearcher_addReader(
{
auto searcher = SEARCHER;
searcher->add_reader(*(Handle<kiwix::Reader>::getHandle(env, reader)), "");
searcher->add_reader(*(Handle<kiwix::Reader>::getHandle(env, reader)));
}
JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIKiwixSearcher_search(

99
src/android/kiwixserver.cpp Normal file
View File

@ -0,0 +1,99 @@
/*
* Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
* Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#include <jni.h>
#include <zim/file.h>
#include <android/log.h>
#include "org_kiwix_kiwixlib_JNIKiwixServer.h"
#include "tools/base64.h"
#include "server.h"
#include "utils.h"
/* Kiwix Reader JNI functions */
JNIEXPORT jlong JNICALL Java_org_kiwix_kiwixlib_JNIKiwixServer_getNativeServer(
JNIEnv* env, jobject obj, jobject jLibrary)
{
__android_log_print(ANDROID_LOG_INFO, "kiwix", "Attempting to create server");
Lock l;
try {
auto library = Handle<kiwix::Library>::getHandle(env, jLibrary);
kiwix::Server* server = new kiwix::Server(*library);
return reinterpret_cast<jlong>(new Handle<kiwix::Server>(server));
} catch (std::exception& e) {
__android_log_print(ANDROID_LOG_WARN, "kiwix", "Error creating the server");
__android_log_print(ANDROID_LOG_WARN, "kiwix", e.what());
return 0;
}
}
JNIEXPORT void JNICALL
Java_org_kiwix_kiwixlib_JNIKiwixServer_dispose(JNIEnv* env, jobject obj)
{
Handle<kiwix::Server>::dispose(env, obj);
}
#define SERVER (Handle<kiwix::Server>::getHandle(env, obj))
/* Kiwix library functions */
JNIEXPORT void JNICALL
Java_org_kiwix_kiwixlib_JNIKiwixServer_setRoot(JNIEnv* env, jobject obj, jstring jRoot)
{
std::string root = jni2c(jRoot, env);
SERVER->setRoot(root);
}
JNIEXPORT void JNICALL
Java_org_kiwix_kiwixlib_JNIKiwixServer_setAddress(JNIEnv* env, jobject obj, jstring jAddress)
{
std::string address = jni2c(jAddress, env);
SERVER->setAddress(address);
}
JNIEXPORT void JNICALL
Java_org_kiwix_kiwixlib_JNIKiwixServer_setPort(JNIEnv* env, jobject obj, int port)
{
SERVER->setPort(port);
}
JNIEXPORT void JNICALL
Java_org_kiwix_kiwixlib_JNIKiwixServer_setNbThreads(JNIEnv* env, jobject obj, int threads)
{
SERVER->setNbThreads(threads);
}
JNIEXPORT void JNICALL
Java_org_kiwix_kiwixlib_JNIKiwixServer_setTaskbar(JNIEnv* env, jobject obj, jboolean withTaskbar, jboolean withLibraryButton)
{
SERVER->setTaskbar(withTaskbar, withLibraryButton);
}
JNIEXPORT jboolean JNICALL
Java_org_kiwix_kiwixlib_JNIKiwixServer_start(JNIEnv* env, jobject obj)
{
return SERVER->start();
}
JNIEXPORT void JNICALL
Java_org_kiwix_kiwixlib_JNIKiwixServer_stop(JNIEnv* env, jobject obj)
{
SERVER->stop();
}

6
src/android/meson.build
View File

@ -2,7 +2,9 @@
kiwix_jni = custom_target('jni',
input: ['org/kiwix/kiwixlib/JNIICU.java',
'org/kiwix/kiwixlib/JNIKiwixReader.java',
'org/kiwix/kiwixlib/JNIKiwixLibrary.java',
'org/kiwix/kiwixlib/JNIKiwixSearcher.java',
'org/kiwix/kiwixlib/JNIKiwixServer.java',
'org/kiwix/kiwixlib/JNIKiwixInt.java',
'org/kiwix/kiwixlib/JNIKiwixString.java',
'org/kiwix/kiwixlib/JNIKiwixBool.java',
@ -10,6 +12,8 @@ kiwix_jni = custom_target('jni',
'org/kiwix/kiwixlib/Pair.java'],
output: ['org_kiwix_kiwixlib_JNIKiwix.h',
'org_kiwix_kiwixlib_JNIKiwixReader.h',
'org_kiwix_kiwixlib_JNIKiwixLibrary.h',
'org_kiwix_kiwixlib_JNIKiwixServer.h',
'org_kiwix_kiwixlib_JNIKiwixSearcher.h',
'org_kiwix_kiwixlib_JNIKiwixSearcher_Result.h'],
command:['javac', '-d', '@OUTDIR@', '-h', '@OUTDIR@', '@INPUT@']
@ -18,7 +22,9 @@ kiwix_jni = custom_target('jni',
kiwix_sources += [
'android/kiwixicu.cpp',
'android/kiwixreader.cpp',
'android/kiwixlibrary.cpp',
'android/kiwixsearcher.cpp',
'android/kiwixserver.cpp',
kiwix_jni]
install_subdir('org', install_dir: 'kiwix-lib/java')

38
src/android/org/kiwix/kiwixlib/JNIKiwixLibrary.java Normal file
View File

@ -0,0 +1,38 @@
/*
* Copyright (C) 2013 Emmanuel Engelhart <kelson@kiwix.org>
* Copyright (C) 2017 Matthieu Gautier <mgautier@kymeria.fr>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
package org.kiwix.kiwixlib;
import org.kiwix.kiwixlib.JNIKiwixException;
public class JNIKiwixLibrary
{
public native boolean addBook(String path) throws JNIKiwixException;
public JNIKiwixLibrary(String filename)
{
nativeHandle = getNativeLibrary();
}
public native void dispose();
private native long getNativeLibrary();
private long nativeHandle;
}

48
src/android/org/kiwix/kiwixlib/JNIKiwixServer.java Normal file
View File

@ -0,0 +1,48 @@
/*
* Copyright (C) 2019 Matthieu Gautier <mgautier@kymeria.fr>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
package org.kiwix.kiwixlib;
import org.kiwix.kiwixlib.JNIKiwixException;
import org.kiwix.kiwixlib.JNIKiwixLibrary;
public class JNIKiwixServer
{
public native void setRoot(String root);
public native void setAddress(String address);
public native void setPort(int port);
public native void setNbThreads(int nbTreads);
public native void setTaskbar(boolean withTaskBar, boolean witLibraryButton);
public native boolean start();
public native void stop();
public JNIKiwixServer(JNIKiwixLibrary library)
{
nativeHandle = getNativeServer(library);
}
private native long getNativeServer(JNIKiwixLibrary library);
private long nativeHandle;
}

4
src/aria2.cpp
View File

@ -2,20 +2,24 @@
#include "aria2.h"
#include "xmlrpc.h"
#include <iostream>
#include <sstream>
#include <thread>
#include <chrono>
#include <tools/otherTools.h>
#include <tools/pathTools.h>
#include <tools/stringTools.h>
#include <downloader.h> // For AriaError
#ifdef _WIN32
# define ARIA2_CMD "aria2c.exe"
#else
# define ARIA2_CMD "aria2c"
# include <unistd.h>
#endif
namespace kiwix {
Aria2::Aria2():

2
src/book.cpp
View File

@ -156,7 +156,7 @@ void Book::updateFromOpds(const pugi::xml_node& node, const std::string& urlHost
}
#undef VALUE
std::string Book::getHumanReadableIdFromPath()
std::string Book::getHumanReadableIdFromPath() const
{
std::string id = m_path;
if (!id.empty()) {

3
src/downloader.cpp
View File

@ -19,6 +19,7 @@
#include "downloader.h"
#include "tools/pathTools.h"
#include "tools/stringTools.h"
#include <algorithm>
#include <thread>
@ -174,7 +175,7 @@ Download* Downloader::getDownload(const std::string& did)
try {
m_knownDownloads.at(did).get()->updateStatus(true);
return m_knownDownloads.at(did).get();
} catch(exception& e) {
} catch(std::exception& e) {
for (auto gid : mp_aria->tellActive()) {
if (gid == did) {
m_knownDownloads[gid] = std::unique_ptr<Download>(new Download(mp_aria, gid));

1
src/kiwixserve.cpp
View File

@ -10,6 +10,7 @@
#endif
#include "tools/pathTools.h"
#include "tools/stringTools.h"
namespace kiwix {

17
src/library.cpp
View File

@ -19,11 +19,13 @@
#include "library.h"
#include "book.h"
#include "reader.h"
#include "libxml_dumper.h"
#include "tools/base64.h"
#include "tools/regexTools.h"
#include "tools/pathTools.h"
#include "tools/stringTools.h"
#include <pugixml.hpp>
#include <algorithm>
@ -31,6 +33,7 @@
namespace kiwix
{
/* Constructor */
Library::Library()
{
@ -82,6 +85,20 @@ Book& Library::getBookById(const std::string& id)
return m_books.at(id);
}
std::shared_ptr<Reader> Library::getReaderById(const std::string& id)
{
try {
return m_readers.at(id);
} catch (std::out_of_range& e) {}
auto book = getBookById(id);
if (!book.isPathValid())
return nullptr;
auto sptr = make_shared<Reader>(book.getPath());
m_readers[id] = sptr;
return sptr;
}
unsigned int Library::getBookCount(const bool localBooks,
const bool remoteBooks)
{

7
src/libxml_dumper.cpp
View File

@ -20,9 +20,10 @@
#include "libxml_dumper.h"
#include "book.h"
#include <tools/base64.h>
#include <tools/stringTools.h>
#include <tools/otherTools.h>
#include "tools/base64.h"
#include "tools/stringTools.h"
#include "tools/otherTools.h"
#include "tools/pathTools.h"
namespace kiwix
{

5
src/meson.build
View File

@ -8,7 +8,9 @@ kiwix_sources = [
'downloader.cpp',
'reader.cpp',
'entry.cpp',
'server.cpp',
'searcher.cpp',
'search_renderer.cpp',
'subprocess.cpp',
'aria2.cpp',
'tools/base64.cpp',
@ -18,6 +20,9 @@ kiwix_sources = [
'tools/networkTools.cpp',
'tools/otherTools.cpp',
'kiwixserve.cpp',
'name_mapper.cpp',
'server/request_context.cpp',
'server/response.cpp'
]
kiwix_sources += lib_resources

59
src/name_mapper.cpp Normal file
View File

@ -0,0 +1,59 @@
/*
* Copyright 2019 Matthieu Gautier <mgautier@kymeria.fr>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#include "name_mapper.h"
#include "library.h"
#include "tools/regexTools.h"
#include <iostream>
namespace kiwix {
HumanReadableNameMapper::HumanReadableNameMapper(kiwix::Library& library, bool withAlias) {
for (auto& bookId: library.filter(kiwix::Filter().local(true).valid(true))) {
auto& currentBook = library.getBookById(bookId);
auto bookName = currentBook.getHumanReadableIdFromPath();
m_idToName[bookId] = bookName;
m_nameToId[bookName] = bookId;
if (!withAlias)
continue;
auto aliasName = replaceRegex(bookName, "", "_[[:digit:]]{4}-[[:digit:]]{2}$");
if (m_nameToId.find(aliasName) == m_nameToId.end()) {
m_nameToId[aliasName] = bookId;
} else {
auto alreadyPresentPath = library.getBookById(aliasName).getPath();
std::cerr << "Path collision: " << alreadyPresentPath
<< " and " << currentBook.getPath()
<< " can't share the same URL path '" << aliasName << "'."
<< " Therefore, only " << alreadyPresentPath
<< " will be served." << std::endl;
}
}
}
std::string HumanReadableNameMapper::getNameForId(const std::string& id) {
return m_idToName.at(id);
}
std::string HumanReadableNameMapper::getIdForName(const std::string& name) {
return m_nameToId.at(name);
}
}

3
src/opds_dumper.cpp
View File

@ -20,7 +20,8 @@
#include "opds_dumper.h"
#include "book.h"
#include <tools/otherTools.h>
#include "tools/otherTools.h"
#include <iomanip>
namespace kiwix
{

151
src/search_renderer.cpp Normal file
View File

@ -0,0 +1,151 @@
/*
* Copyright 2011 Emmanuel Engelhart <kelson@kiwix.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#include <cmath>
#include "search_renderer.h"
#include "searcher.h"
#include "reader.h"
#include "library.h"
#include "name_mapper.h"
#include <zim/search.h>
#include <mustache.hpp>
#include "kiwixlib-resources.h"
namespace kiwix
{
/* Constructor */
SearchRenderer::SearchRenderer(Searcher* searcher, NameMapper* mapper)
: mp_searcher(searcher),
mp_nameMapper(mapper),
protocolPrefix("zim://"),
searchProtocolPrefix("search://?")
{}
/* Destructor */
SearchRenderer::~SearchRenderer() = default;
void SearchRenderer::setSearchPattern(const std::string& pattern)
{
this->searchPattern = pattern;
}
void SearchRenderer::setSearchContent(const std::string& name)
{
this->searchContent = name;
}
void SearchRenderer::setProtocolPrefix(const std::string& prefix)
{
this->protocolPrefix = prefix;
}
void SearchRenderer::setSearchProtocolPrefix(const std::string& prefix)
{
this->searchProtocolPrefix = prefix;
}
std::string SearchRenderer::getHtml()
{
kainjow::mustache::data results{kainjow::mustache::data::type::list};
mp_searcher->restart_search();
Result* p_result = NULL;
while ((p_result = mp_searcher->getNextResult())) {
kainjow::mustache::data result;
result.set("title", p_result->get_title());
result.set("url", p_result->get_url());
result.set("snippet", p_result->get_snippet());
auto readerIndex = p_result->get_readerIndex();
auto reader = mp_searcher->get_reader(readerIndex);
result.set("resultContentId", mp_nameMapper->getNameForId(reader->getId()));
if (p_result->get_wordCount() >= 0) {
result.set("wordCount", kiwix::beautifyInteger(p_result->get_wordCount()));
}
results.push_back(result);
delete p_result;
}
// pages
kainjow::mustache::data pages{kainjow::mustache::data::type::list};
auto resultStart = mp_searcher->getResultStart();
auto resultEnd = mp_searcher->getResultEnd();
auto resultCountPerPage = resultEnd - resultStart;
auto estimatedResultCount = mp_searcher->getEstimatedResultCount();
unsigned int pageStart
= resultStart / resultCountPerPage >= 5
? resultStart / resultCountPerPage - 4
: 0;
unsigned int pageCount
= estimatedResultCount / resultCountPerPage + 1 - pageStart;
if (pageCount > 10) {
pageCount = 10;
} else if (pageCount == 1) {
pageCount = 0;
}
for (unsigned int i = pageStart; i < pageStart + pageCount; i++) {
kainjow::mustache::data page;
page.set("label", to_string(i + 1));
page.set("start", to_string(i * resultCountPerPage));
page.set("end", to_string((i + 1) * resultCountPerPage));
if (i * resultCountPerPage == resultStart) {
page.set("selected", true);
}
pages.push_back(page);
}
std::string template_str = RESOURCE::templates::search_result_html;
kainjow::mustache::mustache tmpl(template_str);
kainjow::mustache::data allData;
allData.set("results", results);
allData.set("pages", pages);
allData.set("hasResult", estimatedResultCount != 0);
allData.set("count", kiwix::beautifyInteger(estimatedResultCount));
allData.set("searchPattern", kiwix::encodeDiples(this->searchPattern));
allData.set("searchPatternEncoded", urlEncode(this->searchPattern));
allData.set("resultStart", to_string(resultStart + 1));
allData.set("resultEnd", to_string(min(resultEnd, estimatedResultCount)));
allData.set("resultRange", to_string(resultCountPerPage));
allData.set("resultLastPageStart", to_string(estimatedResultCount > resultCountPerPage
? round(estimatedResultCount / resultCountPerPage) * resultCountPerPage
: 0));
allData.set("lastResult", to_string(estimatedResultCount));
allData.set("protocolPrefix", this->protocolPrefix);
allData.set("searchProtocolPrefix", this->searchProtocolPrefix);
allData.set("contentId", this->searchContent);
std::stringstream ss;
tmpl.render(allData, [&ss](const std::string& str) { ss << str; });
return ss.str();
}
}

131
src/searcher.cpp
View File

@ -65,16 +65,12 @@ struct SearcherInternal {
};
/* Constructor */
Searcher::Searcher(const std::string& humanReadableName)
Searcher::Searcher()
: internal(new SearcherInternal()),
searchPattern(""),
protocolPrefix("zim://"),
searchProtocolPrefix("search://?"),
resultCountPerPage(0),
estimatedResultCount(0),
resultStart(0),
resultEnd(0),
contentHumanReadableId(humanReadableName)
resultEnd(0)
{
loadICUExternalTables();
}
@ -85,16 +81,21 @@ Searcher::~Searcher()
delete internal;
}
bool Searcher::add_reader(Reader* reader, const std::string& humanReadableName)
bool Searcher::add_reader(Reader* reader)
{
if (!reader->hasFulltextIndex()) {
return false;
}
this->readers.push_back(reader);
this->humanReaderNames.push_back(humanReadableName);
return true;
}
Reader* Searcher::get_reader(int readerIndex)
{
return readers.at(readerIndex);
}
/* Search strings in the database */
void Searcher::search(std::string& search,
unsigned int resultStart,
@ -107,22 +108,8 @@ void Searcher::search(std::string& search,
cout << "Performing query `" << search << "'" << endl;
}
/* If resultEnd & resultStart inverted */
if (resultStart > resultEnd) {
resultEnd += resultStart;
resultStart = resultEnd - resultStart;
resultEnd -= resultStart;
}
/* Try to find results */
if (resultStart != resultEnd) {
/* Avoid big researches */
this->resultCountPerPage = resultEnd - resultStart;
if (this->resultCountPerPage > MAX_SEARCH_LEN) {
resultEnd = resultStart + MAX_SEARCH_LEN;
this->resultCountPerPage = MAX_SEARCH_LEN;
}
/* Perform the search */
this->searchPattern = search;
this->resultStart = resultStart;
@ -159,25 +146,11 @@ void Searcher::geo_search(float latitude, float longitude, float distance,
cout << "Performing geo query `" << distance << "&(" << latitude << ";" << longitude << ")'" << endl;
}
/* If resultEnd & resultStart inverted */
if (resultStart > resultEnd) {
resultEnd += resultStart;
resultStart = resultEnd - resultStart;
resultEnd -= resultStart;
}
/* Try to find results */
if (resultStart == resultEnd) {
return;
}
/* Avoid big researches */
this->resultCountPerPage = resultEnd - resultStart;
if (this->resultCountPerPage > MAX_SEARCH_LEN) {
resultEnd = resultStart + MAX_SEARCH_LEN;
this->resultCountPerPage = MAX_SEARCH_LEN;
}
/* Perform the search */
std::ostringstream oss;
oss << "Articles located less than " << distance << " meters of " << latitude << ";" << longitude;
@ -261,18 +234,6 @@ unsigned int Searcher::getEstimatedResultCount()
return this->estimatedResultCount;
}
bool Searcher::setProtocolPrefix(const std::string prefix)
{
this->protocolPrefix = prefix;
return true;
}
bool Searcher::setSearchProtocolPrefix(const std::string prefix)
{
this->searchProtocolPrefix = prefix;
return true;
}
_Result::_Result(zim::Search::iterator& iterator)
: iterator(iterator)
{
@ -314,79 +275,5 @@ int _Result::get_readerIndex()
return iterator.get_fileIndex();
}
string Searcher::getHtml()
{
kainjow::mustache::data results{kainjow::mustache::data::type::list};
this->restart_search();
Result* p_result = NULL;
while ((p_result = this->getNextResult())) {
kainjow::mustache::data result;
result.set("title", p_result->get_title());
result.set("url", p_result->get_url());
result.set("snippet", p_result->get_snippet());
result.set("resultContentId", humanReaderNames[p_result->get_readerIndex()]);
if (p_result->get_wordCount() >= 0) {
result.set("wordCount", kiwix::beautifyInteger(p_result->get_wordCount()));
}
results.push_back(result);
delete p_result;
}
// pages
kainjow::mustache::data pages{kainjow::mustache::data::type::list};
unsigned int pageStart
= this->resultStart / this->resultCountPerPage >= 5
? this->resultStart / this->resultCountPerPage - 4
: 0;
unsigned int pageCount
= this->estimatedResultCount / this->resultCountPerPage + 1 - pageStart;
if (pageCount > 10) {
pageCount = 10;
} else if (pageCount == 1) {
pageCount = 0;
}
for (unsigned int i = pageStart; i < pageStart + pageCount; i++) {
kainjow::mustache::data page;
page.set("label", to_string(i + 1));
page.set("start", to_string(i * this->resultCountPerPage));
page.set("end", to_string((i + 1) * this->resultCountPerPage));
if (i * this->resultCountPerPage == this->resultStart) {
page.set("selected", true);
}
pages.push_back(page);
}
std::string template_str = RESOURCE::search_result_tmpl;
kainjow::mustache::mustache tmpl(template_str);
kainjow::mustache::data allData;
allData.set("results", results);
allData.set("pages", pages);
allData.set("hasResult", this->estimatedResultCount != 0);
allData.set("count", kiwix::beautifyInteger(this->estimatedResultCount));
allData.set("searchPattern", kiwix::encodeDiples(this->searchPattern));
allData.set("searchPatternEncoded", urlEncode(this->searchPattern));
allData.set("resultStart", to_string(this->resultStart + 1));
allData.set("resultEnd", to_string(min(this->resultEnd, this->estimatedResultCount)));
allData.set("resultRange", to_string(this->resultCountPerPage));
allData.set("resultLastPageStart", to_string(this->estimatedResultCount > this->resultCountPerPage
? round(this->estimatedResultCount / this->resultCountPerPage) * this->resultCountPerPage
: 0));
allData.set("lastResult", to_string(this->estimatedResultCount));
allData.set("protocolPrefix", this->protocolPrefix);
allData.set("searchProtocolPrefix", this->searchProtocolPrefix);
allData.set("contentId", this->contentHumanReadableId);
std::stringstream ss;
tmpl.render(allData, [&ss](const std::string& str) { ss << str; });
return ss.str();
}
}

911
src/server.cpp Normal file
View File

@ -0,0 +1,911 @@
/*
* Copyright 2019 Matthieu Gautier <mgautier@kymeria.fr>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#include "server.h"
#ifdef _WIN32
# if !defined(__MINGW32__) && (_MSC_VER < 1600)
# include "stdint4win.h"
# endif
# include <winsock2.h>
# include <ws2tcpip.h>
# ifdef __GNUC__
// inet_pton is not declared in mingw, even if the function exists.
extern "C" {
WINSOCK_API_LINKAGE INT WSAAPI inet_pton( INT Family, PCSTR pszAddrString, PVOID pAddrBuf);
}
# endif
typedef UINT64 uint64_t;
typedef UINT16 uint16_t;
#endif
extern "C" {
#include <microhttpd.h>
}
#include "tools/otherTools.h"
#include "tools/pathTools.h"
#include "tools/regexTools.h"
#include "tools/stringTools.h"
#include "library.h"
#include "name_mapper.h"
#include "entry.h"
#include "searcher.h"
#include "search_renderer.h"
#include "opds_dumper.h"
#include <zim/uuid.h>
#include <mustache.hpp>
#include <pthread.h>
#include <atomic>
#include <string>
#include <vector>
#include <chrono>
#include "kiwixlib-resources.h"
#ifndef _WIN32
# include <arpa/inet.h>
#endif
#include "server/request_context.h"
#include "server/response.h"
#define MAX_SEARCH_LEN 140
#define KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE 100
namespace kiwix {
static IdNameMapper defaultNameMapper;
static int staticHandlerCallback(void* cls,
struct MHD_Connection* connection,
const char* url,
const char* method,
const char* version,
const char* upload_data,
size_t* upload_data_size,
void** cont_cls);
class InternalServer {
public:
InternalServer(Library* library,
NameMapper* nameMapper,
std::string addr,
int port,
std::string root,
int nbThreads,
bool verbose,
bool withTaskbar,
bool withLibraryButton);
virtual ~InternalServer() = default;
int handlerCallback(struct MHD_Connection* connection,
const char* url,
const char* method,
const char* version,
const char* upload_data,
size_t* upload_data_size,
void** cont_cls);
bool start();
void stop();
private:
Response handle_request(const RequestContext& request);
Response build_500(const std::string& msg);
Response build_404(const RequestContext& request, const std::string& zimName);
Response build_homepage(const RequestContext& request);
Response handle_skin(const RequestContext& request);
Response handle_catalog(const RequestContext& request);
Response handle_meta(const RequestContext& request);
Response handle_search(const RequestContext& request);
Response handle_suggest(const RequestContext& request);
Response handle_random(const RequestContext& request);
Response handle_content(const RequestContext& request);
kainjow::mustache::data get_default_data();
Response get_default_response();
std::string m_addr;
int m_port;
std::string m_root;
int m_nbThreads;
std::atomic_bool m_verbose;
bool m_withTaskbar;
bool m_withLibraryButton;
struct MHD_Daemon* mp_daemon;
Library* mp_library;
NameMapper* mp_nameMapper;
};
Server::Server(Library* library, NameMapper* nameMapper) :
mp_library(library),
mp_nameMapper(nameMapper),
mp_server(nullptr)
{
}
Server::~Server() = default;
bool Server::start() {
mp_server.reset(new InternalServer(
mp_library,
mp_nameMapper,
m_addr,
m_port,
m_root,
m_nbThreads,
m_verbose,
m_withTaskbar,
m_withLibraryButton));
return mp_server->start();
}
void Server::stop() {
mp_server->stop();
mp_server.reset(nullptr);
}
void Server::setRoot(const std::string& root)
{
m_root = root;
if (m_root[0] != '/') {
m_root = "/" + m_root;
}
if (m_root.back() == '/') {
m_root.erase(m_root.size() - 1);
}
}
InternalServer::InternalServer(Library* library,
NameMapper* nameMapper,
std::string addr,
int port,
std::string root,
int nbThreads,
bool verbose,
bool withTaskbar,
bool withLibraryButton) :
m_addr(addr),
m_port(port),
m_root(root),
m_nbThreads(nbThreads),
m_verbose(verbose),
m_withTaskbar(withTaskbar),
m_withLibraryButton(withLibraryButton),
mp_daemon(nullptr),
mp_library(library),
mp_nameMapper(nameMapper ? nameMapper : &defaultNameMapper)
{}
bool InternalServer::start() {
#ifdef _WIN32
int flags = MHD_USE_SELECT_INTERNALLY;
#else
int flags = MHD_USE_POLL_INTERNALLY;
#endif
if (m_verbose.load())
flags |= MHD_USE_DEBUG;
struct sockaddr_in sockAddr;
memset(&sockAddr, 0, sizeof(sockAddr));
sockAddr.sin_family = AF_INET;
sockAddr.sin_port = htons(m_port);
if (m_addr.empty()) {
if (0 != INADDR_ANY)
sockAddr.sin_addr.s_addr = htonl(INADDR_ANY);
} else {
if (inet_pton(AF_INET, m_addr.c_str(), &(sockAddr.sin_addr.s_addr)) == 0) {
std::cerr << "Ip address " << m_addr << " is not a valid ip address" << std::endl;
return false;
}
}
mp_daemon = MHD_start_daemon(flags,
m_port,
NULL,
NULL,
&staticHandlerCallback,
this,
MHD_OPTION_SOCK_ADDR, &sockAddr,
MHD_OPTION_THREAD_POOL_SIZE, m_nbThreads,
MHD_OPTION_END);
if (mp_daemon == nullptr) {
std::cerr << "Unable to instantiate the HTTP daemon. The port " << m_port
<< " is maybe already occupied or need more permissions to be open. "
"Please try as root or with a port number higher or equal to 1024."
<< std::endl;
return false;
}
return true;
}
void InternalServer::stop()
{
MHD_stop_daemon(mp_daemon);
}
static int staticHandlerCallback(void* cls,
struct MHD_Connection* connection,
const char* url,
const char* method,
const char* version,
const char* upload_data,
size_t* upload_data_size,
void** cont_cls)
{
InternalServer* _this = static_cast<InternalServer*>(cls);
return _this->handlerCallback(connection,
url,
method,
version,
upload_data,
upload_data_size,
cont_cls);
}
int InternalServer::handlerCallback(struct MHD_Connection* connection,
const char* url,
const char* method,
const char* version,
const char* upload_data,
size_t* upload_data_size,
void** cont_cls)
{
auto start_time = std::chrono::steady_clock::now();
if (m_verbose.load() ) {
printf("======================\n");
printf("Requesting : \n");
printf("full_url : %s\n", url);
}
RequestContext request(connection, m_root, url, method, version);
if (m_verbose.load() ) {
request.print_debug_info();
}
/* Unexpected method */
if (request.get_method() != RequestMethod::GET
&& request.get_method() != RequestMethod::POST) {
printf("Reject request because of unhandled request method.\n");
printf("----------------------\n");
return MHD_NO;
}
auto response = handle_request(request);
if (response.getReturnCode() == MHD_HTTP_INTERNAL_SERVER_ERROR) {
printf("========== INTERNAL ERROR !! ============\n");
if (!m_verbose.load()) {
printf("Requesting : \n");
printf("full_url : %s\n", url);
request.print_debug_info();
}
}
auto ret = response.send(request, connection);
auto end_time = std::chrono::steady_clock::now();
auto time_span = std::chrono::duration_cast<std::chrono::duration<double>>(end_time - start_time);
if (m_verbose.load()) {
printf("Request time : %fs\n", time_span.count());
printf("----------------------\n");
}
return ret;
}
Response InternalServer::handle_request(const RequestContext& request)
{
try {
if (! request.is_valid_url())
return build_404(request, "");
if (kiwix::startsWith(request.get_url(), "/skin/"))
return handle_skin(request);
if (startsWith(request.get_url(), "/catalog"))
return handle_catalog(request);
if (request.get_url() == "/meta")
return handle_meta(request);
if (request.get_url() == "/search")
return handle_search(request);
if (request.get_url() == "/suggest")
return handle_suggest(request);
if (request.get_url() == "/random")
return handle_random(request);
return handle_content(request);
} catch (std::exception& e) {
fprintf(stderr, "===== Unhandled error : %s\n", e.what());
return build_500(e.what());
} catch (...) {
fprintf(stderr, "===== Unhandled unknown error\n");
return build_500("Unknown error");
}
}
kainjow::mustache::data InternalServer::get_default_data()
{
kainjow::mustache::data data;
data.set("root", m_root);
return data;
}
Response InternalServer::get_default_response()
{
return Response(m_root, m_verbose.load(), m_withTaskbar, m_withLibraryButton);
}
Response InternalServer::build_404(const RequestContext& request,
const std::string& bookName)
{
kainjow::mustache::data results;
results.set("url", request.get_full_url());
auto response = get_default_response();
response.set_template(RESOURCE::templates::_404_html, results);
response.set_mimeType("text/html");
response.set_code(MHD_HTTP_NOT_FOUND);
response.set_compress(true);
response.set_taskbar(bookName, "");
return response;
}
Response InternalServer::build_500(const std::string& msg)
{
kainjow::mustache::data data;
data.set("error", msg);
Response response(m_root, true, false, false);
response.set_template(RESOURCE::templates::_500_html, data);
response.set_mimeType("text/html");
response.set_code(MHD_HTTP_INTERNAL_SERVER_ERROR);
return response;
}
Response InternalServer::build_homepage(const RequestContext& request)
{
auto data = get_default_data();
kainjow::mustache::data books{kainjow::mustache::data::type::list};
for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
auto& currentBook = mp_library->getBookById(bookId);
kainjow::mustache::data book;
book.set("name", mp_nameMapper->getNameForId(bookId));
book.set("title", currentBook.getTitle());
book.set("description", currentBook.getDescription());
book.set("articleCount", beautifyInteger(currentBook.getArticleCount()));
book.set("mediaCount", beautifyInteger(currentBook.getMediaCount()));
books.push_back(book);
}
data.set("books", books);
auto response = get_default_response();
response.set_template(RESOURCE::templates::index_html, data);
response.set_mimeType("text/html; charset=utf-8");
response.set_compress(true);
response.set_taskbar("", "");
return response;
}
Response InternalServer::handle_meta(const RequestContext& request)
{
std::string bookName;
std::string bookId;
std::string meta_name;
std::shared_ptr<Reader> reader;
try {
bookName = request.get_argument("content");
bookId = mp_nameMapper->getIdForName(bookName);
meta_name = request.get_argument("name");
reader = mp_library->getReaderById(bookId);
} catch (const std::out_of_range& e) {
return build_404(request, bookName);
}
if (reader == nullptr) {
return build_404(request, bookName);
}
std::string content;
std::string mimeType = "text";
if (meta_name == "title") {
content = reader->getTitle();
} else if (meta_name == "description") {
content = reader->getDescription();
} else if (meta_name == "language") {
content = reader->getLanguage();
} else if (meta_name == "name") {
content = reader->getName();
} else if (meta_name == "tags") {
content = reader->getTags();
} else if (meta_name == "date") {
content = reader->getDate();
} else if (meta_name == "creator") {
content = reader->getCreator();
} else if (meta_name == "publisher") {
content = reader->getPublisher();
} else if (meta_name == "favicon") {
reader->getFavicon(content, mimeType);
} else {
return build_404(request, bookName);
}
auto response = get_default_response();
response.set_content(content);
response.set_mimeType(mimeType);
response.set_compress(false);
response.set_cache(true);
return response;
}
Response InternalServer::handle_suggest(const RequestContext& request)
{
if (m_verbose.load()) {
printf("** running handle_suggest\n");
}
std::string content;
std::string mimeType;
unsigned int maxSuggestionCount = 10;
unsigned int suggestionCount = 0;
std::string suggestion;
std::string bookName;
std::string bookId;
std::string term;
std::shared_ptr<Reader> reader;
try {
bookName = request.get_argument("content");
bookId = mp_nameMapper->getIdForName(bookName);
term = request.get_argument("term");
reader = mp_library->getReaderById(bookId);
} catch (const std::out_of_range&) {
return build_404(request, bookName);
}
if (m_verbose.load()) {
printf("Searching suggestions for: \"%s\"\n", term.c_str());
}
kainjow::mustache::data results{kainjow::mustache::data::type::list};
bool first = true;
if (reader != nullptr) {
/* Get the suggestions */
reader->searchSuggestionsSmart(term, maxSuggestionCount);
while (reader->getNextSuggestion(suggestion)) {
kainjow::mustache::data result;
result.set("label", suggestion);
result.set("value", suggestion);
result.set("first", first);
first = false;
results.push_back(result);
suggestionCount++;
}
}
/* Propose the fulltext search if possible */
if (reader->hasFulltextIndex()) {
kainjow::mustache::data result;
result.set("label", "containing '" + term + "'...");
result.set("value", term);
result.set("first", first);
results.push_back(result);
}
auto data = get_default_data();
data.set("suggestions", results);
auto response = get_default_response();
response.set_template(RESOURCE::templates::suggestion_json, data);
response.set_mimeType("application/json; charset=utf-8");
response.set_compress(true);
return response;
}
Response InternalServer::handle_skin(const RequestContext& request)
{
if (m_verbose.load()) {
printf("** running handle_skin\n");
}
auto response = get_default_response();
auto resourceName = request.get_url().substr(1);
try {
response.set_content(getResource(resourceName));
} catch (const ResourceNotFound& e) {
return build_404(request, "");
}
response.set_mimeType(getMimeTypeForFile(resourceName));
response.set_compress(true);
response.set_cache(true);
return response;
}
Response InternalServer::handle_search(const RequestContext& request)
{
if (m_verbose.load()) {
printf("** running handle_search\n");
}
std::string bookName;
std::string bookId;
try {
bookName = request.get_argument("content");
bookId = mp_nameMapper->getIdForName(bookName);
} catch (const std::out_of_range&) {}
std::string patternString;
try {
patternString = request.get_argument("pattern");
} catch (const std::out_of_range&) {}
/* Retrive geo search */
bool has_geo_query = false;
float latitude = 0;
float longitude = 0;
float distance = 0;
try {
latitude = request.get_argument<float>("latitude");
longitude = request.get_argument<float>("longitude");
distance = request.get_argument<float>("distance");
has_geo_query = true;
} catch(const std::out_of_range&) {}
catch(const std::invalid_argument&) {}
std::shared_ptr<Reader> reader(nullptr);
try {
reader = mp_library->getReaderById(bookId);
} catch (const std::out_of_range&) {}
/* Try first to load directly the article */
if (reader != nullptr && !patternString.empty()) {
std::string patternCorrespondingUrl;
auto variants = reader->getTitleVariants(patternString);
auto variantsItr = variants.begin();
while (patternCorrespondingUrl.empty() && variantsItr != variants.end()) {
try {
auto entry = reader->getEntryFromTitle(*variantsItr);
entry = entry.getFinalEntry();
patternCorrespondingUrl = entry.getPath();
break;
} catch(kiwix::NoEntry& e) {
variantsItr++;
}
}
/* If article found then redirect directly to it */
if (!patternCorrespondingUrl.empty()) {
auto response = get_default_response();
response.set_redirection(m_root + "/" + bookName + "/" + patternCorrespondingUrl);
return response;
}
}
/* Make the search */
auto response = get_default_response();
response.set_mimeType("text/html; charset=utf-8");
response.set_taskbar(bookName, reader ? reader->getTitle() : "");
response.set_compress(true);
if ( (!reader && !bookName.empty())
|| (patternString.empty() && ! has_geo_query) ) {
auto data = get_default_data();
data.set("pattern", encodeDiples(patternString));
response.set_template(RESOURCE::templates::no_search_result_html, data);
response.set_code(MHD_HTTP_NOT_FOUND);
return response;
}
Searcher searcher;
if (reader) {
searcher.add_reader(reader.get());
} else {
for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
auto currentReader = mp_library->getReaderById(bookId);
if (currentReader) {
searcher.add_reader(currentReader.get());
}
}
}
auto start = 0;
try {
start = request.get_argument<unsigned int>("start");
} catch (const std::exception&) {}
auto end = 25;
try {
end = request.get_argument<unsigned int>("end");
} catch (const std::exception&) {}
if (start>end) {
auto tmp = start;
start = end;
end = tmp;
}
if (end > start + MAX_SEARCH_LEN) {
end = start + MAX_SEARCH_LEN;
}
/* Get the results */
try {
if (patternString.empty()) {
searcher.geo_search(latitude, longitude, distance,
start, end, m_verbose.load());
} else {
searcher.search(patternString,
start, end, m_verbose.load());
}
SearchRenderer renderer(&searcher, mp_nameMapper);
renderer.setSearchPattern(patternString);
renderer.setSearchContent(bookName);
renderer.setProtocolPrefix(m_root + "/");
renderer.setSearchProtocolPrefix(m_root + "/search?");
response.set_content(renderer.getHtml());
} catch (const std::exception& e) {
std::cerr << e.what() << std::endl;
}
return response;
}
Response InternalServer::handle_random(const RequestContext& request)
{
if (m_verbose.load()) {
printf("** running handle_random\n");
}
std::string bookName;
std::string bookId;
std::shared_ptr<Reader> reader;
try {
bookName = request.get_argument("content");
bookId = mp_nameMapper->getIdForName(bookName);
reader = mp_library->getReaderById(bookId);
} catch (const std::out_of_range&) {
return build_404(request, bookName);
}
if (reader == nullptr) {
return build_404(request, bookName);
}
try {
auto entry = reader->getRandomPage();
entry = entry.getFinalEntry();
auto response = get_default_response();
response.set_redirection(m_root + "/" + bookName + "/" + kiwix::urlEncode(entry.getPath()));
return response;
} catch(kiwix::NoEntry& e) {
return build_404(request, bookName);
}
}
Response InternalServer::handle_catalog(const RequestContext& request)
{
if (m_verbose.load()) {
printf("** running handle_catalog");
}
std::string host;
std::string url;
try {
host = request.get_header("Host");
url = request.get_url_part(1);
} catch (const std::out_of_range&) {
return build_404(request, "");
}
if (url != "searchdescription.xml" && url != "root.xml" && url != "search") {
return build_404(request, "");
}
auto response = get_default_response();
response.set_compress(true);
if (url == "searchdescription.xml") {
response.set_template(RESOURCE::opensearchdescription_xml, get_default_data());
response.set_mimeType("application/opensearchdescription+xml");
return response;
}
zim::Uuid uuid;
kiwix::OPDSDumper opdsDumper;
opdsDumper.setRootLocation(m_root);
opdsDumper.setSearchDescriptionUrl("catalog/searchdescription.xml");
opdsDumper.setId(kiwix::to_string(uuid));
opdsDumper.setLibrary(mp_library);
response.set_mimeType("application/atom+xml;profile=opds-catalog;kind=acquisition; charset=utf-8");
std::vector<std::string> bookIdsToDump;
if (url == "root.xml") {
opdsDumper.setTitle("All zims");
uuid = zim::Uuid::generate(host);
bookIdsToDump = mp_library->filter(kiwix::Filter().valid(true).local(true).remote(true));
} else if (url == "search") {
std::string query;
std::string language;
std::vector<std::string> tags;
std::vector<std::string> noTags;
size_t count(10);
size_t startIndex(0);
try {
query = request.get_argument("q");
} catch (const std::out_of_range&) {}
try {
language = request.get_argument("lang");
} catch (const std::out_of_range&) {}
try {
count = extractFromString<unsigned long>(request.get_argument("count"));
} catch (...) {}
try {
startIndex = extractFromString<unsigned long>(request.get_argument("start"));
} catch (...) {}
try {
tags = kiwix::split(request.get_argument("notag"), ";");
} catch (...) {}
try {
noTags = kiwix::split(request.get_argument("notag"), ";");
} catch (...) {}
opdsDumper.setTitle("Search result for " + query);
uuid = zim::Uuid::generate();
bookIdsToDump = mp_library->filter(
kiwix::Filter().valid(true).local(true).remote(true)
.query(query)
.lang(language)
.acceptTags(tags)
.rejectTags(noTags)
);
auto totalResults = bookIdsToDump.size();
bookIdsToDump.erase(bookIdsToDump.begin(), bookIdsToDump.begin()+startIndex);
if (count>0 && bookIdsToDump.size() > count) {
bookIdsToDump.resize(count);
}
opdsDumper.setOpenSearchInfo(totalResults, startIndex, bookIdsToDump.size());
}
response.set_content(opdsDumper.dumpOPDSFeed(bookIdsToDump));
return response;
}
Response InternalServer::handle_content(const RequestContext& request)
{
if (m_verbose.load()) {
printf("** running handle_content\n");
}
std::string baseUrl;
std::string content;
std::string mimeType;
kiwix::Entry entry;
std::string bookName;
try {
bookName = request.get_url_part(0);
} catch (const std::out_of_range& e) {
return build_homepage(request);
}
if (bookName.empty())
return build_homepage(request);
std::string bookId;
std::shared_ptr<Reader> reader;
try {
bookId = mp_nameMapper->getIdForName(bookName);
reader = mp_library->getReaderById(bookId);
} catch (const std::out_of_range& e) {
return build_404(request, bookName);
}
if (reader == nullptr) {
return build_404(request, bookName);
}
auto urlStr = request.get_url().substr(bookName.size()+1);
if (urlStr[0] == '/') {
urlStr = urlStr.substr(1);
}
try {
entry = reader->getEntryFromPath(urlStr);
if (entry.isRedirect() || urlStr.empty()) {
// If urlStr is empty, we want to mainPage.
// We must do a redirection to the real page.
entry = entry.getFinalEntry();
auto response = get_default_response();
response.set_redirection(m_root + "/" + bookName + "/" +
kiwix::urlEncode(entry.getPath()));
return response;
}
} catch(kiwix::NoEntry& e) {
if (m_verbose.load())
printf("Failed to find %s\n", urlStr.c_str());
return build_404(request, bookName);
}
try {
mimeType = entry.getMimetype();
} catch (exception& e) {
mimeType = "application/octet-stream";
}
if (m_verbose.load()) {
printf("Found %s\n", urlStr.c_str());
printf("mimeType: %s\n", mimeType.c_str());
}
if (mimeType.find("text/") != string::npos
|| mimeType.find("application/javascript") != string::npos
|| mimeType.find("application/json") != string::npos) {
zim::Blob raw_content = entry.getBlob();
content = string(raw_content.data(), raw_content.size());
auto response = get_default_response();
response.set_mimeType(mimeType);
/* Special rewrite URL in case of ZIM file use intern *asbolute* url like
* /A/Kiwix */
if (mimeType.find("text/html") != string::npos) {
content = replaceRegex(content,
"$1$2" + m_root + "/" + bookName + "/$3/",
"(href|src)(=[\"|\']{0,1})/([A-Z|\\-])/");
content = replaceRegex(content,
"$1$2" + m_root + "/" + bookName + "/$3/",
"(@import[ ]+)([\"|\']{0,1})/([A-Z|\\-])/");
response.set_taskbar(bookName, reader->getTitle());
} else if (mimeType.find("text/css") != string::npos) {
content = replaceRegex(content,
"$1$2" + m_root + "/" + bookName + "/$3/",
"(url|URL)(\\([\"|\']{0,1})/([A-Z|\\-])/");
}
response.set_content(content);
response.set_compress(true);
response.set_cache(true);
return response;
} else {
int range_len;
if (request.get_range().second == -1) {
range_len = entry.getSize() - request.get_range().first;
} else {
range_len = request.get_range().second - request.get_range().first;
}
auto response = get_default_response();
response.set_entry(entry);
response.set_mimeType(mimeType);
response.set_range_first(request.get_range().first);
response.set_range_len(range_len);
response.set_cache(true);
return response;
}
}
}

211
src/server/request_context.cpp Normal file
View File

@ -0,0 +1,211 @@
/*
* Copyright 2009-2016 Emmanuel Engelhart <kelson@kiwix.org>
* Copyright 2017 Matthieu Gautier<mgautier@kymeria.fr>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#include "request_context.h"
#include <string.h>
#include <stdexcept>
#include <sstream>
#include <cstdio>
#include <atomic>
namespace kiwix {
static std::atomic_ullong s_requestIndex(0);
RequestContext::RequestContext(struct MHD_Connection* connection,
std::string rootLocation,
const std::string& _url,
const std::string& method,
const std::string& version) :
full_url(_url),
url(_url),
valid_url(true),
version(version),
requestIndex(s_requestIndex++),
acceptEncodingDeflate(false),
accept_range(false),
range_pair(0, -1)
{
if (method == "GET") {
this->method = RequestMethod::GET;
} else if (method == "HEAD") {
this->method = RequestMethod::HEAD;
} else if (method == "POST") {
this->method = RequestMethod::POST;
} else if (method == "PUT") {
this->method = RequestMethod::PUT;
} else if (method == "DELETE") {
this->method = RequestMethod::DELETE_;
} else if (method == "CONNECT") {
this->method = RequestMethod::CONNECT;
} else if (method == "OPTIONS") {
this->method = RequestMethod::OPTIONS;
} else if (method == "TRACE") {
this->method = RequestMethod::TRACE;
} else if (method == "PATCH") {
this->method = RequestMethod::PATCH;
} else {
this->method = RequestMethod::OTHER;
}
MHD_get_connection_values(connection, MHD_HEADER_KIND, &RequestContext::fill_header, this);
MHD_get_connection_values(connection, MHD_GET_ARGUMENT_KIND, &RequestContext::fill_argument, this);
valid_url = true;
if (rootLocation.empty()) {
// nothing special to handle.
url = full_url;
} else {
if (full_url == rootLocation) {
url = "/";
} else if (full_url.size() > rootLocation.size() &&
full_url.substr(0, rootLocation.size()+1) == rootLocation + "/") {
url = full_url.substr(rootLocation.size());
} else {
valid_url = false;
}
}
try {
acceptEncodingDeflate =
(get_header(MHD_HTTP_HEADER_ACCEPT_ENCODING).find("deflate") != std::string::npos);
} catch (const std::out_of_range&) {}
/*Check if range is requested. */
try {
auto range = get_header(MHD_HTTP_HEADER_RANGE);
int start = 0;
int end = -1;
std::istringstream iss(range);
char c;
iss >> start >> c;
if (iss.good() && c=='-') {
iss >> end;
if (iss.fail()) {
// Something went wrong will extracting.
end = -1;
}
if (iss.eof()) {
accept_range = true;
range_pair = std::pair<int, int>(start, end);
}
}
} catch (const std::out_of_range&) {}
}
RequestContext::~RequestContext()
{}
int RequestContext::fill_header(void *__this, enum MHD_ValueKind kind,
const char *key, const char *value)
{
RequestContext *_this = static_cast<RequestContext*>(__this);
_this->headers[key] = value;
return MHD_YES;
}
int RequestContext::fill_argument(void *__this, enum MHD_ValueKind kind,
const char *key, const char* value)
{
RequestContext *_this = static_cast<RequestContext*>(__this);
_this->arguments[key] = value == nullptr ? "" : value;
return MHD_YES;
}
void RequestContext::print_debug_info() const {
printf("method : %s (%d)\n", method==RequestMethod::GET ? "GET" :
method==RequestMethod::POST ? "POST" :
"OTHER", (int)method);
printf("version : %s\n", version.c_str());
printf("request# : %lld\n", requestIndex);
printf("headers :\n");
for (auto it=headers.begin(); it!=headers.end(); it++) {
printf(" - %s : '%s'\n", it->first.c_str(), it->second.c_str());
}
printf("arguments :\n");
for (auto it=arguments.begin(); it!=arguments.end(); it++) {
printf(" - %s : '%s'\n", it->first.c_str(), it->second.c_str());
}
printf("Parsed : \n");
printf("url : %s\n", url.c_str());
printf("acceptEncodingDeflate : %d\n", acceptEncodingDeflate);
printf("has_range : %d\n", accept_range);
printf("is_valid_url : %d\n", valid_url);
printf(".............\n");
}
RequestMethod RequestContext::get_method() const {
return method;
}
std::string RequestContext::get_url() const {
return url;
}
std::string RequestContext::get_url_part(int number) const {
size_t start = 1;
while(true) {
auto found = url.find('/', start);
if (number == 0) {
if (found == std::string::npos) {
return url.substr(start);
} else {
return url.substr(start, found-start);
}
} else {
if (found == std::string::npos) {
throw std::out_of_range("No parts");
}
start = found + 1;
number -= 1;
}
}
}
std::string RequestContext::get_full_url() const {
return full_url;
}
bool RequestContext::is_valid_url() const {
return valid_url;
}
bool RequestContext::has_range() const {
return accept_range;
}
std::pair<int, int> RequestContext::get_range() const {
return range_pair;
}
template<>
std::string RequestContext::get_argument(const std::string& name) const {
return arguments.at(name);
}
std::string RequestContext::get_header(const std::string& name) const {
return headers.at(name);
}
}

109
src/server/request_context.h Normal file
View File

@ -0,0 +1,109 @@
/*
* Copyright 2009-2016 Emmanuel Engelhart <kelson@kiwix.org>
* Copyright 2017 Matthieu Gautier<mgautier@kymeria.fr>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#ifndef REQUEST_CONTEXT_H
#define REQUEST_CONTEXT_H
#include <string>
#include <sstream>
#include <map>
#include <stdexcept>
extern "C" {
#include <microhttpd.h>
}
namespace kiwix {
enum class RequestMethod {
GET,
HEAD,
POST,
PUT,
DELETE_,
CONNECT,
OPTIONS,
TRACE,
PATCH,
OTHER
};
class KeyError : public std::runtime_error {};
class IndexError: public std::runtime_error {};
class RequestContext {
public:
RequestContext(struct MHD_Connection* connection,
std::string rootLocation,
const std::string& url,
const std::string& method,
const std::string& version);
~RequestContext();
void print_debug_info() const;
bool is_valid_url() const;
std::string get_header(const std::string& name) const;
template<typename T=std::string>
T get_argument(const std::string& name) const {
std::istringstream stream(arguments.at(name));
T v;
stream >> v;
return v;
}
RequestMethod get_method() const;
std::string get_url() const;
std::string get_url_part(int part) const;
std::string get_full_url() const;
bool has_range() const;
std::pair<int, int> get_range() const;
bool can_compress() const { return acceptEncodingDeflate; }
private:
std::string full_url;
std::string url;
bool valid_url;
RequestMethod method;
std::string version;
unsigned long long requestIndex;
bool acceptEncodingDeflate;
bool accept_range;
std::pair<int, int> range_pair;
std::map<std::string, std::string> headers;
std::map<std::string, std::string> arguments;
static int fill_header(void *, enum MHD_ValueKind, const char*, const char*);
static int fill_argument(void *, enum MHD_ValueKind, const char*, const char*);
};
template<> std::string RequestContext::get_argument(const std::string& name) const;
}
#endif //REQUEST_CONTEXT_H

250
src/server/response.cpp Normal file
View File

@ -0,0 +1,250 @@
#include "response.h"
#include "request_context.h"
#include "kiwixlib-resources.h"
#include "tools/regexTools.h"
#include "tools/stringTools.h"
#include "string.h"
#include <mustache.hpp>
#include <zlib.h>
#define KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE 100
namespace kiwix {
Response::Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton)
: m_verbose(verbose),
m_root(root),
m_content(""),
m_mimeType(""),
m_returnCode(MHD_HTTP_OK),
m_withTaskbar(withTaskbar),
m_withLibraryButton(withLibraryButton),
m_useCache(false),
m_addTaskbar(false),
m_bookName(""),
m_startRange(0),
m_lenRange(0)
{
}
static int print_key_value (void *cls, enum MHD_ValueKind kind,
const char *key, const char *value)
{
printf (" - %s: '%s'\n", key, value);
return MHD_YES;
}
struct RunningResponse {
kiwix::Entry entry;
int range_start;
RunningResponse(kiwix::Entry entry,
int range_start) :
entry(entry),
range_start(range_start)
{}
};
static ssize_t callback_reader_from_entry(void* cls,
uint64_t pos,
char* buf,
size_t max)
{
RunningResponse* response = static_cast<RunningResponse*>(cls);
size_t max_size_to_set = min<size_t>(
max,
response->entry.getSize() - pos - response->range_start);
if (max_size_to_set <= 0) {
return MHD_CONTENT_READER_END_WITH_ERROR;
}
zim::Blob blob = response->entry.getBlob(response->range_start+pos, max_size_to_set);
memcpy(buf, blob.data(), max_size_to_set);
return max_size_to_set;
}
static void callback_free_response(void* cls)
{
RunningResponse* response = static_cast<RunningResponse*>(cls);
delete response;
}
void print_response_info(int retCode, MHD_Response* response)
{
printf("Response :\n");
printf("httpResponseCode : %d\n", retCode);
printf("headers :\n");
MHD_get_response_headers(response, print_key_value, nullptr);
}
std::string render_template(const std::string& template_str, kainjow::mustache::data data)
{
kainjow::mustache::mustache tmpl(template_str);
kainjow::mustache::data urlencode{kainjow::mustache::lambda2{
[](const std::string& str,const kainjow::mustache::renderer& r) { return urlEncode(r(str), true); }}};
data.set("urlencoded", urlencode);
std::stringstream ss;
tmpl.render(data, [&ss](const std::string& str) { ss << str; });
return ss.str();
}
void Response::introduce_taskbar()
{
if (! m_withTaskbar)
// Taskbar is globally disabled.
return;
kainjow::mustache::data data;
data.set("root", m_root);
data.set("content", m_bookName);
data.set("hascontent", !m_bookName.empty());
data.set("title", m_bookTitle);
data.set("withlibrarybutton", m_withLibraryButton);
auto head_content = render_template(RESOURCE::templates::head_part_html, data);
m_content = appendToFirstOccurence(
m_content,
"<head>",
head_content);
auto taskbar_part = render_template(RESOURCE::templates::taskbar_part_html, data);
m_content = appendToFirstOccurence(
m_content,
"<body[^>]*>",
taskbar_part);
}
int Response::send(const RequestContext& request, MHD_Connection* connection)
{
MHD_Response* response = nullptr;
switch (m_mode) {
case ResponseMode::RAW_CONTENT : {
if (m_addTaskbar) {
introduce_taskbar();
}
bool shouldCompress = m_compress && request.can_compress();
shouldCompress &= m_mimeType.find("text/") != string::npos
|| m_mimeType.find("application/javascript") != string::npos
|| m_mimeType.find("application/json") != string::npos;
shouldCompress &= (m_content.size() > KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE);
if (shouldCompress) {
std::vector<Bytef> compr_buffer(compressBound(m_content.size()));
uLongf comprLen = compr_buffer.capacity();
int err = compress(&compr_buffer[0],
&comprLen,
(const Bytef*)(m_content.data()),
m_content.size());
if (err == Z_OK && comprLen > 2 && comprLen < (m_content.size() + 2)) {
/* /!\ Internet Explorer has a bug with deflate compression.
It can not handle the first two bytes (compression headers)
We need to chunk them off (move the content 2bytes)
It has no incidence on other browsers
See http://www.subbu.org/blog/2008/03/ie7-deflate-or-not and comments */
m_content = string((char*)&compr_buffer[2], comprLen - 2);
} else {
shouldCompress = false;
}
}
response = MHD_create_response_from_buffer(
m_content.size(), const_cast<char*>(m_content.data()), MHD_RESPMEM_MUST_COPY);
if (shouldCompress) {
MHD_add_response_header(
response, MHD_HTTP_HEADER_VARY, "Accept-Encoding");
MHD_add_response_header(
response, MHD_HTTP_HEADER_CONTENT_ENCODING, "deflate");
}
MHD_add_response_header(response, MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str());
break;
}
case ResponseMode::REDIRECTION : {
response = MHD_create_response_from_buffer(0, nullptr, MHD_RESPMEM_MUST_COPY);
MHD_add_response_header(response, MHD_HTTP_HEADER_LOCATION, m_content.c_str());
break;
}
case ResponseMode::ENTRY : {
response = MHD_create_response_from_callback(m_entry.getSize(),
16384,
callback_reader_from_entry,
new RunningResponse(m_entry, m_startRange),
callback_free_response);
MHD_add_response_header(response,
MHD_HTTP_HEADER_CONTENT_TYPE, m_mimeType.c_str());
MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes");
std::ostringstream oss;
oss << "bytes " << m_startRange << "-" << m_startRange + m_lenRange - 1
<< "/" << m_entry.getSize();
MHD_add_response_header(response,
MHD_HTTP_HEADER_CONTENT_RANGE, oss.str().c_str());
MHD_add_response_header(response,
MHD_HTTP_HEADER_CONTENT_LENGTH, kiwix::to_string(m_lenRange).c_str());
break;
}
}
MHD_add_response_header(response, "Access-Control-Allow-Origin", "*");
MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL,
m_useCache ? "max-age=2723040, public" : "no-cache, no-store, must-revalidate");
if (m_returnCode == MHD_HTTP_OK && request.has_range())
m_returnCode = MHD_HTTP_PARTIAL_CONTENT;
if (m_verbose)
print_response_info(m_returnCode, response);
auto ret = MHD_queue_response(connection, m_returnCode, response);
MHD_destroy_response(response);
return ret;
}
void Response::set_template(const std::string& template_str, kainjow::mustache::data data) {
set_content(render_template(template_str, data));
}
void Response::set_content(const std::string& content) {
m_content = content;
m_mode = ResponseMode::RAW_CONTENT;
}
void Response::set_redirection(const std::string& url) {
m_content = url;
m_mode = ResponseMode::REDIRECTION;
m_returnCode = MHD_HTTP_FOUND;
}
void Response::set_entry(const Entry& entry) {
m_entry = entry;
m_mode = ResponseMode::ENTRY;
}
void Response::set_taskbar(const std::string& bookName, const std::string& bookTitle)
{
m_addTaskbar = true;
m_bookName = bookName;
m_bookTitle = bookTitle;
}
}

89
src/server/response.h Normal file
View File

@ -0,0 +1,89 @@
/*
* Copyright 2019 Matthieu Gautier<mgautier@kymeria.fr>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
#ifndef KIWIXLIB_SERVER_RESPONSE_H
#define KIWIXLIB_SERVER_RESPONSE_H
#include <string>
#include <mustache.hpp>
#include "entry.h"
extern "C" {
#include <microhttpd.h>
}
namespace kiwix {
enum class ResponseMode {
RAW_CONTENT,
REDIRECTION,
ENTRY
};
class RequestContext;
class Response {
public:
Response(const std::string& root, bool verbose, bool withTaskbar, bool withLibraryButton);
~Response() = default;
int send(const RequestContext& request, MHD_Connection* connection);
void set_template(const std::string& template_str, kainjow::mustache::data data);
void set_content(const std::string& content);
void set_redirection(const std::string& url);
void set_entry(const Entry& entry);
void set_mimeType(const std::string& mimeType) { m_mimeType = mimeType; }
void set_code(int code) { m_returnCode = code; }
void set_cache(bool cache) { m_useCache = cache; }
void set_compress(bool compress) { m_compress = compress; }
void set_taskbar(const std::string& bookName, const std::string& bookTitle);
void set_range_first(uint64_t start) { m_startRange = start; }
void set_range_len(uint64_t len) { m_lenRange = len; }
int getReturnCode() { return m_returnCode; }
void introduce_taskbar();
private:
bool m_verbose;
ResponseMode m_mode;
std::string m_root;
std::string m_content;
Entry m_entry;
std::string m_mimeType;
int m_returnCode;
bool m_withTaskbar;
bool m_withLibraryButton;
bool m_useCache;
bool m_compress;
bool m_addTaskbar;
std::string m_bookName;
std::string m_bookTitle;
uint64_t m_startRange;
uint64_t m_lenRange;
};
}
#endif //KIWIXLIB_SERVER_RESPONSE_H

13
src/tools/otherTools.cpp
View File

@ -17,8 +17,17 @@
* MA 02110-1301, USA.
*/
#include <tools/otherTools.h>
#include "tools/otherTools.h"
#ifdef _WIN32
#include <windows.h>
#else
#include <unistd.h>
#endif
#include <map>
#include <pugixml.hpp>
static std::map<std::string, std::string> codeisomapping {
{ "aa", "aar" },
@ -185,7 +194,7 @@ struct XmlStringWriter: pugi::xml_writer
}
};
std::string kiwix::nodeToString(pugi::xml_node node)
std::string kiwix::nodeToString(const pugi::xml_node& node)
{
XmlStringWriter writer;
node.print(writer, " ");

123
src/tools/pathTools.cpp
View File

@ -17,7 +17,7 @@
* MA 02110-1301, USA.
*/
#include <tools/pathTools.h>
#include "tools/pathTools.h"
#ifdef __APPLE__
#include <limits.h>
@ -29,6 +29,17 @@
#define getcwd _getcwd // stupid MSFT "deprecation" warning
#endif
#include "tools/stringTools.h"
#include <string.h>
#include <map>
#include <vector>
#include <sys/stat.h>
#include <sstream>
#include <fstream>
#include <iomanip>
#include <iostream>
#ifdef _WIN32
const std::string SEPARATOR("\\");
#else
@ -42,7 +53,7 @@ const std::string SEPARATOR("/");
#define PATH_MAX 1024
#endif
bool isRelativePath(const string& path)
bool isRelativePath(const std::string& path)
{
#ifdef _WIN32
return path.empty() || path.substr(1, 2) == ":\\" ? false : true;
@ -51,7 +62,7 @@ bool isRelativePath(const string& path)
#endif
}
string computeRelativePath(const string path, const string absolutePath)
std::string computeRelativePath(const std::string& path, const std::string& absolutePath)
{
std::vector<std::string> pathParts = kiwix::split(path, SEPARATOR);
std::vector<std::string> absolutePathParts
@ -64,7 +75,7 @@ string computeRelativePath(const string path, const string absolutePath)
commonCount++;
}
string relativePath;
std::string relativePath;
#ifdef _WIN32
/* On Windows you have a token more because the root is represented
by a letter */
@ -84,9 +95,9 @@ string computeRelativePath(const string path, const string absolutePath)
}
/* Warning: the relative path must be with slashes */
string computeAbsolutePath(const string path, const string relativePath)
std::string computeAbsolutePath(const std::string& path, const std::string& relativePath)
{
string absolutePath;
std::string absolutePath;
if (path.empty()) {
char* path = NULL;
@ -98,7 +109,7 @@ string computeAbsolutePath(const string path, const string relativePath)
path = getcwd(path, size);
#endif
absolutePath = string(path) + SEPARATOR;
absolutePath = std::string(path) + SEPARATOR;
} else {
absolutePath = path.substr(path.length() - 1, 1) == SEPARATOR
? path
@ -113,11 +124,11 @@ string computeAbsolutePath(const string path, const string relativePath)
char* token = strtok(cRelativePath, "/");
while (token != NULL) {
if (string(token) == "..") {
if (std::string(token) == "..") {
absolutePath = removeLastPathElement(absolutePath, true, false);
token = strtok(NULL, "/");
} else if (strcmp(token, ".") && strcmp(token, "")) {
absolutePath += string(token);
absolutePath += std::string(token);
token = strtok(NULL, "/");
if (token != NULL) {
absolutePath += SEPARATOR;
@ -130,11 +141,11 @@ string computeAbsolutePath(const string path, const string relativePath)
return absolutePath;
}
string removeLastPathElement(const string path,
std::string removeLastPathElement(const std::string& path,
const bool removePreSeparator,
const bool removePostSeparator)
{
string newPath = path;
std::string newPath = path;
size_t offset = newPath.find_last_of(SEPARATOR);
if (removePreSeparator &&
#ifndef _WIN32
@ -149,18 +160,18 @@ string removeLastPathElement(const string path,
return newPath;
}
string appendToDirectory(const string& directoryPath, const string& filename)
std::string appendToDirectory(const std::string& directoryPath, const std::string& filename)
{
string newPath = directoryPath + SEPARATOR + filename;
std::string newPath = directoryPath + SEPARATOR + filename;
return newPath;
}
string getLastPathElement(const string& path)
std::string getLastPathElement(const std::string& path)
{
return path.substr(path.find_last_of(SEPARATOR) + 1);
}
unsigned int getFileSize(const string& path)
unsigned int getFileSize(const std::string& path)
{
#ifdef _WIN32
struct _stat filestatus;
@ -173,14 +184,14 @@ unsigned int getFileSize(const string& path)
return filestatus.st_size / 1024;
}
string getFileSizeAsString(const string& path)
std::string getFileSizeAsString(const std::string& path)
{
ostringstream convert;
std::ostringstream convert;
convert << getFileSize(path);
return convert.str();
}
string getFileContent(const string& path)
std::string getFileContent(const std::string& path)
{
std::ifstream f(path, std::ios::in|std::ios::ate);
std::string content;
@ -194,14 +205,14 @@ string getFileContent(const string& path)
return content;
}
bool fileExists(const string& path)
bool fileExists(const std::string& path)
{
#ifdef _WIN32
return PathFileExists(path.c_str());
#else
bool flag = false;
fstream fin;
fin.open(path.c_str(), ios::in);
std::fstream fin;
fin.open(path.c_str(), std::ios::in);
if (fin.is_open()) {
flag = true;
}
@ -210,7 +221,7 @@ bool fileExists(const string& path)
#endif
}
bool makeDirectory(const string& path)
bool makeDirectory(const std::string& path)
{
#ifdef _WIN32
int status = _mkdir(path.c_str());
@ -220,7 +231,7 @@ bool makeDirectory(const string& path)
return status == 0;
}
string makeTmpDirectory()
std::string makeTmpDirectory()
{
#ifdef _WIN32
char cbase[MAX_PATH];
@ -231,16 +242,16 @@ string makeTmpDirectory()
GetTempFileName(cbase, "kiwix", 0, ctmp);
DeleteFile(ctmp);
_mkdir(ctmp);
return string(ctmp);
return std::string(ctmp);
#else
char _template_array[] = {"/tmp/kiwix-lib_XXXXXX"};
string dir = mkdtemp(_template_array);
std::string dir = mkdtemp(_template_array);
return dir;
#endif
}
/* Try to create a link and if does not work then make a copy */
bool copyFile(const string& sourcePath, const string& destPath)
bool copyFile(const std::string& sourcePath, const std::string& destPath)
{
try {
#ifndef _WIN32
@ -252,15 +263,15 @@ bool copyFile(const string& sourcePath, const string& destPath)
#ifndef _WIN32
}
#endif
} catch (exception& e) {
cerr << e.what() << endl;
} catch (std::exception& e) {
std::cerr << e.what() << std::endl;
return false;
}
return true;
}
string getExecutablePath()
std::string getExecutablePath()
{
char binRootPath[PATH_MAX];
@ -281,7 +292,7 @@ string getExecutablePath()
return "";
}
bool writeTextFile(const string& path, const string& content)
bool writeTextFile(const std::string& path, const std::string& content)
{
std::ofstream file;
file.open(path.c_str());
@ -290,15 +301,15 @@ bool writeTextFile(const string& path, const string& content)
return true;
}
string getCurrentDirectory()
std::string getCurrentDirectory()
{
char* a_cwd = getcwd(NULL, 0);
string s_cwd(a_cwd);
std::string s_cwd(a_cwd);
free(a_cwd);
return s_cwd;
}
string getDataDirectory()
std::string getDataDirectory()
{
#ifdef _WIN32
char* cDataDir = ::getenv("APPDATA");
@ -323,3 +334,49 @@ string getDataDirectory()
#endif
return appendToDirectory(dataDir, "kiwix");
}
static std::map<std::string, std::string> extMimeTypes = {
{ "html", "text/html"},
{ "htm", "text/html"},
{ "png", "image/png"},
{ "tiff", "image/tiff"},
{ "tif", "image/tiff"},
{ "jpeg", "image/jpeg"},
{ "jpg", "image/jpeg"},
{ "gif", "image/gif"},
{ "svg", "image/svg+xml"},
{ "txt", "text/plain"},
{ "xml", "text/xml"},
{ "pdf", "application/pdf"},
{ "ogg", "application/ogg"},
{ "js", "application/javascript"},
{ "css", "text/css"},
{ "otf", "application/vnd.ms-opentype"},
{ "ttf", "application/font-ttf"},
{ "woff", "application/font-woff"},
{ "vtt", "text/vtt"}
};
/* Try to get the mimeType from the file extension */
std::string getMimeTypeForFile(const std::string& filename)
{
std::string mimeType = "text/plain";
auto pos = filename.find_last_of(".");
if (pos != std::string::npos) {
std::string extension = filename.substr(pos + 1);
auto it = extMimeTypes.find(extension);
if (it != extMimeTypes.end()) {
mimeType = it->second;
} else {
it = extMimeTypes.find(kiwix::lcAll(extension));
if (it != extMimeTypes.end()) {
mimeType = it->second;
}
}
}
return mimeType;
}

62
src/tools/regexTools.cpp
View File

@ -18,40 +18,45 @@
*/
#include <tools/regexTools.h>
#include <tools/lock.h>
std::map<std::string, icu::RegexMatcher*> regexCache;
#include <unicode/regex.h>
#include <unicode/ucnv.h>
icu::RegexMatcher* buildRegex(const std::string& regex)
#include <memory>
#include <map>
#include <pthread.h>
std::map<std::string, std::shared_ptr<icu::RegexPattern>> regexCache;
static pthread_mutex_t regexLock = PTHREAD_MUTEX_INITIALIZER;
std::unique_ptr<icu::RegexMatcher> buildMatcher(const std::string& regex, const icu::UnicodeString& content)
{
icu::RegexMatcher* matcher;
auto itr = regexCache.find(regex);
std::shared_ptr<icu::RegexPattern> pattern;
/* Regex is in cache */
if (itr != regexCache.end()) {
matcher = itr->second;
}
/* Regex needs to be parsed (and cached) */
else {
try {
pattern = regexCache.at(regex);
} catch (std::out_of_range&) {
// Redo the search with a lock to avoid race condition.
kiwix::Lock l(&regexLock);
try {
pattern = regexCache.at(regex);
} catch (std::out_of_range&) {
UErrorCode status = U_ZERO_ERROR;
UParseError pe;
icu::UnicodeString uregex(regex.c_str());
matcher = new icu::RegexMatcher(uregex, UREGEX_CASE_INSENSITIVE, status);
regexCache[regex] = matcher;
pattern.reset(icu::RegexPattern::compile(uregex, UREGEX_CASE_INSENSITIVE, pe, status));
regexCache[regex] = pattern;
}
}
UErrorCode status = U_ZERO_ERROR;
return std::unique_ptr<icu::RegexMatcher>(pattern->matcher(content, status));
}
return matcher;
}
/* todo */
void freeRegexCache()
{
}
bool matchRegex(const std::string& content, const std::string& regex)
{
ucnv_setDefaultName("UTF-8");
icu::UnicodeString ucontent(content.c_str());
auto matcher = buildRegex(regex);
matcher->reset(ucontent);
auto matcher = buildMatcher(regex, content.c_str());
return matcher->find();
}
@ -60,10 +65,9 @@ std::string replaceRegex(const std::string& content,
const std::string& regex)
{
ucnv_setDefaultName("UTF-8");
icu::UnicodeString ucontent(content.c_str());
icu::UnicodeString ureplacement(replacement.c_str());
auto matcher = buildRegex(regex);
matcher->reset(ucontent);
icu::UnicodeString ucontent(content.c_str());
auto matcher = buildMatcher(regex, ucontent);
UErrorCode status = U_ZERO_ERROR;
auto uresult = matcher->replaceAll(ureplacement, status);
std::string tmp;
@ -72,15 +76,13 @@ std::string replaceRegex(const std::string& content,
}
std::string appendToFirstOccurence(const std::string& content,
const std::string regex,
const std::string& regex,
const std::string& replacement)
{
ucnv_setDefaultName("UTF-8");
icu::UnicodeString ucontent(content.c_str());
icu::UnicodeString ureplacement(replacement.c_str());
auto matcher = buildRegex(regex);
matcher->reset(ucontent);
auto matcher = buildMatcher(regex, ucontent);
if (matcher->find()) {
UErrorCode status = U_ZERO_ERROR;
ucontent.insert(matcher->end(status), ureplacement);

15
src/tools/stringTools.cpp
View File

@ -19,6 +19,7 @@
#include <tools/stringTools.h>
#include <tools/pathTools.h>
#include <unicode/normlzr.h>
#include <unicode/rep.h>
#include <unicode/translit.h>
@ -26,6 +27,10 @@
#include <unicode/uniset.h>
#include <unicode/ustring.h>
#include <iostream>
#include <iomanip>
/* tell ICU where to find its dat file (tables) */
void kiwix::loadICUExternalTables()
{
@ -36,7 +41,7 @@ void kiwix::loadICUExternalTables()
= computeAbsolutePath(executableDirectory, "icudt58l.dat");
try {
u_setDataDirectory(datPath.c_str());
} catch (exception& e) {
} catch (std::exception& e) {
std::cerr << e.what() << std::endl;
}
#endif
@ -372,3 +377,11 @@ std::string kiwix::normalize(const std::string& word)
{
return kiwix::lcAll(word);
}
bool kiwix::startsWith(const std::string& base, const std::string& start)
{
return start.length() <= base.length()
&& std::equal(start.begin(), start.end(), base.begin());
}

1
src/xmlrpc.h
View File

@ -4,6 +4,7 @@
#define KIWIX_XMLRPC_H_
#include <tools/otherTools.h>
#include <pugixml.hpp>
namespace kiwix {

2
static/meson.build
View File

@ -7,5 +7,5 @@ lib_resources = custom_target('resources',
'--hfile', '@OUTPUT1@',
'--source_dir', '@OUTDIR@',
'@INPUT@'],
depend_files: files('search_result.tmpl')
build_always_stale: true
)

27
static/navigation_feed.tmpl Normal file
View File

@ -0,0 +1,27 @@
<?xml version="1.0" encoding="UTF-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<id>{{id}}</id>
<link rel="self"
href="{{self_url}}"
type="application/atom+xml;profile=opds-catalog;kind=navigation"/>
<link rel="start"
href="{{start_url}}"
type="application/atom+xml;profile=opds-catalog;kind=navigation"/>
<title>{{title}}</title>
<updated>{{updated}}</updated>
<author>
<name>{{author_name}}</name>
<uri>{{author_uri}}</uri>
</author>
{{#entries}}
<entry>
<title>{{title}}</title>
<link rel="subsection"
href="{{href}}"
type="application/atom+xml;profile=opds-catalog;kind=acquisition"/>
<updated>{{updated}}</updated>
<id>{{id}}</id>
</entry>
{{/entries}}
</feed>

9
static/opensearchdescription.xml Normal file
View File

@ -0,0 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/">
<ShortName>Zim catalog search</ShortName>
<Description>Search zim files in the catalog.</Description>
<Url type="application/atom+xml;profile=opds-catalog"
xmlns:atom="http://www.w3.org/2005/Atom"
indexOffset="0"
template="/{{root}}/catalog/search?q={searchTerms}&lang={language}&count={count}&start={startIndex}"/>
</OpenSearchDescription>

32
static/resources_list.txt
View File

@ -1 +1,31 @@
search_result.tmpl
skin/jquery-ui/jquery-ui.structure.min.css
skin/jquery-ui/jquery-ui.min.js
skin/jquery-ui/external/jquery/jquery.js
skin/jquery-ui/images/ui-bg_flat_0_aaaaaa_40x100.png
skin/jquery-ui/images/ui-bg_flat_75_ffffff_40x100.png
skin/jquery-ui/images/ui-icons_222222_256x240.png
skin/jquery-ui/images/ui-bg_glass_55_fbf9ee_1x400.png
skin/jquery-ui/images/ui-bg_highlight-soft_75_cccccc_1x100.png
skin/jquery-ui/images/ui-bg_glass_65_ffffff_1x400.png
skin/jquery-ui/images/ui-icons_2e83ff_256x240.png
skin/jquery-ui/images/ui-icons_cd0a0a_256x240.png
skin/jquery-ui/images/ui-icons_888888_256x240.png
skin/jquery-ui/images/ui-bg_glass_75_e6e6e6_1x400.png
skin/jquery-ui/images/animated-overlay.gif
skin/jquery-ui/images/ui-bg_glass_75_dadada_1x400.png
skin/jquery-ui/images/ui-icons_454545_256x240.png
skin/jquery-ui/images/ui-bg_glass_95_fef1ec_1x400.png
skin/jquery-ui/jquery-ui.theme.min.css
skin/jquery-ui/jquery-ui.min.css
skin/caret.png
skin/taskbar.js
skin/taskbar.css
templates/search_result.html
templates/no_search_result.html
templates/404.html
templates/500.html
templates/index.html
templates/suggestion.json
templates/head_part.html
templates/taskbar_part.html
opensearchdescription.xml

BIN
static/skin/caret.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.4 KiB

9789
static/skin/jquery-ui/external/jquery/jquery.js vendored Normal file
View File

File diff suppressed because it is too large Load Diff

BIN
static/skin/jquery-ui/images/animated-overlay.gif Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.7 KiB

BIN
static/skin/jquery-ui/images/ui-bg_flat_0_aaaaaa_40x100.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 212 B

BIN
static/skin/jquery-ui/images/ui-bg_flat_75_ffffff_40x100.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 208 B

BIN
static/skin/jquery-ui/images/ui-bg_glass_55_fbf9ee_1x400.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 335 B

BIN
static/skin/jquery-ui/images/ui-bg_glass_65_ffffff_1x400.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 207 B

BIN
static/skin/jquery-ui/images/ui-bg_glass_75_dadada_1x400.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 262 B

BIN
static/skin/jquery-ui/images/ui-bg_glass_75_e6e6e6_1x400.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 262 B

BIN
static/skin/jquery-ui/images/ui-bg_glass_95_fef1ec_1x400.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 332 B

BIN
static/skin/jquery-ui/images/ui-bg_highlight-soft_75_cccccc_1x100.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 280 B

BIN
static/skin/jquery-ui/images/ui-icons_222222_256x240.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 6.8 KiB

BIN
static/skin/jquery-ui/images/ui-icons_2e83ff_256x240.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 4.4 KiB

BIN
static/skin/jquery-ui/images/ui-icons_454545_256x240.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 6.8 KiB

BIN
static/skin/jquery-ui/images/ui-icons_888888_256x240.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 6.8 KiB

BIN
static/skin/jquery-ui/images/ui-icons_cd0a0a_256x240.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 4.4 KiB

7
static/skin/jquery-ui/jquery-ui.min.css vendored Normal file
View File

File diff suppressed because one or more lines are too long

13
static/skin/jquery-ui/jquery-ui.min.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

5
static/skin/jquery-ui/jquery-ui.structure.min.css vendored Normal file
View File

File diff suppressed because one or more lines are too long

5
static/skin/jquery-ui/jquery-ui.theme.min.css vendored Normal file
View File

File diff suppressed because one or more lines are too long

186
static/skin/taskbar.css Normal file
View File

@ -0,0 +1,186 @@
#kiwixtoolbar {
position: fixed;
padding: .5em;
left: 0;
right: 0;
top: 0;
z-index: 100;
background-position-y: 0;
transition: 0.3s;
width: 100%;
box-sizing: border-box;
}
#kiwixtoolbar>a {
float: left;
}
#kiwixfooter {
text-align: center;
margin-top: 1em;
}
.height_separator {
height: 3em;
}
.kiwixsearch {
position: relative;
height: 26px;
width: 100%;
left: 0;
margin-bottom: 0;
}
.kiwix_searchform {
width: 20em;
}
#kiwix_serve_taskbar_home_button button {
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
max-width: 160px;
}
.kiwix .kiwix_centered {
max-width: 720px;
margin: 0 auto;
}
#kiwix_button_show_toggle {
display: none;
}
#kiwix_button_show_toggle:checked~label~.kiwix_button_cont,
#kiwix_button_show_toggle:checked~label~.kiwix_button_cont>a {
display: block;
}
#kiwix_button_show_toggle:not(:checked)~label~.kiwix_button_cont {
display: none;
}
label[for="kiwix_button_show_toggle"] {
display: inline-block;
height: 26px;
}
label[for="kiwix_button_show_toggle"] img {
transition: 0.1s;
height: 26px;
}
#kiwix_button_show_toggle:checked~label img {
transform: rotate(-180deg);
}
label[for="kiwix_button_show_toggle"],
.kiwix_button_cont {
display: block;
}
.kiwix .kiwix_searchform {
float: right;
}
.kiwix #kiwixtoolbar button,
.kiwix #kiwixtoolbar input[type="submit"] {
box-sizing: border-box !important;
height: 26px !important;
line-height: 20px !important;
margin-right: 5px !important;
padding: 2px 6px !important;
border: 1px solid #999 !important;
border-radius: 3px !important;
background-color: #ededed !important;
font-weight: normal !important;
cursor: pointer !important;
font-size: 16px !important;
}
.kiwix #kiwixtoolbar #kiwixsearchform input[type='text'] {
position: absolute;
left: 0;
box-sizing: border-box !important;
width: 100%;
height: 26px !important;
line-height: 20px !important;
border: 1px solid #999 !important;
border-radius: 3px !important;
background-color: #fff !important;
padding: 2px 2px 2px 27px !important;
font-size: 16px !important;
}
label[for=kiwixsearchbox] {
z-index: 1;
position: absolute;
height: 100%;
left: 5px;
font-size: 90%;
line-height: 26px;
vertical-align: middle;
}
body {
padding-top: 40px !important;
}
/* Try to fix buggy stuff in jquery-ui autocomplete */
#ui-id-1,
.ui-autocomplete {
background: white !important;
border: solid 1px grey !important;
}
li.ui-state-focus {
font-weight: bold;
}
@media(min-width:420px) {
.kiwix_button_cont {
display: inline-block !important;
}
.kiwix_button_cont>a {
display: inline-block !important;
}
label[for="kiwix_button_show_toggle"] {
display: none;
}
}
@media (max-width: 645px) {
#kiwix_button_show_toggle~label~.kiwix_button_cont.searching {
display: none !important;
}
label[for="kiwix_button_show_toggle"].searching {
display: none !important;
}
.kiwix_searchform.full_width {
width: 100%;
}
.kiwixsearch {
float: none;
}
.kiwix_searchform {
width: 36%;
}
.height_separator {
height: 6em;
}
}
@media(max-width:415px) {
.kiwix_searchform {
width: 80%;
}
}

49
static/skin/taskbar.js Normal file
View File

@ -0,0 +1,49 @@
(function ($) {
if ($(window).width() < 520) {
var didScroll;
var lastScrollTop = 0;
var delta = 5;
// on scroll, let the interval function know the user has scrolled
$(window).scroll(function (event) {
didScroll = true;
});
// run hasScrolled() and reset didScroll status
setInterval(function () {
if (didScroll) {
hasScrolled();
didScroll = false;
}
}, 250);
function hasScrolled() {
var st = $(this).scrollTop();
// Make sure they scroll more than delta
if (Math.abs(lastScrollTop - st) <= delta)
return;
// If they scrolled down and are past the navbar, add class .nav-up.
// This is necessary so you never see what is "behind" the navbar.
if (st > lastScrollTop) {
// Scroll Down
$('#kiwixtoolbar').css({ top: '-100%' });
} else {
// Scroll Up
$('#kiwixtoolbar').css({ top: '0' });
}
lastScrollTop = st;
}
}
$('#kiwixsearchbox').on({
focus: function () {
$('.kiwix_searchform').addClass('full_width');
$('label[for="kiwix_button_show_toggle"], .kiwix_button_cont').addClass('searching');
},
blur: function () {
$('.kiwix_searchform').removeClass('full_width');
$('label[for="kiwix_button_show_toggle"], .kiwix_button_cont').removeClass('searching');
}
});
})(jQuery);

13
static/templates/404.html Normal file
View File

@ -0,0 +1,13 @@
<!DOCTYPE html>
<html>
<head>
<meta content="text/html;charset=UTF-8" http-equiv="content-type" />
<title>Content not found</title>
</head>
<body>
<h1>Not Found</h1>
<p>
The requested URL "{{url}}" was not found on this server.
</p>
</body>
</html>

16
static/templates/500.html Normal file
View File

@ -0,0 +1,16 @@
<!DOCTYPE html>
<html>
<head>
<meta content="text/html;charset=UTF-8" http-equiv="content-type" />
<title>Internal Server Error</title>
</head>
<body>
<h1>Internal Server Error</h1>
<p>
An internal server error occured. We are sorry about that :/
</p>
<p>
{{ error }}
</p>
</body>
</html>

28
static/templates/head_part.html Normal file
View File

@ -0,0 +1,28 @@
<link type="text/css" href="{{root}}/skin/jquery-ui/jquery-ui.min.css" rel="Stylesheet" />
<link type="text/css" href="{{root}}/skin/jquery-ui/jquery-ui.theme.min.css" rel="Stylesheet" />
<link type="text/css" href="{{root}}/skin/taskbar.css" rel="Stylesheet" />
<script type="text/javascript" src="{{root}}/skin/jquery-ui/external/jquery/jquery.js"></script>
<script type="text/javascript" src="{{root}}/skin/jquery-ui/jquery-ui.min.js"></script>
<script>
var jk = jQuery.noConflict();
jk(function() {
jk( "#kiwixsearchbox" ).autocomplete({
source: "{{root}}/suggest?content={{#urlencoded}}{{{content}}}{{/urlencoded}}",
dataType: "json",
cache: false,
select: function(event, ui) {
jk( "#kiwixsearchbox" ).val(ui.item.value);
jk( "#kiwixsearchform" ).submit();
},
});
});
/* cybook hack */
if (navigator.userAgent.indexOf("bookeen/cybook") != -1) {
jk("html").addClass("cybook");
}
</script>
<script type="text/javascript" src="{{root}}/skin/taskbar.js" async></script>

68
static/templates/index.html Normal file
View File

@ -0,0 +1,68 @@
<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta charset="UTF-8" />
<title>Welcome to Kiwix Server</title>
<script type="text/javascript" src="{{root}}/skin/jquery-ui/external/jquery/jquery.js"></script>
<script type="text/javascript" src="{{root}}/skin/jquery-ui/jquery-ui.min.js"></script>
<link type="text/css" href="{{root}}/skin/jquery-ui/jquery-ui.min.css" rel="Stylesheet" />
<link type="text/css" href="{{root}}/skin/jquery-ui/jquery-ui.theme.min.css" rel="Stylesheet" />
<style>
body {
background:
radial-gradient(#EEEEEE 15%, transparent 16%) 0 0,
radial-gradient(#EEEEEE 15%, transparent 16%) 8px 8px,
radial-gradient(rgba(255,255,255,.1) 15%, transparent 20%) 0 1px,
radial-gradient(rgba(255,255,255,.1) 15%, transparent 20%) 8px 9px;
background-color:#E8E8E8;
background-size:16px 16px;
margin-left: auto;
margin-right: auto;
max-width: 1100px;
}
.book__list { text-align: center; }
.book {
display: inline-block; vertical-align: bottom; margin: 8px; padding: 12px 15px; width: 300px;
border: 1px solid #ccc; border-radius: 8px;
text-align: left; color: #000; font-family: sans-serif; font-size: 13px;
background-color:#F1F1F1;
box-shadow: 2px 2px 5px 0px #ccc;
}
.book:hover { background-color: #F9F9F9; box-shadow: none;}
.book__background { background-repeat: no-repeat; background-size: auto; background-position: top right; }
.book__title {
padding: 0 55px 0 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
font-size: 18px; color: #0645ad;
}
.book__description {
padding: 5px 55px 5px 0px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
font-size: 15px;
}
.book__info { line-height: 18px; color: #777; font-weight: bold; font-size: 13px; }
</style>
<script type="text/javascript" src="{{root}}/skin/taskbar.js" async></script>
</head>
<body class="kiwix">
<div class="kiwix">
<div class='book__list'>
{{#books}}
<a href="{{root}}/{{name}}">
<div class='book'>
<div class='book__background' style="background-image: url('{{root}}/meta?content={{#urlencoded}}{{{name}}}{{/urlencoded}}&name=favicon');">
<div class='book__title' title='{{title}}'>{{title}}</div>
<div class='book__description' title='{{description}}'>{{description}}</div>
<div class='book__info'>{{articleCount}} articles, {{mediaCount}} medias</div>
</div>
</div>
</a>
{{/books}}
</div>
</div>
<div id="kiwixfooter">
Powered by <a href="https://kiwix.org">Kiwix</a>
</div>
</body>
</html>

103
static/templates/no_search_result.html Normal file
View File

@ -0,0 +1,103 @@
<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta content="text/html; charset=utf-8" http-equiv="content-type" />
<style type="text/css">
body{
color: #000000;
font: small/normal Arial,Helvetica,Sans-Serif;
margin-top: 0.5em;
font-size: 90%;
}
a{
color: #04c;
}
a:visited {
color: #639
}
a:hover {
text-decoration: underline
}
.header {
font-size: 120%;
}
ul {
margin:0;
padding:0
}
.results {
font-size: 110%;
}
.results li {
list-style-type:none;
margin-top: 0.5em;
}
.results a {
font-size: 110%;
text-decoration: underline
}
cite {
font-style:normal;
word-wrap:break-word;
display: block;
font-size: 100%;
}
.informations {
color: #388222;
font-size: 100%;
}
.footer {
padding: 0;
margin-top: 1em;
width: 100%;
float: left
}
.footer a, .footer span {
display: block;
padding: .3em .7em;
margin: 0 .38em 0 0;
text-align:center;
text-decoration: none;
}
.footer a:hover {
background: #ededed;
}
.footer ul, .footer li {
list-style:none;
margin: 0;
padding: 0;
}
.footer li {
float: left;
}
.selected {
background: #ededed;
}
</style>
<title>Fulltext search unavailable</title>
</head>
<body bgcolor="white">
<div class="header">Not found</div>
<p>
There is no article with the title <b> "{{pattern}}"</b>
and the fulltext search engine is not available for this content.
</p>
</body>
</html>

2
static/search_result.tmpl → static/templates/search_result.html
View File

@ -4,7 +4,7 @@
<meta content="text/html; charset=utf-8" http-equiv="content-type" />
<style type="text/css">
body{
color: #00000;
color: #000000;
font: small/normal Arial,Helvetica,Sans-Serif;
margin-top: 0.5em;
font-size: 90%;

7
static/templates/suggestion.json Normal file
View File

@ -0,0 +1,7 @@
[
{{#suggestions}}{{^first}},{{/first}}
{
"value" : "{{value}}",
"label" : "{{& label}}"
}{{/suggestions}}
]

26
static/templates/taskbar_part.html Normal file
View File

@ -0,0 +1,26 @@
<span class="kiwix">
<span id="kiwixtoolbar" class="ui-widget-header">
<div class="kiwix_centered">
<div class="kiwix_searchform">
<form class="kiwixsearch" method="GET" action="{{root}}/search" id="kiwixsearchform">
{{#hascontent}}<input type="hidden" name="content" value="{{content}}" />{{/hascontent}}
<label for="kiwixsearchbox">&#x1f50d;</label>
<input autocomplete="off" class="ui-autocomplete-input" id="kiwixsearchbox" name="pattern" type="text">
</form>
</div>
{{#hascontent}}
<input type="checkbox" id="kiwix_button_show_toggle">
<label for="kiwix_button_show_toggle"><img src="{{root}}/skin/caret.png" alt=""></label>
<div class="kiwix_button_cont">
{{#withlibrarybutton}}
<a id="kiwix_serve_taskbar_library_button" href="{{root}}/"><button>&#x1f3e0;</button></a>
{{/withlibrarybutton}}
<a id="kiwix_serve_taskbar_home_button" href="{{root}}/{{content}}/"><button>{{title}}</button></a>
<a id="kiwix_serve_taskbar_random_button"
href="{{root}}/random?content={{#urlencoded}}{{{content}}}{{/urlencoded}}"><button>&#x1F3B2;</button></a>
</div>
{{/hascontent}}
</div>
</span>
</span>
<div style="display: block; height: 5em;"></div>