smallsolar
/
libkiwix
mirror of https://github.com/kiwix/libkiwix.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #61 from kiwix/code_format 

Format all the code using clang-format.
This commit is contained in:
Matthieu Gautier 2017-07-11 17:24:19 +02:00 committed by GitHub
parent a205ff00c8 f76e9d2dbf
commit 80f6d0bf46
28 changed files with 2647 additions and 2190 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.clang-format Normal file
View File

@ -0,0 +1,12 @@
BasedOnStyle: Google
BinPackArguments: false
BinPackParameters: false
BreakBeforeBinaryOperators: All
BreakBeforeBraces: Linux
DerivePointerAlignment: false
SpacesInContainerLiterals: false
Standard: Cpp11
AllowShortFunctionsOnASingleLine: Inline
AllowShortIfStatementsOnASingleLine: false
AllowShortLoopsOnASingleLine: false

36
format_code.sh Executable file
View File

@ -0,0 +1,36 @@
#!/usr/bin/bash
files=(
"include/library.h"
"include/common/stringTools.h"
"include/common/pathTools.h"
"include/common/otherTools.h"
"include/common/regexTools.h"
"include/common/networkTools.h"
"include/manager.h"
"include/reader.h"
"include/kiwix.h"
"include/xapianSearcher.h"
"include/searcher.h"
"src/library.cpp"
"src/android/kiwix.cpp"
"src/android/org/kiwix/kiwixlib/JNIKiwixBool.java"
"src/android/org/kiwix/kiwixlib/JNIKiwix.java"
"src/android/org/kiwix/kiwixlib/JNIKiwixString.java"
"src/android/org/kiwix/kiwixlib/JNIKiwixInt.java"
"src/searcher.cpp"
"src/common/pathTools.cpp"
"src/common/regexTools.cpp"
"src/common/otherTools.cpp"
"src/common/networkTools.cpp"
"src/common/stringTools.cpp"
"src/xapianSearcher.cpp"
"src/manager.cpp"
"src/reader.cpp"
)
for i in "${files[@]}"
do
echo $i
clang-format -i -style=file $i
done

13
include/common/networkTools.h
View File

@ -24,23 +24,24 @@
#include <winsock2.h>
#include <ws2tcpip.h>
#else
#include <net/if.h>
#include <netdb.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <net/if.h>
#include <netdb.h>
#include <unistd.h>
#endif
#include <iostream>
#include <vector>
#include <string>
#include <map>
#include <string>
#include <vector>
namespace kiwix {
namespace kiwix
{
std::map<std::string, std::string> getNetworkInterfaces();
std::string getBestPublicIp();
}

3
include/common/otherTools.h
View File

@ -26,7 +26,8 @@
#include <unistd.h>
#endif
namespace kiwix {
namespace kiwix
{
void sleep(unsigned int milliseconds);
}

23
include/common/pathTools.h
View File

@ -20,18 +20,18 @@
#ifndef KIWIX_PATHTOOLS_H
#define KIWIX_PATHTOOLS_H
#include <fcntl.h>
#include <limits.h>
#include <stdio.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <fstream>
#include <ios>
#include <iostream>
#include <sstream>
#include <string>
#include <vector>
#include <sstream>
#include <iostream>
#include <fstream>
#include <string.h>
#include <stdio.h>
#include <sys/types.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <ios>
#include <limits.h>
#ifdef _WIN32
#include <direct.h>
@ -44,7 +44,8 @@ using namespace std;
bool isRelativePath(const string& path);
string computeAbsolutePath(const string path, const string relativePath);
string computeRelativePath(const string path, const string absolutePath);
string removeLastPathElement(const string path, const bool removePreSeparator = false,
string removeLastPathElement(const string path,
const bool removePreSeparator = false,
const bool removePostSeparator = false);
string appendToDirectory(const string& directoryPath, const string& filename);

10
include/common/regexTools.h
View File

@ -22,11 +22,15 @@
#include <unicode/regex.h>
#include <unicode/ucnv.h>
#include <string>
#include <map>
#include <string>
bool matchRegex(const std::string& content, const std::string& regex);
std::string replaceRegex(const std::string &content, const std::string &replacement, const std::string &regex);
std::string appendToFirstOccurence(const std::string &content, const std::string regex, const std::string &replacement);
std::string replaceRegex(const std::string& content,
const std::string& replacement,
const std::string& regex);
std::string appendToFirstOccurence(const std::string& content,
const std::string regex,
const std::string& replacement);
#endif

14
include/common/stringTools.h
View File

@ -22,16 +22,16 @@
#include <unicode/unistr.h>
#include <iostream>
#include <vector>
#include <string>
#include <fstream>
#include <iostream>
#include <sstream>
#include <string>
#include <vector>
#include "pathTools.h"
namespace kiwix {
namespace kiwix
{
#ifndef __ANDROID__
std::string beautifyInteger(const unsigned int number);
@ -39,7 +39,9 @@ namespace kiwix {
std::string urlEncode(const std::string& c);
void printStringInHexadecimal(const char* s);
void printStringInHexadecimal(UnicodeString s);
void stringReplacement(std::string& str, const std::string& oldStr, const std::string& newStr);
void stringReplacement(std::string& str,
const std::string& oldStr,
const std::string& newStr);
std::string encodeDiples(const std::string& str);
#endif

1
include/kiwix.h
View File

@ -22,5 +22,4 @@
#include "library.h"
#endif

19
include/library.h
View File

@ -22,24 +22,24 @@
#include <stdio.h>
#include <stdlib.h>
#include <string>
#include <string.h>
#include <vector>
#include <stack>
#include <string>
#include <vector>
#include "common/stringTools.h"
#include "common/regexTools.h"
#include "common/stringTools.h"
#define KIWIX_LIBRARY_VERSION "20110515"
using namespace std;
namespace kiwix {
namespace kiwix
{
enum supportedIndexType { UNKNOWN, XAPIAN };
class Book {
class Book
{
public:
Book();
~Book();
@ -78,8 +78,8 @@ namespace kiwix {
string faviconMimeType;
};
class Library {
class Library
{
public:
Library();
~Library();
@ -101,7 +101,6 @@ namespace kiwix {
*/
stack<string> current;
};
}
#endif

48
include/manager.h
View File

@ -20,45 +20,55 @@
#ifndef KIWIX_MANAGER_H
#define KIWIX_MANAGER_H
#include <string>
#include <sstream>
#include <time.h>
#include <sstream>
#include <string>
#include <pugixml.hpp>
#include "common/base64.h"
#include "common/regexTools.h"
#include "common/pathTools.h"
#include "common/regexTools.h"
#include "library.h"
#include "reader.h"
using namespace std;
namespace kiwix {
namespace kiwix
{
enum supportedListMode { LASTOPEN, REMOTE, LOCAL };
enum supportedListSortBy { TITLE, SIZE, DATE, CREATOR, PUBLISHER };
class Manager {
class Manager
{
public:
Manager();
~Manager();
bool readFile(const string path, const bool readOnly = true);
bool readFile(const string nativePath, const string UTF8Path, const bool readOnly = true);
bool readXml(const string xml, const bool readOnly = true, const string libraryPath = "");
bool readFile(const string nativePath,
const string UTF8Path,
const bool readOnly = true);
bool readXml(const string xml,
const bool readOnly = true,
const string libraryPath = "");
bool writeFile(const string path);
bool removeBookByIndex(const unsigned int bookIndex);
bool removeBookById(const string id);
bool setCurrentBookId(const string id);
string getCurrentBookId();
bool setBookIndex(const string id, const string path, const supportedIndexType type);
bool setBookIndex(const string id,
const string path,
const supportedIndexType type);
bool setBookIndex(const string id, const string path);
bool setBookPath(const string id, const string path);
string addBookFromPathAndGetId(const string pathToOpen, const string pathToSave = "", const string url = "",
string addBookFromPathAndGetId(const string pathToOpen,
const string pathToSave = "",
const string url = "",
const bool checkMetaData = false);
bool addBookFromPath(const string pathToOpen, const string pathToSave = "", const string url = "",
bool addBookFromPath(const string pathToOpen,
const string pathToSave = "",
const string url = "",
const bool checkMetaData = false);
Library cloneLibrary();
bool getBookById(const string id, Book& book);
@ -66,8 +76,13 @@ namespace kiwix {
unsigned int getBookCount(const bool localBooks, const bool remoteBooks);
bool updateBookLastOpenDateById(const string id);
void removeBookPaths();
bool listBooks(const supportedListMode mode, const supportedListSortBy sortBy, const unsigned int maxSize,
const string language, const string creator, const string publisher, const string search);
bool listBooks(const supportedListMode mode,
const supportedListSortBy sortBy,
const unsigned int maxSize,
const string language,
const string creator,
const string publisher,
const string search);
vector<string> getBooksLanguages();
vector<string> getBooksCreators();
vector<string> getBooksPublishers();
@ -81,12 +96,13 @@ namespace kiwix {
kiwix::Library library;
bool readBookFromPath(const string path, Book* book = NULL);
bool parseXmlDom(const pugi::xml_document &doc, const bool readOnly, const string libraryPath);
bool parseXmlDom(const pugi::xml_document& doc,
const bool readOnly,
const string libraryPath);
private:
void checkAndCleanBookPaths(Book& book, const string& libraryPath);
};
}
#endif

58
include/reader.h
View File

@ -20,24 +20,24 @@
#ifndef KIWIX_READER_H
#define KIWIX_READER_H
#include <zim/zim.h>
#include <zim/file.h>
#include <zim/article.h>
#include <zim/fileiterator.h>
#include <stdio.h>
#include <string>
#include <zim/article.h>
#include <zim/file.h>
#include <zim/fileiterator.h>
#include <zim/zim.h>
#include <exception>
#include <sstream>
#include <map>
#include <sstream>
#include <string>
#include "common/pathTools.h"
#include "common/stringTools.h"
using namespace std;
namespace kiwix {
class Reader {
namespace kiwix
{
class Reader
{
public:
Reader(const string zimFilePath);
~Reader();
@ -64,13 +64,33 @@ namespace kiwix {
bool getFavicon(string& content, string& mimeType) const;
bool getPageUrlFromTitle(const string& title, string& url) const;
bool getMimeTypeByUrl(const string& url, string& mimeType) const;
bool getContentByUrl(const string &url, string &content, unsigned int &contentLength, string &contentType) const;
bool getContentByEncodedUrl(const string &url, string &content, unsigned int &contentLength, string &contentType, string &baseUrl) const;
bool getContentByEncodedUrl(const string &url, string &content, unsigned int &contentLength, string &contentType) const;
bool getContentByDecodedUrl(const string &url, string &content, unsigned int &contentLength, string &contentType, string &baseUrl) const;
bool getContentByDecodedUrl(const string &url, string &content, unsigned int &contentLength, string &contentType) const;
bool searchSuggestions(const string &prefix, unsigned int suggestionsCount, const bool reset = true);
bool searchSuggestionsSmart(const string &prefix, unsigned int suggestionsCount);
bool getContentByUrl(const string& url,
string& content,
unsigned int& contentLength,
string& contentType) const;
bool getContentByEncodedUrl(const string& url,
string& content,
unsigned int& contentLength,
string& contentType,
string& baseUrl) const;
bool getContentByEncodedUrl(const string& url,
string& content,
unsigned int& contentLength,
string& contentType) const;
bool getContentByDecodedUrl(const string& url,
string& content,
unsigned int& contentLength,
string& contentType,
string& baseUrl) const;
bool getContentByDecodedUrl(const string& url,
string& content,
unsigned int& contentLength,
string& contentType) const;
bool searchSuggestions(const string& prefix,
unsigned int suggestionsCount,
const bool reset = true);
bool searchSuggestionsSmart(const string& prefix,
unsigned int suggestionsCount);
bool urlExists(const string& url) const;
bool hasFulltextIndex() const;
std::vector<std::string> getTitleVariants(const std::string& title) const;
@ -81,7 +101,8 @@ namespace kiwix {
bool parseUrl(const string& url, char* ns, string& title) const;
unsigned int getFileSize() const;
zim::File* getZimFileHandler() const;
bool getArticleObjectByDecodedUrl(const string &url, zim::Article &article) const;
bool getArticleObjectByDecodedUrl(const string& url,
zim::Article& article) const;
protected:
zim::File* zimFileHandler;
@ -98,7 +119,6 @@ namespace kiwix {
private:
std::map<const std::string, unsigned int> parseCounterMetadata() const;
};
}
#endif

32
include/searcher.h
View File

@ -22,22 +22,24 @@
#include <stdio.h>
#include <stdlib.h>
#include <string>
#include <unicode/putil.h>
#include <algorithm>
#include <vector>
#include <locale>
#include <cctype>
#include <locale>
#include <string>
#include <vector>
#include <vector>
#include "common/pathTools.h"
#include "common/stringTools.h"
#include <unicode/putil.h>
#include "kiwix_config.h"
using namespace std;
namespace kiwix {
namespace kiwix
{
class Reader;
class Result {
class Result
{
public:
virtual ~Result(){};
virtual std::string get_url() = 0;
@ -48,16 +50,17 @@ namespace kiwix {
virtual int get_size() = 0;
};
struct SearcherInternal;
class Searcher {
class Searcher
{
public:
Searcher(const string& xapianDirectoryPath, Reader* reader);
~Searcher();
void search(std::string &search, unsigned int resultStart,
unsigned int resultEnd, const bool verbose=false);
void search(std::string& search,
unsigned int resultStart,
unsigned int resultEnd,
const bool verbose = false);
Result* getNextResult();
void restart_search();
unsigned int getEstimatedResultCount();
@ -73,8 +76,10 @@ namespace kiwix {
protected:
std::string beautifyInteger(const unsigned int number);
void closeIndex();
void searchInIndex(string &search, const unsigned int resultStart,
const unsigned int resultEnd, const bool verbose=false);
void searchInIndex(string& search,
const unsigned int resultStart,
const unsigned int resultEnd,
const bool verbose = false);
Reader* reader;
SearcherInternal* internal;
@ -88,7 +93,6 @@ namespace kiwix {
unsigned int resultEnd;
std::string contentHumanReadableId;
};
}
#endif

24
include/xapianSearcher.h
View File

@ -21,19 +21,20 @@
#define KIWIX_XAPIAN_SEARCHER_H
#include <xapian.h>
#include "searcher.h"
#include "reader.h"
#include "searcher.h"
#include <map>
#include <string>
using namespace std;
namespace kiwix {
namespace kiwix
{
class XapianSearcher;
class XapianResult : public Result {
class XapianResult : public Result
{
public:
XapianResult(XapianSearcher* searcher, Xapian::MSetIterator& iterator);
virtual ~XapianResult(){};
@ -51,18 +52,24 @@ namespace kiwix {
Xapian::Document document;
};
class NoXapianIndexInZim: public exception {
virtual const char* what() const throw() {
class NoXapianIndexInZim : public exception
{
virtual const char* what() const throw()
{
return "There is no fulltext index in the zim file";
}
};
class XapianSearcher {
class XapianSearcher
{
friend class XapianResult;
public:
XapianSearcher(const string& xapianDirectoryPath, Reader* reader);
virtual ~XapianSearcher(){};
void searchInIndex(string &search, const unsigned int resultStart, const unsigned int resultEnd,
void searchInIndex(string& search,
const unsigned int resultStart,
const unsigned int resultEnd,
const bool verbose = false);
virtual Result* getNextResult();
void restart_search();
@ -84,7 +91,6 @@ namespace kiwix {
Xapian::MSetIterator current_result;
std::map<std::string, int> valuesmap;
};
}
#endif

165
src/android/kiwix.cpp
View File

@ -7,19 +7,19 @@
#include <iostream>
#include <string>
#include "unicode/putil.h"
#include "common/base64.h"
#include "reader.h"
#include "searcher.h"
#include "common/base64.h"
#include "unicode/putil.h"
#include <android/log.h>
#define LOGI(...) __android_log_print(ANDROID_LOG_INFO, "kiwix", __VA_ARGS__)
#include <xapian.h>
#include <zim/zim.h>
#include <zim/file.h>
#include <zim/article.h>
#include <zim/error.h>
#include <zim/file.h>
#include <zim/zim.h>
/* global variables */
kiwix::Reader* reader = NULL;
@ -29,56 +29,63 @@ static pthread_mutex_t readerLock = PTHREAD_MUTEX_INITIALIZER;
static pthread_mutex_t searcherLock = PTHREAD_MUTEX_INITIALIZER;
/* c2jni type conversion functions */
jboolean c2jni(const bool &val) {
jboolean c2jni(const bool& val)
{
return val ? JNI_TRUE : JNI_FALSE;
}
jstring c2jni(const std::string &val, JNIEnv *env) {
jstring c2jni(const std::string& val, JNIEnv* env)
{
return env->NewStringUTF(val.c_str());
}
jint c2jni(const int val) {
jint c2jni(const int val)
{
return (jint)val;
}
jint c2jni(const unsigned val) {
jint c2jni(const unsigned val)
{
return (unsigned)val;
}
/* jni2c type conversion functions */
bool jni2c(const jboolean &val) {
bool jni2c(const jboolean& val)
{
return val == JNI_TRUE;
}
std::string jni2c(const jstring &val, JNIEnv *env) {
std::string jni2c(const jstring& val, JNIEnv* env)
{
return std::string(env->GetStringUTFChars(val, 0));
}
int jni2c(const jint val) {
int jni2c(const jint val)
{
return (int)val;
}
/* Method to deal with variable passed by reference */
void setStringObjValue(const std::string &value, const jobject obj, JNIEnv *env) {
void setStringObjValue(const std::string& value, const jobject obj, JNIEnv* env)
{
jclass objClass = env->GetObjectClass(obj);
jfieldID objFid = env->GetFieldID(objClass, "value", "Ljava/lang/String;");
env->SetObjectField(obj, objFid, c2jni(value, env));
}
void setIntObjValue(const int value, const jobject obj, JNIEnv *env) {
void setIntObjValue(const int value, const jobject obj, JNIEnv* env)
{
jclass objClass = env->GetObjectClass(obj);
jfieldID objFid = env->GetFieldID(objClass, "value", "I");
env->SetIntField(obj, objFid, value);
}
void setBoolObjValue(const bool value, const jobject obj, JNIEnv *env) {
void setBoolObjValue(const bool value, const jobject obj, JNIEnv* env)
{
jclass objClass = env->GetObjectClass(obj);
jfieldID objFid = env->GetFieldID(objClass, "value", "Z");
env->SetIntField(obj, objFid, c2jni(value));
}
/* Kiwix library functions */
JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getMainPage(JNIEnv *env, jobject obj) {
JNIEXPORT jstring JNICALL
Java_org_kiwix_kiwixlib_JNIKiwix_getMainPage(JNIEnv* env, jobject obj)
{
jstring url;
pthread_mutex_lock(&readerLock);
@ -95,7 +102,9 @@ JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getMainPage(JNIEnv *e
return url;
}
JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getId(JNIEnv *env, jobject obj) {
JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getId(JNIEnv* env,
jobject obj)
{
jstring id;
pthread_mutex_lock(&readerLock);
@ -112,7 +121,9 @@ JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getId(JNIEnv *env, jo
return id;
}
JNIEXPORT jint JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getFileSize(JNIEnv *env, jobject obj) {
JNIEXPORT jint JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getFileSize(JNIEnv* env,
jobject obj)
{
jint size;
pthread_mutex_lock(&readerLock);
@ -129,7 +140,9 @@ JNIEXPORT jint JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getFileSize(JNIEnv *env,
return size;
}
JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getCreator(JNIEnv *env, jobject obj) {
JNIEXPORT jstring JNICALL
Java_org_kiwix_kiwixlib_JNIKiwix_getCreator(JNIEnv* env, jobject obj)
{
jstring creator;
pthread_mutex_lock(&readerLock);
@ -146,7 +159,9 @@ JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getCreator(JNIEnv *en
return creator;
}
JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getPublisher(JNIEnv *env, jobject obj) {
JNIEXPORT jstring JNICALL
Java_org_kiwix_kiwixlib_JNIKiwix_getPublisher(JNIEnv* env, jobject obj)
{
jstring publisher;
pthread_mutex_lock(&readerLock);
@ -163,7 +178,9 @@ JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getPublisher(JNIEnv *
return publisher;
}
JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getName(JNIEnv *env, jobject obj) {
JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getName(JNIEnv* env,
jobject obj)
{
jstring name;
pthread_mutex_lock(&readerLock);
@ -180,8 +197,9 @@ JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getName(JNIEnv *env,
return name;
}
JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getFavicon(JNIEnv *env, jobject obj) {
JNIEXPORT jstring JNICALL
Java_org_kiwix_kiwixlib_JNIKiwix_getFavicon(JNIEnv* env, jobject obj)
{
jstring favicon;
pthread_mutex_lock(&readerLock);
@ -190,7 +208,11 @@ JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getFavicon(JNIEnv *en
std::string cContent;
std::string cMime;
reader->getFavicon(cContent, cMime);
favicon = c2jni(base64_encode(reinterpret_cast<const unsigned char*>(cContent.c_str()), cContent.length()), env);
favicon
= c2jni(base64_encode(
reinterpret_cast<const unsigned char*>(cContent.c_str()),
cContent.length()),
env);
} catch (...) {
std::cerr << "Unable to get ZIM favicon" << std::endl;
}
@ -200,7 +222,9 @@ JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getFavicon(JNIEnv *en
return favicon;
}
JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getDate(JNIEnv *env, jobject obj) {
JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getDate(JNIEnv* env,
jobject obj)
{
jstring date;
pthread_mutex_lock(&readerLock);
@ -217,7 +241,9 @@ JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getDate(JNIEnv *env,
return date;
}
JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getLanguage(JNIEnv *env, jobject obj) {
JNIEXPORT jstring JNICALL
Java_org_kiwix_kiwixlib_JNIKiwix_getLanguage(JNIEnv* env, jobject obj)
{
jstring language;
pthread_mutex_lock(&readerLock);
@ -234,7 +260,9 @@ JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getLanguage(JNIEnv *e
return language;
}
JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getMimeType(JNIEnv *env, jobject obj, jstring url) {
JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getMimeType(
JNIEnv* env, jobject obj, jstring url)
{
jstring mimeType;
pthread_mutex_lock(&readerLock);
@ -253,13 +281,17 @@ JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getMimeType(JNIEnv *e
return mimeType;
}
JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_loadZIM(JNIEnv *env, jobject obj, jstring path) {
JNIEXPORT jboolean JNICALL
Java_org_kiwix_kiwixlib_JNIKiwix_loadZIM(JNIEnv* env, jobject obj, jstring path)
{
jboolean retVal = JNI_TRUE;
std::string cPath = jni2c(path, env);
pthread_mutex_lock(&readerLock);
try {
if (reader != NULL) delete reader;
if (reader != NULL) {
delete reader;
}
reader = new kiwix::Reader(cPath);
} catch (...) {
std::cerr << "Unable to load ZIM " << cPath << std::endl;
@ -271,8 +303,9 @@ JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_loadZIM(JNIEnv *env,
return retVal;
}
JNIEXPORT jbyteArray JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getContent(JNIEnv *env, jobject obj, jstring url, jobject mimeTypeObj, jobject sizeObj) {
JNIEXPORT jbyteArray JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getContent(
JNIEnv* env, jobject obj, jstring url, jobject mimeTypeObj, jobject sizeObj)
{
/* Default values */
setStringObjValue("", mimeTypeObj, env);
setIntObjValue(0, sizeObj, env);
@ -289,7 +322,8 @@ JNIEXPORT jbyteArray JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getContent(JNIEnv
try {
if (reader->getContentByUrl(cUrl, cData, cSize, cMimeType)) {
data = env->NewByteArray(cSize);
env->SetByteArrayRegion(data, 0, cSize, reinterpret_cast<const jbyte*>(cData.c_str()));
env->SetByteArrayRegion(
data, 0, cSize, reinterpret_cast<const jbyte*>(cData.c_str()));
setStringObjValue(cMimeType, mimeTypeObj, env);
setIntObjValue(cSize, sizeObj, env);
}
@ -302,8 +336,9 @@ JNIEXPORT jbyteArray JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getContent(JNIEnv
return data;
}
JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_searchSuggestions
(JNIEnv *env, jobject obj, jstring prefix, jint count) {
JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_searchSuggestions(
JNIEnv* env, jobject obj, jstring prefix, jint count)
{
jboolean retVal = JNI_FALSE;
std::string cPrefix = jni2c(prefix, env);
unsigned int cCount = jni2c(count);
@ -316,15 +351,17 @@ JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_searchSuggestions
}
}
} catch (...) {
std::cerr << "Unable to search suggestions for pattern " << cPrefix << std::endl;
std::cerr << "Unable to search suggestions for pattern " << cPrefix
<< std::endl;
}
pthread_mutex_unlock(&readerLock);
return retVal;
}
JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getNextSuggestion
(JNIEnv *env, jobject obj, jobject titleObj) {
JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getNextSuggestion(
JNIEnv* env, jobject obj, jobject titleObj)
{
jboolean retVal = JNI_FALSE;
std::string cTitle;
@ -344,8 +381,9 @@ JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getNextSuggestion
return retVal;
}
JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getPageUrlFromTitle
(JNIEnv *env, jobject obj, jstring title, jobject urlObj) {
JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getPageUrlFromTitle(
JNIEnv* env, jobject obj, jstring title, jobject urlObj)
{
jboolean retVal = JNI_FALSE;
std::string cTitle = jni2c(title, env);
std::string cUrl;
@ -366,8 +404,9 @@ JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getPageUrlFromTitle
return retVal;
}
JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getTitle
(JNIEnv *env , jobject obj, jobject titleObj) {
JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getTitle(
JNIEnv* env, jobject obj, jobject titleObj)
{
jboolean retVal = JNI_FALSE;
std::string cTitle;
@ -384,10 +423,11 @@ JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getTitle
pthread_mutex_unlock(&readerLock);
return retVal;
}
JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getDescription(JNIEnv *env, jobject obj) {
JNIEXPORT jstring JNICALL
Java_org_kiwix_kiwixlib_JNIKiwix_getDescription(JNIEnv* env, jobject obj)
{
jstring description;
pthread_mutex_lock(&readerLock);
@ -404,8 +444,9 @@ JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getDescription(JNIEnv
return description;
}
JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getRandomPage
(JNIEnv *env, jobject obj, jobject urlObj) {
JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getRandomPage(
JNIEnv* env, jobject obj, jobject urlObj)
{
jboolean retVal = JNI_FALSE;
std::string cUrl;
@ -424,8 +465,9 @@ JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_getRandomPage
return retVal;
}
JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_setDataDirectory
(JNIEnv *env, jobject obj, jstring dirStr) {
JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_setDataDirectory(
JNIEnv* env, jobject obj, jstring dirStr)
{
std::string cPath = jni2c(dirStr, env);
pthread_mutex_lock(&readerLock);
@ -437,14 +479,18 @@ JNIEXPORT void JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_setDataDirectory
pthread_mutex_unlock(&readerLock);
}
JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_loadFulltextIndex(JNIEnv *env, jobject obj, jstring path) {
JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_loadFulltextIndex(
JNIEnv* env, jobject obj, jstring path)
{
jboolean retVal = JNI_TRUE;
std::string cPath = jni2c(path, env);
pthread_mutex_lock(&searcherLock);
searcher = NULL;
try {
if (searcher != NULL) delete searcher;
if (searcher != NULL) {
delete searcher;
}
searcher = new kiwix::Searcher(cPath, reader);
} catch (...) {
searcher = NULL;
@ -456,8 +502,9 @@ JNIEXPORT jboolean JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_loadFulltextIndex(JN
return retVal;
}
JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_indexedQuery
(JNIEnv *env, jclass obj, jstring query, jint count) {
JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_indexedQuery(
JNIEnv* env, jclass obj, jstring query, jint count)
{
std::string cQuery = jni2c(query, env);
unsigned int cCount = jni2c(count);
kiwix::Result* p_result;
@ -467,9 +514,9 @@ JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_indexedQuery
try {
if (searcher != NULL) {
searcher->search(cQuery, 0, count);
while ( (p_result = searcher->getNextResult()) &&
!(p_result->get_title().empty()) &&
!(p_result->get_url().empty())) {
while ((p_result = searcher->getNextResult())
&& !(p_result->get_title().empty())
&& !(p_result->get_url().empty())) {
result += p_result->get_title() + "\n";
delete p_result;
}
@ -481,5 +528,3 @@ JNIEXPORT jstring JNICALL Java_org_kiwix_kiwixlib_JNIKiwix_indexedQuery
return env->NewStringUTF(result.c_str());
}

9
src/android/org/kiwix/kiwixlib/JNIKiwix.java
View File

@ -23,12 +23,9 @@ import org.kiwix.kiwixlib.JNIKiwixString;
import org.kiwix.kiwixlib.JNIKiwixBool;
import org.kiwix.kiwixlib.JNIKiwixInt;
public class JNIKiwix {
static {
System.loadLibrary("kiwix");
}
public class JNIKiwix
{
static { System.loadLibrary("kiwix"); }
public native String getMainPage();
public native String getId();

4
src/android/org/kiwix/kiwixlib/JNIKiwixBool.java
View File

@ -19,7 +19,7 @@
package org.kiwix.kiwixlib;
public class JNIKiwixBool {
public class JNIKiwixBool
{
public boolean value;
}

5
src/android/org/kiwix/kiwixlib/JNIKiwixInt.java
View File

@ -19,8 +19,7 @@
package org.kiwix.kiwixlib;
public class JNIKiwixInt {
public class JNIKiwixInt
{
public int value;
}

4
src/android/org/kiwix/kiwixlib/JNIKiwixString.java
View File

@ -19,7 +19,7 @@
package org.kiwix.kiwixlib;
public class JNIKiwixString {
public class JNIKiwixString
{
public String value;
}

71
src/common/networkTools.cpp
View File

@ -19,23 +19,32 @@
#include <common/networkTools.h>
std::map<std::string, std::string> kiwix::getNetworkInterfaces() {
std::map<std::string, std::string> kiwix::getNetworkInterfaces()
{
std::map<std::string, std::string> interfaces;
#ifdef _WIN32
SOCKET sd = WSASocket(AF_INET, SOCK_DGRAM, 0, 0, 0, 0);
if (sd == SOCKET_ERROR) {
std::cerr << "Failed to get a socket. Error " << WSAGetLastError() <<
std::endl;
std::cerr << "Failed to get a socket. Error " << WSAGetLastError()
<< std::endl;
return interfaces;
}
INTERFACE_INFO InterfaceList[20];
unsigned long nBytesReturned;
if (WSAIoctl(sd, SIO_GET_INTERFACE_LIST, 0, 0, &InterfaceList,
sizeof(InterfaceList), &nBytesReturned, 0, 0) == SOCKET_ERROR) {
std::cerr << "Failed calling WSAIoctl: error " << WSAGetLastError() <<
std::endl;
if (WSAIoctl(sd,
SIO_GET_INTERFACE_LIST,
0,
0,
&InterfaceList,
sizeof(InterfaceList),
&nBytesReturned,
0,
0)
== SOCKET_ERROR) {
std::cerr << "Failed calling WSAIoctl: error " << WSAGetLastError()
<< std::endl;
return interfaces;
}
@ -47,7 +56,8 @@ std::map<std::string, std::string> kiwix::getNetworkInterfaces() {
/* Add to the map */
std::string interfaceName = std::string(inet_ntoa(pAddress->sin_addr));
std::string interfaceIp = std::string(inet_ntoa(pAddress->sin_addr));
interfaces.insert(std::pair<std::string, std::string>(interfaceName, interfaceIp));
interfaces.insert(
std::pair<std::string, std::string>(interfaceName, interfaceIp));
}
#else
/* Get Network interfaces information */
@ -70,14 +80,19 @@ std::map<std::string, std::string> kiwix::getNetworkInterfaces() {
if (ifreq->ifr_addr.sa_family == AF_INET) {
/* Get the network interface ip */
char host[128] = {0};
const int error = getnameinfo(&(ifreq->ifr_addr), sizeof ifreq->ifr_addr,
host, sizeof host,
0, 0, NI_NUMERICHOST);
const int error = getnameinfo(&(ifreq->ifr_addr),
sizeof ifreq->ifr_addr,
host,
sizeof host,
0,
0,
NI_NUMERICHOST);
if (!error) {
std::string interfaceName = std::string(ifreq->ifr_name);
std::string interfaceIp = std::string(host);
/* Add to the map */
interfaces.insert(std::pair<std::string, std::string>(interfaceName, interfaceIp));
interfaces.insert(
std::pair<std::string, std::string>(interfaceName, interfaceIp));
} else {
perror("getnameinfo()");
}
@ -97,41 +112,49 @@ std::map<std::string, std::string> kiwix::getNetworkInterfaces() {
return interfaces;
}
std::string kiwix::getBestPublicIp() {
std::string kiwix::getBestPublicIp()
{
std::map<std::string, std::string> interfaces = kiwix::getNetworkInterfaces();
#ifndef _WIN32
const char* const prioritizedNames[] =
{ "eth0", "eth1", "wlan0", "wlan1", "en0", "en1" };
const char* const prioritizedNames[]
= {"eth0", "eth1", "wlan0", "wlan1", "en0", "en1"};
const int count = (sizeof prioritizedNames) / (sizeof prioritizedNames[0]);
for (int i = 0; i < count; ++i) {
std::map<std::string, std::string>::const_iterator it =
interfaces.find(prioritizedNames[i]);
if (it != interfaces.end())
std::map<std::string, std::string>::const_iterator it
= interfaces.find(prioritizedNames[i]);
if (it != interfaces.end()) {
return it->second;
}
}
#endif
for (std::map<std::string, std::string>::iterator iter = interfaces.begin();
iter != interfaces.end(); ++iter) {
iter != interfaces.end();
++iter) {
std::string interfaceIp = iter->second;
if (interfaceIp.length() >= 7 && interfaceIp.substr(0, 7) == "192.168")
if (interfaceIp.length() >= 7 && interfaceIp.substr(0, 7) == "192.168") {
return interfaceIp;
}
}
for (std::map<std::string, std::string>::iterator iter = interfaces.begin();
iter != interfaces.end(); ++iter) {
iter != interfaces.end();
++iter) {
std::string interfaceIp = iter->second;
if (interfaceIp.length() >= 7 && interfaceIp.substr(0, 7) == "172.16.")
if (interfaceIp.length() >= 7 && interfaceIp.substr(0, 7) == "172.16.") {
return interfaceIp;
}
}
for (std::map<std::string, std::string>::iterator iter = interfaces.begin();
iter != interfaces.end(); ++iter) {
iter != interfaces.end();
++iter) {
std::string interfaceIp = iter->second;
if (interfaceIp.length() >= 3 && interfaceIp.substr(0, 3) == "10.")
if (interfaceIp.length() >= 3 && interfaceIp.substr(0, 3) == "10.") {
return interfaceIp;
}
}
return "127.0.0.1";
}

3
src/common/otherTools.cpp
View File

@ -19,7 +19,8 @@
#include <common/otherTools.h>
void kiwix::sleep(unsigned int milliseconds) {
void kiwix::sleep(unsigned int milliseconds)
{
#ifdef _WIN32
Sleep(milliseconds);
#else

70
src/common/pathTools.cpp
View File

@ -20,12 +20,12 @@
#include <common/pathTools.h>
#ifdef __APPLE__
#include <mach-o/dyld.h>
#include <limits.h>
#include <mach-o/dyld.h>
#elif _WIN32
#include <direct.h>
#include <windows.h>
#include "shlwapi.h"
#include <direct.h>
#define getcwd _getcwd // stupid MSFT "deprecation" warning
#endif
@ -47,7 +47,8 @@
#define PATH_MAX 1024
#endif
bool isRelativePath(const string &path) {
bool isRelativePath(const string& path)
{
#ifdef _WIN32
return path.empty() || path.substr(1, 2) == ":\\" ? false : true;
#else
@ -55,14 +56,16 @@ bool isRelativePath(const string &path) {
#endif
}
string computeRelativePath(const string path, const string absolutePath) {
string computeRelativePath(const string path, const string absolutePath)
{
std::vector<std::string> pathParts = kiwix::split(path, SEPARATOR);
std::vector<std::string> absolutePathParts = kiwix::split(absolutePath, SEPARATOR);
std::vector<std::string> absolutePathParts
= kiwix::split(absolutePath, SEPARATOR);
unsigned int commonCount = 0;
while (commonCount < pathParts.size() &&
commonCount < absolutePathParts.size() &&
pathParts[commonCount] == absolutePathParts[commonCount]) {
while (commonCount < pathParts.size()
&& commonCount < absolutePathParts.size()
&& pathParts[commonCount] == absolutePathParts[commonCount]) {
if (!pathParts[commonCount].empty()) {
commonCount++;
}
@ -89,7 +92,8 @@ string computeRelativePath(const string path, const string absolutePath) {
}
/* Warning: the relative path must be with slashes */
string computeAbsolutePath(const string path, const string relativePath) {
string computeAbsolutePath(const string path, const string relativePath)
{
string absolutePath;
if (path.empty()) {
@ -104,7 +108,9 @@ string computeAbsolutePath(const string path, const string relativePath) {
absolutePath = string(path) + SEPARATOR;
} else {
absolutePath = path.substr(path.length() - 1, 1) == SEPARATOR ? path : path + SEPARATOR;
absolutePath = path.substr(path.length() - 1, 1) == SEPARATOR
? path
: path + SEPARATOR;
}
#if _WIN32
@ -121,8 +127,9 @@ string computeAbsolutePath(const string path, const string relativePath) {
} else if (strcmp(token, ".") && strcmp(token, "")) {
absolutePath += string(token);
token = strtok(NULL, "/");
if (token != NULL)
if (token != NULL) {
absolutePath += SEPARATOR;
}
} else {
token = strtok(NULL, "/");
}
@ -131,7 +138,10 @@ string computeAbsolutePath(const string path, const string relativePath) {
return absolutePath;
}
string removeLastPathElement(const string path, const bool removePreSeparator, const bool removePostSeparator) {
string removeLastPathElement(const string path,
const bool removePreSeparator,
const bool removePostSeparator)
{
string newPath = path;
size_t offset = newPath.find_last_of(SEPARATOR);
if (removePreSeparator &&
@ -142,20 +152,24 @@ string removeLastPathElement(const string path, const bool removePreSeparator, c
newPath = newPath.substr(0, offset);
offset = newPath.find_last_of(SEPARATOR);
}
newPath = removePostSeparator ? newPath.substr(0, offset) : newPath.substr(0, offset+1);
newPath = removePostSeparator ? newPath.substr(0, offset)
: newPath.substr(0, offset + 1);
return newPath;
}
string appendToDirectory(const string &directoryPath, const string &filename) {
string appendToDirectory(const string& directoryPath, const string& filename)
{
string newPath = directoryPath + SEPARATOR + filename;
return newPath;
}
string getLastPathElement(const string &path) {
string getLastPathElement(const string& path)
{
return path.substr(path.find_last_of(SEPARATOR) + 1);
}
unsigned int getFileSize(const string &path) {
unsigned int getFileSize(const string& path)
{
#ifdef _WIN32
struct _stat filestatus;
_stat(path.c_str(), &filestatus);
@ -167,12 +181,15 @@ unsigned int getFileSize(const string &path) {
return filestatus.st_size / 1024;
}
string getFileSizeAsString(const string &path) {
ostringstream convert; convert << getFileSize(path);
string getFileSizeAsString(const string& path)
{
ostringstream convert;
convert << getFileSize(path);
return convert.str();
}
bool fileExists(const string &path) {
bool fileExists(const string& path)
{
#ifdef _WIN32
return PathFileExists(path.c_str());
#else
@ -187,7 +204,8 @@ bool fileExists(const string &path) {
#endif
}
bool makeDirectory(const string &path) {
bool makeDirectory(const string& path)
{
#ifdef _WIN32
int status = _mkdir(path.c_str());
#else
@ -197,7 +215,8 @@ bool makeDirectory(const string &path) {
}
/* Try to create a link and if does not work then make a copy */
bool copyFile(const string &sourcePath, const string &destPath) {
bool copyFile(const string& sourcePath, const string& destPath)
{
try {
#ifndef _WIN32
if (link(sourcePath.c_str(), destPath.c_str()) != 0) {
@ -216,7 +235,8 @@ bool copyFile(const string &sourcePath, const string &destPath) {
return true;
}
string getExecutablePath() {
string getExecutablePath()
{
char binRootPath[PATH_MAX];
#ifdef _WIN32
@ -236,7 +256,8 @@ string getExecutablePath() {
return "";
}
bool writeTextFile(const string &path, const string &content) {
bool writeTextFile(const string& path, const string& content)
{
std::ofstream file;
file.open(path.c_str());
file << content;
@ -244,7 +265,8 @@ bool writeTextFile(const string &path, const string &content) {
return true;
}
string getCurrentDirectory() {
string getCurrentDirectory()
{
char* a_cwd = getcwd(NULL, 0);
string s_cwd(a_cwd);
free(a_cwd);

21
src/common/regexTools.cpp
View File

@ -21,7 +21,8 @@
std::map<std::string, RegexMatcher*> regexCache;
RegexMatcher *buildRegex(const std::string &regex) {
RegexMatcher* buildRegex(const std::string& regex)
{
RegexMatcher* matcher;
std::map<std::string, RegexMatcher*>::iterator itr = regexCache.find(regex);
@ -42,10 +43,11 @@ RegexMatcher *buildRegex(const std::string &regex) {
}
/* todo */
void freeRegexCache() {
void freeRegexCache()
{
}
bool matchRegex(const std::string &content, const std::string &regex) {
bool matchRegex(const std::string& content, const std::string& regex)
{
ucnv_setDefaultName("UTF-8");
UnicodeString ucontent = UnicodeString(content.c_str());
RegexMatcher* matcher = buildRegex(regex);
@ -53,7 +55,10 @@ bool matchRegex(const std::string &content, const std::string &regex) {
return matcher->find();
}
std::string replaceRegex(const std::string &content, const std::string &replacement, const std::string &regex) {
std::string replaceRegex(const std::string& content,
const std::string& replacement,
const std::string& regex)
{
ucnv_setDefaultName("UTF-8");
UnicodeString ucontent = UnicodeString(content.c_str());
UnicodeString ureplacement = UnicodeString(replacement.c_str());
@ -66,7 +71,10 @@ std::string replaceRegex(const std::string &content, const std::string &replacem
return tmp;
}
std::string appendToFirstOccurence(const std::string &content, const std::string regex, const std::string &replacement) {
std::string appendToFirstOccurence(const std::string& content,
const std::string regex,
const std::string& replacement)
{
ucnv_setDefaultName("UTF-8");
UnicodeString ucontent = UnicodeString(content.c_str());
UnicodeString ureplacement = UnicodeString(replacement.c_str());
@ -83,4 +91,3 @@ std::string appendToFirstOccurence(const std::string &content, const std::strin
return content;
}

147
src/common/stringTools.cpp
View File

@ -19,19 +19,21 @@
#include <common/stringTools.h>
#include <unicode/translit.h>
#include <unicode/normlzr.h>
#include <unicode/ustring.h>
#include <unicode/rep.h>
#include <unicode/uniset.h>
#include <unicode/translit.h>
#include <unicode/ucnv.h>
#include <unicode/uniset.h>
#include <unicode/ustring.h>
/* tell ICU where to find its dat file (tables) */
void kiwix::loadICUExternalTables() {
void kiwix::loadICUExternalTables()
{
#ifdef __APPLE__
std::string executablePath = getExecutablePath();
std::string executableDirectory = removeLastPathElement(executablePath);
std::string datPath = computeAbsolutePath(executableDirectory, "icudt49l.dat");
std::string datPath
= computeAbsolutePath(executableDirectory, "icudt49l.dat");
try {
u_setDataDirectory(datPath.c_str());
} catch (exception& e) {
@ -40,11 +42,13 @@ void kiwix::loadICUExternalTables() {
#endif
}
std::string kiwix::removeAccents(const std::string &text) {
std::string kiwix::removeAccents(const std::string& text)
{
loadICUExternalTables();
ucnv_setDefaultName("UTF-8");
UErrorCode status = U_ZERO_ERROR;
Transliterator *removeAccentsTrans = Transliterator::createInstance("Lower; NFD; [:M:] remove; NFC", UTRANS_FORWARD, status);
Transliterator* removeAccentsTrans = Transliterator::createInstance(
"Lower; NFD; [:M:] remove; NFC", UTRANS_FORWARD, status);
UnicodeString ustring = UnicodeString(text.c_str());
removeAccentsTrans->transliterate(ustring);
delete removeAccentsTrans;
@ -56,7 +60,8 @@ std::string kiwix::removeAccents(const std::string &text) {
#ifndef __ANDROID__
/* Prepare integer for display */
std::string kiwix::beautifyInteger(const unsigned int number) {
std::string kiwix::beautifyInteger(const unsigned int number)
{
std::stringstream numberStream;
numberStream << number;
std::string numberString = numberStream.str();
@ -70,40 +75,48 @@ std::string kiwix::beautifyInteger(const unsigned int number) {
return numberString;
}
std::string kiwix::beautifyFileSize(const unsigned int number) {
std::string kiwix::beautifyFileSize(const unsigned int number)
{
if (number > 1024 * 1024) {
return kiwix::beautifyInteger(number / (1024 * 1024)) + " GB";
} else {
return kiwix::beautifyInteger(number/1024 !=
0 ? number/1024 : 1) + " MB";
return kiwix::beautifyInteger(number / 1024 != 0 ? number / 1024 : 1)
+ " MB";
}
}
void kiwix::printStringInHexadecimal(UnicodeString s) {
void kiwix::printStringInHexadecimal(UnicodeString s)
{
std::cout << std::showbase << std::hex;
for (int i = 0; i < s.length(); i++) {
char c = (char)((s.getTerminatedBuffer())[i]);
if (c & 0x80)
if (c & 0x80) {
std::cout << (c & 0xffff) << " ";
else
} else {
std::cout << c << " ";
}
}
std::cout << std::endl;
}
void kiwix::printStringInHexadecimal(const char *s) {
void kiwix::printStringInHexadecimal(const char* s)
{
std::cout << std::showbase << std::hex;
for (char const* pc = s; *pc; ++pc) {
if (*pc & 0x80)
if (*pc & 0x80) {
std::cout << (*pc & 0xffff);
else
} else {
std::cout << *pc;
}
std::cout << ' ';
}
std::cout << std::endl;
}
void kiwix::stringReplacement(std::string& str, const std::string& oldStr, const std::string& newStr) {
void kiwix::stringReplacement(std::string& str,
const std::string& oldStr,
const std::string& newStr)
{
size_t pos = 0;
while ((pos = str.find(oldStr, pos)) != std::string::npos) {
str.replace(pos, oldStr.length(), newStr);
@ -112,7 +125,8 @@ void kiwix::stringReplacement(std::string& str, const std::string& oldStr, const
}
/* Encode string to avoid XSS attacks */
std::string kiwix::encodeDiples(const std::string& str) {
std::string kiwix::encodeDiples(const std::string& str)
{
std::string result = str;
kiwix::stringReplacement(result, "<", "&lt;");
kiwix::stringReplacement(result, ">", "&gt;");
@ -122,13 +136,22 @@ std::string kiwix::encodeDiples(const std::string& str) {
// Urlencode
// based on javascript encodeURIComponent()
std::string char2hex(char dec) {
std::string char2hex(char dec)
{
char dig1 = (dec & 0xF0) >> 4;
char dig2 = (dec & 0x0F);
if ( 0<= dig1 && dig1<= 9) dig1+=48; //0,48inascii
if (10<= dig1 && dig1<=15) dig1+=97-10; //a,97inascii
if ( 0<= dig2 && dig2<= 9) dig2+=48;
if (10<= dig2 && dig2<=15) dig2+=97-10;
if (0 <= dig1 && dig1 <= 9) {
dig1 += 48; // 0,48inascii
}
if (10 <= dig1 && dig1 <= 15) {
dig1 += 97 - 10; // a,97inascii
}
if (0 <= dig2 && dig2 <= 9) {
dig2 += 48;
}
if (10 <= dig2 && dig2 <= 15) {
dig2 += 97 - 10;
}
std::string r;
r.append(&dig1, 1);
@ -136,21 +159,20 @@ std::string char2hex(char dec) {
return r;
}
std::string kiwix::urlEncode(const std::string &c) {
std::string kiwix::urlEncode(const std::string& c)
{
std::string escaped = "";
int max = c.length();
for(int i=0; i<max; i++)
{
for (int i = 0; i < max; i++) {
if ((48 <= c[i] && c[i] <= 57) || // 0-9
(65 <= c[i] && c[i] <= 90) ||//abc...xyz
(97 <= c[i] && c[i] <= 122) || //ABC...XYZ
(c[i]=='~' || c[i]=='!' || c[i]=='*' || c[i]=='(' || c[i]==')' || c[i]=='\'')
)
{
(65 <= c[i] && c[i] <= 90)
|| // abc...xyz
(97 <= c[i] && c[i] <= 122)
|| // ABC...XYZ
(c[i] == '~' || c[i] == '!' || c[i] == '*' || c[i] == '(' || c[i] == ')'
|| c[i] == '\'')) {
escaped.append(&c[i], 1);
}
else
{
} else {
escaped.append("%");
escaped.append(char2hex(c[i])); // converts char 255 to string "ff"
}
@ -160,18 +182,20 @@ std::string kiwix::urlEncode(const std::string &c) {
#endif
static char charFromHex(std::string a) {
static char charFromHex(std::string a)
{
std::istringstream Blat(a);
int Z;
Blat >> std::hex >> Z;
return char(Z);
}
std::string kiwix::urlDecode(const std::string &originalUrl) {
std::string kiwix::urlDecode(const std::string& originalUrl)
{
std::string url = originalUrl;
std::string::size_type pos = 0;
while ((pos = url.find('%', pos)) != std::string::npos &&
pos + 2 < url.length()) {
while ((pos = url.find('%', pos)) != std::string::npos
&& pos + 2 < url.length()) {
url.replace(pos, 3, 1, charFromHex(url.substr(pos + 1, 2)));
++pos;
}
@ -186,8 +210,7 @@ std::vector<std::string> kiwix::split(const std::string & str,
std::string::size_type pos = str.find_first_of(delims, lastPos);
std::vector<std::string> tokens;
while (std::string::npos != pos || std::string::npos != lastPos)
{
while (std::string::npos != pos || std::string::npos != lastPos) {
tokens.push_back(str.substr(lastPos, pos - lastPos));
lastPos = str.find_first_not_of(delims, pos);
pos = str.find_first_of(delims, lastPos);
@ -196,22 +219,27 @@ std::vector<std::string> kiwix::split(const std::string & str,
return tokens;
}
std::vector<std::string> kiwix::split(const char* lhs, const char* rhs){
std::vector<std::string> kiwix::split(const char* lhs, const char* rhs)
{
const std::string m1(lhs), m2(rhs);
return split(m1, m2);
}
std::vector<std::string> kiwix::split(const char* lhs, const std::string& rhs){
std::vector<std::string> kiwix::split(const char* lhs, const std::string& rhs)
{
return split(lhs, rhs.c_str());
}
std::vector<std::string> kiwix::split(const std::string& lhs, const char* rhs){
std::vector<std::string> kiwix::split(const std::string& lhs, const char* rhs)
{
return split(lhs.c_str(), rhs);
}
std::string kiwix::ucFirst (const std::string &word) {
if (word.empty())
std::string kiwix::ucFirst(const std::string& word)
{
if (word.empty()) {
return "";
}
std::string result;
@ -223,9 +251,11 @@ std::string kiwix::ucFirst (const std::string &word) {
return result;
}
std::string kiwix::ucAll (const std::string &word) {
if (word.empty())
std::string kiwix::ucAll(const std::string& word)
{
if (word.empty()) {
return "";
}
std::string result;
@ -235,9 +265,11 @@ std::string kiwix::ucAll (const std::string &word) {
return result;
}
std::string kiwix::lcFirst (const std::string &word) {
if (word.empty())
std::string kiwix::lcFirst(const std::string& word)
{
if (word.empty()) {
return "";
}
std::string result;
@ -249,9 +281,11 @@ std::string kiwix::lcFirst (const std::string &word) {
return result;
}
std::string kiwix::lcAll (const std::string &word) {
if (word.empty())
std::string kiwix::lcAll(const std::string& word)
{
if (word.empty()) {
return "";
}
std::string result;
@ -261,9 +295,11 @@ std::string kiwix::lcAll (const std::string &word) {
return result;
}
std::string kiwix::toTitle (const std::string &word) {
if (word.empty())
std::string kiwix::toTitle(const std::string& word)
{
if (word.empty()) {
return "";
}
std::string result;
@ -274,6 +310,7 @@ std::string kiwix::toTitle (const std::string &word) {
return result;
}
std::string kiwix::normalize (const std::string &word) {
std::string kiwix::normalize(const std::string& word)
{
return kiwix::lcAll(word);
}

69
src/library.cpp
View File

@ -19,47 +19,54 @@
#include "library.h"
namespace kiwix {
namespace kiwix
{
/* Constructor */
Book::Book():
readOnly(false) {
Book::Book() : readOnly(false)
{
}
/* Destructor */
Book::~Book() {
Book::~Book()
{
}
/* Sort functions */
bool Book::sortByLastOpen(const kiwix::Book &a, const kiwix::Book &b) {
bool Book::sortByLastOpen(const kiwix::Book& a, const kiwix::Book& b)
{
return atoi(a.last.c_str()) > atoi(b.last.c_str());
}
bool Book::sortByTitle(const kiwix::Book &a, const kiwix::Book &b) {
bool Book::sortByTitle(const kiwix::Book& a, const kiwix::Book& b)
{
return strcmp(a.title.c_str(), b.title.c_str()) < 0;
}
bool Book::sortByDate(const kiwix::Book &a, const kiwix::Book &b) {
bool Book::sortByDate(const kiwix::Book& a, const kiwix::Book& b)
{
return strcmp(a.date.c_str(), b.date.c_str()) > 0;
}
bool Book::sortBySize(const kiwix::Book &a, const kiwix::Book &b) {
bool Book::sortBySize(const kiwix::Book& a, const kiwix::Book& b)
{
return atoi(a.size.c_str()) < atoi(b.size.c_str());
}
bool Book::sortByPublisher(const kiwix::Book &a, const kiwix::Book &b) {
bool Book::sortByPublisher(const kiwix::Book& a, const kiwix::Book& b)
{
return strcmp(a.publisher.c_str(), b.publisher.c_str()) < 0;
}
bool Book::sortByCreator(const kiwix::Book &a, const kiwix::Book &b) {
bool Book::sortByCreator(const kiwix::Book& a, const kiwix::Book& b)
{
return strcmp(a.creator.c_str(), b.creator.c_str()) < 0;
}
bool Book::sortByLanguage(const kiwix::Book &a, const kiwix::Book &b) {
bool Book::sortByLanguage(const kiwix::Book& a, const kiwix::Book& b)
{
return strcmp(a.language.c_str(), b.language.c_str()) < 0;
}
std::string Book::getHumanReadableIdFromPath() {
std::string Book::getHumanReadableIdFromPath()
{
std::string id = pathAbsolute;
if (!id.empty()) {
kiwix::removeAccents(id);
@ -78,16 +85,15 @@ namespace kiwix {
}
/* Constructor */
Library::Library():
version(KIWIX_LIBRARY_VERSION) {
Library::Library() : version(KIWIX_LIBRARY_VERSION)
{
}
/* Destructor */
Library::~Library() {
Library::~Library()
{
}
bool Library::addBook(const Book &book) {
bool Library::addBook(const Book& book)
{
/* Try to find it */
std::vector<kiwix::Book>::iterator itr;
for (itr = this->books.begin(); itr != this->books.end(); ++itr) {
@ -95,20 +101,25 @@ namespace kiwix {
if (!itr->readOnly) {
itr->readOnly = book.readOnly;
if (itr->path.empty())
if (itr->path.empty()) {
itr->path = book.path;
}
if (itr->pathAbsolute.empty())
if (itr->pathAbsolute.empty()) {
itr->pathAbsolute = book.pathAbsolute;
}
if (itr->url.empty())
if (itr->url.empty()) {
itr->url = book.url;
}
if (itr->tags.empty())
if (itr->tags.empty()) {
itr->tags = book.tags;
}
if (itr->name.empty())
if (itr->name.empty()) {
itr->name = book.name;
}
if (itr->indexPath.empty()) {
itr->indexPath = book.indexPath;
@ -135,9 +146,9 @@ namespace kiwix {
return true;
}
bool Library::removeBookByIndex(const unsigned int bookIndex) {
bool Library::removeBookByIndex(const unsigned int bookIndex)
{
books.erase(books.begin() + bookIndex);
return true;
}
}

290
src/manager.cpp
View File

@ -19,18 +19,20 @@
#include "manager.h"
namespace kiwix {
namespace kiwix
{
/* Constructor */
Manager::Manager() :
writableLibraryPath("") {
Manager::Manager() : writableLibraryPath("")
{
}
/* Destructor */
Manager::~Manager() {
Manager::~Manager()
{
}
bool Manager::parseXmlDom(const pugi::xml_document &doc, const bool readOnly, const string libraryPath) {
bool Manager::parseXmlDom(const pugi::xml_document& doc,
const bool readOnly,
const string libraryPath)
{
pugi::xml_node libraryNode = doc.child("library");
if (strlen(libraryNode.attribute("current").value()))
@ -38,15 +40,17 @@ namespace kiwix {
string libraryVersion = libraryNode.attribute("version").value();
for (pugi::xml_node bookNode = libraryNode.child("book"); bookNode; bookNode = bookNode.next_sibling("book")) {
for (pugi::xml_node bookNode = libraryNode.child("book"); bookNode;
bookNode = bookNode.next_sibling("book")) {
bool ok = true;
kiwix::Book book;
book.readOnly = readOnly;
book.id = bookNode.attribute("id").value();
book.path = bookNode.attribute("path").value();
book.last = (std::string(bookNode.attribute("last").value()) != "undefined" ?
bookNode.attribute("last").value() : "");
book.last = (std::string(bookNode.attribute("last").value()) != "undefined"
? bookNode.attribute("last").value()
: "");
book.indexPath = bookNode.attribute("indexPath").value();
book.indexType = XAPIAN;
book.title = bookNode.attribute("title").value();
@ -69,7 +73,8 @@ namespace kiwix {
this->checkAndCleanBookPaths(book, libraryPath);
/* Update the book properties with the new importer */
if (libraryVersion.empty() || atoi(libraryVersion.c_str()) <= atoi(KIWIX_LIBRARY_VERSION)) {
if (libraryVersion.empty()
|| atoi(libraryVersion.c_str()) <= atoi(KIWIX_LIBRARY_VERSION)) {
if (!book.path.empty()) {
ok = this->readBookFromPath(book.pathAbsolute);
}
@ -83,9 +88,13 @@ namespace kiwix {
return true;
}
bool Manager::readXml(const string xml, const bool readOnly, const string libraryPath) {
bool Manager::readXml(const string xml,
const bool readOnly,
const string libraryPath)
{
pugi::xml_document doc;
pugi::xml_parse_result result = doc.load_buffer_inplace((void*)xml.data(), xml.size());
pugi::xml_parse_result result
= doc.load_buffer_inplace((void*)xml.data(), xml.size());
if (result) {
this->parseXmlDom(doc, readOnly, libraryPath);
@ -94,11 +103,15 @@ namespace kiwix {
return true;
}
bool Manager::readFile(const string path, const bool readOnly) {
bool Manager::readFile(const string path, const bool readOnly)
{
return this->readFile(path, path, readOnly);
}
bool Manager::readFile(const string nativePath, const string UTF8Path, const bool readOnly) {
bool Manager::readFile(const string nativePath,
const string UTF8Path,
const bool readOnly)
{
bool retVal = true;
pugi::xml_document doc;
pugi::xml_parse_result result = doc.load_file(nativePath.c_str());
@ -119,7 +132,8 @@ namespace kiwix {
return retVal;
}
bool Manager::writeFile(const string path) {
bool Manager::writeFile(const string path)
{
pugi::xml_document doc;
/* Add the library node */
@ -135,15 +149,15 @@ namespace kiwix {
/* Add each book */
std::vector<kiwix::Book>::iterator itr;
for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
if (!itr->readOnly) {
this->checkAndCleanBookPaths(*itr, path);
pugi::xml_node bookNode = libraryNode.append_child("book");
bookNode.append_attribute("id") = itr->id.c_str();
if (!itr->path.empty())
if (!itr->path.empty()) {
bookNode.append_attribute("path") = itr->path.c_str();
}
if (!itr->last.empty() && itr->last != "undefined") {
bookNode.append_attribute("last") = itr->last.c_str();
@ -153,9 +167,10 @@ namespace kiwix {
bookNode.append_attribute("indexPath") = itr->indexPath.c_str();
if (!itr->indexPath.empty() || !itr->indexPathAbsolute.empty()) {
if (itr->indexType == XAPIAN)
if (itr->indexType == XAPIAN) {
bookNode.append_attribute("indexType") = "xapian";
}
}
if (itr->origId.empty()) {
if (!itr->title.empty())
@ -183,14 +198,17 @@ namespace kiwix {
bookNode.append_attribute("favicon") = itr->favicon.c_str();
if (!itr->faviconMimeType.empty())
bookNode.append_attribute("faviconMimeType") = itr->faviconMimeType.c_str();
bookNode.append_attribute("faviconMimeType")
= itr->faviconMimeType.c_str();
}
if (!itr->date.empty())
if (!itr->date.empty()) {
bookNode.append_attribute("date") = itr->date.c_str();
}
if (!itr->url.empty())
if (!itr->url.empty()) {
bookNode.append_attribute("url") = itr->url.c_str();
}
if (!itr->origId.empty())
bookNode.append_attribute("origId") = itr->origId.c_str();
@ -201,10 +219,11 @@ namespace kiwix {
if (!itr->mediaCount.empty())
bookNode.append_attribute("mediaCount") = itr->mediaCount.c_str();
if (!itr->size.empty())
if (!itr->size.empty()) {
bookNode.append_attribute("size") = itr->size.c_str();
}
}
}
/* saving file */
doc.save_file(path.c_str());
@ -212,36 +231,46 @@ namespace kiwix {
return true;
}
bool Manager::setCurrentBookId(const string id) {
bool Manager::setCurrentBookId(const string id)
{
if (library.current.empty() || library.current.top() != id) {
if (id.empty() && !library.current.empty())
if (id.empty() && !library.current.empty()) {
library.current.pop();
else
} else {
library.current.push(id);
}
}
return true;
}
string Manager::getCurrentBookId() {
return library.current.empty() ?
"" : library.current.top();
string Manager::getCurrentBookId()
{
return library.current.empty() ? "" : library.current.top();
}
/* Add a book to the library. Return empty string if failed, book id otherwise */
string Manager::addBookFromPathAndGetId(const string pathToOpen, const string pathToSave,
const string url, const bool checkMetaData) {
/* Add a book to the library. Return empty string if failed, book id otherwise
*/
string Manager::addBookFromPathAndGetId(const string pathToOpen,
const string pathToSave,
const string url,
const bool checkMetaData)
{
kiwix::Book book;
if (this->readBookFromPath(pathToOpen, &book)) {
if (pathToSave != pathToOpen) {
book.path = pathToSave;
book.pathAbsolute = isRelativePath(pathToSave) ?
computeAbsolutePath(removeLastPathElement(writableLibraryPath, true, false), pathToSave) : pathToSave;
book.pathAbsolute
= isRelativePath(pathToSave)
? computeAbsolutePath(
removeLastPathElement(writableLibraryPath, true, false),
pathToSave)
: pathToSave;
}
if (!checkMetaData ||
(checkMetaData && !book.title.empty() && !book.language.empty() && !book.date.empty())) {
if (!checkMetaData
|| (checkMetaData && !book.title.empty() && !book.language.empty()
&& !book.date.empty())) {
book.url = url;
library.addBook(book);
return book.id;
@ -251,12 +280,20 @@ namespace kiwix {
return "";
}
/* Wrapper over Manager::addBookFromPath which return a bool instead of a string */
bool Manager::addBookFromPath(const string pathToOpen, const string pathToSave, const string url, const bool checkMetaData) {
return !(this->addBookFromPathAndGetId(pathToOpen, pathToSave, url, checkMetaData).empty());
/* Wrapper over Manager::addBookFromPath which return a bool instead of a string
*/
bool Manager::addBookFromPath(const string pathToOpen,
const string pathToSave,
const string url,
const bool checkMetaData)
{
return !(
this->addBookFromPathAndGetId(pathToOpen, pathToSave, url, checkMetaData)
.empty());
}
bool Manager::readBookFromPath(const string path, kiwix::Book *book) {
bool Manager::readBookFromPath(const string path, kiwix::Book* book)
{
try {
kiwix::Reader* reader = new kiwix::Reader(path);
@ -281,13 +318,16 @@ namespace kiwix {
mediaCountStream << reader->getMediaCount();
book->mediaCount = mediaCountStream.str();
ostringstream convert; convert << reader->getFileSize();
ostringstream convert;
convert << reader->getFileSize();
book->size = convert.str();
string favicon;
string faviconMimeType;
if (reader->getFavicon(favicon, faviconMimeType)) {
book->favicon = base64_encode(reinterpret_cast<const unsigned char*>(favicon.c_str()), favicon.length());
book->favicon = base64_encode(
reinterpret_cast<const unsigned char*>(favicon.c_str()),
favicon.length());
book->faviconMimeType = faviconMimeType;
}
}
@ -301,11 +341,13 @@ namespace kiwix {
return true;
}
bool Manager::removeBookByIndex(const unsigned int bookIndex) {
bool Manager::removeBookByIndex(const unsigned int bookIndex)
{
return this->library.removeBookByIndex(bookIndex);
}
bool Manager::removeBookById(const string id) {
bool Manager::removeBookById(const string id)
{
unsigned int bookIndex = 0;
std::vector<kiwix::Book>::iterator itr;
for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
@ -317,12 +359,14 @@ namespace kiwix {
return false;
}
vector<string> Manager::getBooksLanguages() {
vector<string> Manager::getBooksLanguages()
{
std::vector<string> booksLanguages;
std::vector<kiwix::Book>::iterator itr;
std::map<string, bool> booksLanguagesMap;
std::sort(library.books.begin(), library.books.end(), kiwix::Book::sortByLanguage);
std::sort(
library.books.begin(), library.books.end(), kiwix::Book::sortByLanguage);
for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
if (booksLanguagesMap.find(itr->language) == booksLanguagesMap.end()) {
if (itr->origId.empty()) {
@ -335,12 +379,14 @@ namespace kiwix {
return booksLanguages;
}
vector<string> Manager::getBooksCreators() {
vector<string> Manager::getBooksCreators()
{
std::vector<string> booksCreators;
std::vector<kiwix::Book>::iterator itr;
std::map<string, bool> booksCreatorsMap;
std::sort(library.books.begin(), library.books.end(), kiwix::Book::sortByCreator);
std::sort(
library.books.begin(), library.books.end(), kiwix::Book::sortByCreator);
for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
if (booksCreatorsMap.find(itr->creator) == booksCreatorsMap.end()) {
if (itr->origId.empty()) {
@ -353,8 +399,8 @@ namespace kiwix {
return booksCreators;
}
vector<string> Manager::getBooksIds() {
vector<string> Manager::getBooksIds()
{
std::vector<string> booksIds;
std::vector<kiwix::Book>::iterator itr;
@ -365,12 +411,14 @@ namespace kiwix {
return booksIds;
}
vector<string> Manager::getBooksPublishers() {
vector<string> Manager::getBooksPublishers()
{
std::vector<string> booksPublishers;
std::vector<kiwix::Book>::iterator itr;
std::map<string, bool> booksPublishersMap;
std::sort(library.books.begin(), library.books.end(), kiwix::Book::sortByPublisher);
std::sort(
library.books.begin(), library.books.end(), kiwix::Book::sortByPublisher);
for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
if (booksPublishersMap.find(itr->publisher) == booksPublishersMap.end()) {
if (itr->origId.empty()) {
@ -383,11 +431,12 @@ namespace kiwix {
return booksPublishers;
}
kiwix::Library Manager::cloneLibrary() {
kiwix::Library Manager::cloneLibrary()
{
return this->library;
}
bool Manager::getCurrentBook(Book &book) {
bool Manager::getCurrentBook(Book& book)
{
string currentBookId = getCurrentBookId();
if (currentBookId.empty()) {
return false;
@ -397,7 +446,8 @@ namespace kiwix {
}
}
bool Manager::getBookById(const string id, Book &book) {
bool Manager::getBookById(const string id, Book& book)
{
std::vector<kiwix::Book>::iterator itr;
for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
if (itr->id == id) {
@ -408,7 +458,8 @@ namespace kiwix {
return false;
}
bool Manager::updateBookLastOpenDateById(const string id) {
bool Manager::updateBookLastOpenDateById(const string id)
{
std::vector<kiwix::Book>::iterator itr;
for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
if (itr->id == id) {
@ -422,13 +473,20 @@ namespace kiwix {
return false;
}
bool Manager::setBookIndex(const string id, const string path, const supportedIndexType type) {
bool Manager::setBookIndex(const string id,
const string path,
const supportedIndexType type)
{
std::vector<kiwix::Book>::iterator itr;
for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
if (itr->id == id) {
itr->indexPath = path;
itr->indexPathAbsolute = isRelativePath(path) ?
computeAbsolutePath(removeLastPathElement(writableLibraryPath, true, false), path) : path;
itr->indexPathAbsolute
= isRelativePath(path)
? computeAbsolutePath(
removeLastPathElement(writableLibraryPath, true, false),
path)
: path;
itr->indexType = type;
return true;
}
@ -437,17 +495,23 @@ namespace kiwix {
return false;
}
bool Manager::setBookIndex(const string id, const string path) {
bool Manager::setBookIndex(const string id, const string path)
{
return this->setBookIndex(id, path, XAPIAN);
}
bool Manager::setBookPath(const string id, const string path) {
bool Manager::setBookPath(const string id, const string path)
{
std::vector<kiwix::Book>::iterator itr;
for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
if (itr->id == id) {
itr->path = path;
itr->pathAbsolute = isRelativePath(path) ?
computeAbsolutePath(removeLastPathElement(writableLibraryPath, true, false), path) : path;
itr->pathAbsolute
= isRelativePath(path)
? computeAbsolutePath(
removeLastPathElement(writableLibraryPath, true, false),
path)
: path;
return true;
}
}
@ -455,7 +519,8 @@ namespace kiwix {
return false;
}
void Manager::removeBookPaths() {
void Manager::removeBookPaths()
{
std::vector<kiwix::Book>::iterator itr;
for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
itr->path = "";
@ -463,70 +528,98 @@ namespace kiwix {
}
}
unsigned int Manager::getBookCount(const bool localBooks, const bool remoteBooks) {
unsigned int Manager::getBookCount(const bool localBooks,
const bool remoteBooks)
{
unsigned int result = 0;
std::vector<kiwix::Book>::iterator itr;
for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
if ((!itr->path.empty() && localBooks) || (itr->path.empty() && remoteBooks))
if ((!itr->path.empty() && localBooks)
|| (itr->path.empty() && remoteBooks)) {
result++;
}
}
return result;
}
bool Manager::listBooks(const supportedListMode mode, const supportedListSortBy sortBy,
const unsigned int maxSize, const string language, const string creator,
const string publisher, const string search) {
bool Manager::listBooks(const supportedListMode mode,
const supportedListSortBy sortBy,
const unsigned int maxSize,
const string language,
const string creator,
const string publisher,
const string search)
{
this->bookIdList.clear();
std::vector<kiwix::Book>::iterator itr;
/* Sort */
if (sortBy == TITLE) {
std::sort(library.books.begin(), library.books.end(), kiwix::Book::sortByTitle);
std::sort(
library.books.begin(), library.books.end(), kiwix::Book::sortByTitle);
} else if (sortBy == SIZE) {
std::sort(library.books.begin(), library.books.end(), kiwix::Book::sortBySize);
std::sort(
library.books.begin(), library.books.end(), kiwix::Book::sortBySize);
} else if (sortBy == DATE) {
std::sort(library.books.begin(), library.books.end(), kiwix::Book::sortByDate);
std::sort(
library.books.begin(), library.books.end(), kiwix::Book::sortByDate);
} else if (sortBy == CREATOR) {
std::sort(library.books.begin(), library.books.end(), kiwix::Book::sortByCreator);
std::sort(
library.books.begin(), library.books.end(), kiwix::Book::sortByCreator);
} else if (sortBy == PUBLISHER) {
std::sort(library.books.begin(), library.books.end(), kiwix::Book::sortByPublisher);
std::sort(library.books.begin(),
library.books.end(),
kiwix::Book::sortByPublisher);
}
/* Special sort for LASTOPEN */
if (mode == LASTOPEN) {
std::sort(library.books.begin(), library.books.end(), kiwix::Book::sortByLastOpen);
std::sort(library.books.begin(),
library.books.end(),
kiwix::Book::sortByLastOpen);
for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
if (!itr->last.empty())
if (!itr->last.empty()) {
this->bookIdList.push_back(itr->id);
}
}
} else {
/* Generate the list of book id */
for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
bool ok = true;
if (mode == LOCAL && itr->path.empty())
if (mode == LOCAL && itr->path.empty()) {
ok = false;
}
if (ok == true && mode == REMOTE && (!itr->path.empty() || itr->url.empty()))
if (ok == true && mode == REMOTE
&& (!itr->path.empty() || itr->url.empty())) {
ok = false;
}
if (ok == true && maxSize != 0 && (unsigned int)atoi(itr->size.c_str()) > maxSize * 1024 * 1024)
if (ok == true && maxSize != 0
&& (unsigned int)atoi(itr->size.c_str()) > maxSize * 1024 * 1024) {
ok = false;
}
if (ok == true && !language.empty() && !matchRegex(itr->language, language))
if (ok == true && !language.empty()
&& !matchRegex(itr->language, language)) {
ok = false;
}
if (ok == true && !creator.empty() && itr->creator != creator)
if (ok == true && !creator.empty() && itr->creator != creator) {
ok = false;
}
if (ok == true && !publisher.empty() && itr->publisher != publisher)
if (ok == true && !publisher.empty() && itr->publisher != publisher) {
ok = false;
}
if ((ok == true && !search.empty()) && !(matchRegex(itr->title, "\\Q" + search + "\\E") ||
matchRegex(itr->description, "\\Q" + search + "\\E") ||
matchRegex(itr->language, "\\Q" + search + "\\E")
))
if ((ok == true && !search.empty())
&& !(matchRegex(itr->title, "\\Q" + search + "\\E")
|| matchRegex(itr->description, "\\Q" + search + "\\E")
|| matchRegex(itr->language, "\\Q" + search + "\\E"))) {
ok = false;
}
if (ok == true) {
this->bookIdList.push_back(itr->id);
@ -537,26 +630,29 @@ namespace kiwix {
return true;
}
void Manager::checkAndCleanBookPaths(Book &book, const string &libraryPath) {
void Manager::checkAndCleanBookPaths(Book& book, const string& libraryPath)
{
if (!book.path.empty()) {
if (isRelativePath(book.path)) {
book.pathAbsolute = computeAbsolutePath(removeLastPathElement(libraryPath, true, false), book.path);
book.pathAbsolute = computeAbsolutePath(
removeLastPathElement(libraryPath, true, false), book.path);
} else {
book.pathAbsolute = book.path;
book.path = computeRelativePath(removeLastPathElement(libraryPath, true, false), book.pathAbsolute);
book.path = computeRelativePath(
removeLastPathElement(libraryPath, true, false), book.pathAbsolute);
}
}
if (!book.indexPath.empty()) {
if (isRelativePath(book.indexPath)) {
book.indexPathAbsolute =
computeAbsolutePath(removeLastPathElement(libraryPath, true, false), book.indexPath);
book.indexPathAbsolute = computeAbsolutePath(
removeLastPathElement(libraryPath, true, false), book.indexPath);
} else {
book.indexPathAbsolute = book.indexPath;
book.indexPath =
computeRelativePath(removeLastPathElement(libraryPath, true, false), book.indexPathAbsolute);
book.indexPath
= computeRelativePath(removeLastPathElement(libraryPath, true, false),
book.indexPathAbsolute);
}
}
}
}

333
src/reader.cpp
View File

@ -20,55 +20,63 @@
#include "reader.h"
#include <time.h>
inline char hi(char v) {
inline char hi(char v)
{
char hex[] = "0123456789abcdef";
return hex[(v >> 4) & 0xf];
}
inline char lo(char v) {
inline char lo(char v)
{
char hex[] = "0123456789abcdef";
return hex[v & 0xf];
}
std::string hexUUID (std::string in) {
std::string hexUUID(std::string in)
{
std::ostringstream out;
for (unsigned n = 0; n < 4; ++n)
for (unsigned n = 0; n < 4; ++n) {
out << hi(in[n]) << lo(in[n]);
}
out << '-';
for (unsigned n = 4; n < 6; ++n)
for (unsigned n = 4; n < 6; ++n) {
out << hi(in[n]) << lo(in[n]);
}
out << '-';
for (unsigned n = 6; n < 8; ++n)
for (unsigned n = 6; n < 8; ++n) {
out << hi(in[n]) << lo(in[n]);
}
out << '-';
for (unsigned n = 8; n < 10; ++n)
for (unsigned n = 8; n < 10; ++n) {
out << hi(in[n]) << lo(in[n]);
}
out << '-';
for (unsigned n = 10; n < 16; ++n)
for (unsigned n = 10; n < 16; ++n) {
out << hi(in[n]) << lo(in[n]);
}
std::string op = out.str();
return op;
}
namespace kiwix {
namespace kiwix
{
/* Constructor */
Reader::Reader(const string zimFilePath)
: zimFileHandler(NULL) {
Reader::Reader(const string zimFilePath) : zimFileHandler(NULL)
{
string tmpZimFilePath = zimFilePath;
/* Remove potential trailing zimaa */
size_t found = tmpZimFilePath.rfind("zimaa");
if (found != string::npos &&
tmpZimFilePath.size() > 5 &&
found == tmpZimFilePath.size() - 5) {
if (found != string::npos && tmpZimFilePath.size() > 5
&& found == tmpZimFilePath.size() - 5) {
tmpZimFilePath.resize(tmpZimFilePath.size() - 2);
}
this->zimFileHandler = new zim::File(tmpZimFilePath);
if (this->zimFileHandler != NULL) {
this->firstArticleOffset = this->zimFileHandler->getNamespaceBeginOffset('A');
this->firstArticleOffset
= this->zimFileHandler->getNamespaceBeginOffset('A');
this->lastArticleOffset = this->zimFileHandler->getNamespaceEndOffset('A');
this->currentArticleOffset = this->firstArticleOffset;
this->nsACount = this->zimFileHandler->getNamespaceCount('A');
@ -81,22 +89,24 @@ namespace kiwix {
}
/* Destructor */
Reader::~Reader() {
Reader::~Reader()
{
if (this->zimFileHandler != NULL) {
delete this->zimFileHandler;
}
}
zim::File* Reader::getZimFileHandler() const {
zim::File* Reader::getZimFileHandler() const
{
return this->zimFileHandler;
}
/* Reset the cursor for GetNextArticle() */
void Reader::reset() {
void Reader::reset()
{
this->currentArticleOffset = this->firstArticleOffset;
}
std::map<const std::string, unsigned int> Reader::parseCounterMetadata() const {
std::map<const std::string, unsigned int> Reader::parseCounterMetadata() const
{
std::map<const std::string, unsigned int> counters;
string mimeType, item, counterString;
unsigned int counter;
@ -121,64 +131,73 @@ namespace kiwix {
}
/* Get the count of articles which can be indexed/displayed */
unsigned int Reader::getArticleCount() const {
std::map<const std::string, unsigned int> counterMap = this->parseCounterMetadata();
unsigned int Reader::getArticleCount() const
{
std::map<const std::string, unsigned int> counterMap
= this->parseCounterMetadata();
unsigned int counter = 0;
if (counterMap.empty()) {
counter = this->nsACount;
} else {
auto it = counterMap.find("text/html");
if (it != counterMap.end())
if (it != counterMap.end()) {
counter = it->second;
}
}
return counter;
}
/* Get the count of medias content in the ZIM file */
unsigned int Reader::getMediaCount() const {
std::map<const std::string, unsigned int> counterMap = this->parseCounterMetadata();
unsigned int Reader::getMediaCount() const
{
std::map<const std::string, unsigned int> counterMap
= this->parseCounterMetadata();
unsigned int counter = 0;
if (counterMap.empty())
if (counterMap.empty()) {
counter = this->nsICount;
else {
} else {
auto it = counterMap.find("image/jpeg");
if (it != counterMap.end())
counter += it->second;
it = counterMap.find("image/gif");
if (it != counterMap.end())
counter += it->second;
it = counterMap.find("image/png");
if (it != counterMap.end())
if (it != counterMap.end()) {
counter += it->second;
}
it = counterMap.find("image/gif");
if (it != counterMap.end()) {
counter += it->second;
}
it = counterMap.find("image/png");
if (it != counterMap.end()) {
counter += it->second;
}
}
return counter;
}
/* Get the total of all items of a ZIM file, redirects included */
unsigned int Reader::getGlobalCount() const {
unsigned int Reader::getGlobalCount() const
{
return this->zimFileHandler->getCountArticles();
}
/* Return the UID of the ZIM file */
string Reader::getId() const {
string Reader::getId() const
{
std::ostringstream s;
s << this->zimFileHandler->getFileheader().getUuid();
return s.str();
}
/* Return a page url from a title */
bool Reader::getPageUrlFromTitle(const string &title, string &url) const {
bool Reader::getPageUrlFromTitle(const string& title, string& url) const
{
/* Extract the content from the zim file */
zim::Article article = this->zimFileHandler->getArticleByTitle('A', title);
if ( ! article.good() )
{
if (!article.good()) {
return false;
}
@ -192,14 +211,16 @@ namespace kiwix {
}
/* Return an URL from a title */
string Reader::getRandomPageUrl() const {
string Reader::getRandomPageUrl() const
{
zim::Article article;
zim::size_type idx;
std::string mainPageUrl = this->getMainPageUrl();
do {
idx = this->firstArticleOffset +
(zim::size_type)((double)rand() / ((double)RAND_MAX + 1) * this->nsACount);
idx = this->firstArticleOffset
+ (zim::size_type)((double)rand() / ((double)RAND_MAX + 1)
* this->nsACount);
article = zimFileHandler->getArticle(idx);
} while (article.getLongUrl() == mainPageUrl);
@ -207,11 +228,13 @@ namespace kiwix {
}
/* Return the welcome page URL */
string Reader::getMainPageUrl() const {
string Reader::getMainPageUrl() const
{
string url = "";
if (this->zimFileHandler->getFileheader().hasMainPage()) {
zim::Article article = zimFileHandler->getArticle(this->zimFileHandler->getFileheader().getMainPage());
zim::Article article = zimFileHandler->getArticle(
this->zimFileHandler->getFileheader().getMainPage());
url = article.getLongUrl();
if (url.empty()) {
@ -224,24 +247,20 @@ namespace kiwix {
return url;
}
bool Reader::getFavicon(string &content, string &mimeType) const {
bool Reader::getFavicon(string& content, string& mimeType) const
{
unsigned int contentLength = 0;
this->getContentByUrl( "/-/favicon.png", content,
contentLength, mimeType);
this->getContentByUrl("/-/favicon.png", content, contentLength, mimeType);
if (content.empty()) {
this->getContentByUrl( "/I/favicon.png", content,
contentLength, mimeType);
this->getContentByUrl("/I/favicon.png", content, contentLength, mimeType);
if (content.empty()) {
this->getContentByUrl( "/I/favicon", content,
contentLength, mimeType);
this->getContentByUrl("/I/favicon", content, contentLength, mimeType);
if (content.empty()) {
this->getContentByUrl( "/-/favicon", content,
contentLength, mimeType);
this->getContentByUrl("/-/favicon", content, contentLength, mimeType);
}
}
}
@ -249,20 +268,21 @@ namespace kiwix {
return content.empty() ? false : true;
}
string Reader::getZimFilePath() const {
string Reader::getZimFilePath() const
{
return this->zimFilePath;
}
/* Return a metatag value */
bool Reader::getMetatag(const string &name, string &value) const {
bool Reader::getMetatag(const string& name, string& value) const
{
unsigned int contentLength = 0;
string contentType = "";
return this->getContentByUrl( "/M/" + name, value,
contentLength, contentType);
return this->getContentByUrl("/M/" + name, value, contentLength, contentType);
}
string Reader::getTitle() const {
string Reader::getTitle() const
{
string value;
this->getMetatag("Title", value);
if (value.empty()) {
@ -274,19 +294,22 @@ namespace kiwix {
return value;
}
string Reader::getName() const {
string Reader::getName() const
{
string value;
this->getMetatag("Name", value);
return value;
}
string Reader::getTags() const {
string Reader::getTags() const
{
string value;
this->getMetatag("Tags", value);
return value;
}
string Reader::getDescription() const{
string Reader::getDescription() const
{
string value;
this->getMetatag("Description", value);
@ -298,50 +321,52 @@ namespace kiwix {
return value;
}
string Reader::getLanguage() const {
string Reader::getLanguage() const
{
string value;
this->getMetatag("Language", value);
return value;
}
string Reader::getDate() const {
string Reader::getDate() const
{
string value;
this->getMetatag("Date", value);
return value;
}
string Reader::getCreator() const {
string Reader::getCreator() const
{
string value;
this->getMetatag("Creator", value);
return value;
}
string Reader::getPublisher() const {
string Reader::getPublisher() const
{
string value;
this->getMetatag("Publisher", value);
return value;
}
string Reader::getOrigId() const {
string Reader::getOrigId() const
{
string value;
this->getMetatag("startfileuid", value);
if(value.empty())
if (value.empty()) {
return "";
}
std::string id = value;
std::string origID;
std::string temp = "";
unsigned int k = 0;
char tempArray[16] = "";
for(unsigned int i=0; i<id.size(); i++)
{
if(id[i]=='\n')
{
for (unsigned int i = 0; i < id.size(); i++) {
if (id[i] == '\n') {
tempArray[k] = atoi(temp.c_str());
temp = "";
k++;
}
else
{
} else {
temp += id[i];
}
}
@ -350,19 +375,23 @@ namespace kiwix {
}
/* Return the first page URL */
string Reader::getFirstPageUrl() const {
string Reader::getFirstPageUrl() const
{
zim::size_type firstPageOffset = zimFileHandler->getNamespaceBeginOffset('A');
zim::Article article = zimFileHandler->getArticle(firstPageOffset);
return article.getLongUrl();
}
bool Reader::parseUrl(const string &url, char *ns, string &title) const {
bool Reader::parseUrl(const string& url, char* ns, string& title) const
{
/* Offset to visit the url */
unsigned int urlLength = url.size();
unsigned int offset = 0;
/* Ignore the '/' */
while ((offset < urlLength) && (url[offset] == '/')) offset++;
while ((offset < urlLength) && (url[offset] == '/')) {
offset++;
}
/* Get namespace */
while ((offset < urlLength) && (url[offset] != '/')) {
@ -371,7 +400,9 @@ namespace kiwix {
}
/* Ignore the '/' */
while ((offset < urlLength) && (url[offset] == '/')) offset++;
while ((offset < urlLength) && (url[offset] == '/')) {
offset++;
}
/* Get content title */
unsigned int titleOffset = offset;
@ -386,7 +417,9 @@ namespace kiwix {
}
/* Return article by url */
bool Reader::getArticleObjectByDecodedUrl(const string &url, zim::Article &article) const {
bool Reader::getArticleObjectByDecodedUrl(const string& url,
zim::Article& article) const
{
if (this->zimFileHandler == NULL) {
return false;
}
@ -407,7 +440,8 @@ namespace kiwix {
}
/* Return the mimeType without the content */
bool Reader::getMimeTypeByUrl(const string &url, string &mimeType) const {
bool Reader::getMimeTypeByUrl(const string& url, string& mimeType) const
{
if (this->zimFileHandler == NULL) {
return false;
}
@ -417,7 +451,8 @@ namespace kiwix {
try {
mimeType = article.getMimeType();
} catch (exception& e) {
cerr << "Unable to get the mimetype for " << url << ":" << e.what() << endl;
cerr << "Unable to get the mimetype for " << url << ":" << e.what()
<< endl;
mimeType = "application/octet-stream";
}
return true;
@ -428,25 +463,56 @@ namespace kiwix {
}
/* Get a content from a zim file */
bool Reader::getContentByUrl(const string &url, string &content, unsigned int &contentLength, string &contentType) const {
bool Reader::getContentByUrl(const string& url,
string& content,
unsigned int& contentLength,
string& contentType) const
{
return this->getContentByEncodedUrl(url, content, contentLength, contentType);
}
bool Reader::getContentByEncodedUrl(const string &url, string &content, unsigned int &contentLength, string &contentType, string &baseUrl) const {
return this->getContentByDecodedUrl(kiwix::urlDecode(url), content, contentLength, contentType, baseUrl);
bool Reader::getContentByEncodedUrl(const string& url,
string& content,
unsigned int& contentLength,
string& contentType,
string& baseUrl) const
{
return this->getContentByDecodedUrl(
kiwix::urlDecode(url), content, contentLength, contentType, baseUrl);
}
bool Reader::getContentByEncodedUrl(const string &url, string &content, unsigned int &contentLength, string &contentType) const {
bool Reader::getContentByEncodedUrl(const string& url,
string& content,
unsigned int& contentLength,
string& contentType) const
{
std::string stubRedirectUrl;
return this->getContentByEncodedUrl(kiwix::urlDecode(url), content, contentLength, contentType, stubRedirectUrl);
return this->getContentByEncodedUrl(kiwix::urlDecode(url),
content,
contentLength,
contentType,
stubRedirectUrl);
}
bool Reader::getContentByDecodedUrl(const string &url, string &content, unsigned int &contentLength, string &contentType) const {
bool Reader::getContentByDecodedUrl(const string& url,
string& content,
unsigned int& contentLength,
string& contentType) const
{
std::string stubRedirectUrl;
return this->getContentByDecodedUrl(kiwix::urlDecode(url), content, contentLength, contentType, stubRedirectUrl);
return this->getContentByDecodedUrl(kiwix::urlDecode(url),
content,
contentLength,
contentType,
stubRedirectUrl);
}
bool Reader::getContentByDecodedUrl(const string &url, string &content, unsigned int &contentLength, string &contentType, string &baseUrl) const {
bool Reader::getContentByDecodedUrl(const string& url,
string& content,
unsigned int& contentLength,
string& contentType,
string& baseUrl) const
{
content = "";
contentType = "";
contentLength = 0;
@ -464,13 +530,16 @@ namespace kiwix {
if (loopCounter < 42) {
/* Compute base url (might be different from the url if redirects */
baseUrl = "/" + std::string(1, article.getNamespace()) + "/" + article.getUrl();
baseUrl
= "/" + std::string(1, article.getNamespace()) + "/" + article.getUrl();
/* Get the content mime-type */
try {
contentType = string(article.getMimeType().data(), article.getMimeType().size());
contentType
= string(article.getMimeType().data(), article.getMimeType().size());
} catch (exception& e) {
cerr << "Unable to get the mimetype for "<< baseUrl<< ":" << e.what() << endl;
cerr << "Unable to get the mimetype for " << baseUrl << ":" << e.what()
<< endl;
contentType = "application/octet-stream";
}
@ -479,10 +548,13 @@ namespace kiwix {
}
/* Try to set a stub HTML header/footer if necesssary */
if (contentType.find("text/html") != string::npos &&
content.find("<body") == std::string::npos &&
content.find("<BODY") == std::string::npos) {
content = "<html><head><title>" + article.getTitle() + "</title><meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\" /></head><body>" + content + "</body></html>";
if (contentType.find("text/html") != string::npos
&& content.find("<body") == std::string::npos
&& content.find("<BODY") == std::string::npos) {
content = "<html><head><title>" + article.getTitle() +
"</title><meta http-equiv=\"Content-Type\" content=\"text/html; "
"charset=utf-8\" /></head><body>" +
content + "</body></html>";
}
/* Get the data length */
@ -492,7 +564,8 @@ namespace kiwix {
}
/* Check if an article exists */
bool Reader::urlExists(const string &url) const {
bool Reader::urlExists(const string& url) const
{
char ns = 0;
string titleStr;
this->parseUrl(url, &ns, titleStr);
@ -502,16 +575,21 @@ namespace kiwix {
}
/* Does the ZIM file has a fulltext index */
bool Reader::hasFulltextIndex() const {
bool Reader::hasFulltextIndex() const
{
return this->urlExists("/Z/fulltextIndex/xapian");
}
/* Search titles by prefix */
bool Reader::searchSuggestions(const string &prefix, unsigned int suggestionsCount, const bool reset) {
bool Reader::searchSuggestions(const string& prefix,
unsigned int suggestionsCount,
const bool reset)
{
bool retVal = false;
zim::File::const_iterator articleItr;
/* Reset the suggestions otherwise check if the suggestions number is less than the suggestionsCount */
/* Reset the suggestions otherwise check if the suggestions number is less
* than the suggestionsCount */
if (reset) {
this->suggestions.clear();
this->suggestionsOffset = this->suggestions.begin();
@ -527,13 +605,13 @@ namespace kiwix {
}
for (articleItr = zimFileHandler->findByTitle('A', prefix);
articleItr != zimFileHandler->end() &&
articleItr->getTitle().compare(0, prefix.size(), prefix) == 0 &&
this->suggestions.size() < suggestionsCount ;
articleItr != zimFileHandler->end()
&& articleItr->getTitle().compare(0, prefix.size(), prefix) == 0
&& this->suggestions.size() < suggestionsCount;
++articleItr) {
/* Extract the interesting part of article title & url */
std::string normalizedArticleTitle = kiwix::normalize(articleItr->getTitle());
std::string normalizedArticleTitle
= kiwix::normalize(articleItr->getTitle());
std::string articleFinalUrl = "/A/" + articleItr->getUrl();
if (articleItr->isRedirect()) {
zim::Article article = *articleItr;
@ -549,7 +627,9 @@ namespace kiwix {
title) */
bool insert = true;
std::vector<std::vector<std::string>>::iterator suggestionItr;
for (suggestionItr = this->suggestions.begin(); suggestionItr != this->suggestions.end(); suggestionItr++) {
for (suggestionItr = this->suggestions.begin();
suggestionItr != this->suggestions.end();
suggestionItr++) {
int result = normalizedArticleTitle.compare((*suggestionItr)[2]);
if (result == 0 && articleFinalUrl.compare((*suggestionItr)[1]) == 0) {
insert = false;
@ -578,7 +658,9 @@ namespace kiwix {
return retVal;
}
std::vector<std::string> Reader::getTitleVariants(const std::string &title) const {
std::vector<std::string> Reader::getTitleVariants(
const std::string& title) const
{
std::vector<std::string> variants;
variants.push_back(title);
variants.push_back(kiwix::ucFirst(title));
@ -588,7 +670,9 @@ namespace kiwix {
}
/* Try also a few variations of the prefix to have better results */
bool Reader::searchSuggestionsSmart(const string &prefix, unsigned int suggestionsCount) {
bool Reader::searchSuggestionsSmart(const string& prefix,
unsigned int suggestionsCount)
{
std::vector<std::string> variants = this->getTitleVariants(prefix);
bool retVal;
@ -597,14 +681,16 @@ namespace kiwix {
for (std::vector<std::string>::iterator variantsItr = variants.begin();
variantsItr != variants.end();
variantsItr++) {
retVal = this->searchSuggestions(*variantsItr, suggestionsCount, false) || retVal;
retVal = this->searchSuggestions(*variantsItr, suggestionsCount, false)
|| retVal;
}
return retVal;
}
/* Get next suggestion */
bool Reader::getNextSuggestion(string &title) {
bool Reader::getNextSuggestion(string& title)
{
if (this->suggestionsOffset != this->suggestions.end()) {
/* title */
title = (*(this->suggestionsOffset))[0];
@ -618,7 +704,8 @@ namespace kiwix {
return false;
}
bool Reader::getNextSuggestion(string &title, string &url) {
bool Reader::getNextSuggestion(string& title, string& url)
{
if (this->suggestionsOffset != this->suggestions.end()) {
/* title */
title = (*(this->suggestionsOffset))[0];
@ -634,15 +721,18 @@ namespace kiwix {
}
/* Check if the file has as checksum */
bool Reader::canCheckIntegrity() const {
bool Reader::canCheckIntegrity() const
{
return this->zimFileHandler->getChecksum() != "";
}
/* Return true if corrupted, false otherwise */
bool Reader::isCorrupted() const {
bool Reader::isCorrupted() const
{
try {
if (this->zimFileHandler->verify() == true)
if (this->zimFileHandler->verify() == true) {
return false;
}
} catch (exception& e) {
cerr << e.what() << endl;
return true;
@ -652,7 +742,8 @@ namespace kiwix {
}
/* Return the file size, works also for splitted files */
unsigned int Reader::getFileSize() const {
unsigned int Reader::getFileSize() const
{
zim::File* file = this->getZimFileHandler();
zim::offset_type size = 0;

141
src/searcher.cpp
View File

@ -18,9 +18,9 @@
*/
#include "searcher.h"
#include "xapianSearcher.h"
#include "reader.h"
#include "kiwixlib-resources.h"
#include "reader.h"
#include "xapianSearcher.h"
#include <zim/search.h>
@ -33,10 +33,10 @@
using namespace CTPP;
#endif
namespace kiwix {
class _Result : public Result {
namespace kiwix
{
class _Result : public Result
{
public:
_Result(Searcher* searcher, zim::Search::iterator& iterator);
virtual ~_Result(){};
@ -58,23 +58,21 @@ namespace kiwix {
XapianSearcher* _xapianSearcher;
zim::Search::iterator current_iterator;
SearcherInternal() :
_search(NULL),
_xapianSearcher(NULL)
{}
~SearcherInternal() {
if ( _search != NULL )
SearcherInternal() : _search(NULL), _xapianSearcher(NULL) {}
~SearcherInternal()
{
if (_search != NULL) {
delete _search;
if ( _xapianSearcher != NULL )
}
if (_xapianSearcher != NULL) {
delete _xapianSearcher;
}
}
};
/* Constructor */
Searcher::Searcher(const string &xapianDirectoryPath, Reader* reader) :
reader(reader),
Searcher::Searcher(const string& xapianDirectoryPath, Reader* reader)
: reader(reader),
internal(new SearcherInternal()),
searchPattern(""),
protocolPrefix("zim://"),
@ -92,13 +90,16 @@ namespace kiwix {
}
/* Destructor */
Searcher::~Searcher() {
Searcher::~Searcher()
{
delete internal;
}
/* Search strings in the database */
void Searcher::search(std::string &search, unsigned int resultStart,
unsigned int resultEnd, const bool verbose) {
void Searcher::search(std::string& search,
unsigned int resultStart,
unsigned int resultEnd,
const bool verbose)
{
this->reset();
if (verbose == true) {
@ -114,7 +115,6 @@ namespace kiwix {
/* Try to find results */
if (resultStart != resultEnd) {
/* Avoid big researches */
this->resultCountPerPage = resultEnd - resultStart;
if (this->resultCountPerPage > 70) {
@ -128,10 +128,13 @@ namespace kiwix {
this->resultEnd = resultEnd;
string unaccentedSearch = removeAccents(search);
if (internal->_xapianSearcher) {
internal->_xapianSearcher->searchInIndex(unaccentedSearch, resultStart, resultEnd, verbose);
this->estimatedResultCount = internal->_xapianSearcher->results.get_matches_estimated();
internal->_xapianSearcher->searchInIndex(
unaccentedSearch, resultStart, resultEnd, verbose);
this->estimatedResultCount
= internal->_xapianSearcher->results.get_matches_estimated();
} else {
internal->_search = this->reader->getZimFileHandler()->search(unaccentedSearch, resultStart, resultEnd);
internal->_search = this->reader->getZimFileHandler()->search(
unaccentedSearch, resultStart, resultEnd);
internal->current_iterator = internal->_search->begin();
this->estimatedResultCount = internal->_search->get_matches_estimated();
}
@ -140,7 +143,8 @@ namespace kiwix {
return;
}
void Searcher::restart_search() {
void Searcher::restart_search()
{
if (internal->_xapianSearcher) {
internal->_xapianSearcher->restart_search();
} else {
@ -148,7 +152,8 @@ namespace kiwix {
}
}
Result* Searcher::getNextResult() {
Result* Searcher::getNextResult()
{
if (internal->_xapianSearcher) {
return internal->_xapianSearcher->getNextResult();
} else if (internal->current_iterator != internal->_search->end()) {
@ -159,67 +164,70 @@ namespace kiwix {
return NULL;
}
/* Reset the results */
void Searcher::reset() {
void Searcher::reset()
{
this->estimatedResultCount = 0;
this->searchPattern = "";
return;
}
/* Return the result count estimation */
unsigned int Searcher::getEstimatedResultCount() {
unsigned int Searcher::getEstimatedResultCount()
{
return this->estimatedResultCount;
}
bool Searcher::setProtocolPrefix(const std::string prefix) {
bool Searcher::setProtocolPrefix(const std::string prefix)
{
this->protocolPrefix = prefix;
return true;
}
bool Searcher::setSearchProtocolPrefix(const std::string prefix) {
bool Searcher::setSearchProtocolPrefix(const std::string prefix)
{
this->searchProtocolPrefix = prefix;
return true;
}
void Searcher::setContentHumanReadableId(const string &contentHumanReadableId) {
void Searcher::setContentHumanReadableId(const string& contentHumanReadableId)
{
this->contentHumanReadableId = contentHumanReadableId;
}
_Result::_Result(Searcher* searcher, zim::Search::iterator& iterator):
searcher(searcher),
iterator(iterator)
_Result::_Result(Searcher* searcher, zim::Search::iterator& iterator)
: searcher(searcher), iterator(iterator)
{
}
std::string _Result::get_url() {
std::string _Result::get_url()
{
return iterator.get_url();
}
std::string _Result::get_title() {
std::string _Result::get_title()
{
return iterator.get_title();
}
int _Result::get_score() {
int _Result::get_score()
{
return iterator.get_score();
}
std::string _Result::get_snippet() {
std::string _Result::get_snippet()
{
return iterator.get_snippet();
}
int _Result::get_size() {
int _Result::get_size()
{
return iterator.get_size();
}
int _Result::get_wordCount() {
int _Result::get_wordCount()
{
return iterator.get_wordCount();
}
#ifdef ENABLE_CTPP2
string Searcher::getHtml() {
string Searcher::getHtml()
{
SimpleVM oSimpleVM;
// Fill data
@ -234,11 +242,13 @@ namespace kiwix {
result["url"] = p_result->get_url();
result["snippet"] = p_result->get_snippet();
if (p_result->get_size() >= 0)
if (p_result->get_size() >= 0) {
result["size"] = kiwix::beautifyInteger(p_result->get_size());
}
if (p_result->get_wordCount() >= 0)
if (p_result->get_wordCount() >= 0) {
result["wordCount"] = kiwix::beautifyInteger(p_result->get_wordCount());
}
resultsCDT.PushBack(result);
delete p_result;
@ -249,13 +259,18 @@ namespace kiwix {
// pages
CDT pagesCDT(CDT::ARRAY_VAL);
unsigned int pageStart = this->resultStart / this->resultCountPerPage >= 5 ? this->resultStart / this->resultCountPerPage - 4 : 0;
unsigned int pageCount = this->estimatedResultCount / this->resultCountPerPage + 1 - pageStart;
unsigned int pageStart
= this->resultStart / this->resultCountPerPage >= 5
? this->resultStart / this->resultCountPerPage - 4
: 0;
unsigned int pageCount
= this->estimatedResultCount / this->resultCountPerPage + 1 - pageStart;
if (pageCount > 10)
if (pageCount > 10) {
pageCount = 10;
else if (pageCount == 1)
} else if (pageCount == 1) {
pageCount = 0;
}
for (unsigned int i = pageStart; i < pageStart + pageCount; i++) {
CDT page;
@ -263,8 +278,9 @@ namespace kiwix {
page["start"] = i * this->resultCountPerPage;
page["end"] = (i + 1) * this->resultCountPerPage;
if (i * this->resultCountPerPage == this->resultStart)
if (i * this->resultCountPerPage == this->resultStart) {
page["selected"] = true;
}
pagesCDT.PushBack(page);
}
@ -274,9 +290,14 @@ namespace kiwix {
oData["searchPattern"] = kiwix::encodeDiples(this->searchPattern);
oData["searchPatternEncoded"] = urlEncode(this->searchPattern);
oData["resultStart"] = this->resultStart + 1;
oData["resultEnd"] = (this->resultEnd > this->estimatedResultCount ? this->estimatedResultCount : this->resultEnd);
oData["resultEnd"] = (this->resultEnd > this->estimatedResultCount
? this->estimatedResultCount
: this->resultEnd);
oData["resultRange"] = this->resultCountPerPage;
oData["resultLastPageStart"] = this->estimatedResultCount > this->resultCountPerPage ? this->estimatedResultCount - this->resultCountPerPage : 0;
oData["resultLastPageStart"]
= this->estimatedResultCount > this->resultCountPerPage
? this->estimatedResultCount - this->resultCountPerPage
: 0;
oData["protocolPrefix"] = this->protocolPrefix;
oData["searchProtocolPrefix"] = this->searchProtocolPrefix;
oData["contentId"] = this->contentHumanReadableId;
@ -292,9 +313,7 @@ namespace kiwix {
oSimpleVM.Run(oData, oLoader, sResult, oLogger);
return sResult;
}
#endif
}

128
src/xapianSearcher.cpp
View File

@ -18,58 +18,61 @@
*/
#include "xapianSearcher.h"
#include "xapian/myhtmlparse.h"
#include <zim/zim.h>
#include <zim/file.h>
#include <sys/types.h>
#include <unicode/locid.h>
#include <unistd.h>
#include <zim/article.h>
#include <zim/error.h>
#include <sys/types.h>
#include <unistd.h>
#include <unicode/locid.h>
#include <zim/file.h>
#include <zim/zim.h>
#include "xapian/myhtmlparse.h"
#include <vector>
namespace kiwix {
std::map<std::string, int> read_valuesmap(const std::string &s) {
namespace kiwix
{
std::map<std::string, int> read_valuesmap(const std::string& s)
{
std::map<std::string, int> result;
std::vector<std::string> elems = split(s, ";");
for (std::vector<std::string>::iterator elem = elems.begin();
elem != elems.end();
elem++)
{
elem++) {
std::vector<std::string> tmp_elems = split(*elem, ":");
result.insert( std::pair<std::string, int>(tmp_elems[0], atoi(tmp_elems[1].c_str())) );
result.insert(
std::pair<std::string, int>(tmp_elems[0], atoi(tmp_elems[1].c_str())));
}
return result;
}
/* Constructor */
XapianSearcher::XapianSearcher(const string &xapianDirectoryPath, Reader* reader)
XapianSearcher::XapianSearcher(const string& xapianDirectoryPath,
Reader* reader)
: reader(reader)
{
this->openIndex(xapianDirectoryPath);
}
/* Open Xapian readable database */
void XapianSearcher::openIndex(const string &directoryPath) {
void XapianSearcher::openIndex(const string& directoryPath)
{
this->readableDatabase = Xapian::Database(directoryPath);
this->valuesmap = read_valuesmap(this->readableDatabase.get_metadata("valuesmap"));
this->valuesmap
= read_valuesmap(this->readableDatabase.get_metadata("valuesmap"));
this->language = this->readableDatabase.get_metadata("language");
this->stopwords = this->readableDatabase.get_metadata("stopwords");
setup_queryParser();
}
/* Close Xapian writable database */
void XapianSearcher::closeIndex() {
void XapianSearcher::closeIndex()
{
return;
}
void XapianSearcher::setup_queryParser()
{
queryParser.set_database(readableDatabase);
if ( ! language.empty() )
{
if (!language.empty()) {
/* Build ICU Local object to retrieve ISO-639 language code (from
ISO-639-3) */
icu::Locale languageLocale(language.c_str());
@ -80,12 +83,12 @@ std::map<std::string, int> read_valuesmap(const std::string &s) {
queryParser.set_stemmer(stemmer);
queryParser.set_stemming_strategy(Xapian::QueryParser::STEM_ALL);
} catch (...) {
std::cout << "No steemming for language '" << languageLocale.getLanguage() << "'" << std::endl;
std::cout << "No steemming for language '" << languageLocale.getLanguage()
<< "'" << std::endl;
}
}
if ( ! stopwords.empty() )
{
if (!stopwords.empty()) {
std::string stopWord;
std::istringstream file(this->stopwords);
while (std::getline(file, stopWord, '\n')) {
@ -96,8 +99,11 @@ std::map<std::string, int> read_valuesmap(const std::string &s) {
}
/* Search strings in the database */
void XapianSearcher::searchInIndex(string &search, const unsigned int resultStart,
const unsigned int resultEnd, const bool verbose) {
void XapianSearcher::searchInIndex(string& search,
const unsigned int resultStart,
const unsigned int resultEnd,
const bool verbose)
{
/* Create the query */
Xapian::Query query = queryParser.parse_query(search);
@ -111,7 +117,8 @@ std::map<std::string, int> read_valuesmap(const std::string &s) {
}
/* Get next result */
Result* XapianSearcher::getNextResult() {
Result* XapianSearcher::getNextResult()
{
if (this->current_result != this->results.end()) {
XapianResult* result = new XapianResult(this, this->current_result);
this->current_result++;
@ -120,54 +127,52 @@ std::map<std::string, int> read_valuesmap(const std::string &s) {
return NULL;
}
void XapianSearcher::restart_search() {
void XapianSearcher::restart_search()
{
this->current_result = this->results.begin();
}
XapianResult::XapianResult(XapianSearcher* searcher, Xapian::MSetIterator& iterator):
searcher(searcher),
iterator(iterator),
document(iterator.get_document())
XapianResult::XapianResult(XapianSearcher* searcher,
Xapian::MSetIterator& iterator)
: searcher(searcher), iterator(iterator), document(iterator.get_document())
{
}
std::string XapianResult::get_url() {
std::string XapianResult::get_url()
{
return document.get_data();
}
std::string XapianResult::get_title() {
if ( searcher->valuesmap.empty() )
std::string XapianResult::get_title()
{
if (searcher->valuesmap.empty()) {
/* This is the old legacy version. Guess and try */
return document.get_value(0);
}
else if ( searcher->valuesmap.find("title") != searcher->valuesmap.end() )
{
} else if (searcher->valuesmap.find("title") != searcher->valuesmap.end()) {
return document.get_value(searcher->valuesmap["title"]);
}
return "";
}
int XapianResult::get_score() {
int XapianResult::get_score()
{
return iterator.get_percent();
}
std::string XapianResult::get_snippet() {
if ( searcher->valuesmap.empty() )
std::string XapianResult::get_snippet()
{
if (searcher->valuesmap.empty()) {
/* This is the old legacy version. Guess and try */
std::string stored_snippet = document.get_value(1);
if ( ! stored_snippet.empty() )
if (!stored_snippet.empty()) {
return stored_snippet;
/* Let's continue here, and see if we can genenate one */
}
else if ( searcher->valuesmap.find("snippet") != searcher->valuesmap.end() )
{
/* Let's continue here, and see if we can genenate one */
} else if (searcher->valuesmap.find("snippet") != searcher->valuesmap.end()) {
return document.get_value(searcher->valuesmap["snippet"]);
}
/* No reader, no snippet */
if ( ! searcher->reader )
if (!searcher->reader) {
return "";
}
/* Get the content of the article to generate a snippet.
We parse it and use the html dump to avoid remove html tags in the
content and be able to nicely cut the text at random place. */
@ -175,21 +180,23 @@ std::map<std::string, int> read_valuesmap(const std::string &s) {
std::string content;
unsigned int contentLength;
std::string contentType;
searcher->reader->getContentByUrl(get_url(), content, contentLength, contentType);
searcher->reader->getContentByUrl(
get_url(), content, contentLength, contentType);
try {
htmlParser.parse_html(content, "UTF-8", true);
} catch (...) {}
} catch (...) {
}
return searcher->results.snippet(htmlParser.dump, 500);
}
int XapianResult::get_size() {
if ( searcher->valuesmap.empty() )
int XapianResult::get_size()
{
if (searcher->valuesmap.empty()) {
/* This is the old legacy version. Guess and try */
return document.get_value(2).empty() == true ? -1 : atoi(document.get_value(2).c_str());
}
else if ( searcher->valuesmap.find("size") != searcher->valuesmap.end() )
{
return document.get_value(2).empty() == true
? -1
: atoi(document.get_value(2).c_str());
} else if (searcher->valuesmap.find("size") != searcher->valuesmap.end()) {
return atoi(document.get_value(searcher->valuesmap["size"]).c_str());
}
/* The size is never used. Do we really want to get the content and
@ -197,14 +204,15 @@ std::map<std::string, int> read_valuesmap(const std::string &s) {
return -1;
}
int XapianResult::get_wordCount() {
if ( searcher->valuesmap.empty() )
int XapianResult::get_wordCount()
{
if (searcher->valuesmap.empty()) {
/* This is the old legacy version. Guess and try */
return document.get_value(3).empty() == true ? -1 : atoi(document.get_value(3).c_str());
}
else if ( searcher->valuesmap.find("wordcount") != searcher->valuesmap.end() )
{
return document.get_value(3).empty() == true
? -1
: atoi(document.get_value(3).c_str());
} else if (searcher->valuesmap.find("wordcount")
!= searcher->valuesmap.end()) {
return atoi(document.get_value(searcher->valuesmap["wordcount"]).c_str());
}
return -1;